ardaatahan commited on
Commit
37ef5f5
β€’
1 Parent(s): 8f6a603

add sku names to device support columns

Browse files
dashboard_data/performance_data.json CHANGED
The diff for this file is too large to render. See raw diff
 
dashboard_data/support_data.csv CHANGED
@@ -1,4 +1,4 @@
1
- ,Model,Apple M2 Pro,Apple M2 Ultra,Apple M2,Apple M3 Max,Apple M3 Max,Apple M1,iPad Air 11-inch (M2),iPad mini (A17 Pro),iPad Pro 11-inch (M4),iPhone 11,iPhone 12 mini,iPhone 12 Pro Max,iPhone 13 Pro,iPhone 13,iPhone 14,iPhone 15 Pro,iPhone 15 Pro Max,iPhone 16 Pro,iPhone 16
2
  distil-whisper_distil-large-v3,distil-whisper_distil-large-v3,βœ… macOS 15.0.1,βœ… macOS 15.0.1,βœ… macOS 15.1,βœ… macOS 15.2,βœ… macOS 15.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/MacBookAir10%2C1_summary_2024-10-30T012535.json>macOS 15.1</a>,βœ… iPadOS 17.6.1,Not Supported,βœ… iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
3
  distil-whisper_distil-large-v3_594MB,distil-whisper_distil-large-v3_594MB,βœ… macOS 15.0.1,βœ… macOS 15.0.1,βœ… macOS 15.1,βœ… macOS 15.2,βœ… macOS 15.1,βœ… macOS 15.1,βœ… iPadOS 17.6.1,βœ… iPadOS 18.0.1,βœ… iPadOS 18.1,Not Supported,Not Supported,Not Supported,βœ… iOS 18.1,βœ… iOS 17.3,βœ… iOS 17.3,βœ… iOS 18.1,βœ… iOS 18.2,βœ… iOS 18.0,βœ… iOS 18.0.1
4
  distil-whisper_distil-large-v3_turbo,distil-whisper_distil-large-v3_turbo,βœ… macOS 15.0.1,βœ… macOS 15.0.1,βœ… macOS 15.1,βœ… macOS 15.2,βœ… macOS 15.1,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad14%2C8_summary_2024-10-30T022440.json>iPadOS 17.6.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
 
1
+ ,Model,"Apple M2 Pro (Mac14,12)","Apple M2 Ultra (Mac14,14)","Apple M2 (Mac14,2)","Apple M3 Max (Mac15,10)","Apple M3 Max (Mac15,9)","Apple M1 (MacBookAir10,1)","iPad Air 11-inch (M2) (iPad14,8)","iPad mini (A17 Pro) (iPad16,1)","iPad Pro 11-inch (M4) (iPad16,3)","iPhone 11 (iPhone12,1)","iPhone 12 mini (iPhone13,1)","iPhone 12 Pro Max (iPhone13,4)","iPhone 13 Pro (iPhone14,2)","iPhone 13 (iPhone14,5)","iPhone 14 (iPhone14,7)","iPhone 15 Pro (iPhone16,1)","iPhone 15 Pro Max (iPhone16,2)","iPhone 16 Pro (iPhone17,1)","iPhone 16 (iPhone17,3)"
2
  distil-whisper_distil-large-v3,distil-whisper_distil-large-v3,βœ… macOS 15.0.1,βœ… macOS 15.0.1,βœ… macOS 15.1,βœ… macOS 15.2,βœ… macOS 15.1,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/MacBookAir10%2C1_summary_2024-10-30T012535.json>macOS 15.1</a>,βœ… iPadOS 17.6.1,Not Supported,βœ… iPadOS 18.1,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
3
  distil-whisper_distil-large-v3_594MB,distil-whisper_distil-large-v3_594MB,βœ… macOS 15.0.1,βœ… macOS 15.0.1,βœ… macOS 15.1,βœ… macOS 15.2,βœ… macOS 15.1,βœ… macOS 15.1,βœ… iPadOS 17.6.1,βœ… iPadOS 18.0.1,βœ… iPadOS 18.1,Not Supported,Not Supported,Not Supported,βœ… iOS 18.1,βœ… iOS 17.3,βœ… iOS 17.3,βœ… iOS 18.1,βœ… iOS 18.2,βœ… iOS 18.0,βœ… iOS 18.0.1
4
  distil-whisper_distil-large-v3_turbo,distil-whisper_distil-large-v3_turbo,βœ… macOS 15.0.1,βœ… macOS 15.0.1,βœ… macOS 15.1,βœ… macOS 15.2,βœ… macOS 15.1,Not Supported,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad14%2C8_summary_2024-10-30T022440.json>iPadOS 17.6.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C1_summary_2024-10-29T234233.json>iPadOS 18.0.1</a>,⚠️ <a style='color: #3B82F6; text-decoration: underline; text-decoration-style: dotted;' href=https://huggingface.co/datasets/argmaxinc/whisperkit-evals-dataset/blob/main/benchmark_data/2024-10-29T190053_a9b92c4/iPad16%2C3_summary_2024-10-30T073931.json>iPadOS 18.1</a>,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported,Not Supported
performance_generate.py CHANGED
@@ -299,7 +299,7 @@ def calculate_and_save_support_results(
299
  indicate support status.
300
  """
301
  all_models = sorted(support_results["modelsTested"])
302
- all_devices = sorted(set(support_results["devices"]))
303
 
304
  df = pd.DataFrame(index=all_models, columns=["Model"] + all_devices)
305
 
@@ -347,7 +347,7 @@ def calculate_and_save_support_results(
347
 
348
  cols = df.columns.tolist()
349
  cols = ["Model"] + [
350
- get_device_name(col).replace("_", " ") for col in cols if col != "Model"
351
  ]
352
  df.columns = cols
353
 
@@ -455,7 +455,7 @@ def main():
455
  process_summary_file(file_path, support_results, releases)
456
  else:
457
  process_benchmark_file(file_path, dataset_dfs, performance_results, releases)
458
-
459
  not_supported = calculate_and_save_performance_results(
460
  performance_results, "dashboard_data/performance_data.json"
461
  )
 
299
  indicate support status.
300
  """
301
  all_models = sorted(support_results["modelsTested"])
302
+ all_devices = sorted(support_results["devices"])
303
 
304
  df = pd.DataFrame(index=all_models, columns=["Model"] + all_devices)
305
 
 
347
 
348
  cols = df.columns.tolist()
349
  cols = ["Model"] + [
350
+ f"""{get_device_name(col).replace("_", " ")} ({col})""" for col in cols if col != "Model"
351
  ]
352
  df.columns = cols
353
 
 
455
  process_summary_file(file_path, support_results, releases)
456
  else:
457
  process_benchmark_file(file_path, dataset_dfs, performance_results, releases)
458
+
459
  not_supported = calculate_and_save_performance_results(
460
  performance_results, "dashboard_data/performance_data.json"
461
  )