Updated with evaluation for ASR hands-on

app.py

@@ -17,26 +17,66 @@ def pass_emoji(passed):
 api = HfApi()
 
 
-def get_user_audio_classification_models(hf_username):
+def get_user_models(hf_username, task):
     """
-    List the user's Audio Classification models
+    List the user's models for a given task
     :param hf_username: User HF username
     """
-
-    models = api.list_models(author=hf_username, filter=["audio-classification"])
+    models = api.list_models(author=hf_username, filter=[task])
     user_model_ids = [x.modelId for x in models]
-    models_gtzan = []
 
+    match task:
+      case "audio-classification":
+        dataset = 'marsyas/gtzan'
+      case "automatic-speech-recognition":
+        dataset = 'PolyAI/minds14'
+      case _:
+        print("Unsupported task")
+
+    dataset_specific_models = []
+    
     for model in user_model_ids:
         meta = get_metadata(model)
         if meta is None:
             continue
         try:
-          if meta["datasets"] == ['marsyas/gtzan']:
-              models_gtzan.append(model)
+          if meta["datasets"] == [dataset]:
+              dataset_specific_models.append(model)
         except: continue
-    return models_gtzan
+    return dataset_specific_models
 
+def calculate_best_result(user_models, task):
+  """
+  Calculate the best results of a unit for a given task
+  :param user_model_ids: models of a user
+  """
+  
+  best_model = ""
+  
+  if task == "audio-classification":
+    best_result = -100
+    larger_is_better = True
+  elif task == "automatic-speech-recognition":
+    best_result = 100
+    larger_is_better = False  
+
+  for model in user_models:
+    meta = get_metadata(model)
+    if meta is None:
+      continue 
+    metric = parse_metrics(model, task) 
+
+    if larger_is_better:
+     if metric > best_result:
+      best_result = metric
+      best_model = meta['model-index'][0]["name"]
+    else:
+      if metric < best_result:
+        best_result = metric
+        best_model = meta['model-index'][0]["name"]
+
+  return best_result, best_model
+  
 
 def get_metadata(model_id):
   """
@@ -51,48 +91,34 @@ def get_metadata(model_id):
     return None
 
 
-def extract_accuracy(model_card_content):
+def extract_metric(model_card_content, task):
     """
-    Extract the accuracy value from the models' model card
+    Extract the metric value from the models' model card
     :param model_card_content: model card content
     """
     accuracy_pattern = r"Accuracy: (\d+\.\d+)"
-    match = re.search(accuracy_pattern, model_card_content)
+    wer_pattern = r"Wer: (\d+\.\d+)" 
+    
+    if task == "audio-classification":
+      pattern = accuracy_pattern
+    elif task == "automatic-speech-recognition":
+      pattern = wer_pattern
+    
+    match = re.search(pattern, model_card_content)
     if match:
-        accuracy = match.group(1)
-        return float(accuracy)
+        metric = match.group(1)
+        return float(metric)
     else:
-        return None
+        return None    
 
 
-def parse_metrics_accuracy(model_id):
+def parse_metrics(model, task):
   """
   Get model card and parse it
   :param model_id: model id
   """
-  card = ModelCard.load(model_id)
-  return extract_accuracy(card.content)
-
-  
-def calculate_best_acc_result(user_model_ids):
-  """
-  Calculate the best results of a unit
-  :param user_model_ids: RL models of a user
-  """
-
-  best_result = -100
-  best_model = ""
-
-  for model in user_model_ids:
-    meta = get_metadata(model)
-    if meta is None:
-      continue
-    accuracy = parse_metrics_accuracy(model)
-    if accuracy > best_result:
-      best_result = accuracy
-      best_model = meta['model-index'][0]["name"]
-      
-  return best_result, best_model
+  card = ModelCard.load(model)
+  return extract_metric(card.content, task)
 
 
 def certification(hf_username):
@@ -106,9 +132,9 @@ def certification(hf_username):
           "passed_": False
       },
   {
-          "unit": "Unit 5: TBD",
-          "task": "TBD",          
-          "baseline_metric": 0.99,
+          "unit": "Unit 5: Automatic Speech Recognition",
+          "task": "automatic-speech-recognition",          
+          "baseline_metric": 0.37,
           "best_result": 0,
           "best_model_id": "",
           "passed_": False
@@ -133,17 +159,32 @@ def certification(hf_username):
 
   for unit in results_certification:
     unit["passed"] = pass_emoji(unit["passed_"])
-    if unit["task"] == "audio-classification":
-      user_models = get_user_audio_classification_models(hf_username)
-      best_result, best_model_id = calculate_best_acc_result(user_models)
-      unit["best_result"] = best_result
-      unit["best_model_id"] = best_model_id
-      if unit["best_result"] >= unit["baseline_metric"]: 
-        unit["passed_"] = True
-        unit["passed"] = pass_emoji(unit["passed_"])
-    else: 
-      # TBD for other units
-      continue        
+
+    match unit["task"]:
+      case "audio-classification":
+        try:
+          user_ac_models = get_user_models(hf_username, task = "audio-classification")
+          best_result, best_model_id = calculate_best_result(user_ac_models, task = "audio-classification")
+          unit["best_result"] = best_result
+          unit["best_model_id"] = best_model_id
+          if unit["best_result"] >= unit["baseline_metric"]: 
+            unit["passed_"] = True
+            unit["passed"] = pass_emoji(unit["passed_"])
+        except: print("Either no relevant models found, or no metrics in the model card for audio classificaiton")      
+      case "automatic-speech-recognition":
+        try:
+          user_asr_models = get_user_models(hf_username, task = "automatic-speech-recognition")
+          best_result, best_model_id = calculate_best_result(user_asr_models, task = "automatic-speech-recognition")
+          unit["best_result"] = best_result
+          unit["best_model_id"] = best_model_id
+          if unit["best_result"] <= unit["baseline_metric"]: 
+            unit["passed_"] = True
+            unit["passed"] = pass_emoji(unit["passed_"])
+        except: print("Either no relevant models found, or no metrics in the model card for automatic speech recognition")      
+      case "TBD":
+        print("Evaluation for this unit is work in progress")
+      case _:
+        print("Unknown task")
     
   print(results_certification)
  
@@ -151,7 +192,6 @@ def certification(hf_username):
   df = df[['passed', 'unit', 'task', 'baseline_metric', 'best_result', 'best_model_id']]
   return df
 
-
 with gr.Blocks() as demo:
     gr.Markdown(f"""
     # 🏆 Check your progress in the Audio Course 🏆