Updates for Audio course

app.py

@@ -1,44 +1,45 @@
 import gradio as gr
 from huggingface_hub import HfApi, hf_hub_download
 from huggingface_hub.repocard import metadata_load
-
+import requests
+import re
 import pandas as pd
+from huggingface_hub import ModelCard
+
+def make_clickable_model(model_name):
+    # remove user from model name
+    model_name_show = ' '.join(model_name.split('/')[1:])
 
-from utils import *
+    link = "https://huggingface.co/" + model_name
+    return f'<a target="_blank" href="{link}">{model_name_show}</a>'
+
+def pass_emoji(passed):
+    if passed is True:
+        passed = "✅"
+    else:
+        passed = "❌"
+    return passed
 
 api = HfApi()
 
-def get_user_models(hf_username, env_tag, lib_tag):
+def get_user_audio_classification_models(hf_username):
     """
-    List the Reinforcement Learning models
-    from user given environment and lib
+    List the user's Audio Classification models
     :param hf_username: User HF username
-    :param env_tag: Environment tag
-    :param lib_tag: Library tag
     """
-    api = HfApi()
-    models = api.list_models(author=hf_username, filter=["reinforcement-learning", env_tag, lib_tag])
-
-    user_model_ids = [x.modelId for x in models]
-    return user_model_ids
-
-
-def get_user_sf_models(hf_username, env_tag, lib_tag):
-    api = HfApi()
-    models_sf = []
-    models = api.list_models(author=hf_username, filter=["reinforcement-learning", lib_tag])
 
+    models = api.list_models(author=hf_username, filter=["audio-classification"])
     user_model_ids = [x.modelId for x in models]
+    models_gtzan = []
 
     for model in user_model_ids:
         meta = get_metadata(model)
         if meta is None:
             continue
-        result = meta["model-index"][0]["results"][0]["dataset"]["name"]
-        if result == env_tag:
-            models_sf.append(model)
-            
-    return models_sf
+        if meta["datasets"] == ['marsyas/gtzan']:
+            models_gtzan.append(model)
+    
+    return models_gtzan
 
 
 def get_metadata(model_id):
@@ -54,232 +55,125 @@ def get_metadata(model_id):
     return None
 
 
-def parse_metrics_accuracy(meta):
-  """
-  Get model results and parse it
-  :param meta: model metadata
-  """
-  if "model-index" not in meta:
-    return None
-  result = meta["model-index"][0]["results"]
-  metrics = result[0]["metrics"]
-  accuracy = metrics[0]["value"]
-  
-  return accuracy
+def extract_accuracy(model_card_content):
+    """
+    Extract the accuracy value from the models' model card
+    :param model_card_content: model card content
+    """
+    accuracy_pattern = r"Accuracy: (\d+\.\d+)"
+    match = re.search(accuracy_pattern, model_card_content)
+    if match:
+        accuracy = match.group(1)
+        return float(accuracy)
+    else:
+        return None
 
 
-def parse_rewards(accuracy):
+def parse_metrics_accuracy(model_id):
   """
-  Parse mean_reward and std_reward
-  :param accuracy: model results
+  Get model card and parse it
+  :param model_id: model id
   """
-  default_std = -1000
-  default_reward= -1000
-  if accuracy !=  None:
-      accuracy = str(accuracy)
-      parsed =  accuracy.split(' +/- ')
-      if len(parsed)>1:
-          mean_reward = float(parsed[0])
-          std_reward =  float(parsed[1])
-      elif len(parsed)==1: #only mean reward   
-          mean_reward = float(parsed[0])
-          std_reward =  float(0)
-      else: 
-          mean_reward = float(default_std)
-          std_reward = float(default_reward)
-  else:
-      mean_reward = float(default_std)
-      std_reward = float(default_reward)
-  
-  return mean_reward, std_reward
+  card = ModelCard.load(model_id)
+  return extract_accuracy(card.content)
 
-def calculate_best_result(user_model_ids):
+  
+def calculate_best_acc_result(user_model_ids):
   """
   Calculate the best results of a unit
-  best_result = mean_reward - std_reward
   :param user_model_ids: RL models of a user
   """
-  best_result = -1000
-  best_model_id = ""
+
+  best_result = -100
+  best_model = ""
+
   for model in user_model_ids:
     meta = get_metadata(model)
     if meta is None:
       continue
-    accuracy = parse_metrics_accuracy(meta)
-    mean_reward, std_reward = parse_rewards(accuracy)
-    result = mean_reward - std_reward
-    if result > best_result:
-      best_result = result
-      best_model_id = model
+    accuracy = parse_metrics_accuracy(model)
+    if accuracy > best_result:
+      best_result = accuracy
+      best_model = meta['model-index'][0]["name"]
       
-  return best_result, best_model_id
+  return best_result, best_model
 
-def check_if_passed(model):
-  """
-  Check if result >= baseline
-  to know if you pass
-  :param model: user model
-  """
-  if model["best_result"] >= model["min_result"]:
-    model["passed_"] = True
 
 def certification(hf_username):
   results_certification = [
       {
-          "unit": "Unit 1",
-          "env": "LunarLander-v2",
-          "library": "stable-baselines3",
-          "min_result": 200,
+          "unit": "Unit 4: Audio Classification",
+          "task": "audio-classification",
+          "baseline_metric": 0.87,
           "best_result": 0,
           "best_model_id": "",
           "passed_": False
       },
   {
-          "unit": "Unit 2",
-          "env": "Taxi-v3",
-          "library": "q-learning",
-          "min_result": 4,
+          "unit": "Unit 5: TBD",
+          "task": "TBD",          
+          "baseline_metric": 0.99,
           "best_result": 0,
           "best_model_id": "",
           "passed_": False
   },
   {
-          "unit": "Unit 3",
-          "env": "SpaceInvadersNoFrameskip-v4",
-          "library": "stable-baselines3",
-          "min_result": 200,
+          "unit": "Unit 6: TBD",
+          "task": "TBD",
+          "baseline_metric": 0.99,
           "best_result": 0,
           "best_model_id": "",
           "passed_": False
   },
   {
-          "unit": "Unit 4",
-          "env": "CartPole-v1",
-          "library": "reinforce",
-          "min_result": 350,
+          "unit": "Unit 7: TBD",
+          "task": "TBD",
+          "baseline_metric": 0.99,
           "best_result": 0,
           "best_model_id": "",
           "passed_": False
   },
-    {
-          "unit": "Unit 4",
-          "env": "Pixelcopter-PLE-v0",
-          "library": "reinforce",
-          "min_result": 5,
-          "best_result": 0,
-          "best_model_id": "",
-          "passed_": False
-    },
-      {
-          "unit": "Unit 5",
-          "env": "ML-Agents-SnowballTarget",
-          "library": "ml-agents",
-          "min_result": -100,
-          "best_result": 0,
-          "best_model_id": "",
-          "passed_": False
-    },
-      {
-          "unit": "Unit 5",
-          "env": "ML-Agents-Pyramids",
-          "library": "ml-agents",
-          "min_result": -100,
-          "best_result": 0,
-          "best_model_id": "",
-          "passed_": False
-    },
-      {
-          "unit": "Unit 6",
-          "env": "AntBulletEnv-v0",
-          "library": "stable-baselines3",
-          "min_result": 650,
-          "best_result": 0,
-          "best_model_id": "",
-          "passed_": False
-    },
-      {
-          "unit": "Unit 6",
-          "env": "PandaReachDense-v2",
-          "library": "stable-baselines3",
-          "min_result": -3.5,
-          "best_result": 0,
-          "best_model_id": "",
-          "passed_": False
-    },
-      {
-          "unit": "Unit 7",
-          "env": "ML-Agents-SoccerTwos",
-          "library": "ml-agents",
-          "min_result": -100,
-          "best_result": 0,
-          "best_model_id": "",
-          "passed_": False
-    },
-      {
-          "unit": "Unit 8 PI",
-          "env": "LunarLander-v2",
-          "library": "deep-rl-course",
-          "min_result": -500,
-          "best_result": 0,
-          "best_model_id": "",
-          "passed_": False
-    },
-      {
-          "unit": "Unit 8 PII",
-          "env": "doom_health_gathering_supreme",
-          "library": "sample-factory",
-          "min_result": 5,
-          "best_result": 0,
-          "best_model_id": "",
-          "passed_": False
-    },
   ] 
+
   for unit in results_certification:
-    if unit["unit"] != "Unit 8 PII":
-      # Get user model
-      user_models = get_user_models(hf_username, unit['env'], unit['library'])
-    # For sample factory vizdoom we don't have env tag for now
+    if unit["task"] == "audio-classification":
+      user_models = get_user_audio_classification_models(hf_username)
+      best_result, best_model_id = calculate_best_acc_result(user_models)
+      unit["best_result"] = best_result
+      unit["best_model_id"] = make_clickable_model(best_model_id)
+      if unit["best_result"] >= unit["baseline_metric"]: 
+        unit["passed_"] = True
+        unit["passed"] = pass_emoji(unit["passed_"])
     else: 
-      user_models = get_user_sf_models(hf_username, unit['env'], unit['library'])
-        
-
-    # Calculate the best result and get the best_model_id
-    best_result, best_model_id = calculate_best_result(user_models)
-
-    # Save best_result and best_model_id
-    unit["best_result"] = best_result
-    unit["best_model_id"] = make_clickable_model(best_model_id)
-
-    # Based on best_result do we pass the unit?
-    check_if_passed(unit)
-    unit["passed"] = pass_emoji(unit["passed_"])
+      # TBD for other units
+      unit["passed"] = pass_emoji(unit["passed_"])
+      continue        
     
   print(results_certification)
  
   df = pd.DataFrame(results_certification)
-  df = df[['passed', 'unit', 'env', 'min_result', 'best_result', 'best_model_id']]
+  df = df[['passed', 'unit', 'task', 'baseline_metric', 'best_result', 'best_model_id']]
   return df
 
 
 with gr.Blocks() as demo:
     gr.Markdown(f"""
-    # 🏆 Check your progress in the Deep Reinforcement Learning Course 🏆
+    # 🏆 Check your progress in the Audio Course 🏆
     You can check your progress here.
     
-    - To get a certificate of completion, you must **pass 80% of the assignments before June 1st 2023**.
-    - To get an honors certificate, you must **pass 100% of the assignments before June 1st 2023**.
+    - To get a certificate of completion, you must **pass 3 out of 4 assignments before July 31st 2023**.
+    - To get an honors certificate, you must **pass 4 out of 4 assignments before July 31st 2023**.
 
-    To pass an assignment your model result (mean_reward - std_reward) must be >= min_result
+    To pass an assignment, your model's metric should be equal or higher than the baseline metric
 
-    **When min_result = -100 it means that you just need to push a model to pass this hands-on. No need to reach a certain result.**
+    **When min_result = -100 it means that you just need to push a model to pass this hands-on.**
     
-    Just type your Hugging Face Username 🤗 (in my case ThomasSimonini)
+    Just type your Hugging Face Username 🤗 (in my case MariaK)
     """)
     
-    hf_username = gr.Textbox(placeholder="ThomasSimonini", label="Your Hugging Face Username")
-    #email = gr.Textbox(placeholder="thomas.simonini@huggingface.co", label="Your Email (to receive your certificate)")
+    hf_username = gr.Textbox(placeholder="MariaK", label="Your Hugging Face Username")
     check_progress_button = gr.Button(value="Check my progress")
-    output = gr.components.Dataframe(value= certification(hf_username), headers=["Pass?", "Unit", "Environment", "Baseline", "Your best result", "Your best model id"], datatype=["markdown", "markdown", "markdown", "number", "number", "markdown", "bool"])
+    output = gr.components.Dataframe(value=certification(hf_username))    
     check_progress_button.click(fn=certification, inputs=hf_username, outputs=output)
 
 demo.launch()