Spaces:
Runtime error
Runtime error
| import gradio as gr | |
| from huggingface_hub import HfApi, hf_hub_download | |
| from huggingface_hub.repocard import metadata_load | |
| from huggingface_hub import HfApi, Repository | |
| import time | |
| import os | |
| import pandas as pd | |
| from utils import * | |
| api = HfApi() | |
| DATASET_REPO_URL = "https://huggingface.co/datasets/huggingface-projects/Deep-RL-Course-Certification" | |
| CERTIFIED_USERS_FILENAME = "certified_users.csv" | |
| CERTIFIED_USERS_DIR = "certified_users" | |
| HF_TOKEN = os.environ.get("HF_TOKEN") | |
| repo = Repository( | |
| local_dir=CERTIFIED_USERS_DIR, clone_from=DATASET_REPO_URL, use_auth_token=HF_TOKEN | |
| ) | |
| def get_user_models(hf_username, env_tag, lib_tag): | |
| """ | |
| List the Reinforcement Learning models | |
| from user given environment and lib | |
| :param hf_username: User HF username | |
| :param env_tag: Environment tag | |
| :param lib_tag: Library tag | |
| """ | |
| api = HfApi() | |
| models = api.list_models(author=hf_username, filter=["reinforcement-learning", env_tag, lib_tag]) | |
| user_model_ids = [x.modelId for x in models] | |
| return user_model_ids | |
| def get_metadata(model_id): | |
| """ | |
| Get model metadata (contains evaluation data) | |
| :param model_id | |
| """ | |
| try: | |
| readme_path = hf_hub_download(model_id, filename="README.md") | |
| return metadata_load(readme_path) | |
| except requests.exceptions.HTTPError: | |
| # 404 README.md not found | |
| return None | |
| def parse_metrics_accuracy(meta): | |
| """ | |
| Get model results and parse it | |
| :param meta: model metadata | |
| """ | |
| if "model-index" not in meta: | |
| return None | |
| result = meta["model-index"][0]["results"] | |
| metrics = result[0]["metrics"] | |
| accuracy = metrics[0]["value"] | |
| return accuracy | |
| def parse_rewards(accuracy): | |
| """ | |
| Parse mean_reward and std_reward | |
| :param accuracy: model results | |
| """ | |
| default_std = -1000 | |
| default_reward= -1000 | |
| if accuracy != None: | |
| accuracy = str(accuracy) | |
| parsed = accuracy.split(' +/- ') | |
| if len(parsed)>1: | |
| mean_reward = float(parsed[0]) | |
| std_reward = float(parsed[1]) | |
| elif len(parsed)==1: #only mean reward | |
| mean_reward = float(parsed[0]) | |
| std_reward = float(0) | |
| else: | |
| mean_reward = float(default_std) | |
| std_reward = float(default_reward) | |
| else: | |
| mean_reward = float(default_std) | |
| std_reward = float(default_reward) | |
| return mean_reward, std_reward | |
| def calculate_best_result(user_model_ids): | |
| """ | |
| Calculate the best results of a unit | |
| best_result = mean_reward - std_reward | |
| :param user_model_ids: RL models of a user | |
| """ | |
| best_result = -100 | |
| best_model_id = "" | |
| for model in user_model_ids: | |
| meta = get_metadata(model) | |
| if meta is None: | |
| continue | |
| accuracy = parse_metrics_accuracy(meta) | |
| mean_reward, std_reward = parse_rewards(accuracy) | |
| result = mean_reward - std_reward | |
| if result > best_result: | |
| best_result = result | |
| best_model_id = model | |
| return best_result, best_model_id | |
| def check_if_passed(model): | |
| """ | |
| Check if result >= baseline | |
| to know if you pass | |
| :param model: user model | |
| """ | |
| if model["best_result"] >= model["min_result"]: | |
| model["passed_"] = True | |
| def certification(hf_username): | |
| results_certification = [ | |
| { | |
| "unit": "Unit 1", | |
| "env": "LunarLander-v2", | |
| "library": "stable-baselines3", | |
| "min_result": 200, | |
| "best_result": 0, | |
| "best_model_id": "", | |
| "passed_": False | |
| }, | |
| { | |
| "unit": "Unit 2", | |
| "env": "Taxi-v3", | |
| "library": "q-learning", | |
| "min_result": 4, | |
| "best_result": 0, | |
| "best_model_id": "", | |
| "passed_": False | |
| }, | |
| { | |
| "unit": "Unit 3", | |
| "env": "SpaceInvadersNoFrameskip-v4", | |
| "library": "stable-baselines3", | |
| "min_result": 200, | |
| "best_result": 0, | |
| "best_model_id": "", | |
| "passed_": False | |
| }, | |
| { | |
| "unit": "Unit 4", | |
| "env": "CartPole-v1", | |
| "library": "reinforce", | |
| "min_result": 350, | |
| "best_result": 0, | |
| "best_model_id": "", | |
| "passed_": False | |
| }, | |
| { | |
| "unit": "Unit 4", | |
| "env": "Pixelcopter-PLE-v0", | |
| "library": "reinforce", | |
| "min_result": 5, | |
| "best_result": 0, | |
| "best_model_id": "", | |
| "passed_": False | |
| }, | |
| { | |
| "unit": "Unit 5", | |
| "env": "ML-Agents-SnowballTarget", | |
| "library": "ml-agents", | |
| "min_result": -100, | |
| "best_result": 0, | |
| "best_model_id": "", | |
| "passed_": False | |
| }, | |
| { | |
| "unit": "Unit 5", | |
| "env": "ML-Agents-Pyramids", | |
| "library": "ml-agents", | |
| "min_result": -100, | |
| "best_result": 0, | |
| "best_model_id": "", | |
| "passed_": False | |
| }, | |
| { | |
| "unit": "Unit 6", | |
| "env": "AntBulletEnv-v0", | |
| "library": "stable-baselines3", | |
| "min_result": 650, | |
| "best_result": 0, | |
| "best_model_id": "", | |
| "passed_": False | |
| }, | |
| { | |
| "unit": "Unit 6", | |
| "env": "PandaReachDense-v2", | |
| "library": "stable-baselines3", | |
| "min_result": -3.5, | |
| "best_result": 0, | |
| "best_model_id": "", | |
| "passed_": False | |
| }, | |
| { | |
| "unit": "Unit 7", | |
| "env": "ML-Agents-SoccerTwos", | |
| "library": "ml-agents", | |
| "min_result": -100, | |
| "best_result": 0, | |
| "best_model_id": "", | |
| "passed_": False | |
| }, | |
| { | |
| "unit": "Unit 8 PI", | |
| "env": "GodotRL-JumperHard", | |
| "library": "cleanrl", | |
| "min_result": -100, | |
| "best_result": 0, | |
| "best_model_id": "", | |
| "passed_": False | |
| }, | |
| { | |
| "unit": "Unit 8 PII", | |
| "env": "Vizdoom-Battle", | |
| "library": "cleanrl", | |
| "min_result": -100, | |
| "best_result": 0, | |
| "best_model_id": "", | |
| "passed_": False | |
| }, | |
| ] | |
| for unit in results_certification: | |
| # Get user model | |
| user_models = get_user_models(hf_username, unit['env'], unit['library']) | |
| # Calculate the best result and get the best_model_id | |
| best_result, best_model_id = calculate_best_result(user_models) | |
| # Save best_result and best_model_id | |
| unit["best_result"] = best_result | |
| unit["best_model_id"] = make_clickable_model(best_model_id) | |
| # Based on best_result do we pass the unit? | |
| check_if_passed(unit) | |
| unit["passed"] = pass_emoji(unit["passed_"]) | |
| print(results_certification) | |
| df1 = pd.DataFrame(results_certification) | |
| df = df1[['passed', 'unit', 'env', 'min_result', 'best_result', 'best_model_id']] | |
| verify_certification(results_certification, hf_username, first_name, last_name) | |
| return df | |
| """ | |
| Verify that the user pass. | |
| If yes: | |
| - Generate the certification | |
| - Send an email | |
| - Print the certification | |
| If no: | |
| - Explain why the user didn't pass yet | |
| """ | |
| def verify_certification(df, hf_username, first_name, last_name): | |
| # Check that we pass | |
| model_pass_nb = 0 | |
| pass_percentage = 0 | |
| for unit in df: | |
| if unit["passed_"] is True: | |
| model_pass_nb += 1 | |
| pass_percentage = (model_pass_nb/12) * 100 | |
| print("pass_percentage", pass_percentage) | |
| if pass_percentage == 100: | |
| # Generate a certificate of excellence | |
| generate_certificate("./certificate_models/certificate-excellence.png", first_name, last_name) | |
| # Add this user to our database | |
| add_certified_user(hf_username, first_name, last_name, pass_percentage) | |
| # Output everything in gradio | |
| elif pass_percentage < 100 and pass_percentage >= 80: | |
| # Certificate of completion | |
| generate_certificate("./certificate_models/certificate-completion.png", first_name, last_name) | |
| # Add this user to our database | |
| add_certified_user(hf_username, first_name, last_name, pass_percentage) | |
| # Output everything in gradio | |
| else: | |
| # Not pass yet | |
| print ("not pass yet") | |
| def generate_certificate(certificate_model, first_name, last_name): | |
| im = Image.open(certificate_model) | |
| d = ImageDraw.Draw(im) | |
| name_font = ImageFont.truetype("Quattrocento-Regular.ttf", 100) | |
| date_font = ImageFont.truetype("Quattrocento-Regular.ttf", 48) | |
| name = first_name + " " + last_name | |
| # Debug line name | |
| #d.line(((200, 740), (1800, 740)), "gray") | |
| #d.line(((1000, 0), (1000, 1400)), "gray") | |
| # Name | |
| d.text((1000, 740), name, fill="black", anchor="mm", font=name_font) | |
| # Debug line date | |
| #d.line(((1500, 0), (1500, 1400)), "gray") | |
| # Date of certification | |
| d.text((1480, 1170), str(date.today()), fill="black", anchor="mm", font=date_font) | |
| im.save("certificate_"+".png") | |
| def add_certified_user(hf_username, first_name, last_name, pass_percentage): | |
| repo.git_pull() | |
| history = pd.read_csv(os.path.join(CERTIFIED_USERS_DIR, CERTIFIED_USERS_FILENAME)) | |
| new_row = {'hf_username': hf_username, 'first_name': first_name, 'last_name': last_name, 'pass_percentage': pass_percentage, 'datetime': time.time()} | |
| new_history = pd.DataFrame(new_row) | |
| history = pd.concat([history, new_history]) | |
| print("HISTORY", history) | |
| history.to_csv(os.path.join(CERTIFIED_USERS_DIR, CERTIFIED_USERS_FILENAME), index=False) | |
| df.to_csv(os.path.join(CERTIFIED_USERS_DIR, CERTIFIED_USERS_FILENAME), index=False) | |
| repo.push_to_hub(commit_message="Update certified users list") | |
| with gr.Blocks() as demo: | |
| gr.Markdown(f""" | |
| # 🏆 Check your progress in the Deep Reinforcement Learning Course 🏆 | |
| You can check your progress here. | |
| - To get a certificate of completion, you must **pass 80% of the assignments before the end of April 2023**. | |
| - To get an honors certificate, you must **pass 100% of the assignments before the end of April 2023**. | |
| To pass an assignment your model result (mean_reward - std_reward) must be >= min_result | |
| **When min_result = -100 it means that you just need to push a model to pass this hands-on. No need to reach a certain result.** | |
| Just type your Hugging Face Username 🤗 (in my case ThomasSimonini) | |
| """) | |
| hf_username = gr.Textbox(placeholder="ThomasSimonini", label="Your Hugging Face Username (case sensitive)") | |
| first_name = gr.Textbox(placeholder="Jane", label="Your First Name") | |
| last_name = gr.Textbox(placeholder="Doe", label="Your Last Name") | |
| #email = gr.Textbox(placeholder="jane.doe@gmail.com", label="Your Email (to receive your certificate)") | |
| check_progress_button = gr.Button(value="Check my progress") | |
| output = gr.components.Dataframe(value= certification(hf_username), headers=["Pass?", "Unit", "Environment", "Baseline", "Your best result", "Your best model id"], datatype=["markdown", "markdown", "markdown", "number", "number", "markdown", "bool"]) | |
| check_progress_button.click(fn=certification, inputs=hf_username, outputs=output) | |
| demo.launch(debug=True) |