Spaces:
Runtime error
Runtime error
| import json | |
| import re | |
| from pathlib import Path | |
| import requests | |
| import streamlit as st | |
| import yaml | |
| from huggingface_hub import hf_hub_download | |
| from streamlit_ace import st_ace | |
| from streamlit_tags import st_tags | |
| # exact same regex as in the Hub server. Please keep in sync. | |
| REGEX_YAML_BLOCK = re.compile(r"---[\n\r]+([\S\s]*?)[\n\r]+---[\n\r]") | |
| with open("languages.json") as f: | |
| lang2name = json.load(f) | |
| def try_parse_yaml(yaml_block): | |
| try: | |
| metadata = yaml.load(yaml_block, yaml.SafeLoader) | |
| except yaml.YAMLError as e: | |
| print("Error while parsing the metadata YAML:") | |
| if hasattr(e, "problem_mark"): | |
| if e.context is not None: | |
| st.error( | |
| str(e.problem_mark) | |
| + "\n " | |
| + str(e.problem) | |
| + " " | |
| + str(e.context) | |
| + "\nPlease correct the README.md and retry." | |
| ) | |
| else: | |
| st.error( | |
| str(e.problem_mark) | |
| + "\n " | |
| + str(e.problem) | |
| + "\nPlease correct the README.md and retry." | |
| ) | |
| else: | |
| st.error( | |
| "Something went wrong while parsing the metadata. " | |
| "Make sure it's written according to the YAML spec!" | |
| ) | |
| return None | |
| return metadata | |
| def main(): | |
| st.markdown("# The 🤗 Speech Bench Metrics Editor") | |
| st.markdown("This tool will help you report the evaluation metrics for all of your speech recognition models. " | |
| "Follow the steps and watch your models appear on the [Speech Bench Leaderboard](https://huggingface.co/spaces/huggingface/hf-speech-bench)!") | |
| st.markdown("## 1. Load your model's metadata") | |
| st.markdown("Enter your model's path below.") | |
| model_id = st.text_input("", placeholder="<username>/<model>") | |
| if not model_id.strip(): | |
| st.stop() | |
| try: | |
| readme_path = hf_hub_download(model_id, filename="README.md") | |
| except requests.exceptions.HTTPError: | |
| st.error( | |
| f"ERROR: https://huggingface.co/{model_id}/blob/main/README.md " | |
| f"not found, make sure you've entered a correct model path and created a model card for it!" | |
| ) | |
| st.stop() | |
| content = Path(readme_path).read_text() | |
| match = REGEX_YAML_BLOCK.search(content) | |
| if match: | |
| meta_yaml = match.group(1) | |
| else: | |
| st.error( | |
| "ERROR: Couldn't find the metadata section inside your model's `README.md`. Do you have some basic metadata " | |
| "enclosed in `---` as described in [the Hub documentation](https://huggingface.co/docs/hub/model-repos#model-card-metadata)?" | |
| ) | |
| st.stop() | |
| metadata = try_parse_yaml(meta_yaml) | |
| if metadata is None: | |
| st.stop() | |
| return | |
| else: | |
| st.success("Successfully loaded the metadata!") | |
| with st.expander("Inspect the parsed metadata for debugging"): | |
| st.json(metadata) | |
| st.markdown("## 2. Edit the data") | |
| if "tags" not in metadata: | |
| metadata["tags"] = [] | |
| metadata["tags"].append("hf-asr-leaderboard") | |
| ############################ | |
| # LANGUAGES | |
| ############################ | |
| st.markdown("### Language(s)") | |
| st.markdown( | |
| "For each spoken language that your model handles, enter an " | |
| "[ISO 639-1](https://en.wikipedia.org/wiki/List_of_ISO_639-1_codes) language code, or " | |
| "find an appropriate alternative from " | |
| "[our list here](https://huggingface.co/spaces/huggingface/hf-speech-bench/blob/main/languages.json). " | |
| "When in doubt, use the most generic language code, e.g. `en` instead of `en-GB` and `en-US`." | |
| ) | |
| st.markdown("*Example*: `en, gsw, pt-BR`") | |
| metadata["language"] = metadata["language"] if "language" in metadata else [] | |
| metadata["language"] = ( | |
| metadata["language"] | |
| if isinstance(metadata["language"], list) | |
| else [metadata["language"]] | |
| ) | |
| languages = st_tags( | |
| label="", text="add more if needed, and press enter", value=metadata["language"], | |
| key=model_id+"_langs" | |
| ) | |
| if not languages: | |
| st.stop() | |
| return | |
| lang_names = [lang2name[lang] if lang in lang2name else lang for lang in languages] | |
| st.markdown("These languages will be parsed by the leaderboard as: ") | |
| st.code(", ".join(lang_names)) | |
| metadata["language"] = languages | |
| ############################ | |
| # TRAIN DATASETS | |
| ############################ | |
| st.markdown("### Training dataset(s)") | |
| st.markdown( | |
| "List the datasets that your model was **trained** on. " | |
| "If the datasets aren't published on the Hub yet, just add their names anyway." | |
| ) | |
| st.markdown( | |
| "*Example*: `librispeech_asr, mozilla-foundation/common_voice_8_0, my_custom_youtube_dataset`" | |
| ) | |
| if "datasets" not in metadata: | |
| metadata["datasets"] = [] | |
| train_datasets = st_tags( | |
| label="", text="add more if needed, and press enter", value=metadata["datasets"], | |
| key=model_id+"_train_dataset" | |
| ) | |
| if not train_datasets: | |
| st.stop() | |
| return | |
| if "common_voice" in train_datasets: | |
| st.warning( | |
| "WARNING: `common_voice` is deprecated, please replace it with its equivalent: " | |
| "`mozilla-foundation/common_voice_6_1`" | |
| ) | |
| metadata["datasets"] = train_datasets | |
| ############################ | |
| # MODEL NAME | |
| ############################ | |
| st.markdown("### Model name") | |
| st.markdown("Enter a pretty name for your model.") | |
| st.markdown("*Example*: `XLS-R Wav2Vec2 LM Spanish by Jane Doe`") | |
| if "model-index" not in metadata: | |
| metadata["model-index"] = [{}] | |
| if "name" not in ["model-index"][0]: | |
| metadata["model-index"][0]["name"] = model_id.split("/")[-1] | |
| model_name = st.text_input("", value=metadata["model-index"][0]["name"]) | |
| if not model_name: | |
| st.stop() | |
| return | |
| metadata["model-index"][0]["name"] = model_name | |
| ############################ | |
| # EVAL RESULTS | |
| ############################ | |
| st.markdown("### Evaluation results") | |
| st.markdown( | |
| "To edit the metrics, you can either use the YAML editor below, or add new metrics using the handy " | |
| "form under it." | |
| ) | |
| if "results" not in metadata["model-index"][0]: | |
| metadata["model-index"][0]["results"] = [] | |
| results_editor = st.empty() | |
| with results_editor: | |
| results_yaml = yaml.dump( | |
| metadata["model-index"][0]["results"], sort_keys=False, line_break="\n" | |
| ) | |
| results_yaml = st_ace(value=results_yaml, language="yaml") | |
| metadata["model-index"][0]["results"] = try_parse_yaml(results_yaml) | |
| dataset_path_kwargs = {} | |
| dataset_name_kwargs = {} | |
| if ( | |
| len(metadata["model-index"][0]["results"]) > 0 | |
| and "dataset" in metadata["model-index"][0]["results"][0] | |
| ): | |
| if "type" in metadata["model-index"][0]["results"][0]["dataset"]: | |
| dataset_path_kwargs["value"] = metadata["model-index"][0]["results"][0][ | |
| "dataset" | |
| ]["type"] | |
| if "name" in metadata["model-index"][0]["results"][0]["dataset"]: | |
| dataset_name_kwargs["value"] = metadata["model-index"][0]["results"][0][ | |
| "dataset" | |
| ]["name"] | |
| with st.form(key="eval_form"): | |
| dataset_path = st.text_input( | |
| label="Dataset path / id", | |
| placeholder="mozilla-foundation/common_voice_8_0", | |
| **dataset_path_kwargs, | |
| ) | |
| dataset_name = st.text_input( | |
| label="A pretty name for the dataset. Examples: 'Common Voice 9.0 (French)', 'LibriSpeech (clean)'", | |
| placeholder="Common Voice 9.0 (French)", | |
| **dataset_name_kwargs, | |
| ) | |
| dataset_config = st.text_input( | |
| label="Dataset configuration. Examples: clean, other, en, pt-BR", | |
| placeholder="en", | |
| ) | |
| dataset_language_kwargs = {"value": languages[0]} if len(languages) > 0 else {} | |
| dataset_language = st.text_input( | |
| label="Dataset language. Examples: en, pt-BR", | |
| placeholder="en", | |
| **dataset_language_kwargs | |
| ) | |
| dataset_split = st.text_input( | |
| label="Dataset split. Examples: test, validation", | |
| value="test", | |
| placeholder="test", | |
| ) | |
| metric2name = {"wer": "Word Error Rate", "cer": "Character Error Rate"} | |
| metric_type = st.selectbox( | |
| label="Metric", | |
| options=["wer", "cer"], | |
| format_func=lambda key: metric2name[key], | |
| ) | |
| metric_name = st.text_input( | |
| label="A pretty name for the metric. Example: Test WER (+LM)", | |
| placeholder="Test WER", | |
| value="Test WER", | |
| ) | |
| metric_value = st.text_input( | |
| label="Metric value. Use values in range 0.0 to 100.0.", | |
| placeholder="12.34", | |
| ) | |
| # try: | |
| # metric_value = float(metric_value) | |
| # except ValueError: | |
| # st.error( | |
| # f"Couldn't parse `{metric_value}`. Make sure it's a number from 0.0 to 100.0" | |
| # ) | |
| submitted = st.form_submit_button("Add metric") | |
| if ( | |
| submitted | |
| and dataset_name | |
| and dataset_path | |
| and dataset_config | |
| and dataset_split | |
| and dataset_language | |
| and metric_name | |
| and metric_type | |
| and metric_value | |
| ): | |
| metric = { | |
| "name": metric_name, | |
| "type": metric_type, | |
| "value": metric_value, | |
| } | |
| # first, try to find an existing dataset+config record to add a new metric to it | |
| updated_existing = False | |
| for existing_result in metadata["model-index"][0]["results"]: | |
| existing_dataset = existing_result["dataset"] | |
| if ( | |
| existing_dataset["type"] == dataset_path | |
| and "config" in existing_dataset | |
| and existing_dataset["config"] == dataset_config | |
| and "split" in existing_dataset | |
| and existing_dataset["split"] == dataset_split | |
| ): | |
| if "metrics" not in existing_result: | |
| existing_result["metrics"] = [] | |
| existing_result["metrics"].append(metric) | |
| updated_existing = True | |
| break | |
| # if no dataset+config results found, create a new one | |
| if not updated_existing: | |
| result = { | |
| "task": { | |
| "name": "Automatic Speech Recognition", | |
| "type": "automatic-speech-recognition", | |
| }, | |
| "dataset": { | |
| "name": dataset_name, | |
| "type": dataset_path, | |
| "config": dataset_config, | |
| "split": dataset_split, | |
| "args": {"language": dataset_language}, | |
| }, | |
| "metrics": [metric], | |
| } | |
| metadata["model-index"][0]["results"].append(result) | |
| # update the code editor | |
| with results_editor: | |
| results_yaml = yaml.dump( | |
| metadata["model-index"][0]["results"], | |
| sort_keys=False, | |
| line_break="\n", | |
| ) | |
| results_yaml = st_ace(value=results_yaml, language="yaml") | |
| metadata["model-index"][0]["results"] = try_parse_yaml(results_yaml) | |
| st.success( | |
| f"Added the metric for {dataset_path} - {dataset_config}! " | |
| f"Check the result in the YAML editor above." | |
| ) | |
| elif submitted: | |
| st.error( | |
| f"Make sure that you've filled the whole form before clicking 'Add metric'!" | |
| ) | |
| ############################ | |
| # FINAL YAML | |
| ############################ | |
| st.markdown("## 3. Copy the generated metadata") | |
| st.markdown( | |
| "Copy the YAML from below and replace the metadata at the top of your model's README.md here: " | |
| f"https://huggingface.co/{model_id}/edit/main/README.md" | |
| ) | |
| st.markdown("For more info on the metadata schema please refer to " | |
| "https://raw.githubusercontent.com/huggingface/hub-docs/main/modelcard.md") | |
| new_yaml = yaml.dump(metadata, sort_keys=False, line_break="\n") | |
| st.markdown(f"```yaml\n---\n{new_yaml}---\n```") | |
| if __name__ == "__main__": | |
| main() | |