Spaces:
Sleeping
Sleeping
| import gradio as gr | |
| import pandas as pd | |
| title = """ | |
| # hmLeaderboard | |
|  | |
| """ | |
| description = """ | |
| ## Space for tracking and ranking models on Historic NER Datasets. | |
| At the moment the following models are supported: | |
| * hmBERT: [Historical Multilingual Language Models for Named Entity Recognition](https://huggingface.co/hmbert). | |
| * hmTEAMS: [Historic Multilingual TEAMS Models](https://huggingface.co/hmteams). | |
| """ | |
| footer = "Made from Bavarian Oberland with ❤️ and 🥨." | |
| model_selection_file_names = { | |
| "Best Configuration": "best_model_configurations.csv", | |
| "Best Model": "best_models.csv" | |
| } | |
| df_init = pd.read_csv(model_selection_file_names["Best Configuration"]) | |
| dataset_names = df_init.columns.values[1:].tolist() | |
| languages = list(set([dataset_name.split(" ")[0] for dataset_name in dataset_names])) | |
| def perform_evaluation_for_datasets(model_selection, selected_datasets): | |
| df = pd.read_csv(model_selection_file_names.get(model_selection)) | |
| selected_indices = [] | |
| for selected_dataset in selected_datasets: | |
| selected_indices.append(dataset_names.index(selected_dataset) + 1) | |
| mean_column = df.iloc[:, selected_indices].mean(axis=1).round(2) | |
| # Include column with column name | |
| result_df = df.iloc[:, [0] + selected_indices] | |
| result_df["Average"] = mean_column | |
| return result_df | |
| def perform_evaluation_for_languages(model_selection, selected_languages): | |
| df = pd.read_csv(model_selection_file_names.get(model_selection)) | |
| selected_indices = [] | |
| for selected_language in selected_languages: | |
| selected_language = selected_language.lower() | |
| found_indices = [i for i, column_name in enumerate(df.columns) if selected_language in column_name.lower()] | |
| for found_index in found_indices: | |
| selected_indices.append(found_index) | |
| mean_column = df.iloc[:, selected_indices].mean(axis=1).round(2) | |
| # Include column with column name | |
| result_df = df.iloc[:, [0] + selected_indices] | |
| result_df["Average"] = mean_column | |
| return result_df | |
| with gr.Blocks() as demo: | |
| gr.Markdown(title) | |
| gr.Markdown(description) | |
| with gr.Tab("Overview"): | |
| gr.Markdown("### Best Configuration\nThe best hyper-parameter configuration for each model is used and average F1-score over runs with different seeds is reported here:") | |
| df_result = perform_evaluation_for_datasets("Best Configuration", dataset_names) | |
| gr.Dataframe(value=df_result) | |
| gr.Markdown("### Best Model\nThe best hyper-parameter configuration for each model is used and the model with highest F1-score is used and its performance is reported here:") | |
| df_result = perform_evaluation_for_datasets("Best Model", dataset_names) | |
| gr.Dataframe(value=df_result) | |
| with gr.Tab("Filtering"): | |
| gr.Markdown("### Filtering\nSwiss-knife filtering for single datasets and languages is possible.") | |
| model_selection = gr.Radio(choices=["Best Configuration", "Best Model"], | |
| label="Model Selection", | |
| info="Defines if best configuration or best model should be used for evaluation. When 'Best Configuration' is used, the best hyper-parameter configuration is used and then averaged F1-score over all runs is calculated. When 'Best Model' is chosen, the best hyper-parameter configuration and model with highest F1-score on development dataset is used (best model).", | |
| value="Best Configuration") | |
| with gr.Tab("Dataset Selection"): | |
| datasets_selection = gr.CheckboxGroup( | |
| dataset_names, label="Datasets", info="Select datasets for evaluation" | |
| ) | |
| output_df = gr.Dataframe() | |
| evaluation_button = gr.Button("Evaluate") | |
| evaluation_button.click(fn=perform_evaluation_for_datasets, inputs=[model_selection, datasets_selection], outputs=output_df) | |
| with gr.Tab("Language Selection"): | |
| language_selection = gr.CheckboxGroup( | |
| languages, label="Languages", info="Select languages for evaluation" | |
| ) | |
| output_df = gr.Dataframe() | |
| evaluation_button = gr.Button("Evaluate") | |
| evaluation_button.click(fn=perform_evaluation_for_languages, inputs=[model_selection, language_selection], outputs=output_df) | |
| gr.Markdown(footer) | |
| demo.launch() | |