Spaces:
Running
Running
| import gradio as gr | |
| import pandas as pd | |
| from huggingface_hub import HfApi | |
| DATASETS = [ | |
| "mMARCO-fr", | |
| "BSARD", | |
| ] | |
| SINGLE_VECTOR_MODELS = [ | |
| "antoinelouis/biencoder-camemberta-base-mmarcoFR", | |
| "antoinelouis/biencoder-camembert-base-mmarcoFR", | |
| "antoinelouis/biencoder-distilcamembert-mmarcoFR", | |
| "antoinelouis/biencoder-camembert-L10-mmarcoFR", | |
| "antoinelouis/biencoder-camembert-L8-mmarcoFR", | |
| "antoinelouis/biencoder-camembert-L6-mmarcoFR", | |
| "antoinelouis/biencoder-camembert-L4-mmarcoFR", | |
| "antoinelouis/biencoder-camembert-L2-mmarcoFR", | |
| "antoinelouis/biencoder-electra-base-mmarcoFR", | |
| "antoinelouis/biencoder-mMiniLMv2-L12-mmarcoFR", | |
| "antoinelouis/biencoder-mMiniLMv2-L6-mmarcoFR", | |
| "OrdalieTech/Solon-embeddings-large-0.1", | |
| "OrdalieTech/Solon-embeddings-base-0.1", | |
| ] | |
| MULTI_VECTOR_MODELS = [ | |
| "antoinelouis/colbertv1-camembert-base-mmarcoFR", | |
| "antoinelouis/colbertv2-camembert-L4-mmarcoFR", | |
| "antoinelouis/colbert-xm", | |
| ] | |
| SPARSE_LEXICAL_MODELS = [ | |
| "antoinelouis/spladev2-camembert-base-mmarcoFR", | |
| ] | |
| CROSS_ENCODER_MODELS = [ | |
| "antoinelouis/crossencoder-camemberta-L2-mmarcoFR", | |
| "antoinelouis/crossencoder-camemberta-L4-mmarcoFR", | |
| "antoinelouis/crossencoder-camemberta-L6-mmarcoFR", | |
| "antoinelouis/crossencoder-camemberta-L8-mmarcoFR", | |
| "antoinelouis/crossencoder-camemberta-L10-mmarcoFR", | |
| "antoinelouis/crossencoder-camemberta-base-mmarcoFR", | |
| "antoinelouis/crossencoder-camembert-L2-mmarcoFR", | |
| "antoinelouis/crossencoder-camembert-L4-mmarcoFR", | |
| "antoinelouis/crossencoder-camembert-L6-mmarcoFR", | |
| "antoinelouis/crossencoder-camembert-L8-mmarcoFR", | |
| "antoinelouis/crossencoder-camembert-L10-mmarcoFR", | |
| "antoinelouis/crossencoder-camembert-base-mmarcoFR", | |
| "antoinelouis/crossencoder-camembert-large-mmarcoFR", | |
| "antoinelouis/crossencoder-distilcamembert-mmarcoFR", | |
| "antoinelouis/crossencoder-electra-base-mmarcoFR", | |
| "antoinelouis/crossencoder-me5-base-mmarcoFR", | |
| "antoinelouis/crossencoder-me5-small-mmarcoFR", | |
| "antoinelouis/crossencoder-t5-base-mmarcoFR", | |
| "antoinelouis/crossencoder-t5-small-mmarcoFR", | |
| "antoinelouis/crossencoder-mt5-base-mmarcoFR", | |
| "antoinelouis/crossencoder-mt5-small-mmarcoFR", | |
| "antoinelouis/crossencoder-xlm-roberta-base-mmarcoFR", | |
| "antoinelouis/crossencoder-mdebertav3-base-mmarcoFR", | |
| "antoinelouis/crossencoder-mMiniLMv2-L12-mmarcoFR", | |
| "antoinelouis/crossencoder-mMiniLMv2-L6-mmarcoFR", | |
| ] | |
| COLUMNS = { | |
| "Model": "html", | |
| "#Params (M)": "number", | |
| "Type": "str", | |
| "Dataset": "str", | |
| "Recall@1000": "number", | |
| "Recall@500": "number", | |
| "Recall@100": "number", | |
| "Recall@10": "number", | |
| "MRR@10": "number", | |
| "nDCG@10": "number", | |
| "MAP@10": "number", | |
| } | |
| def get_model_info(model_id: str, model_type: str) -> pd.DataFrame: | |
| data = {} | |
| api = HfApi() | |
| model_info = api.model_info(model_id) | |
| for result in model_info.card_data.eval_results: | |
| if result.dataset_name in DATASETS and result.dataset_name not in data: | |
| data[result.dataset_name] = {key: None for key in COLUMNS.keys()} | |
| data[result.dataset_name]["Model"] = f'<a href="https://huggingface.co/{model_id}" target="_blank" style="color: blue; text-decoration: none;">{model_id}</a>' | |
| data[result.dataset_name]["#Params (M)"] = round(model_info.safetensors.total/1e6, 0) if model_info.safetensors else None | |
| data[result.dataset_name]["Type"] = model_type | |
| data[result.dataset_name]["Dataset"] = result.dataset_name | |
| if result.dataset_name in DATASETS and result.metric_name in data[result.dataset_name]: | |
| data[result.dataset_name][result.metric_name] = result.metric_value | |
| return pd.DataFrame(list(data.values())) | |
| def load_all_results() -> pd.DataFrame: | |
| # Load results from external baseline models. | |
| df = pd.read_csv('./baselines.csv') | |
| # Load results from own Hugging Face models. | |
| for model_id in SINGLE_VECTOR_MODELS: | |
| df = pd.concat([df, get_model_info(model_id, model_type="SINGLE")]) | |
| for model_id in MULTI_VECTOR_MODELS: | |
| df = pd.concat([df, get_model_info(model_id, model_type="MULTI")]) | |
| for model_id in SPARSE_LEXICAL_MODELS: | |
| df = pd.concat([df, get_model_info(model_id, model_type="SPARSE")]) | |
| for model_id in CROSS_ENCODER_MODELS: | |
| df = pd.concat([df, get_model_info(model_id, model_type="CROSS")]) | |
| # Round all metrics to 1 decimal. | |
| for col in df.columns: | |
| if "Recall" in col or "MRR" in col or "nDCG" in col or "MAP" in col: | |
| df[col] = df[col].round(1) | |
| return df | |
| def filter_dataf_by_dataset(dataf: pd.DataFrame, dataset_name: str, sort_by: str) -> pd.DataFrame: | |
| return (dataf | |
| .loc[dataf["Dataset"] == dataset_name] | |
| .drop(columns=["Dataset"]) | |
| .sort_values(by=sort_by, ascending=False) | |
| ) | |
| def update_table(dataf: pd.DataFrame, query: str, selected_types: list, selected_sizes: list) -> pd.DataFrame: | |
| filtered_df = dataf.copy() | |
| if selected_types: | |
| filtered_df = filtered_df[filtered_df['Type'].isin([t.split()[-1][1:-1] for t in selected_types])] | |
| size_conditions = [] | |
| for val in selected_sizes: | |
| if val == 'Small (< 100M)': | |
| size_conditions.append(filtered_df['#Params (M)'] < 100) | |
| elif val == 'Base (100M-300M)': | |
| size_conditions.append((filtered_df['#Params (M)'] >= 100) & (filtered_df['#Params (M)'] <= 300)) | |
| elif val == 'Large (300M-500M)': | |
| size_conditions.append((filtered_df['#Params (M)'] >= 300) & (filtered_df['#Params (M)'] <= 500)) | |
| elif val == 'Extra-large (500M+)': | |
| size_conditions.append(filtered_df['#Params (M)'] > 500) | |
| if size_conditions: | |
| filtered_df = filtered_df[pd.concat(size_conditions, axis=1).any(axis=1)] | |
| if query: | |
| filtered_df = filtered_df[filtered_df['Model'].str.contains(query, case=False)] | |
| return filtered_df | |
| with gr.Blocks() as demo: | |
| gr.HTML(""" | |
| <div style="display: flex; flex-direction: column; align-items: center;"> | |
| <div style="align-self: flex-start;"> | |
| <a href="mailto:antoiloui@gmail.com" target="_blank" style="color: blue; text-decoration: none;">Contact/Submissions</a> | |
| </div> | |
| <h1 style="margin: 0;">🥇 DécouvrIR\n</h1>A Benchmark for Evaluating the Robustness of Information Retrieval Models in French</h1> | |
| </div> | |
| """) | |
| # Create the Pandas dataframes (one per dataset) | |
| all_df = load_all_results() | |
| mmarco_df = filter_dataf_by_dataset(all_df, dataset_name="mMARCO-fr", sort_by="Recall@500") | |
| bsard_df = filter_dataf_by_dataset(all_df, dataset_name="BSARD", sort_by="Recall@500") | |
| # Search and filter widgets | |
| with gr.Column(): | |
| with gr.Row(): | |
| search_bar = gr.Textbox(placeholder=" 🔍 Search for a model...", show_label=False, elem_id="search-bar") | |
| with gr.Row(): | |
| filter_type = gr.CheckboxGroup( | |
| label="Model type", | |
| choices=[ | |
| 'Single-vector dense bi-encoder (SINGLE)', | |
| 'Multi-vector dense bi-encoder (MULTI)', | |
| 'Sparse lexical model (SPARSE)', | |
| 'Cross-encoder (CROSS)', | |
| ], | |
| value=[], | |
| interactive=True, | |
| elem_id="filter-type", | |
| ) | |
| with gr.Row(): | |
| filter_size = gr.CheckboxGroup( | |
| label="Model size", | |
| choices=['Small (< 100M)', 'Base (100M-300M)', 'Large (300M-500M)', 'Extra-large (500M+)'], | |
| value=[], | |
| interactive=True, | |
| elem_id="filter-size", | |
| ) | |
| # Leaderboard tables | |
| with gr.Tabs(): | |
| with gr.TabItem("🌐 mMARCO-fr"): | |
| gr.HTML(""" | |
| <p>The <a href="https://huggingface.co/datasets/unicamp-dl/mmarco" target="_blank" style="color: blue; text-decoration: none;">mMARCO</a> dataset is a machine-translated version of | |
| the widely popular MS MARCO dataset across 13 languages (including French) for studying <strong> domain-general</strong> passage retrieval.</p> | |
| <p>The evaluation is performed on <strong>6,980 dev questions</strong> labeled with relevant passages to be retrieved from a corpus of <strong>8,841,823 candidates</strong>.</p> | |
| """) | |
| mmarco_table = gr.Dataframe( | |
| value=mmarco_df, | |
| datatype=[COLUMNS[col] for col in mmarco_df.columns], | |
| interactive=False, | |
| elem_classes="text-sm", | |
| ) | |
| with gr.TabItem("⚖️ BSARD"): | |
| gr.HTML(""" | |
| <p>The <a href="https://huggingface.co/datasets/maastrichtlawtech/bsard" target="_blank" style="color: blue; text-decoration: none;">Belgian Statutory Article Retrieval Dataset (BSARD)</a> is a | |
| French native dataset for studying <strong>legal</strong> document retrieval.</p> | |
| <p>The evaluation is performed on <strong>222 test questions</strong> labeled by experienced jurists with relevant Belgian law articles to be retrieved from a corpus of <strong>22,633 candidates</strong>.</p> | |
| <i>[Coming soon...]</i> | |
| """) | |
| # bsard_table = gr.Dataframe( | |
| # value=bsard_df, | |
| # datatype=[COLUMNS[col] for col in bsard_df.columns], | |
| # interactive=False, | |
| # elem_classes="text-sm", | |
| # ) | |
| # Update tables on filter widgets change. | |
| widgets = [search_bar, filter_type, filter_size] | |
| for w in widgets: | |
| w.change(fn=lambda q, t, s: update_table(dataf=mmarco_df, query=q, selected_types=t, selected_sizes=s), inputs=widgets, outputs=[mmarco_table]) | |
| #w.change(fn=lambda q, t, s: update_table(dataf=bsard_df, query=q, selected_types=t, selected_sizes=s), inputs=widgets, outputs=[bsard_table]) | |
| # Citation | |
| with gr.Column(): | |
| with gr.Row(): | |
| gr.HTML(""" | |
| <h2>Citation</h2> | |
| <p>For attribution in academic contexts, please cite this benchmark and any of the models released by <a href="https://huggingface.co/antoinelouis" target="_blank" style="color: blue; text-decoration: none;">@antoinelouis</a> as follows:</p> | |
| """) | |
| with gr.Row(): | |
| citation_block = ( | |
| "@online{louis2024decouvrir,\n" | |
| "\tauthor = 'Antoine Louis',\n" | |
| "\ttitle = 'DécouvrIR: A Benchmark for Evaluating the Robustness of Information Retrieval Models in French',\n" | |
| "\tpublisher = 'Hugging Face',\n" | |
| "\tmonth = 'mar',\n" | |
| "\tyear = '2024',\n" | |
| "\turl = 'https://huggingface.co/spaces/antoinelouis/decouvrir',\n" | |
| "}\n" | |
| ) | |
| gr.Code(citation_block, language=None, show_label=False) | |
| demo.launch() |