update.

app.py

@@ -1,5 +1,5 @@
-import os
-os.system("pip install gradio==3.28.0 pydantic==1.10.15")
+# import os
+# os.system("pip install gradio==3.28.0 pydantic==1.10.15")
 
 import gradio as gr
 import pandas as pd
@@ -262,7 +262,7 @@ with demo:
                         interactive=True,
                         elem_id="filter-columns-precision",
                     )
-                    with gr.Box() as config:
+                    with gr.Group() as config:
                         gr.HTML("""<p style='padding-bottom: 0.5rem; '>Quantization config</p>""")
                         with gr.Row():
                             filter_columns_computeDtype = gr.Dropdown(choices=[i.value.name for i in ComputeDtype], label="Compute Dtype", multiselect=False, value="float16", interactive=True,)

requirements.txt

@@ -16,3 +16,4 @@ tokenizers>=0.15.0
 gradio-space-ci @ git+https://huggingface.co/spaces/Wauplin/gradio-space-ci@0.2.1 # CI !!!
 gradio==3.28.0
 GitPython==3.1.40
+pydantic==1.10.15

src/display/about.py

@@ -50,20 +50,21 @@ python main.py --model=hf-causal-experimental \
     --num_fewshot=<n_few_shot> \
     --batch_size=1 \
     --output_path=<output_path>
+
 ```
 
 **Note:** You can expect results to vary slightly for different batch sizes because of padding.
 
 The tasks and few shots parameters are:
-- ARC-C: 0-shot, *arc_challenge* (`acc_norm`)
-- ARC-E: 0-shot, *arc_easy* (`acc_norm`)
-- HellaSwag: 0-shot, *hellaswag* (`acc_norm`)
+- ARC-C: 0-shot, *arc_challenge* (`acc`)
+- ARC-E: 0-shot, *arc_easy* (`acc`)
+- HellaSwag: 0-shot, *hellaswag* (`acc`)
 - TruthfulQA: 0-shot, *truthfulqa_mc2* (`acc`)
 - MMLU: 0-shot, *hendrycksTest-abstract_algebra,hendrycksTest-anatomy,hendrycksTest-astronomy,hendrycksTest-business_ethics,hendrycksTest-clinical_knowledge,hendrycksTest-college_biology,hendrycksTest-college_chemistry,hendrycksTest-college_computer_science,hendrycksTest-college_mathematics,hendrycksTest-college_medicine,hendrycksTest-college_physics,hendrycksTest-computer_security,hendrycksTest-conceptual_physics,hendrycksTest-econometrics,hendrycksTest-electrical_engineering,hendrycksTest-elementary_mathematics,hendrycksTest-formal_logic,hendrycksTest-global_facts,hendrycksTest-high_school_biology,hendrycksTest-high_school_chemistry,hendrycksTest-high_school_computer_science,hendrycksTest-high_school_european_history,hendrycksTest-high_school_geography,hendrycksTest-high_school_government_and_politics,hendrycksTest-high_school_macroeconomics,hendrycksTest-high_school_mathematics,hendrycksTest-high_school_microeconomics,hendrycksTest-high_school_physics,hendrycksTest-high_school_psychology,hendrycksTest-high_school_statistics,hendrycksTest-high_school_us_history,hendrycksTest-high_school_world_history,hendrycksTest-human_aging,hendrycksTest-human_sexuality,hendrycksTest-international_law,hendrycksTest-jurisprudence,hendrycksTest-logical_fallacies,hendrycksTest-machine_learning,hendrycksTest-management,hendrycksTest-marketing,hendrycksTest-medical_genetics,hendrycksTest-miscellaneous,hendrycksTest-moral_disputes,hendrycksTest-moral_scenarios,hendrycksTest-nutrition,hendrycksTest-philosophy,hendrycksTest-prehistory,hendrycksTest-professional_accounting,hendrycksTest-professional_law,hendrycksTest-professional_medicine,hendrycksTest-professional_psychology,hendrycksTest-public_relations,hendrycksTest-security_studies,hendrycksTest-sociology,hendrycksTest-us_foreign_policy,hendrycksTest-virology,hendrycksTest-world_religions* (average of all the results `acc`)
 - Winogrande: 0-shot, *winogrande* (`acc`)
 - Lambada_Openai: 0-shot, *lambada_openai* (`acc`)
-- PIQA: 0-shot, *piqa* (`acc_norm`)
-- OpenBookQA: 0-shot, *openbookqa* (`acc_norm`)
+- PIQA: 0-shot, *piqa* (`acc`)
+- OpenBookQA: 0-shot, *openbookqa* (`acc`)
 - BoolQ: 0-shot, *boolq* (`acc`)
 
 Side note on the baseline scores: 

src/display/utils.py

@@ -14,14 +14,14 @@ class Task:
     col_name: str
 
 class Tasks(Enum):
-    arc = Task("arc:challenge", "acc_norm,none", "ARC-c")
-    arc_easy = Task("arc:easy", "acc_norm,none", "ARC-e")
+    arc = Task("arc:challenge", "acc,none", "ARC-c")
+    arc_easy = Task("arc:easy", "acc,none", "ARC-e")
     boolq = Task("boolq", "acc,none", "Boolq")
-    hellaswag = Task("hellaswag", "acc_norm,none", "HellaSwag")
+    hellaswag = Task("hellaswag", "acc,none", "HellaSwag")
     lambada_openai = Task("lambada:openai", "acc,none", "Lambada_openai")
     mmlu = Task("mmlu", "acc,none", "MMLU")
-    openbookqa = Task("openbookqa", "acc_norm,none", "Openbookqa")
-    piqa = Task("piqa", "acc_norm,none", "Piqa")
+    openbookqa = Task("openbookqa", "acc,none", "Openbookqa")
+    piqa = Task("piqa", "acc,none", "Piqa")
     # truthfulqa:mc1 / truthfulqa:mc2 -- ?
     truthfulqa_mc = Task("truthfulqa:mc1", "acc,none", "Truthfulqa_mc1")
     # arc:challenge ?