Merge branch 'main' of https://huggingface.co/spaces/BramVanroy/open_dutch_llm_leaderboard

app.py

@@ -80,7 +80,7 @@ class ResultSet:
                 "num_parameters": "Size",
                 "average": "Avg.",
                 "arc": "ARC (25-shot)",
-                "hellaswag": "HellaSwag (10-shot)️",
+                "hellaswag": "HellaSwag (10-shot)",
                 "mmlu": "MMLU (5-shot)",
                 "truthfulqa": "TruthfulQA (0-shot)",
             }
@@ -91,7 +91,7 @@ class ResultSet:
                 "Size": "str",
                 "Avg.": "number",
                 "ARC (25-shot)": "number",
-                "HellaSwag (10-shot)️": "number",
+                "HellaSwag (10-shot)": "number",
                 "MMLU (5-shot)": "number",
                 "TruthfulQA (0-shot)": "number",
             }

content.py

@@ -21,7 +21,9 @@ DISCLAIMER = """## Disclaimer
 I did not verify the (translation) quality of the benchmarks. If you encounter issues with the benchmark contents, please contact the original authors.
 
 I am aware that benchmarking models on *translated* data is not ideal. However, for Dutch there are no other options for generative models at the moment. Because the benchmarks were automatically translated, some translationese effects may occur: the translations may not be fluent Dutch or still contain artifacts of the source text (like word order, literal translation, certain vocabulary items). Because of that, an unfair advantage may be given to the non-Dutch models: Dutch is closely related to English, so if the benchmarks are in automatically translated Dutch that still has English properties, those English models may not have too many issues with that. If the benchmarks were to have been manually translated or, even better, created from scratch in Dutch, those non-Dutch models may have a harder time. Maybe not. We cannot know for sure until we have high-quality, manually crafted benchmarks for Dutch.
- 
+
+Another shortcoming is that we do not calculate significancy scores or confidence intervals. When results are close together in the leaderboard I therefore urge caution when interpreting the model ranks.
+
 If you have any suggestions for other Dutch benchmarks, please [let me know](https://twitter.com/BramVanroy) so I can add them!
 """
 

evals/models.json

@@ -59,7 +59,7 @@
         "compute_dtype": "bfloat16",
         "dutch_coverage": "none",
         "model_name": "meta-llama/Llama-2-13b-chat-hf",
-        "model_type": "instruction-tuned",
+        "model_type": "RL-tuned",
         "num_parameters": 13015864320,
         "quantization": "8-bit"
     },
@@ -75,7 +75,7 @@
         "compute_dtype": "bfloat16",
         "dutch_coverage": "none",
         "model_name": "meta-llama/Llama-2-7b-chat-hf",
-        "model_type": "instruction-tuned",
+        "model_type": "RL-tuned",
         "num_parameters": 6738415616,
         "quantization": "8-bit"
     },
@@ -123,7 +123,7 @@
         "compute_dtype": "bfloat16",
         "dutch_coverage": "none",
         "model_name": "microsoft/Orca-2-13b",
-        "model_type": "fine-tuned",
+        "model_type": "instruction-tuned",
         "num_parameters": 13015895040,
         "quantization": "8-bit"
     },
@@ -131,7 +131,7 @@
         "compute_dtype": "bfloat16",
         "dutch_coverage": "none",
         "model_name": "microsoft/Orca-2-7b",
-        "model_type": "fine-tuned",
+        "model_type": "instruction-tuned",
         "num_parameters": 6738440192,
         "quantization": "8-bit"
     },

evals/truthfulqa/truthfulqa_nl_Mixtral-8x7B-v0.1.json

@@ -1,23 +0,0 @@
-{
-  "results": {
-    "truthfulqa_nl": {
-      "mc1": 0.310828025477707,
-      "mc1_stderr": 0.016529733724696277,
-      "mc2": 0.4629846929417042,
-      "mc2_stderr": 0.01508660067127546
-    }
-  },
-  "versions": {
-    "truthfulqa_nl": 1
-  },
-  "config": {
-    "model": "hf-auto",
-    "model_args": "pretrained=mistralai/Mixtral-8x7B-v0.1,use_accelerate=True,device_map_option=auto,dtype=auto,load_in_8bit=True",
-    "batch_size": 2,
-    "device": "cuda",
-    "no_cache": false,
-    "limit": null,
-    "bootstrap_iters": 100000,
-    "description_dict": {}
-  }
-}