fix retention of fields

evals/models.json

@@ -3,96 +3,120 @@
         "compute_dtype": "bfloat16",
         "model_name": "yhavinga/gpt-neo-1.3B-dutch",
         "num_parameters": 1315575808,
-        "quantization": "8-bit"
+        "quantization": "8-bit",
+        "model_type": "pretrained",
+        "dutch_coverage": "pretrained"
     },
     "gpt-neo-125m-dutch": {
         "compute_dtype": "bfloat16",
         "model_name": "yhavinga/gpt-neo-125M-dutch",
         "num_parameters": 125198592,
-        "quantization": "8-bit"
+        "quantization": "8-bit",
+        "model_type": "pretrained",
+        "dutch_coverage": "pretrained"
     },
     "gpt2-large-dutch": {
         "compute_dtype": "bfloat16",
         "model_name": "yhavinga/gpt2-large-dutch",
         "num_parameters": 774030080,
-        "quantization": "8-bit"
+        "quantization": "8-bit",
+        "model_type": "pretrained",
+        "dutch_coverage": "pretrained"
     },
     "gpt2-medium-dutch": {
         "compute_dtype": "bfloat16",
         "model_name": "yhavinga/gpt2-medium-dutch",
         "num_parameters": 354823168,
-        "quantization": "8-bit"
+        "quantization": "8-bit",
+        "model_type": "pretrained",
+        "dutch_coverage": "pretrained"
     },
     "llama-2-13b-chat-dutch": {
         "compute_dtype": "bfloat16",
         "model_name": "BramVanroy/Llama-2-13b-chat-dutch",
         "num_parameters": 13015864320,
-        "quantization": "8-bit"
+        "quantization": "8-bit",
+        "model_type": "instruction-tuned",
+        "dutch_coverage": "fine-tuned"
     },
     "llama-2-13b-chat-hf": {
         "compute_dtype": "bfloat16",
         "model_name": "meta-llama/Llama-2-13b-chat-hf",
         "num_parameters": 13015864320,
-        "quantization": "8-bit"
+        "quantization": "8-bit",
+        "model_type": "instruction-tuned",
+        "dutch_coverage": "none"
     },
     "llama-2-13b-hf": {
         "compute_dtype": "bfloat16",
         "model_name": "meta-llama/Llama-2-13b-hf",
         "num_parameters": 13015864320,
-        "quantization": "8-bit"
+        "quantization": "8-bit",
+        "model_type": "pretrained",
+        "dutch_coverage": "none"
     },
     "llama-2-7b-chat-hf": {
         "compute_dtype": "bfloat16",
         "model_name": "meta-llama/Llama-2-7b-chat-hf",
         "num_parameters": 6738415616,
-        "quantization": "8-bit"
+        "quantization": "8-bit",
+        "model_type": "instruction-tuned",
+        "dutch_coverage": "none"
     },
     "llama-2-7b-hf": {
         "compute_dtype": "bfloat16",
         "model_name": "meta-llama/Llama-2-7b-hf",
         "num_parameters": 6738415616,
-        "quantization": "8-bit"
+        "quantization": "8-bit",
+        "model_type": "pretrained",
+        "dutch_coverage": "none"
     },
     "llama2-13b-ft-mc4_nl_cleaned_tiny": {
         "compute_dtype": "bfloat16",
         "model_name": "BramVanroy/llama2-13b-ft-mc4_nl_cleaned_tiny",
         "num_parameters": 13015864320,
-        "quantization": "8-bit"
+        "quantization": "8-bit",
+        "model_type": "fine-tuned",
+        "dutch_coverage": "fine-tuned"
     },
     "mistral-7b-v0.1": {
         "compute_dtype": "bfloat16",
         "model_name": "mistralai/Mistral-7B-v0.1",
         "num_parameters": 7241732096,
-        "quantization": "8-bit"
-    },
-    "mixtral-8x7b-v0.1": {
-        "compute_dtype": "auto",
-        "model_name": "mistralai/Mixtral-8x7B-v0.1",
-        "num_parameters": 46702792704,
-        "quantization": null
+        "quantization": "8-bit",
+        "model_type": "pretrained",
+        "dutch_coverage": "none"
     },
     "neural-chat-7b-v3-1": {
         "compute_dtype": "bfloat16",
         "model_name": "Intel/neural-chat-7b-v3-1",
         "num_parameters": 7241732096,
-        "quantization": "8-bit"
+        "quantization": "8-bit",
+        "model_type": "RL-tuned",
+        "dutch_coverage": "none"
     },
     "orca-2-13b": {
         "compute_dtype": "bfloat16",
         "model_name": "microsoft/Orca-2-13b",
         "num_parameters": 13015895040,
-        "quantization": "8-bit"
+        "quantization": "8-bit",
+        "model_type": "fine-tuned",
+        "dutch_coverage": "none"
     },
     "orca-2-7b": {
         "compute_dtype": "bfloat16",
         "model_name": "microsoft/Orca-2-7b",
         "num_parameters": 6738440192,
-        "quantization": "8-bit"
+        "quantization": "8-bit",
+        "model_type": "fine-tuned",
+        "dutch_coverage": "none"
     },
     "zephyr-7b-beta": {
         "compute_dtype": "bfloat16",
         "model_name": "HuggingFaceH4/zephyr-7b-beta",
         "num_parameters": 7241732096,
-        "quantization": "8-bit"
+        "quantization": "8-bit",
+        "model_type": "RL-tuned",
+        "dutch_coverage": "none"
     }
 }

generate_overview_json.py

@@ -37,6 +37,11 @@ def main():
             "num_parameters": results[short_name]["num_parameters"]
             if short_name in results and "num_parameters" in results[short_name]
             else get_num_parameters(model_args["pretrained"]),
+            "model_type": results[short_name]["model_type"]
+            if short_name in results and "model_type" in results[short_name]
+            else "not-given",
+            "dutch_coverage": results[short_name]["dutch_coverage"] if short_name in results and "dutch_coverage" in results[short_name]
+            else "not-given",
         }
 
         if "load_in_8bit" in model_args: