add data collection script

content.py

@@ -13,6 +13,7 @@ We test the models on the following benchmarks **for the Dutch version only!!**,
 
 I do not maintain those datasets, I only run benchmarks and add the results to this space. For questions regarding the test sets or running them yourself, see [the original Github repository](https://github.com/laiviet/lm-evaluation-harness).
 
+All models are benchmarked in 8-bit precision.
 """
 
 CREDIT = f"""

evals/models.json

@@ -1,6 +1,70 @@
 {
-  "gpt-neo-1.3B-dutch": "PT",
-  "gpt-neo-125M-dutch": "PT",
-  "gpt2-large-dutch": "PT",
-  "gpt2-medium-dutch": "PT"
+  "bloom-7b1": {
+    "model_name": "pretrained=bigscience/bloom-7b1",
+    "args": "pretrained=bigscience/bloom-7b1"
+  },
+  "gpt-neo-1.3b-dutch": {
+    "model_name": "yhavinga/gpt-neo-1.3B-dutch",
+    "args": "use_accelerate=True,device_map_option=auto,dtype=bfloat16,load_in_8bit=True"
+  },
+  "gpt-neo-125m-dutch": {
+    "model_name": "yhavinga/gpt-neo-125M-dutch",
+    "args": "use_accelerate=True,device_map_option=auto,dtype=bfloat16,load_in_8bit=True"
+  },
+  "gpt2-large-dutch": {
+    "model_name": "yhavinga/gpt2-large-dutch",
+    "args": "use_accelerate=True,device_map_option=auto,dtype=bfloat16,load_in_8bit=True"
+  },
+  "gpt2-medium-dutch": {
+    "model_name": "yhavinga/gpt2-medium-dutch",
+    "args": "use_accelerate=True,device_map_option=auto,dtype=bfloat16,load_in_8bit=True"
+  },
+  "llama-2-13b-chat-dutch": {
+    "model_name": "BramVanroy/Llama-2-13b-chat-dutch",
+    "args": "use_accelerate=True,device_map_option=auto,dtype=bfloat16,load_in_8bit=True"
+  },
+  "llama-2-13b-chat-hf": {
+    "model_name": "meta-llama/Llama-2-13b-chat-hf",
+    "args": "use_accelerate=True,device_map_option=auto,dtype=bfloat16,load_in_8bit=True"
+  },
+  "llama-2-13b-hf": {
+    "model_name": "meta-llama/Llama-2-13b-hf",
+    "args": "use_accelerate=True,device_map_option=auto,dtype=bfloat16,load_in_8bit=True"
+  },
+  "llama-2-7b-chat-hf": {
+    "model_name": "meta-llama/Llama-2-7b-chat-hf",
+    "args": "use_accelerate=True,device_map_option=auto,dtype=bfloat16,load_in_8bit=True"
+  },
+  "llama-2-7b-hf": {
+    "model_name": "meta-llama/Llama-2-7b-hf",
+    "args": "use_accelerate=True,device_map_option=auto,dtype=bfloat16,load_in_8bit=True"
+  },
+  "llama-7b": {
+    "model_name": "pretrained=/sensei-fs/users/daclai/uoChatGPT/llama-7B",
+    "args": "pretrained=/sensei-fs/users/daclai/uoChatGPT/llama-7B"
+  },
+  "llama2-13b-ft-mc4_nl_cleaned_tiny": {
+    "model_name": "BramVanroy/llama2-13b-ft-mc4_nl_cleaned_tiny",
+    "args": "use_accelerate=True,device_map_option=auto,dtype=bfloat16,load_in_8bit=True"
+  },
+  "mistral-7b-v0.1": {
+    "model_name": "mistralai/Mistral-7B-v0.1",
+    "args": "use_accelerate=True,device_map_option=auto,dtype=bfloat16,load_in_8bit=True"
+  },
+  "neural-chat-7b-v3-1": {
+    "model_name": "Intel/neural-chat-7b-v3-1",
+    "args": "use_accelerate=True,device_map_option=auto,dtype=bfloat16,load_in_8bit=True"
+  },
+  "orca-2-13b": {
+    "model_name": "microsoft/Orca-2-13b",
+    "args": "use_accelerate=True,device_map_option=auto,dtype=bfloat16,load_in_8bit=True"
+  },
+  "orca-2-7b": {
+    "model_name": "microsoft/Orca-2-7b",
+    "args": "use_accelerate=True,device_map_option=auto,dtype=bfloat16,load_in_8bit=True"
+  },
+  "zephyr-7b-beta": {
+    "model_name": "HuggingFaceH4/zephyr-7b-beta",
+    "args": "use_accelerate=True,device_map_option=auto,dtype=bfloat16,load_in_8bit=True"
+  }
 }

generate_overview_json.py

@@ -0,0 +1,40 @@
+from pathlib import Path
+import json
+from pprint import pprint
+
+
+def main():
+    results = {}
+    for pfin in Path(__file__).parent.joinpath("evals").rglob("*.json"):
+        if pfin.stem == "models":
+            continue
+        short_name = pfin.stem.split("_")[2]
+        if short_name not in results:
+            results[short_name] = {}
+
+        data = json.loads(pfin.read_text(encoding="utf-8"))
+        if "config" not in data:
+            continue
+
+        config = data["config"]
+        if "model_args" not in config:
+            continue
+
+        model_args = dict(params.split("=") for params in config["model_args"].split(","))
+        if "pretrained" not in model_args:
+            continue
+
+        results[short_name]["model_name"] = model_args["pretrained"]
+        results[short_name]["compute_dtype"] = model_args.get("dtype", None)
+        results[short_name]["quantization"] = None
+        if "load_in_8bit" in model_args:
+            results[short_name]["quantization"] = "8-bit"
+        elif "load_in_4bit" in model_args:
+            results[short_name]["quantization"] = "4-bit"
+
+
+    pprint(results)
+
+
+if __name__ == '__main__':
+    main()