add geitje

evals/arc/arc_nl_GEITje-7B-chat.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_nl": {
+      "acc": 0.41488451668092385,
+      "acc_stderr": 0.014416604608618254,
+      "acc_norm": 0.42001710863986313,
+      "acc_norm_stderr": 0.014441744604939771
+    }
+  },
+  "versions": {
+    "arc_nl": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=Rijgersberg/GEITje-7B-chat,use_accelerate=True,device_map_option=auto,dtype=auto,load_in_8bit=True",
+    "batch_size": 4,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/arc/arc_nl_GEITje-7B.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "arc_nl": {
+      "acc": 0.3857998289136014,
+      "acc_stderr": 0.014243429178521566,
+      "acc_norm": 0.38152266894781867,
+      "acc_norm_stderr": 0.014213487288956106
+    }
+  },
+  "versions": {
+    "arc_nl": 0
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=Rijgersberg/GEITje-7B,use_accelerate=True,device_map_option=auto,dtype=auto,load_in_8bit=True",
+    "batch_size": 4,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/hellaswag/hellaswag_nl_GEITje-7B-chat.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "hellaswag_nl": {
+      "acc": 0.5085806799784134,
+      "acc_stderr": 0.005194057692409769,
+      "acc_norm": 0.6658391797085806,
+      "acc_norm_stderr": 0.004900756947736894
+    }
+  },
+  "versions": {
+    "hellaswag_nl": 1
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=Rijgersberg/GEITje-7B-chat,use_accelerate=True,device_map_option=auto,dtype=auto,load_in_8bit=True",
+    "batch_size": 4,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/hellaswag/hellaswag_nl_GEITje-7B.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "hellaswag_nl": {
+      "acc": 0.4991905018888289,
+      "acc_stderr": 0.005194815910065167,
+      "acc_norm": 0.6545062061521857,
+      "acc_norm_stderr": 0.004940577731224625
+    }
+  },
+  "versions": {
+    "hellaswag_nl": 1
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=Rijgersberg/GEITje-7B,use_accelerate=True,device_map_option=auto,dtype=auto,load_in_8bit=True",
+    "batch_size": 4,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/models.json

@@ -1,4 +1,20 @@
 {
+    "geitje-7b": {
+        "compute_dtype": "auto",
+        "dutch_coverage": "fine-tuned",
+        "model_name": "Rijgersberg/GEITje-7B",
+        "model_type": "fine-tuned",
+        "num_parameters": 7241732096,
+        "quantization": "8-bit"
+    },
+    "geitje-7b-chat": {
+        "compute_dtype": "auto",
+        "dutch_coverage": "fine-tuned",
+        "model_name": "Rijgersberg/GEITje-7B-chat",
+        "model_type": "instruction-tuned",
+        "num_parameters": 7241732096,
+        "quantization": "8-bit"
+    },
     "gpt-neo-1.3b-dutch": {
         "compute_dtype": "bfloat16",
         "dutch_coverage": "pretrained",
@@ -93,7 +109,7 @@
         "model_name": "mistralai/Mixtral-8x7B-v0.1",
         "model_type": "not-given",
         "num_parameters": 46702792704,
-        "quantization": null
+        "quantization": "8-bit"
     },
     "neural-chat-7b-v3-1": {
         "compute_dtype": "bfloat16",
@@ -127,4 +143,4 @@
         "num_parameters": 7241732096,
         "quantization": "8-bit"
     }
-}
+}

evals/truthfulqa/truthfulqa_nl_GEITje-7B-chat.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "truthfulqa_nl": {
+      "mc1": 0.2802547770700637,
+      "mc1_stderr": 0.016040129721386853,
+      "mc2": 0.46430395880030356,
+      "mc2_stderr": 0.015383562770588449
+    }
+  },
+  "versions": {
+    "truthfulqa_nl": 1
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=Rijgersberg/GEITje-7B-chat,use_accelerate=True,device_map_option=auto,dtype=auto,load_in_8bit=True",
+    "batch_size": 4,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/truthfulqa/truthfulqa_nl_GEITje-7B.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "truthfulqa_nl": {
+      "mc1": 0.27388535031847133,
+      "mc1_stderr": 0.01592681625497211,
+      "mc2": 0.4321017012137338,
+      "mc2_stderr": 0.01473465985240878
+    }
+  },
+  "versions": {
+    "truthfulqa_nl": 1
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=Rijgersberg/GEITje-7B,use_accelerate=True,device_map_option=auto,dtype=auto,load_in_8bit=True",
+    "batch_size": 4,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}

evals/truthfulqa/truthfulqa_nl_Mixtral-8x7B-v0.1.json

@@ -0,0 +1,23 @@
+{
+  "results": {
+    "truthfulqa_nl": {
+      "mc1": 0.310828025477707,
+      "mc1_stderr": 0.016529733724696277,
+      "mc2": 0.4629846929417042,
+      "mc2_stderr": 0.01508660067127546
+    }
+  },
+  "versions": {
+    "truthfulqa_nl": 1
+  },
+  "config": {
+    "model": "hf-auto",
+    "model_args": "pretrained=mistralai/Mixtral-8x7B-v0.1,use_accelerate=True,device_map_option=auto,dtype=auto,load_in_8bit=True",
+    "batch_size": 2,
+    "device": "cuda",
+    "no_cache": false,
+    "limit": null,
+    "bootstrap_iters": 100000,
+    "description_dict": {}
+  }
+}