Update config.json

config.json

@@ -1,19 +1,18 @@
 {
-  "_name_or_path": "./mergekit/merged",
+  "_name_or_path": "Crystalcareai/GemMoE-Beta-1",
   "architectures": [
     "GemmoeForCausalLM"
   ],
   "attention_bias": false,
   "attention_dropout": 0.0,
   "auto_map": {
-    "AutoConfig": "configuration_gemmoe.GemmoeConfig",
-    "AutoModelForCausalLM": "modeling_gemmoe.GemmoeForCausalLM"
+    "AutoConfig": "Crystalcareai/GemMoE-Medium-v0.4--configuration_gemmoe.GemmoeConfig",
+    "AutoModelForCausalLM": "Crystalcareai/GemMoE-Medium-v0.4--modeling_gemmoe.GemmoeForCausalLM"
   },
   "bos_token_id": 2,
   "eos_token_id": 1,
   "head_dim": 256,
   "hidden_act": "gelu",
-  "hidden_activation": null,
   "hidden_size": 3072,
   "initializer_range": 0.02,
   "intermediate_size": 24576,
@@ -31,8 +30,8 @@
   "rope_theta": 10000.0,
   "router_aux_loss_coef": 0.02,
   "sliding_window": null,
-  "torch_dtype": "float16",
-  "transformers_version": "4.39.0.dev0",
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.38.2",
   "use_cache": false,
   "vocab_size": 256000
 }