Crystalcareai
/

GemMoE-Beta-1

Text Generation

Model card Files Files and versions Community

Crystalcareai commited on Mar 13

Commit

e71515f

•

1 Parent(s): d678a70

Update config.json

Files changed (1) hide show

config.json +2 -2

config.json CHANGED Viewed

@@ -1,7 +1,7 @@
 {
   "_name_or_path": "Crystalcareai/GemMoE-Beta",
   "architectures": [
-    "GemmaForCausalLM"
   ],
   "attention_bias": false,
   "attention_dropout": 0.0,
@@ -13,7 +13,7 @@
   "initializer_range": 0.02,
   "intermediate_size": 24576,
   "max_position_embeddings": 8192,
-  "model_type": "gemma",
   "num_attention_heads": 16,
   "num_experts_per_tok": 2,
   "num_hidden_layers": 28,

 {
   "_name_or_path": "Crystalcareai/GemMoE-Beta",
   "architectures": [
+    "GemmoeForCausalLM"
   ],
   "attention_bias": false,
   "attention_dropout": 0.0,
   "initializer_range": 0.02,
   "intermediate_size": 24576,
   "max_position_embeddings": 8192,
+  "model_type": "gemmoe",
   "num_attention_heads": 16,
   "num_experts_per_tok": 2,
   "num_hidden_layers": 28,