Update cognitivess_model/configuration_cognitivess.py

cognitivess_model/configuration_cognitivess.py

@@ -1,48 +1,33 @@
-from transformers import LlamaConfig
+# cognitivess_model/configuration_cognitivess.py
 
-class CognitivessConfig(LlamaConfig):
+from transformers import PretrainedConfig
+
+class CognitivessConfig(PretrainedConfig):
     model_type = "cognitivess"
 
-    def __init__(
-        self,
-        vocab_size=128256,
-        hidden_size=4096,
-        intermediate_size=14336,
-        num_hidden_layers=32,
-        num_attention_heads=32,
-        num_key_value_heads=8,
-        hidden_act="silu",
-        max_position_embeddings=8192,
-        initializer_range=0.02,
-        rms_norm_eps=1e-5,
-        use_cache=True,
-        pad_token_id=0,
-        bos_token_id=128000,
-        eos_token_id=128001,
-        tie_word_embeddings=False,
-        attention_dropout=0.0,
-        pretraining_tp=1,
-        rope_theta=500000.0,
-        **kwargs
-    ):
-        super().__init__(
-            vocab_size=vocab_size,
-            hidden_size=hidden_size,
-            intermediate_size=intermediate_size,
-            num_hidden_layers=num_hidden_layers,
-            num_attention_heads=num_attention_heads,
-            num_key_value_heads=num_key_value_heads,
-            hidden_act=hidden_act,
-            max_position_embeddings=max_position_embeddings,
-            initializer_range=initializer_range,
-            rms_norm_eps=rms_norm_eps,
-            use_cache=use_cache,
-            pad_token_id=pad_token_id,
-            bos_token_id=bos_token_id,
-            eos_token_id=eos_token_id,
-            tie_word_embeddings=tie_word_embeddings,
-            pretraining_tp=pretraining_tp,
-            rope_theta=rope_theta,
-            **kwargs
-        )
+    def __init__(self, hidden_size=4096, num_hidden_layers=32, num_attention_heads=32, intermediate_size=14336,
+                 hidden_act="silu", layer_norm_eps=1e-05, max_position_embeddings=8192, vocab_size=128256,
+                 bos_token_id=128000, eos_token_id=128001, pad_token_id=0, attention_dropout=0.0, 
+                 attention_bias=False, tie_word_embeddings=False, mlp_bias=False, pretraining_tp=1, 
+                 rope_scaling=None, rope_theta=500000.0, num_key_value_heads=8, use_cache=True, **kwargs):
+        super().__init__(**kwargs)
+        self.hidden_size = hidden_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.intermediate_size = intermediate_size
+        self.hidden_act = hidden_act
+        self.layer_norm_eps = layer_norm_eps
+        self.max_position_embeddings = max_position_embeddings
+        self.vocab_size = vocab_size
+        self.bos_token_id = bos_token_id
+        self.eos_token_id = eos_token_id
+        self.pad_token_id = pad_token_id
         self.attention_dropout = attention_dropout
+        self.attention_bias = attention_bias
+        self.tie_word_embeddings = tie_word_embeddings
+        self.mlp_bias = mlp_bias
+        self.pretraining_tp = pretraining_tp
+        self.rope_scaling = rope_scaling
+        self.rope_theta = rope_theta
+        self.num_key_value_heads = num_key_value_heads
+        self.use_cache = use_cache