Update configuration_phi.py

configuration_phi.py

@@ -116,13 +116,13 @@ class PhiConfig(PretrainedConfig):
 
     def __init__(
         self,
-        vocab_size=51200,
-        hidden_size=2048,
-        intermediate_size=8192,
-        num_hidden_layers=24,
+        vocab_size=50295,
+        hidden_size=2560,
+        intermediate_size=10240,
+        num_hidden_layers=32,
         num_attention_heads=32,
-        num_key_value_heads=None,
-        resid_pdrop=0.0,
+        num_key_value_heads=32,
+        resid_pdrop=0.1,
         embd_pdrop=0.0,
         attention_dropout=0.0,
         hidden_act="gelu_new",
@@ -133,10 +133,10 @@ class PhiConfig(PretrainedConfig):
         tie_word_embeddings=False,
         rope_theta=10000.0,
         rope_scaling=None,
-        partial_rotary_factor=0.5,
+        partial_rotary_factor=0.4,
         qk_layernorm=False,
-        bos_token_id=1,
-        eos_token_id=2,
+        bos_token_id=0,
+        eos_token_id=0,
         **kwargs,
     ):
         self.vocab_size = vocab_size