Update configuration_hf_nomic_bert.py

configuration_hf_nomic_bert.py

@@ -4,7 +4,8 @@ from transformers import GPT2Config
 class NomicBertConfig(GPT2Config):
     model_type = "nomic_bert"
 
-    def __init__(self,
+    def __init__(
+        self,
         prenorm=False,
         parallel_block=False,
         parallel_block_tied_norm=False,
@@ -14,7 +15,7 @@ class NomicBertConfig(GPT2Config):
         use_flash_attn=False,
         use_xentropy=False,
         qkv_proj_bias=True,
-        rotary_emb_base=1000,
+        rotary_emb_base=10_000,
         rotary_emb_scale_base=None,
         rotary_emb_interleaved=False,
         mlp_fc1_bias=True,
@@ -25,6 +26,8 @@ class NomicBertConfig(GPT2Config):
         dense_seq_output=True,
         pad_vocab_size_multiple=1,
         tie_word_embeddings=True,
+        rotary_scaling_factor=1.0,
+        max_trained_positions=2048,
         **kwargs,
     ):
         self.prenorm = prenorm
@@ -47,5 +50,7 @@ class NomicBertConfig(GPT2Config):
         self.type_vocab_size = type_vocab_size
         self.dense_seq_output = dense_seq_output
         self.pad_vocab_size_multiple = pad_vocab_size_multiple
+        self.rotary_scaling_factor = rotary_scaling_factor
+        self.max_trained_positions = max_trained_positions
 
-        super().__init__(**kwargs)
+        super().__init__(**kwargs)