ddh0
/

OrcaMaid-v2-FIX-13b-32k

Text Generation

Inference Endpoints

text-generation-inference

Model card Files Files and versions Community

ddh0 commited on Dec 20, 2023

Commit

c0e63be

•

1 Parent(s): a281267

Update context length

Files changed (1) hide show

configuration_llama.py +2 -2

configuration_llama.py CHANGED Viewed

@@ -114,7 +114,7 @@ class LlamaConfig(PretrainedConfig):
         num_attention_heads=40,
         num_key_value_heads=40,
         hidden_act="silu",
-        max_position_embeddings=10240,
         initializer_range=0.02,
         rms_norm_eps=1e-05,
         use_cache=True,
@@ -124,7 +124,7 @@ class LlamaConfig(PretrainedConfig):
         pretraining_tp=1,
         tie_word_embeddings=False,
         rope_theta=10000,
-        rope_scaling="yarn",
         attention_bias=False,
         **kwargs,
     ):

         num_attention_heads=40,
         num_key_value_heads=40,
         hidden_act="silu",
+        max_position_embeddings=32768,
         initializer_range=0.02,
         rms_norm_eps=1e-05,
         use_cache=True,
         pretraining_tp=1,
         tie_word_embeddings=False,
         rope_theta=10000,
+        rope_scaling=None,
         attention_bias=False,
         **kwargs,
     ):