Nitral commited on
Commit
1a843bf
1 Parent(s): afce0a5

Update config.json

Browse files
Files changed (1) hide show
  1. config.json +14 -5
config.json CHANGED
@@ -3,24 +3,33 @@
3
  "architectures": [
4
  "MistralForCausalLM"
5
  ],
6
- "attention_dropout": 0.0,
 
 
 
7
  "bos_token_id": 1,
8
  "eos_token_id": 2,
9
  "hidden_act": "silu",
10
- "hidden_size": 4096,
11
  "initializer_range": 0.02,
12
  "intermediate_size": 14336,
13
- "max_position_embeddings": 8192,
 
14
  "model_type": "mistral",
15
  "num_attention_heads": 32,
16
  "num_hidden_layers": 32,
17
  "num_key_value_heads": 8,
18
  "rms_norm_eps": 1e-05,
 
 
 
 
 
 
19
  "rope_theta": 10000.0,
20
- "sliding_window": 4096,
21
  "tie_word_embeddings": false,
22
  "torch_dtype": "bfloat16",
23
- "transformers_version": "4.35.2",
24
  "use_cache": true,
25
  "vocab_size": 32000
26
  }
 
3
  "architectures": [
4
  "MistralForCausalLM"
5
  ],
6
+ "auto_map": {
7
+ "AutoConfig": "configuration_mistral.MistralConfig",
8
+ "AutoModelForCausalLM": "modeling_mistral_yarn.MistralForCausalLM"
9
+ },
10
  "bos_token_id": 1,
11
  "eos_token_id": 2,
12
  "hidden_act": "silu",
 
13
  "initializer_range": 0.02,
14
  "intermediate_size": 14336,
15
+ "max_position_embeddings": 32768,
16
+ "max_sequence_length": 131072,
17
  "model_type": "mistral",
18
  "num_attention_heads": 32,
19
  "num_hidden_layers": 32,
20
  "num_key_value_heads": 8,
21
  "rms_norm_eps": 1e-05,
22
+ "rope_scaling": {
23
+ "factor": 16.0,
24
+ "finetuned": true,
25
+ "original_max_position_embeddings": 8192,
26
+ "type": "yarn"
27
+ },
28
  "rope_theta": 10000.0,
29
+ "sliding_window": 131072,
30
  "tie_word_embeddings": false,
31
  "torch_dtype": "bfloat16",
32
+ "transformers_version": "4.35.0.dev0",
33
  "use_cache": true,
34
  "vocab_size": 32000
35
  }