nroggendorff commited on
Commit
27f08e4
·
verified ·
1 Parent(s): 8b23c80

Training loss: 1.9723

Browse files
Files changed (2) hide show
  1. config.json +6 -6
  2. model.safetensors +2 -2
config.json CHANGED
@@ -8,18 +8,18 @@
8
  "bos_token_id": 0,
9
  "eos_token_id": 2,
10
  "hidden_act": "silu",
11
- "hidden_size": 660,
12
  "initializer_range": 0.02,
13
- "intermediate_size": 2640,
14
  "max_position_embeddings": 128,
15
  "mlp_bias": false,
16
  "model_type": "llama",
17
- "num_attention_heads": 10,
18
- "num_hidden_layers": 20,
19
- "num_key_value_heads": 10,
20
  "pad_token_id": 1,
21
  "pretraining_tp": 1,
22
- "rms_norm_eps": 1e-06,
23
  "rope_scaling": null,
24
  "rope_theta": 10000.0,
25
  "tie_word_embeddings": false,
 
8
  "bos_token_id": 0,
9
  "eos_token_id": 2,
10
  "hidden_act": "silu",
11
+ "hidden_size": 576,
12
  "initializer_range": 0.02,
13
+ "intermediate_size": 2304,
14
  "max_position_embeddings": 128,
15
  "mlp_bias": false,
16
  "model_type": "llama",
17
+ "num_attention_heads": 12,
18
+ "num_hidden_layers": 12,
19
+ "num_key_value_heads": 12,
20
  "pad_token_id": 1,
21
  "pretraining_tp": 1,
22
+ "rms_norm_eps": 1e-05,
23
  "rope_scaling": null,
24
  "rope_theta": 10000.0,
25
  "tie_word_embeddings": false,
model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:c60e384df37cde5ab0a49601666823b0b17f01f69767171300bea47141368311
3
- size 726672464
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:056d05bc279b5d845bd5ed65449e3c5dfb608cea55932b0d6ca304a9db9d3135
3
+ size 402343712