Hugging Face's logo

Nekochu
/
Llama-2-13B-fp16-french

Text Generation
Transformers
PyTorch
GGUF
French
English
llama
Generated from Trainer
text-generation-inference
Inference Endpoints
Model card Files Community
2
Nekochu commited on
Commit
719771a
•
1 Parent(s): b0b36d0

QLoRA merge

Browse files
Files changed (20) hide show
  1. Llama-2-13B-fp16FR-llama7b-4bit-128g.safetensors +3 -0
  2. QLoRA/README.md +20 -0
  3. QLoRA/adapter_config.json +21 -0
  4. QLoRA/adapter_model.bin +3 -0
  5. QLoRA/all_results.json +7 -0
  6. QLoRA/finetuning_args.json +13 -0
  7. QLoRA/train_results.json +7 -0
  8. QLoRA/trainer_log.jsonl +71 -0
  9. QLoRA/trainer_state.json +445 -0
  10. QLoRA/training_args.bin +3 -0
  11. README.md +15 -0
  12. config.json +27 -0
  13. generation_config.json +8 -0
  14. pytorch_model-00001-of-00003.bin +3 -0
  15. pytorch_model-00002-of-00003.bin +3 -0
  16. pytorch_model-00003-of-00003.bin +3 -0
  17. pytorch_model.bin.index.json +410 -0
  18. special_tokens_map.json +24 -0
  19. tokenizer.model +3 -0
  20. tokenizer_config.json +35 -0
Llama-2-13B-fp16FR-llama7b-4bit-128g.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f7bbc67c4f01fdb744dffbafc23d37d226818ba9713ac4e8dad6b7f9c9fbb24c
3
+ size 7255144832
QLoRA/README.md ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ ---
4
+ ## Training procedure
5
+
6
+
7
+ The following `bitsandbytes` quantization config was used during training:
8
+ - load_in_8bit: False
9
+ - load_in_4bit: True
10
+ - llm_int8_threshold: 6.0
11
+ - llm_int8_skip_modules: None
12
+ - llm_int8_enable_fp32_cpu_offload: False
13
+ - llm_int8_has_fp16_weight: False
14
+ - bnb_4bit_quant_type: nf4
15
+ - bnb_4bit_use_double_quant: True
16
+ - bnb_4bit_compute_dtype: float16
17
+ ### Framework versions
18
+
19
+
20
+ - PEFT 0.4.0
QLoRA/adapter_config.json ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "auto_mapping": null,
3
+ "base_model_name_or_path": "Llama-2-13B-Chat-fp16",
4
+ "bias": "none",
5
+ "fan_in_fan_out": false,
6
+ "inference_mode": true,
7
+ "init_lora_weights": true,
8
+ "layers_pattern": null,
9
+ "layers_to_transform": null,
10
+ "lora_alpha": 32.0,
11
+ "lora_dropout": 0,
12
+ "modules_to_save": null,
13
+ "peft_type": "LORA",
14
+ "r": 8,
15
+ "revision": null,
16
+ "target_modules": [
17
+ "q_proj",
18
+ "v_proj"
19
+ ],
20
+ "task_type": "CAUSAL_LM"
21
+ }
QLoRA/adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:17a593b81d12032525cb31b1035d73ea7266226bcdb3a15fcecf7a08cc68db14
3
+ size 26269517
QLoRA/all_results.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 2.98,
3
+ "train_loss": 3.3144316109496983,
4
+ "train_runtime": 2560.536,
5
+ "train_samples_per_second": 2.207,
6
+ "train_steps_per_second": 0.137
7
+ }
QLoRA/finetuning_args.json ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "finetuning_type": "lora",
3
+ "lora_alpha": 32.0,
4
+ "lora_dropout": 0,
5
+ "lora_rank": 8,
6
+ "lora_target": [
7
+ "q_proj",
8
+ "v_proj"
9
+ ],
10
+ "name_module_trainable": "mlp",
11
+ "num_hidden_layers": 32,
12
+ "num_layer_trainable": 3
13
+ }
QLoRA/train_results.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 2.98,
3
+ "train_loss": 3.3144316109496983,
4
+ "train_runtime": 2560.536,
5
+ "train_samples_per_second": 2.207,
6
+ "train_steps_per_second": 0.137
7
+ }
QLoRA/trainer_log.jsonl ADDED
@@ -0,0 +1,71 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {"current_steps": 5, "total_steps": 351, "loss": 4.067, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.997496987311431e-05, "epoch": 0.04, "percentage": 1.42, "elapsed_time": "0:00:58", "remaining_time": "1:07:26"}
2
+ {"current_steps": 10, "total_steps": 351, "loss": 4.1952, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.989992961303738e-05, "epoch": 0.08, "percentage": 2.85, "elapsed_time": "0:01:37", "remaining_time": "0:55:21"}
3
+ {"current_steps": 15, "total_steps": 351, "loss": 3.8727, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.977502948114772e-05, "epoch": 0.13, "percentage": 4.27, "elapsed_time": "0:02:13", "remaining_time": "0:49:41"}
4
+ {"current_steps": 20, "total_steps": 351, "loss": 3.894, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.960051957873725e-05, "epoch": 0.17, "percentage": 5.7, "elapsed_time": "0:02:49", "remaining_time": "0:46:49"}
5
+ {"current_steps": 25, "total_steps": 351, "loss": 3.7174, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.9376749346206006e-05, "epoch": 0.21, "percentage": 7.12, "elapsed_time": "0:03:23", "remaining_time": "0:44:19"}
6
+ {"current_steps": 30, "total_steps": 351, "loss": 3.7136, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.910416686333906e-05, "epoch": 0.25, "percentage": 8.55, "elapsed_time": "0:03:59", "remaining_time": "0:42:38"}
7
+ {"current_steps": 35, "total_steps": 351, "loss": 3.4754, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.878331795206705e-05, "epoch": 0.3, "percentage": 9.97, "elapsed_time": "0:04:34", "remaining_time": "0:41:21"}
8
+ {"current_steps": 40, "total_steps": 351, "loss": 3.6711, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.841484508350679e-05, "epoch": 0.34, "percentage": 11.4, "elapsed_time": "0:05:07", "remaining_time": "0:39:50"}
9
+ {"current_steps": 45, "total_steps": 351, "loss": 3.4796, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.799948609147061e-05, "epoch": 0.38, "percentage": 12.82, "elapsed_time": "0:05:40", "remaining_time": "0:38:36"}
10
+ {"current_steps": 50, "total_steps": 351, "loss": 3.4441, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.753807269502041e-05, "epoch": 0.42, "percentage": 14.25, "elapsed_time": "0:06:20", "remaining_time": "0:38:11"}
11
+ {"current_steps": 55, "total_steps": 351, "loss": 3.603, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.7031528833024976e-05, "epoch": 0.47, "percentage": 15.67, "elapsed_time": "0:06:55", "remaining_time": "0:37:17"}
12
+ {"current_steps": 60, "total_steps": 351, "loss": 3.4614, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.6480868814055424e-05, "epoch": 0.51, "percentage": 17.09, "elapsed_time": "0:07:31", "remaining_time": "0:36:30"}
13
+ {"current_steps": 65, "total_steps": 351, "loss": 3.4613, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.588719528532342e-05, "epoch": 0.55, "percentage": 18.52, "elapsed_time": "0:08:09", "remaining_time": "0:35:55"}
14
+ {"current_steps": 70, "total_steps": 351, "loss": 3.55, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.5251697024729165e-05, "epoch": 0.59, "percentage": 19.94, "elapsed_time": "0:08:46", "remaining_time": "0:35:12"}
15
+ {"current_steps": 75, "total_steps": 351, "loss": 3.3352, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.457564656044056e-05, "epoch": 0.64, "percentage": 21.37, "elapsed_time": "0:09:21", "remaining_time": "0:34:26"}
16
+ {"current_steps": 80, "total_steps": 351, "loss": 3.3781, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.3860397622769756e-05, "epoch": 0.68, "percentage": 22.79, "elapsed_time": "0:09:57", "remaining_time": "0:33:44"}
17
+ {"current_steps": 85, "total_steps": 351, "loss": 3.3805, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.310738243344996e-05, "epoch": 0.72, "percentage": 24.22, "elapsed_time": "0:10:34", "remaining_time": "0:33:07"}
18
+ {"current_steps": 90, "total_steps": 351, "loss": 3.2995, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.231810883773999e-05, "epoch": 0.76, "percentage": 25.64, "elapsed_time": "0:11:13", "remaining_time": "0:32:32"}
19
+ {"current_steps": 95, "total_steps": 351, "loss": 3.1648, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.149415728509971e-05, "epoch": 0.81, "percentage": 27.07, "elapsed_time": "0:11:49", "remaining_time": "0:31:51"}
20
+ {"current_steps": 100, "total_steps": 351, "loss": 3.3295, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.063717766448194e-05, "epoch": 0.85, "percentage": 28.49, "elapsed_time": "0:12:23", "remaining_time": "0:31:05"}
21
+ {"current_steps": 105, "total_steps": 351, "loss": 3.2847, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.974888600057808e-05, "epoch": 0.89, "percentage": 29.91, "elapsed_time": "0:13:00", "remaining_time": "0:30:28"}
22
+ {"current_steps": 110, "total_steps": 351, "loss": 3.25, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.883106101763285e-05, "epoch": 0.93, "percentage": 31.34, "elapsed_time": "0:13:37", "remaining_time": "0:29:51"}
23
+ {"current_steps": 115, "total_steps": 351, "loss": 3.2399, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.7885540577708804e-05, "epoch": 0.98, "percentage": 32.76, "elapsed_time": "0:14:13", "remaining_time": "0:29:11"}
24
+ {"current_steps": 120, "total_steps": 351, "loss": 3.2125, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.69142180005327e-05, "epoch": 1.02, "percentage": 34.19, "elapsed_time": "0:14:50", "remaining_time": "0:28:33"}
25
+ {"current_steps": 125, "total_steps": 351, "loss": 3.3815, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.591903827229282e-05, "epoch": 1.06, "percentage": 35.61, "elapsed_time": "0:15:26", "remaining_time": "0:27:55"}
26
+ {"current_steps": 130, "total_steps": 351, "loss": 3.2365, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.490199415097892e-05, "epoch": 1.1, "percentage": 37.04, "elapsed_time": "0:16:04", "remaining_time": "0:27:20"}
27
+ {"current_steps": 135, "total_steps": 351, "loss": 3.2755, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.386512217606339e-05, "epoch": 1.15, "percentage": 38.46, "elapsed_time": "0:16:38", "remaining_time": "0:26:38"}
28
+ {"current_steps": 140, "total_steps": 351, "loss": 3.17, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.281049859051394e-05, "epoch": 1.19, "percentage": 39.89, "elapsed_time": "0:17:15", "remaining_time": "0:26:00"}
29
+ {"current_steps": 145, "total_steps": 351, "loss": 3.1125, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.17402351833036e-05, "epoch": 1.23, "percentage": 41.31, "elapsed_time": "0:17:53", "remaining_time": "0:25:25"}
30
+ {"current_steps": 150, "total_steps": 351, "loss": 3.3456, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.0656475060743055e-05, "epoch": 1.27, "percentage": 42.74, "elapsed_time": "0:18:30", "remaining_time": "0:24:48"}
31
+ {"current_steps": 155, "total_steps": 351, "loss": 3.294, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.956138835510282e-05, "epoch": 1.32, "percentage": 44.16, "elapsed_time": "0:19:06", "remaining_time": "0:24:10"}
32
+ {"current_steps": 160, "total_steps": 351, "loss": 3.1764, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.845716787911833e-05, "epoch": 1.36, "percentage": 45.58, "elapsed_time": "0:19:43", "remaining_time": "0:23:32"}
33
+ {"current_steps": 165, "total_steps": 351, "loss": 3.3249, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.7346024735079486e-05, "epoch": 1.4, "percentage": 47.01, "elapsed_time": "0:20:20", "remaining_time": "0:22:55"}
34
+ {"current_steps": 170, "total_steps": 351, "loss": 3.1499, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.6230183887296955e-05, "epoch": 1.44, "percentage": 48.43, "elapsed_time": "0:20:57", "remaining_time": "0:22:18"}
35
+ {"current_steps": 175, "total_steps": 351, "loss": 3.1101, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.5111879706811087e-05, "epoch": 1.49, "percentage": 49.86, "elapsed_time": "0:21:31", "remaining_time": "0:21:39"}
36
+ {"current_steps": 180, "total_steps": 351, "loss": 3.266, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.399335149726463e-05, "epoch": 1.53, "percentage": 51.28, "elapsed_time": "0:22:06", "remaining_time": "0:20:59"}
37
+ {"current_steps": 185, "total_steps": 351, "loss": 3.2329, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.2876839010898377e-05, "epoch": 1.57, "percentage": 52.71, "elapsed_time": "0:22:44", "remaining_time": "0:20:24"}
38
+ {"current_steps": 190, "total_steps": 351, "loss": 3.2465, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.1764577963648614e-05, "epoch": 1.61, "percentage": 54.13, "elapsed_time": "0:23:21", "remaining_time": "0:19:47"}
39
+ {"current_steps": 195, "total_steps": 351, "loss": 3.2541, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.0658795558326743e-05, "epoch": 1.66, "percentage": 55.56, "elapsed_time": "0:23:57", "remaining_time": "0:19:10"}
40
+ {"current_steps": 200, "total_steps": 351, "loss": 3.3382, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.956170602484582e-05, "epoch": 1.7, "percentage": 56.98, "elapsed_time": "0:24:33", "remaining_time": "0:18:32"}
41
+ {"current_steps": 205, "total_steps": 351, "loss": 3.3049, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.8475506186424074e-05, "epoch": 1.74, "percentage": 58.4, "elapsed_time": "0:25:08", "remaining_time": "0:17:54"}
42
+ {"current_steps": 210, "total_steps": 351, "loss": 3.048, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.740237106064383e-05, "epoch": 1.78, "percentage": 59.83, "elapsed_time": "0:25:46", "remaining_time": "0:17:18"}
43
+ {"current_steps": 215, "total_steps": 351, "loss": 3.2414, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.6344449504174193e-05, "epoch": 1.83, "percentage": 61.25, "elapsed_time": "0:26:21", "remaining_time": "0:16:40"}
44
+ {"current_steps": 220, "total_steps": 351, "loss": 3.291, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.530385990987863e-05, "epoch": 1.87, "percentage": 62.68, "elapsed_time": "0:26:57", "remaining_time": "0:16:03"}
45
+ {"current_steps": 225, "total_steps": 351, "loss": 3.2234, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.4282685964923642e-05, "epoch": 1.91, "percentage": 64.1, "elapsed_time": "0:27:32", "remaining_time": "0:15:25"}
46
+ {"current_steps": 230, "total_steps": 351, "loss": 3.3281, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.328297247838241e-05, "epoch": 1.95, "percentage": 65.53, "elapsed_time": "0:28:04", "remaining_time": "0:14:46"}
47
+ {"current_steps": 235, "total_steps": 351, "loss": 3.0808, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.230672128668831e-05, "epoch": 2.0, "percentage": 66.95, "elapsed_time": "0:28:40", "remaining_time": "0:14:09"}
48
+ {"current_steps": 240, "total_steps": 351, "loss": 3.0843, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.1355887245137383e-05, "epoch": 2.04, "percentage": 68.38, "elapsed_time": "0:29:16", "remaining_time": "0:13:32"}
49
+ {"current_steps": 245, "total_steps": 351, "loss": 3.263, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.043237431346622e-05, "epoch": 2.08, "percentage": 69.8, "elapsed_time": "0:29:52", "remaining_time": "0:12:55"}
50
+ {"current_steps": 250, "total_steps": 351, "loss": 3.0803, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 9.538031743343628e-06, "epoch": 2.12, "percentage": 71.23, "elapsed_time": "0:30:28", "remaining_time": "0:12:18"}
51
+ {"current_steps": 255, "total_steps": 351, "loss": 3.1773, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 8.67465037541038e-06, "epoch": 2.17, "percentage": 72.65, "elapsed_time": "0:31:03", "remaining_time": "0:11:41"}
52
+ {"current_steps": 260, "total_steps": 351, "loss": 3.2543, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 7.843959053281663e-06, "epoch": 2.21, "percentage": 74.07, "elapsed_time": "0:31:37", "remaining_time": "0:11:04"}
53
+ {"current_steps": 265, "total_steps": 351, "loss": 3.1697, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 7.047621161693152e-06, "epoch": 2.25, "percentage": 75.5, "elapsed_time": "0:32:13", "remaining_time": "0:10:27"}
54
+ {"current_steps": 270, "total_steps": 351, "loss": 3.2204, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 6.28723129572247e-06, "epoch": 2.29, "percentage": 76.92, "elapsed_time": "0:32:52", "remaining_time": "0:09:51"}
55
+ {"current_steps": 275, "total_steps": 351, "loss": 3.1626, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 5.564312067755856e-06, "epoch": 2.34, "percentage": 78.35, "elapsed_time": "0:33:26", "remaining_time": "0:09:14"}
56
+ {"current_steps": 280, "total_steps": 351, "loss": 3.2122, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.880311058593617e-06, "epoch": 2.38, "percentage": 79.77, "elapsed_time": "0:34:02", "remaining_time": "0:08:37"}
57
+ {"current_steps": 285, "total_steps": 351, "loss": 3.2038, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.236597918799706e-06, "epoch": 2.42, "percentage": 81.2, "elapsed_time": "0:34:39", "remaining_time": "0:08:01"}
58
+ {"current_steps": 290, "total_steps": 351, "loss": 3.0183, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.6344616260994946e-06, "epoch": 2.46, "percentage": 82.62, "elapsed_time": "0:35:16", "remaining_time": "0:07:25"}
59
+ {"current_steps": 295, "total_steps": 351, "loss": 3.3333, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.07510790431767e-06, "epoch": 2.51, "percentage": 84.05, "elapsed_time": "0:35:51", "remaining_time": "0:06:48"}
60
+ {"current_steps": 300, "total_steps": 351, "loss": 3.1501, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.5596568090246548e-06, "epoch": 2.55, "percentage": 85.47, "elapsed_time": "0:36:29", "remaining_time": "0:06:12"}
61
+ {"current_steps": 305, "total_steps": 351, "loss": 3.2576, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.0891404847259267e-06, "epoch": 2.59, "percentage": 86.89, "elapsed_time": "0:37:06", "remaining_time": "0:05:35"}
62
+ {"current_steps": 310, "total_steps": 351, "loss": 3.2306, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.664501098085408e-06, "epoch": 2.63, "percentage": 88.32, "elapsed_time": "0:37:41", "remaining_time": "0:04:59"}
63
+ {"current_steps": 315, "total_steps": 351, "loss": 2.9963, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.286588951321363e-06, "epoch": 2.68, "percentage": 89.74, "elapsed_time": "0:38:20", "remaining_time": "0:04:22"}
64
+ {"current_steps": 320, "total_steps": 351, "loss": 3.2718, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 9.561607795526007e-07, "epoch": 2.72, "percentage": 91.17, "elapsed_time": "0:38:54", "remaining_time": "0:03:46"}
65
+ {"current_steps": 325, "total_steps": 351, "loss": 3.2774, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 6.738782355044049e-07, "epoch": 2.76, "percentage": 92.59, "elapsed_time": "0:39:26", "remaining_time": "0:03:09"}
66
+ {"current_steps": 330, "total_steps": 351, "loss": 3.1737, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 4.403065646083809e-07, "epoch": 2.8, "percentage": 94.02, "elapsed_time": "0:40:02", "remaining_time": "0:02:32"}
67
+ {"current_steps": 335, "total_steps": 351, "loss": 3.2095, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 2.559134731492857e-07, "epoch": 2.85, "percentage": 95.44, "elapsed_time": "0:40:40", "remaining_time": "0:01:56"}
68
+ {"current_steps": 340, "total_steps": 351, "loss": 3.2731, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.2106819172520434e-07, "epoch": 2.89, "percentage": 96.87, "elapsed_time": "0:41:16", "remaining_time": "0:01:20"}
69
+ {"current_steps": 345, "total_steps": 351, "loss": 3.0926, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 3.604073589645596e-08, "epoch": 2.93, "percentage": 98.29, "elapsed_time": "0:41:54", "remaining_time": "0:00:43"}
70
+ {"current_steps": 350, "total_steps": 351, "loss": 3.0457, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": 1.0013655036916759e-09, "epoch": 2.97, "percentage": 99.72, "elapsed_time": "0:42:33", "remaining_time": "0:00:07"}
71
+ {"current_steps": 351, "total_steps": 351, "loss": null, "eval_loss": null, "predict_loss": null, "reward": null, "learning_rate": null, "epoch": 2.98, "percentage": 100.0, "elapsed_time": "0:42:40", "remaining_time": "0:00:00"}
QLoRA/trainer_state.json ADDED
@@ -0,0 +1,445 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 2.980891719745223,
5
+ "global_step": 351,
6
+ "is_hyper_param_search": false,
7
+ "is_local_process_zero": true,
8
+ "is_world_process_zero": true,
9
+ "log_history": [
10
+ {
11
+ "epoch": 0.04,
12
+ "learning_rate": 4.997496987311431e-05,
13
+ "loss": 4.067,
14
+ "step": 5
15
+ },
16
+ {
17
+ "epoch": 0.08,
18
+ "learning_rate": 4.989992961303738e-05,
19
+ "loss": 4.1952,
20
+ "step": 10
21
+ },
22
+ {
23
+ "epoch": 0.13,
24
+ "learning_rate": 4.977502948114772e-05,
25
+ "loss": 3.8727,
26
+ "step": 15
27
+ },
28
+ {
29
+ "epoch": 0.17,
30
+ "learning_rate": 4.960051957873725e-05,
31
+ "loss": 3.894,
32
+ "step": 20
33
+ },
34
+ {
35
+ "epoch": 0.21,
36
+ "learning_rate": 4.9376749346206006e-05,
37
+ "loss": 3.7174,
38
+ "step": 25
39
+ },
40
+ {
41
+ "epoch": 0.25,
42
+ "learning_rate": 4.910416686333906e-05,
43
+ "loss": 3.7136,
44
+ "step": 30
45
+ },
46
+ {
47
+ "epoch": 0.3,
48
+ "learning_rate": 4.878331795206705e-05,
49
+ "loss": 3.4754,
50
+ "step": 35
51
+ },
52
+ {
53
+ "epoch": 0.34,
54
+ "learning_rate": 4.841484508350679e-05,
55
+ "loss": 3.6711,
56
+ "step": 40
57
+ },
58
+ {
59
+ "epoch": 0.38,
60
+ "learning_rate": 4.799948609147061e-05,
61
+ "loss": 3.4796,
62
+ "step": 45
63
+ },
64
+ {
65
+ "epoch": 0.42,
66
+ "learning_rate": 4.753807269502041e-05,
67
+ "loss": 3.4441,
68
+ "step": 50
69
+ },
70
+ {
71
+ "epoch": 0.47,
72
+ "learning_rate": 4.7031528833024976e-05,
73
+ "loss": 3.603,
74
+ "step": 55
75
+ },
76
+ {
77
+ "epoch": 0.51,
78
+ "learning_rate": 4.6480868814055424e-05,
79
+ "loss": 3.4614,
80
+ "step": 60
81
+ },
82
+ {
83
+ "epoch": 0.55,
84
+ "learning_rate": 4.588719528532342e-05,
85
+ "loss": 3.4613,
86
+ "step": 65
87
+ },
88
+ {
89
+ "epoch": 0.59,
90
+ "learning_rate": 4.5251697024729165e-05,
91
+ "loss": 3.55,
92
+ "step": 70
93
+ },
94
+ {
95
+ "epoch": 0.64,
96
+ "learning_rate": 4.457564656044056e-05,
97
+ "loss": 3.3352,
98
+ "step": 75
99
+ },
100
+ {
101
+ "epoch": 0.68,
102
+ "learning_rate": 4.3860397622769756e-05,
103
+ "loss": 3.3781,
104
+ "step": 80
105
+ },
106
+ {
107
+ "epoch": 0.72,
108
+ "learning_rate": 4.310738243344996e-05,
109
+ "loss": 3.3805,
110
+ "step": 85
111
+ },
112
+ {
113
+ "epoch": 0.76,
114
+ "learning_rate": 4.231810883773999e-05,
115
+ "loss": 3.2995,
116
+ "step": 90
117
+ },
118
+ {
119
+ "epoch": 0.81,
120
+ "learning_rate": 4.149415728509971e-05,
121
+ "loss": 3.1648,
122
+ "step": 95
123
+ },
124
+ {
125
+ "epoch": 0.85,
126
+ "learning_rate": 4.063717766448194e-05,
127
+ "loss": 3.3295,
128
+ "step": 100
129
+ },
130
+ {
131
+ "epoch": 0.89,
132
+ "learning_rate": 3.974888600057808e-05,
133
+ "loss": 3.2847,
134
+ "step": 105
135
+ },
136
+ {
137
+ "epoch": 0.93,
138
+ "learning_rate": 3.883106101763285e-05,
139
+ "loss": 3.25,
140
+ "step": 110
141
+ },
142
+ {
143
+ "epoch": 0.98,
144
+ "learning_rate": 3.7885540577708804e-05,
145
+ "loss": 3.2399,
146
+ "step": 115
147
+ },
148
+ {
149
+ "epoch": 1.02,
150
+ "learning_rate": 3.69142180005327e-05,
151
+ "loss": 3.2125,
152
+ "step": 120
153
+ },
154
+ {
155
+ "epoch": 1.06,
156
+ "learning_rate": 3.591903827229282e-05,
157
+ "loss": 3.3815,
158
+ "step": 125
159
+ },
160
+ {
161
+ "epoch": 1.1,
162
+ "learning_rate": 3.490199415097892e-05,
163
+ "loss": 3.2365,
164
+ "step": 130
165
+ },
166
+ {
167
+ "epoch": 1.15,
168
+ "learning_rate": 3.386512217606339e-05,
169
+ "loss": 3.2755,
170
+ "step": 135
171
+ },
172
+ {
173
+ "epoch": 1.19,
174
+ "learning_rate": 3.281049859051394e-05,
175
+ "loss": 3.17,
176
+ "step": 140
177
+ },
178
+ {
179
+ "epoch": 1.23,
180
+ "learning_rate": 3.17402351833036e-05,
181
+ "loss": 3.1125,
182
+ "step": 145
183
+ },
184
+ {
185
+ "epoch": 1.27,
186
+ "learning_rate": 3.0656475060743055e-05,
187
+ "loss": 3.3456,
188
+ "step": 150
189
+ },
190
+ {
191
+ "epoch": 1.32,
192
+ "learning_rate": 2.956138835510282e-05,
193
+ "loss": 3.294,
194
+ "step": 155
195
+ },
196
+ {
197
+ "epoch": 1.36,
198
+ "learning_rate": 2.845716787911833e-05,
199
+ "loss": 3.1764,
200
+ "step": 160
201
+ },
202
+ {
203
+ "epoch": 1.4,
204
+ "learning_rate": 2.7346024735079486e-05,
205
+ "loss": 3.3249,
206
+ "step": 165
207
+ },
208
+ {
209
+ "epoch": 1.44,
210
+ "learning_rate": 2.6230183887296955e-05,
211
+ "loss": 3.1499,
212
+ "step": 170
213
+ },
214
+ {
215
+ "epoch": 1.49,
216
+ "learning_rate": 2.5111879706811087e-05,
217
+ "loss": 3.1101,
218
+ "step": 175
219
+ },
220
+ {
221
+ "epoch": 1.53,
222
+ "learning_rate": 2.399335149726463e-05,
223
+ "loss": 3.266,
224
+ "step": 180
225
+ },
226
+ {
227
+ "epoch": 1.57,
228
+ "learning_rate": 2.2876839010898377e-05,
229
+ "loss": 3.2329,
230
+ "step": 185
231
+ },
232
+ {
233
+ "epoch": 1.61,
234
+ "learning_rate": 2.1764577963648614e-05,
235
+ "loss": 3.2465,
236
+ "step": 190
237
+ },
238
+ {
239
+ "epoch": 1.66,
240
+ "learning_rate": 2.0658795558326743e-05,
241
+ "loss": 3.2541,
242
+ "step": 195
243
+ },
244
+ {
245
+ "epoch": 1.7,
246
+ "learning_rate": 1.956170602484582e-05,
247
+ "loss": 3.3382,
248
+ "step": 200
249
+ },
250
+ {
251
+ "epoch": 1.74,
252
+ "learning_rate": 1.8475506186424074e-05,
253
+ "loss": 3.3049,
254
+ "step": 205
255
+ },
256
+ {
257
+ "epoch": 1.78,
258
+ "learning_rate": 1.740237106064383e-05,
259
+ "loss": 3.048,
260
+ "step": 210
261
+ },
262
+ {
263
+ "epoch": 1.83,
264
+ "learning_rate": 1.6344449504174193e-05,
265
+ "loss": 3.2414,
266
+ "step": 215
267
+ },
268
+ {
269
+ "epoch": 1.87,
270
+ "learning_rate": 1.530385990987863e-05,
271
+ "loss": 3.291,
272
+ "step": 220
273
+ },
274
+ {
275
+ "epoch": 1.91,
276
+ "learning_rate": 1.4282685964923642e-05,
277
+ "loss": 3.2234,
278
+ "step": 225
279
+ },
280
+ {
281
+ "epoch": 1.95,
282
+ "learning_rate": 1.328297247838241e-05,
283
+ "loss": 3.3281,
284
+ "step": 230
285
+ },
286
+ {
287
+ "epoch": 2.0,
288
+ "learning_rate": 1.230672128668831e-05,
289
+ "loss": 3.0808,
290
+ "step": 235
291
+ },
292
+ {
293
+ "epoch": 2.04,
294
+ "learning_rate": 1.1355887245137383e-05,
295
+ "loss": 3.0843,
296
+ "step": 240
297
+ },
298
+ {
299
+ "epoch": 2.08,
300
+ "learning_rate": 1.043237431346622e-05,
301
+ "loss": 3.263,
302
+ "step": 245
303
+ },
304
+ {
305
+ "epoch": 2.12,
306
+ "learning_rate": 9.538031743343628e-06,
307
+ "loss": 3.0803,
308
+ "step": 250
309
+ },
310
+ {
311
+ "epoch": 2.17,
312
+ "learning_rate": 8.67465037541038e-06,
313
+ "loss": 3.1773,
314
+ "step": 255
315
+ },
316
+ {
317
+ "epoch": 2.21,
318
+ "learning_rate": 7.843959053281663e-06,
319
+ "loss": 3.2543,
320
+ "step": 260
321
+ },
322
+ {
323
+ "epoch": 2.25,
324
+ "learning_rate": 7.047621161693152e-06,
325
+ "loss": 3.1697,
326
+ "step": 265
327
+ },
328
+ {
329
+ "epoch": 2.29,
330
+ "learning_rate": 6.28723129572247e-06,
331
+ "loss": 3.2204,
332
+ "step": 270
333
+ },
334
+ {
335
+ "epoch": 2.34,
336
+ "learning_rate": 5.564312067755856e-06,
337
+ "loss": 3.1626,
338
+ "step": 275
339
+ },
340
+ {
341
+ "epoch": 2.38,
342
+ "learning_rate": 4.880311058593617e-06,
343
+ "loss": 3.2122,
344
+ "step": 280
345
+ },
346
+ {
347
+ "epoch": 2.42,
348
+ "learning_rate": 4.236597918799706e-06,
349
+ "loss": 3.2038,
350
+ "step": 285
351
+ },
352
+ {
353
+ "epoch": 2.46,
354
+ "learning_rate": 3.6344616260994946e-06,
355
+ "loss": 3.0183,
356
+ "step": 290
357
+ },
358
+ {
359
+ "epoch": 2.51,
360
+ "learning_rate": 3.07510790431767e-06,
361
+ "loss": 3.3333,
362
+ "step": 295
363
+ },
364
+ {
365
+ "epoch": 2.55,
366
+ "learning_rate": 2.5596568090246548e-06,
367
+ "loss": 3.1501,
368
+ "step": 300
369
+ },
370
+ {
371
+ "epoch": 2.59,
372
+ "learning_rate": 2.0891404847259267e-06,
373
+ "loss": 3.2576,
374
+ "step": 305
375
+ },
376
+ {
377
+ "epoch": 2.63,
378
+ "learning_rate": 1.664501098085408e-06,
379
+ "loss": 3.2306,
380
+ "step": 310
381
+ },
382
+ {
383
+ "epoch": 2.68,
384
+ "learning_rate": 1.286588951321363e-06,
385
+ "loss": 2.9963,
386
+ "step": 315
387
+ },
388
+ {
389
+ "epoch": 2.72,
390
+ "learning_rate": 9.561607795526007e-07,
391
+ "loss": 3.2718,
392
+ "step": 320
393
+ },
394
+ {
395
+ "epoch": 2.76,
396
+ "learning_rate": 6.738782355044049e-07,
397
+ "loss": 3.2774,
398
+ "step": 325
399
+ },
400
+ {
401
+ "epoch": 2.8,
402
+ "learning_rate": 4.403065646083809e-07,
403
+ "loss": 3.1737,
404
+ "step": 330
405
+ },
406
+ {
407
+ "epoch": 2.85,
408
+ "learning_rate": 2.559134731492857e-07,
409
+ "loss": 3.2095,
410
+ "step": 335
411
+ },
412
+ {
413
+ "epoch": 2.89,
414
+ "learning_rate": 1.2106819172520434e-07,
415
+ "loss": 3.2731,
416
+ "step": 340
417
+ },
418
+ {
419
+ "epoch": 2.93,
420
+ "learning_rate": 3.604073589645596e-08,
421
+ "loss": 3.0926,
422
+ "step": 345
423
+ },
424
+ {
425
+ "epoch": 2.97,
426
+ "learning_rate": 1.0013655036916759e-09,
427
+ "loss": 3.0457,
428
+ "step": 350
429
+ },
430
+ {
431
+ "epoch": 2.98,
432
+ "step": 351,
433
+ "total_flos": 1.0467902716796928e+17,
434
+ "train_loss": 3.3144316109496983,
435
+ "train_runtime": 2560.536,
436
+ "train_samples_per_second": 2.207,
437
+ "train_steps_per_second": 0.137
438
+ }
439
+ ],
440
+ "max_steps": 351,
441
+ "num_train_epochs": 3,
442
+ "total_flos": 1.0467902716796928e+17,
443
+ "trial_name": null,
444
+ "trial_params": null
445
+ }
QLoRA/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:113464d0d27a0fce5b9c5ec85454396616e1b7c8fdb71ecb97aeb01516de67e3
3
+ size 3489
README.md CHANGED
@@ -1,3 +1,18 @@
1
  ---
2
  license: apache-2.0
 
 
 
 
 
 
3
  ---
 
 
 
 
 
 
 
 
 
 
1
  ---
2
  license: apache-2.0
3
+ pipeline_tag: text-generation
4
+ datasets:
5
+ - Nekochu/novel17_train_alpaca_format
6
+ language:
7
+ - fr
8
+ - en
9
  ---
10
+
11
+ Tool used:
12
+ - hiyouga/LLaMA-Efficient-Tuning
13
+
14
+ - qwopqwop200/GPTQ-for-LLaMa
15
+
16
+ Note: QLoRA training os environ Windows, Python 3.11, CUDA 11.8 on 24GB VRAM:
17
+
18
+ Know issue: load 4bit version in oobabooga/text-generation-webui give gibberish prompt, use ExLlama instead of AutoGPTQ
config.json ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "Llama-2-13B-Chat-fp16",
3
+ "architectures": [
4
+ "LlamaForCausalLM"
5
+ ],
6
+ "bos_token_id": 1,
7
+ "eos_token_id": 2,
8
+ "hidden_act": "silu",
9
+ "hidden_size": 5120,
10
+ "initializer_range": 0.02,
11
+ "intermediate_size": 13824,
12
+ "max_length": 4096,
13
+ "max_position_embeddings": 4096,
14
+ "model_type": "llama",
15
+ "num_attention_heads": 40,
16
+ "num_hidden_layers": 40,
17
+ "num_key_value_heads": 40,
18
+ "pad_token_id": 0,
19
+ "pretraining_tp": 1,
20
+ "rms_norm_eps": 1e-05,
21
+ "rope_scaling": null,
22
+ "tie_word_embeddings": false,
23
+ "torch_dtype": "float16",
24
+ "transformers_version": "4.31.0",
25
+ "use_cache": true,
26
+ "vocab_size": 32000
27
+ }
generation_config.json ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": 2,
5
+ "max_length": 4096,
6
+ "pad_token_id": 0,
7
+ "transformers_version": "4.31.0"
8
+ }
pytorch_model-00001-of-00003.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b161c2b92028915de80e27e0a59b959928b3bc22282ee6a6577c0991fb1b9e55
3
+ size 9948728622
pytorch_model-00002-of-00003.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:77a9b1d0c3ba1a8f1db602929a353c4ad79a484a401bbf94bc6a3f414a4edcc0
3
+ size 9904165088
pytorch_model-00003-of-00003.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:b3c250eaf32a7d9684ca41494d6a4812830b7b3f4984749696f28cd535a3ddcf
3
+ size 6178983817
pytorch_model.bin.index.json ADDED
@@ -0,0 +1,410 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "metadata": {
3
+ "total_size": 26031733760
4
+ },
5
+ "weight_map": {
6
+ "lm_head.weight": "pytorch_model-00003-of-00003.bin",
7
+ "model.embed_tokens.weight": "pytorch_model-00001-of-00003.bin",
8
+ "model.layers.0.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
9
+ "model.layers.0.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
10
+ "model.layers.0.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
11
+ "model.layers.0.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
12
+ "model.layers.0.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
13
+ "model.layers.0.self_attn.k_proj.weight": "pytorch_model-00001-of-00003.bin",
14
+ "model.layers.0.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
15
+ "model.layers.0.self_attn.q_proj.weight": "pytorch_model-00001-of-00003.bin",
16
+ "model.layers.0.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00003.bin",
17
+ "model.layers.0.self_attn.v_proj.weight": "pytorch_model-00001-of-00003.bin",
18
+ "model.layers.1.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
19
+ "model.layers.1.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
20
+ "model.layers.1.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
21
+ "model.layers.1.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
22
+ "model.layers.1.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
23
+ "model.layers.1.self_attn.k_proj.weight": "pytorch_model-00001-of-00003.bin",
24
+ "model.layers.1.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
25
+ "model.layers.1.self_attn.q_proj.weight": "pytorch_model-00001-of-00003.bin",
26
+ "model.layers.1.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00003.bin",
27
+ "model.layers.1.self_attn.v_proj.weight": "pytorch_model-00001-of-00003.bin",
28
+ "model.layers.10.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
29
+ "model.layers.10.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
30
+ "model.layers.10.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
31
+ "model.layers.10.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
32
+ "model.layers.10.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
33
+ "model.layers.10.self_attn.k_proj.weight": "pytorch_model-00001-of-00003.bin",
34
+ "model.layers.10.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
35
+ "model.layers.10.self_attn.q_proj.weight": "pytorch_model-00001-of-00003.bin",
36
+ "model.layers.10.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00003.bin",
37
+ "model.layers.10.self_attn.v_proj.weight": "pytorch_model-00001-of-00003.bin",
38
+ "model.layers.11.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
39
+ "model.layers.11.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
40
+ "model.layers.11.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
41
+ "model.layers.11.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
42
+ "model.layers.11.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
43
+ "model.layers.11.self_attn.k_proj.weight": "pytorch_model-00001-of-00003.bin",
44
+ "model.layers.11.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
45
+ "model.layers.11.self_attn.q_proj.weight": "pytorch_model-00001-of-00003.bin",
46
+ "model.layers.11.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00003.bin",
47
+ "model.layers.11.self_attn.v_proj.weight": "pytorch_model-00001-of-00003.bin",
48
+ "model.layers.12.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
49
+ "model.layers.12.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
50
+ "model.layers.12.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
51
+ "model.layers.12.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
52
+ "model.layers.12.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
53
+ "model.layers.12.self_attn.k_proj.weight": "pytorch_model-00001-of-00003.bin",
54
+ "model.layers.12.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
55
+ "model.layers.12.self_attn.q_proj.weight": "pytorch_model-00001-of-00003.bin",
56
+ "model.layers.12.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00003.bin",
57
+ "model.layers.12.self_attn.v_proj.weight": "pytorch_model-00001-of-00003.bin",
58
+ "model.layers.13.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
59
+ "model.layers.13.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
60
+ "model.layers.13.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
61
+ "model.layers.13.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
62
+ "model.layers.13.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
63
+ "model.layers.13.self_attn.k_proj.weight": "pytorch_model-00001-of-00003.bin",
64
+ "model.layers.13.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
65
+ "model.layers.13.self_attn.q_proj.weight": "pytorch_model-00001-of-00003.bin",
66
+ "model.layers.13.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00003.bin",
67
+ "model.layers.13.self_attn.v_proj.weight": "pytorch_model-00001-of-00003.bin",
68
+ "model.layers.14.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
69
+ "model.layers.14.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
70
+ "model.layers.14.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
71
+ "model.layers.14.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
72
+ "model.layers.14.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
73
+ "model.layers.14.self_attn.k_proj.weight": "pytorch_model-00001-of-00003.bin",
74
+ "model.layers.14.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
75
+ "model.layers.14.self_attn.q_proj.weight": "pytorch_model-00001-of-00003.bin",
76
+ "model.layers.14.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00003.bin",
77
+ "model.layers.14.self_attn.v_proj.weight": "pytorch_model-00001-of-00003.bin",
78
+ "model.layers.15.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
79
+ "model.layers.15.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
80
+ "model.layers.15.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
81
+ "model.layers.15.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
82
+ "model.layers.15.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
83
+ "model.layers.15.self_attn.k_proj.weight": "pytorch_model-00001-of-00003.bin",
84
+ "model.layers.15.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
85
+ "model.layers.15.self_attn.q_proj.weight": "pytorch_model-00001-of-00003.bin",
86
+ "model.layers.15.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00003.bin",
87
+ "model.layers.15.self_attn.v_proj.weight": "pytorch_model-00002-of-00003.bin",
88
+ "model.layers.16.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
89
+ "model.layers.16.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
90
+ "model.layers.16.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
91
+ "model.layers.16.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
92
+ "model.layers.16.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
93
+ "model.layers.16.self_attn.k_proj.weight": "pytorch_model-00002-of-00003.bin",
94
+ "model.layers.16.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
95
+ "model.layers.16.self_attn.q_proj.weight": "pytorch_model-00002-of-00003.bin",
96
+ "model.layers.16.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00003.bin",
97
+ "model.layers.16.self_attn.v_proj.weight": "pytorch_model-00002-of-00003.bin",
98
+ "model.layers.17.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
99
+ "model.layers.17.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
100
+ "model.layers.17.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
101
+ "model.layers.17.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
102
+ "model.layers.17.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
103
+ "model.layers.17.self_attn.k_proj.weight": "pytorch_model-00002-of-00003.bin",
104
+ "model.layers.17.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
105
+ "model.layers.17.self_attn.q_proj.weight": "pytorch_model-00002-of-00003.bin",
106
+ "model.layers.17.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00003.bin",
107
+ "model.layers.17.self_attn.v_proj.weight": "pytorch_model-00002-of-00003.bin",
108
+ "model.layers.18.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
109
+ "model.layers.18.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
110
+ "model.layers.18.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
111
+ "model.layers.18.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
112
+ "model.layers.18.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
113
+ "model.layers.18.self_attn.k_proj.weight": "pytorch_model-00002-of-00003.bin",
114
+ "model.layers.18.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
115
+ "model.layers.18.self_attn.q_proj.weight": "pytorch_model-00002-of-00003.bin",
116
+ "model.layers.18.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00003.bin",
117
+ "model.layers.18.self_attn.v_proj.weight": "pytorch_model-00002-of-00003.bin",
118
+ "model.layers.19.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
119
+ "model.layers.19.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
120
+ "model.layers.19.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
121
+ "model.layers.19.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
122
+ "model.layers.19.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
123
+ "model.layers.19.self_attn.k_proj.weight": "pytorch_model-00002-of-00003.bin",
124
+ "model.layers.19.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
125
+ "model.layers.19.self_attn.q_proj.weight": "pytorch_model-00002-of-00003.bin",
126
+ "model.layers.19.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00003.bin",
127
+ "model.layers.19.self_attn.v_proj.weight": "pytorch_model-00002-of-00003.bin",
128
+ "model.layers.2.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
129
+ "model.layers.2.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
130
+ "model.layers.2.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
131
+ "model.layers.2.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
132
+ "model.layers.2.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
133
+ "model.layers.2.self_attn.k_proj.weight": "pytorch_model-00001-of-00003.bin",
134
+ "model.layers.2.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
135
+ "model.layers.2.self_attn.q_proj.weight": "pytorch_model-00001-of-00003.bin",
136
+ "model.layers.2.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00003.bin",
137
+ "model.layers.2.self_attn.v_proj.weight": "pytorch_model-00001-of-00003.bin",
138
+ "model.layers.20.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
139
+ "model.layers.20.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
140
+ "model.layers.20.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
141
+ "model.layers.20.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
142
+ "model.layers.20.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
143
+ "model.layers.20.self_attn.k_proj.weight": "pytorch_model-00002-of-00003.bin",
144
+ "model.layers.20.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
145
+ "model.layers.20.self_attn.q_proj.weight": "pytorch_model-00002-of-00003.bin",
146
+ "model.layers.20.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00003.bin",
147
+ "model.layers.20.self_attn.v_proj.weight": "pytorch_model-00002-of-00003.bin",
148
+ "model.layers.21.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
149
+ "model.layers.21.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
150
+ "model.layers.21.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
151
+ "model.layers.21.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
152
+ "model.layers.21.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
153
+ "model.layers.21.self_attn.k_proj.weight": "pytorch_model-00002-of-00003.bin",
154
+ "model.layers.21.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
155
+ "model.layers.21.self_attn.q_proj.weight": "pytorch_model-00002-of-00003.bin",
156
+ "model.layers.21.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00003.bin",
157
+ "model.layers.21.self_attn.v_proj.weight": "pytorch_model-00002-of-00003.bin",
158
+ "model.layers.22.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
159
+ "model.layers.22.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
160
+ "model.layers.22.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
161
+ "model.layers.22.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
162
+ "model.layers.22.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
163
+ "model.layers.22.self_attn.k_proj.weight": "pytorch_model-00002-of-00003.bin",
164
+ "model.layers.22.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
165
+ "model.layers.22.self_attn.q_proj.weight": "pytorch_model-00002-of-00003.bin",
166
+ "model.layers.22.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00003.bin",
167
+ "model.layers.22.self_attn.v_proj.weight": "pytorch_model-00002-of-00003.bin",
168
+ "model.layers.23.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
169
+ "model.layers.23.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
170
+ "model.layers.23.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
171
+ "model.layers.23.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
172
+ "model.layers.23.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
173
+ "model.layers.23.self_attn.k_proj.weight": "pytorch_model-00002-of-00003.bin",
174
+ "model.layers.23.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
175
+ "model.layers.23.self_attn.q_proj.weight": "pytorch_model-00002-of-00003.bin",
176
+ "model.layers.23.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00003.bin",
177
+ "model.layers.23.self_attn.v_proj.weight": "pytorch_model-00002-of-00003.bin",
178
+ "model.layers.24.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
179
+ "model.layers.24.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
180
+ "model.layers.24.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
181
+ "model.layers.24.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
182
+ "model.layers.24.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
183
+ "model.layers.24.self_attn.k_proj.weight": "pytorch_model-00002-of-00003.bin",
184
+ "model.layers.24.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
185
+ "model.layers.24.self_attn.q_proj.weight": "pytorch_model-00002-of-00003.bin",
186
+ "model.layers.24.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00003.bin",
187
+ "model.layers.24.self_attn.v_proj.weight": "pytorch_model-00002-of-00003.bin",
188
+ "model.layers.25.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
189
+ "model.layers.25.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
190
+ "model.layers.25.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
191
+ "model.layers.25.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
192
+ "model.layers.25.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
193
+ "model.layers.25.self_attn.k_proj.weight": "pytorch_model-00002-of-00003.bin",
194
+ "model.layers.25.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
195
+ "model.layers.25.self_attn.q_proj.weight": "pytorch_model-00002-of-00003.bin",
196
+ "model.layers.25.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00003.bin",
197
+ "model.layers.25.self_attn.v_proj.weight": "pytorch_model-00002-of-00003.bin",
198
+ "model.layers.26.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
199
+ "model.layers.26.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
200
+ "model.layers.26.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
201
+ "model.layers.26.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
202
+ "model.layers.26.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
203
+ "model.layers.26.self_attn.k_proj.weight": "pytorch_model-00002-of-00003.bin",
204
+ "model.layers.26.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
205
+ "model.layers.26.self_attn.q_proj.weight": "pytorch_model-00002-of-00003.bin",
206
+ "model.layers.26.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00003.bin",
207
+ "model.layers.26.self_attn.v_proj.weight": "pytorch_model-00002-of-00003.bin",
208
+ "model.layers.27.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
209
+ "model.layers.27.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
210
+ "model.layers.27.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
211
+ "model.layers.27.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
212
+ "model.layers.27.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
213
+ "model.layers.27.self_attn.k_proj.weight": "pytorch_model-00002-of-00003.bin",
214
+ "model.layers.27.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
215
+ "model.layers.27.self_attn.q_proj.weight": "pytorch_model-00002-of-00003.bin",
216
+ "model.layers.27.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00003.bin",
217
+ "model.layers.27.self_attn.v_proj.weight": "pytorch_model-00002-of-00003.bin",
218
+ "model.layers.28.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
219
+ "model.layers.28.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
220
+ "model.layers.28.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
221
+ "model.layers.28.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
222
+ "model.layers.28.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
223
+ "model.layers.28.self_attn.k_proj.weight": "pytorch_model-00002-of-00003.bin",
224
+ "model.layers.28.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
225
+ "model.layers.28.self_attn.q_proj.weight": "pytorch_model-00002-of-00003.bin",
226
+ "model.layers.28.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00003.bin",
227
+ "model.layers.28.self_attn.v_proj.weight": "pytorch_model-00002-of-00003.bin",
228
+ "model.layers.29.input_layernorm.weight": "pytorch_model-00002-of-00003.bin",
229
+ "model.layers.29.mlp.down_proj.weight": "pytorch_model-00002-of-00003.bin",
230
+ "model.layers.29.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
231
+ "model.layers.29.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
232
+ "model.layers.29.post_attention_layernorm.weight": "pytorch_model-00002-of-00003.bin",
233
+ "model.layers.29.self_attn.k_proj.weight": "pytorch_model-00002-of-00003.bin",
234
+ "model.layers.29.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
235
+ "model.layers.29.self_attn.q_proj.weight": "pytorch_model-00002-of-00003.bin",
236
+ "model.layers.29.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00003.bin",
237
+ "model.layers.29.self_attn.v_proj.weight": "pytorch_model-00002-of-00003.bin",
238
+ "model.layers.3.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
239
+ "model.layers.3.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
240
+ "model.layers.3.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
241
+ "model.layers.3.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
242
+ "model.layers.3.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
243
+ "model.layers.3.self_attn.k_proj.weight": "pytorch_model-00001-of-00003.bin",
244
+ "model.layers.3.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
245
+ "model.layers.3.self_attn.q_proj.weight": "pytorch_model-00001-of-00003.bin",
246
+ "model.layers.3.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00003.bin",
247
+ "model.layers.3.self_attn.v_proj.weight": "pytorch_model-00001-of-00003.bin",
248
+ "model.layers.30.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
249
+ "model.layers.30.mlp.down_proj.weight": "pytorch_model-00003-of-00003.bin",
250
+ "model.layers.30.mlp.gate_proj.weight": "pytorch_model-00002-of-00003.bin",
251
+ "model.layers.30.mlp.up_proj.weight": "pytorch_model-00002-of-00003.bin",
252
+ "model.layers.30.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
253
+ "model.layers.30.self_attn.k_proj.weight": "pytorch_model-00002-of-00003.bin",
254
+ "model.layers.30.self_attn.o_proj.weight": "pytorch_model-00002-of-00003.bin",
255
+ "model.layers.30.self_attn.q_proj.weight": "pytorch_model-00002-of-00003.bin",
256
+ "model.layers.30.self_attn.rotary_emb.inv_freq": "pytorch_model-00002-of-00003.bin",
257
+ "model.layers.30.self_attn.v_proj.weight": "pytorch_model-00002-of-00003.bin",
258
+ "model.layers.31.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
259
+ "model.layers.31.mlp.down_proj.weight": "pytorch_model-00003-of-00003.bin",
260
+ "model.layers.31.mlp.gate_proj.weight": "pytorch_model-00003-of-00003.bin",
261
+ "model.layers.31.mlp.up_proj.weight": "pytorch_model-00003-of-00003.bin",
262
+ "model.layers.31.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
263
+ "model.layers.31.self_attn.k_proj.weight": "pytorch_model-00003-of-00003.bin",
264
+ "model.layers.31.self_attn.o_proj.weight": "pytorch_model-00003-of-00003.bin",
265
+ "model.layers.31.self_attn.q_proj.weight": "pytorch_model-00003-of-00003.bin",
266
+ "model.layers.31.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00003.bin",
267
+ "model.layers.31.self_attn.v_proj.weight": "pytorch_model-00003-of-00003.bin",
268
+ "model.layers.32.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
269
+ "model.layers.32.mlp.down_proj.weight": "pytorch_model-00003-of-00003.bin",
270
+ "model.layers.32.mlp.gate_proj.weight": "pytorch_model-00003-of-00003.bin",
271
+ "model.layers.32.mlp.up_proj.weight": "pytorch_model-00003-of-00003.bin",
272
+ "model.layers.32.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
273
+ "model.layers.32.self_attn.k_proj.weight": "pytorch_model-00003-of-00003.bin",
274
+ "model.layers.32.self_attn.o_proj.weight": "pytorch_model-00003-of-00003.bin",
275
+ "model.layers.32.self_attn.q_proj.weight": "pytorch_model-00003-of-00003.bin",
276
+ "model.layers.32.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00003.bin",
277
+ "model.layers.32.self_attn.v_proj.weight": "pytorch_model-00003-of-00003.bin",
278
+ "model.layers.33.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
279
+ "model.layers.33.mlp.down_proj.weight": "pytorch_model-00003-of-00003.bin",
280
+ "model.layers.33.mlp.gate_proj.weight": "pytorch_model-00003-of-00003.bin",
281
+ "model.layers.33.mlp.up_proj.weight": "pytorch_model-00003-of-00003.bin",
282
+ "model.layers.33.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
283
+ "model.layers.33.self_attn.k_proj.weight": "pytorch_model-00003-of-00003.bin",
284
+ "model.layers.33.self_attn.o_proj.weight": "pytorch_model-00003-of-00003.bin",
285
+ "model.layers.33.self_attn.q_proj.weight": "pytorch_model-00003-of-00003.bin",
286
+ "model.layers.33.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00003.bin",
287
+ "model.layers.33.self_attn.v_proj.weight": "pytorch_model-00003-of-00003.bin",
288
+ "model.layers.34.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
289
+ "model.layers.34.mlp.down_proj.weight": "pytorch_model-00003-of-00003.bin",
290
+ "model.layers.34.mlp.gate_proj.weight": "pytorch_model-00003-of-00003.bin",
291
+ "model.layers.34.mlp.up_proj.weight": "pytorch_model-00003-of-00003.bin",
292
+ "model.layers.34.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
293
+ "model.layers.34.self_attn.k_proj.weight": "pytorch_model-00003-of-00003.bin",
294
+ "model.layers.34.self_attn.o_proj.weight": "pytorch_model-00003-of-00003.bin",
295
+ "model.layers.34.self_attn.q_proj.weight": "pytorch_model-00003-of-00003.bin",
296
+ "model.layers.34.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00003.bin",
297
+ "model.layers.34.self_attn.v_proj.weight": "pytorch_model-00003-of-00003.bin",
298
+ "model.layers.35.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
299
+ "model.layers.35.mlp.down_proj.weight": "pytorch_model-00003-of-00003.bin",
300
+ "model.layers.35.mlp.gate_proj.weight": "pytorch_model-00003-of-00003.bin",
301
+ "model.layers.35.mlp.up_proj.weight": "pytorch_model-00003-of-00003.bin",
302
+ "model.layers.35.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
303
+ "model.layers.35.self_attn.k_proj.weight": "pytorch_model-00003-of-00003.bin",
304
+ "model.layers.35.self_attn.o_proj.weight": "pytorch_model-00003-of-00003.bin",
305
+ "model.layers.35.self_attn.q_proj.weight": "pytorch_model-00003-of-00003.bin",
306
+ "model.layers.35.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00003.bin",
307
+ "model.layers.35.self_attn.v_proj.weight": "pytorch_model-00003-of-00003.bin",
308
+ "model.layers.36.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
309
+ "model.layers.36.mlp.down_proj.weight": "pytorch_model-00003-of-00003.bin",
310
+ "model.layers.36.mlp.gate_proj.weight": "pytorch_model-00003-of-00003.bin",
311
+ "model.layers.36.mlp.up_proj.weight": "pytorch_model-00003-of-00003.bin",
312
+ "model.layers.36.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
313
+ "model.layers.36.self_attn.k_proj.weight": "pytorch_model-00003-of-00003.bin",
314
+ "model.layers.36.self_attn.o_proj.weight": "pytorch_model-00003-of-00003.bin",
315
+ "model.layers.36.self_attn.q_proj.weight": "pytorch_model-00003-of-00003.bin",
316
+ "model.layers.36.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00003.bin",
317
+ "model.layers.36.self_attn.v_proj.weight": "pytorch_model-00003-of-00003.bin",
318
+ "model.layers.37.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
319
+ "model.layers.37.mlp.down_proj.weight": "pytorch_model-00003-of-00003.bin",
320
+ "model.layers.37.mlp.gate_proj.weight": "pytorch_model-00003-of-00003.bin",
321
+ "model.layers.37.mlp.up_proj.weight": "pytorch_model-00003-of-00003.bin",
322
+ "model.layers.37.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
323
+ "model.layers.37.self_attn.k_proj.weight": "pytorch_model-00003-of-00003.bin",
324
+ "model.layers.37.self_attn.o_proj.weight": "pytorch_model-00003-of-00003.bin",
325
+ "model.layers.37.self_attn.q_proj.weight": "pytorch_model-00003-of-00003.bin",
326
+ "model.layers.37.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00003.bin",
327
+ "model.layers.37.self_attn.v_proj.weight": "pytorch_model-00003-of-00003.bin",
328
+ "model.layers.38.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
329
+ "model.layers.38.mlp.down_proj.weight": "pytorch_model-00003-of-00003.bin",
330
+ "model.layers.38.mlp.gate_proj.weight": "pytorch_model-00003-of-00003.bin",
331
+ "model.layers.38.mlp.up_proj.weight": "pytorch_model-00003-of-00003.bin",
332
+ "model.layers.38.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
333
+ "model.layers.38.self_attn.k_proj.weight": "pytorch_model-00003-of-00003.bin",
334
+ "model.layers.38.self_attn.o_proj.weight": "pytorch_model-00003-of-00003.bin",
335
+ "model.layers.38.self_attn.q_proj.weight": "pytorch_model-00003-of-00003.bin",
336
+ "model.layers.38.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00003.bin",
337
+ "model.layers.38.self_attn.v_proj.weight": "pytorch_model-00003-of-00003.bin",
338
+ "model.layers.39.input_layernorm.weight": "pytorch_model-00003-of-00003.bin",
339
+ "model.layers.39.mlp.down_proj.weight": "pytorch_model-00003-of-00003.bin",
340
+ "model.layers.39.mlp.gate_proj.weight": "pytorch_model-00003-of-00003.bin",
341
+ "model.layers.39.mlp.up_proj.weight": "pytorch_model-00003-of-00003.bin",
342
+ "model.layers.39.post_attention_layernorm.weight": "pytorch_model-00003-of-00003.bin",
343
+ "model.layers.39.self_attn.k_proj.weight": "pytorch_model-00003-of-00003.bin",
344
+ "model.layers.39.self_attn.o_proj.weight": "pytorch_model-00003-of-00003.bin",
345
+ "model.layers.39.self_attn.q_proj.weight": "pytorch_model-00003-of-00003.bin",
346
+ "model.layers.39.self_attn.rotary_emb.inv_freq": "pytorch_model-00003-of-00003.bin",
347
+ "model.layers.39.self_attn.v_proj.weight": "pytorch_model-00003-of-00003.bin",
348
+ "model.layers.4.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
349
+ "model.layers.4.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
350
+ "model.layers.4.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
351
+ "model.layers.4.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
352
+ "model.layers.4.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
353
+ "model.layers.4.self_attn.k_proj.weight": "pytorch_model-00001-of-00003.bin",
354
+ "model.layers.4.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
355
+ "model.layers.4.self_attn.q_proj.weight": "pytorch_model-00001-of-00003.bin",
356
+ "model.layers.4.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00003.bin",
357
+ "model.layers.4.self_attn.v_proj.weight": "pytorch_model-00001-of-00003.bin",
358
+ "model.layers.5.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
359
+ "model.layers.5.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
360
+ "model.layers.5.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
361
+ "model.layers.5.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
362
+ "model.layers.5.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
363
+ "model.layers.5.self_attn.k_proj.weight": "pytorch_model-00001-of-00003.bin",
364
+ "model.layers.5.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
365
+ "model.layers.5.self_attn.q_proj.weight": "pytorch_model-00001-of-00003.bin",
366
+ "model.layers.5.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00003.bin",
367
+ "model.layers.5.self_attn.v_proj.weight": "pytorch_model-00001-of-00003.bin",
368
+ "model.layers.6.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
369
+ "model.layers.6.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
370
+ "model.layers.6.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
371
+ "model.layers.6.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
372
+ "model.layers.6.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
373
+ "model.layers.6.self_attn.k_proj.weight": "pytorch_model-00001-of-00003.bin",
374
+ "model.layers.6.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
375
+ "model.layers.6.self_attn.q_proj.weight": "pytorch_model-00001-of-00003.bin",
376
+ "model.layers.6.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00003.bin",
377
+ "model.layers.6.self_attn.v_proj.weight": "pytorch_model-00001-of-00003.bin",
378
+ "model.layers.7.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
379
+ "model.layers.7.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
380
+ "model.layers.7.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
381
+ "model.layers.7.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
382
+ "model.layers.7.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
383
+ "model.layers.7.self_attn.k_proj.weight": "pytorch_model-00001-of-00003.bin",
384
+ "model.layers.7.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
385
+ "model.layers.7.self_attn.q_proj.weight": "pytorch_model-00001-of-00003.bin",
386
+ "model.layers.7.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00003.bin",
387
+ "model.layers.7.self_attn.v_proj.weight": "pytorch_model-00001-of-00003.bin",
388
+ "model.layers.8.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
389
+ "model.layers.8.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
390
+ "model.layers.8.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
391
+ "model.layers.8.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
392
+ "model.layers.8.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
393
+ "model.layers.8.self_attn.k_proj.weight": "pytorch_model-00001-of-00003.bin",
394
+ "model.layers.8.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
395
+ "model.layers.8.self_attn.q_proj.weight": "pytorch_model-00001-of-00003.bin",
396
+ "model.layers.8.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00003.bin",
397
+ "model.layers.8.self_attn.v_proj.weight": "pytorch_model-00001-of-00003.bin",
398
+ "model.layers.9.input_layernorm.weight": "pytorch_model-00001-of-00003.bin",
399
+ "model.layers.9.mlp.down_proj.weight": "pytorch_model-00001-of-00003.bin",
400
+ "model.layers.9.mlp.gate_proj.weight": "pytorch_model-00001-of-00003.bin",
401
+ "model.layers.9.mlp.up_proj.weight": "pytorch_model-00001-of-00003.bin",
402
+ "model.layers.9.post_attention_layernorm.weight": "pytorch_model-00001-of-00003.bin",
403
+ "model.layers.9.self_attn.k_proj.weight": "pytorch_model-00001-of-00003.bin",
404
+ "model.layers.9.self_attn.o_proj.weight": "pytorch_model-00001-of-00003.bin",
405
+ "model.layers.9.self_attn.q_proj.weight": "pytorch_model-00001-of-00003.bin",
406
+ "model.layers.9.self_attn.rotary_emb.inv_freq": "pytorch_model-00001-of-00003.bin",
407
+ "model.layers.9.self_attn.v_proj.weight": "pytorch_model-00001-of-00003.bin",
408
+ "model.norm.weight": "pytorch_model-00003-of-00003.bin"
409
+ }
410
+ }
special_tokens_map.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": true,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": true,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": "<unk>",
17
+ "unk_token": {
18
+ "content": "<unk>",
19
+ "lstrip": false,
20
+ "normalized": true,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ }
24
+ }
tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
3
+ size 499723
tokenizer_config.json ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": true,
3
+ "add_eos_token": false,
4
+ "bos_token": {
5
+ "__type": "AddedToken",
6
+ "content": "<s>",
7
+ "lstrip": false,
8
+ "normalized": true,
9
+ "rstrip": false,
10
+ "single_word": false
11
+ },
12
+ "clean_up_tokenization_spaces": false,
13
+ "eos_token": {
14
+ "__type": "AddedToken",
15
+ "content": "</s>",
16
+ "lstrip": false,
17
+ "normalized": true,
18
+ "rstrip": false,
19
+ "single_word": false
20
+ },
21
+ "legacy": true,
22
+ "model_max_length": 1000000000000000019884624838656,
23
+ "pad_token": null,
24
+ "padding_side": "left",
25
+ "sp_model_kwargs": {},
26
+ "tokenizer_class": "LlamaTokenizer",
27
+ "unk_token": {
28
+ "__type": "AddedToken",
29
+ "content": "<unk>",
30
+ "lstrip": false,
31
+ "normalized": true,
32
+ "rstrip": false,
33
+ "single_word": false
34
+ }
35
+ }