tsunayoshi21 commited on
Commit
8500a95
1 Parent(s): e22a146

Subiendo mi modelo entrenado

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. README.md +143 -0
  2. adapter_config.json +34 -0
  3. adapter_model.bin +3 -0
  4. checkpoint-116/README.md +202 -0
  5. checkpoint-116/adapter_config.json +34 -0
  6. checkpoint-116/adapter_model.safetensors +3 -0
  7. checkpoint-116/optimizer.pt +3 -0
  8. checkpoint-116/rng_state.pth +3 -0
  9. checkpoint-116/scheduler.pt +3 -0
  10. checkpoint-116/special_tokens_map.json +24 -0
  11. checkpoint-116/tokenizer.model +3 -0
  12. checkpoint-116/tokenizer_config.json +44 -0
  13. checkpoint-116/trainer_state.json +965 -0
  14. checkpoint-116/training_args.bin +3 -0
  15. checkpoint-29/README.md +202 -0
  16. checkpoint-29/adapter_config.json +34 -0
  17. checkpoint-29/adapter_model.safetensors +3 -0
  18. checkpoint-29/optimizer.pt +3 -0
  19. checkpoint-29/rng_state.pth +3 -0
  20. checkpoint-29/scheduler.pt +3 -0
  21. checkpoint-29/special_tokens_map.json +24 -0
  22. checkpoint-29/tokenizer.model +3 -0
  23. checkpoint-29/tokenizer_config.json +44 -0
  24. checkpoint-29/trainer_state.json +268 -0
  25. checkpoint-29/training_args.bin +3 -0
  26. checkpoint-58/README.md +202 -0
  27. checkpoint-58/adapter_config.json +34 -0
  28. checkpoint-58/adapter_model.safetensors +3 -0
  29. checkpoint-58/optimizer.pt +3 -0
  30. checkpoint-58/rng_state.pth +3 -0
  31. checkpoint-58/scheduler.pt +3 -0
  32. checkpoint-58/special_tokens_map.json +24 -0
  33. checkpoint-58/tokenizer.model +3 -0
  34. checkpoint-58/tokenizer_config.json +44 -0
  35. checkpoint-58/trainer_state.json +503 -0
  36. checkpoint-58/training_args.bin +3 -0
  37. checkpoint-87/README.md +202 -0
  38. checkpoint-87/adapter_config.json +34 -0
  39. checkpoint-87/adapter_model.safetensors +3 -0
  40. checkpoint-87/optimizer.pt +3 -0
  41. checkpoint-87/rng_state.pth +3 -0
  42. checkpoint-87/scheduler.pt +3 -0
  43. checkpoint-87/special_tokens_map.json +24 -0
  44. checkpoint-87/tokenizer.model +3 -0
  45. checkpoint-87/tokenizer_config.json +44 -0
  46. checkpoint-87/trainer_state.json +730 -0
  47. checkpoint-87/training_args.bin +3 -0
  48. config.json +44 -0
  49. content/outputs/qlora-out/README.md +143 -0
  50. content/outputs/qlora-out/adapter_config.json +34 -0
README.md ADDED
@@ -0,0 +1,143 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: apache-2.0
3
+ library_name: peft
4
+ tags:
5
+ - generated_from_trainer
6
+ base_model: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
7
+ model-index:
8
+ - name: outputs/qlora-out
9
+ results: []
10
+ ---
11
+
12
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
13
+ should probably proofread and complete it, then remove this comment. -->
14
+
15
+ [<img src="https://raw.githubusercontent.com/OpenAccess-AI-Collective/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/OpenAccess-AI-Collective/axolotl)
16
+ <details><summary>See axolotl config</summary>
17
+
18
+ axolotl version: `0.4.1`
19
+ ```yaml
20
+ adapter: qlora
21
+ base_model: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
22
+ bf16: false
23
+ dataset_prepared_path: null
24
+ datasets:
25
+ - ds_tipe: json
26
+ path: pubmed_continual_pretraning_dataset.jsonl
27
+ type: completion
28
+ debug: null
29
+ deepspeed: null
30
+ early_stopping_patience: null
31
+ eval_sample_packing: false
32
+ evals_per_epoch: 4
33
+ flash_attention: false
34
+ fp16: null
35
+ fsdp: null
36
+ fsdp_config: null
37
+ gradient_accumulation_steps: 4
38
+ gradient_checkpointing: true
39
+ group_by_length: false
40
+ learning_rate: 0.0002
41
+ load_in_4bit: true
42
+ load_in_8bit: false
43
+ local_rank: null
44
+ logging_steps: 1
45
+ lora_alpha: 16
46
+ lora_dropout: 0.05
47
+ lora_fan_in_fan_out: null
48
+ lora_model_dir: null
49
+ lora_r: 32
50
+ lora_target_linear: true
51
+ lora_target_modules: null
52
+ lr_scheduler: cosine
53
+ micro_batch_size: 8
54
+ model_type: LlamaForCausalLM
55
+ num_epochs: 4
56
+ optimizer: paged_adamw_32bit
57
+ output_dir: ./outputs/qlora-out
58
+ pad_to_sequence_len: false
59
+ resume_from_checkpoint: null
60
+ sample_packing: false
61
+ saves_per_epoch: 1
62
+ sequence_len: 4096
63
+ special_tokens: null
64
+ strict: false
65
+ tf32: false
66
+ tokenizer_type: LlamaTokenizer
67
+ train_on_inputs: false
68
+ val_set_size: 0.05
69
+ wandb_entity: null
70
+ wandb_log_model: null
71
+ wandb_name: null
72
+ wandb_project: null
73
+ wandb_watch: null
74
+ warmup_steps: 10
75
+ weight_decay: 0.0
76
+ xformers_attention: null
77
+
78
+ ```
79
+
80
+ </details><br>
81
+
82
+ # outputs/qlora-out
83
+
84
+ This model is a fine-tuned version of [TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T](https://huggingface.co/TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T) on the None dataset.
85
+ It achieves the following results on the evaluation set:
86
+ - Loss: 1.7613
87
+
88
+ ## Model description
89
+
90
+ More information needed
91
+
92
+ ## Intended uses & limitations
93
+
94
+ More information needed
95
+
96
+ ## Training and evaluation data
97
+
98
+ More information needed
99
+
100
+ ## Training procedure
101
+
102
+ ### Training hyperparameters
103
+
104
+ The following hyperparameters were used during training:
105
+ - learning_rate: 0.0002
106
+ - train_batch_size: 8
107
+ - eval_batch_size: 8
108
+ - seed: 42
109
+ - gradient_accumulation_steps: 4
110
+ - total_train_batch_size: 32
111
+ - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
112
+ - lr_scheduler_type: cosine
113
+ - lr_scheduler_warmup_steps: 10
114
+ - num_epochs: 4
115
+
116
+ ### Training results
117
+
118
+ | Training Loss | Epoch | Step | Validation Loss |
119
+ |:-------------:|:------:|:----:|:---------------:|
120
+ | 1.768 | 0.0336 | 1 | 1.8649 |
121
+ | 1.8084 | 0.2689 | 8 | 1.8317 |
122
+ | 1.633 | 0.5378 | 16 | 1.7833 |
123
+ | 1.6737 | 0.8067 | 24 | 1.7644 |
124
+ | 1.6722 | 1.0756 | 32 | 1.7601 |
125
+ | 1.7162 | 1.3445 | 40 | 1.7571 |
126
+ | 1.7046 | 1.6134 | 48 | 1.7558 |
127
+ | 1.6714 | 1.8824 | 56 | 1.7564 |
128
+ | 1.6249 | 2.1513 | 64 | 1.7566 |
129
+ | 1.5604 | 2.4202 | 72 | 1.7599 |
130
+ | 1.7003 | 2.6891 | 80 | 1.7614 |
131
+ | 1.7115 | 2.9580 | 88 | 1.7605 |
132
+ | 1.5937 | 3.2269 | 96 | 1.7609 |
133
+ | 1.655 | 3.4958 | 104 | 1.7612 |
134
+ | 1.5829 | 3.7647 | 112 | 1.7613 |
135
+
136
+
137
+ ### Framework versions
138
+
139
+ - PEFT 0.11.1
140
+ - Transformers 4.41.1
141
+ - Pytorch 2.1.2+cu121
142
+ - Datasets 2.19.1
143
+ - Tokenizers 0.19.1
adapter_config.json ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T",
5
+ "bias": "none",
6
+ "fan_in_fan_out": null,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layer_replication": null,
10
+ "layers_pattern": null,
11
+ "layers_to_transform": null,
12
+ "loftq_config": {},
13
+ "lora_alpha": 16,
14
+ "lora_dropout": 0.05,
15
+ "megatron_config": null,
16
+ "megatron_core": "megatron.core",
17
+ "modules_to_save": null,
18
+ "peft_type": "LORA",
19
+ "r": 32,
20
+ "rank_pattern": {},
21
+ "revision": null,
22
+ "target_modules": [
23
+ "gate_proj",
24
+ "up_proj",
25
+ "down_proj",
26
+ "v_proj",
27
+ "o_proj",
28
+ "q_proj",
29
+ "k_proj"
30
+ ],
31
+ "task_type": "CAUSAL_LM",
32
+ "use_dora": false,
33
+ "use_rslora": false
34
+ }
adapter_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a07fceb0f4a213b192376861226c495015857f3a4db9a7662c58acf847cac9b7
3
+ size 50573978
checkpoint-116/README.md ADDED
@@ -0,0 +1,202 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ base_model: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
4
+ ---
5
+
6
+ # Model Card for Model ID
7
+
8
+ <!-- Provide a quick summary of what the model is/does. -->
9
+
10
+
11
+
12
+ ## Model Details
13
+
14
+ ### Model Description
15
+
16
+ <!-- Provide a longer summary of what this model is. -->
17
+
18
+
19
+
20
+ - **Developed by:** [More Information Needed]
21
+ - **Funded by [optional]:** [More Information Needed]
22
+ - **Shared by [optional]:** [More Information Needed]
23
+ - **Model type:** [More Information Needed]
24
+ - **Language(s) (NLP):** [More Information Needed]
25
+ - **License:** [More Information Needed]
26
+ - **Finetuned from model [optional]:** [More Information Needed]
27
+
28
+ ### Model Sources [optional]
29
+
30
+ <!-- Provide the basic links for the model. -->
31
+
32
+ - **Repository:** [More Information Needed]
33
+ - **Paper [optional]:** [More Information Needed]
34
+ - **Demo [optional]:** [More Information Needed]
35
+
36
+ ## Uses
37
+
38
+ <!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
39
+
40
+ ### Direct Use
41
+
42
+ <!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
43
+
44
+ [More Information Needed]
45
+
46
+ ### Downstream Use [optional]
47
+
48
+ <!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
49
+
50
+ [More Information Needed]
51
+
52
+ ### Out-of-Scope Use
53
+
54
+ <!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
55
+
56
+ [More Information Needed]
57
+
58
+ ## Bias, Risks, and Limitations
59
+
60
+ <!-- This section is meant to convey both technical and sociotechnical limitations. -->
61
+
62
+ [More Information Needed]
63
+
64
+ ### Recommendations
65
+
66
+ <!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
67
+
68
+ Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
69
+
70
+ ## How to Get Started with the Model
71
+
72
+ Use the code below to get started with the model.
73
+
74
+ [More Information Needed]
75
+
76
+ ## Training Details
77
+
78
+ ### Training Data
79
+
80
+ <!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
81
+
82
+ [More Information Needed]
83
+
84
+ ### Training Procedure
85
+
86
+ <!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
87
+
88
+ #### Preprocessing [optional]
89
+
90
+ [More Information Needed]
91
+
92
+
93
+ #### Training Hyperparameters
94
+
95
+ - **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
96
+
97
+ #### Speeds, Sizes, Times [optional]
98
+
99
+ <!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
100
+
101
+ [More Information Needed]
102
+
103
+ ## Evaluation
104
+
105
+ <!-- This section describes the evaluation protocols and provides the results. -->
106
+
107
+ ### Testing Data, Factors & Metrics
108
+
109
+ #### Testing Data
110
+
111
+ <!-- This should link to a Dataset Card if possible. -->
112
+
113
+ [More Information Needed]
114
+
115
+ #### Factors
116
+
117
+ <!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
118
+
119
+ [More Information Needed]
120
+
121
+ #### Metrics
122
+
123
+ <!-- These are the evaluation metrics being used, ideally with a description of why. -->
124
+
125
+ [More Information Needed]
126
+
127
+ ### Results
128
+
129
+ [More Information Needed]
130
+
131
+ #### Summary
132
+
133
+
134
+
135
+ ## Model Examination [optional]
136
+
137
+ <!-- Relevant interpretability work for the model goes here -->
138
+
139
+ [More Information Needed]
140
+
141
+ ## Environmental Impact
142
+
143
+ <!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
144
+
145
+ Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
146
+
147
+ - **Hardware Type:** [More Information Needed]
148
+ - **Hours used:** [More Information Needed]
149
+ - **Cloud Provider:** [More Information Needed]
150
+ - **Compute Region:** [More Information Needed]
151
+ - **Carbon Emitted:** [More Information Needed]
152
+
153
+ ## Technical Specifications [optional]
154
+
155
+ ### Model Architecture and Objective
156
+
157
+ [More Information Needed]
158
+
159
+ ### Compute Infrastructure
160
+
161
+ [More Information Needed]
162
+
163
+ #### Hardware
164
+
165
+ [More Information Needed]
166
+
167
+ #### Software
168
+
169
+ [More Information Needed]
170
+
171
+ ## Citation [optional]
172
+
173
+ <!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
174
+
175
+ **BibTeX:**
176
+
177
+ [More Information Needed]
178
+
179
+ **APA:**
180
+
181
+ [More Information Needed]
182
+
183
+ ## Glossary [optional]
184
+
185
+ <!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
186
+
187
+ [More Information Needed]
188
+
189
+ ## More Information [optional]
190
+
191
+ [More Information Needed]
192
+
193
+ ## Model Card Authors [optional]
194
+
195
+ [More Information Needed]
196
+
197
+ ## Model Card Contact
198
+
199
+ [More Information Needed]
200
+ ### Framework versions
201
+
202
+ - PEFT 0.11.1
checkpoint-116/adapter_config.json ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T",
5
+ "bias": "none",
6
+ "fan_in_fan_out": null,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layer_replication": null,
10
+ "layers_pattern": null,
11
+ "layers_to_transform": null,
12
+ "loftq_config": {},
13
+ "lora_alpha": 16,
14
+ "lora_dropout": 0.05,
15
+ "megatron_config": null,
16
+ "megatron_core": "megatron.core",
17
+ "modules_to_save": null,
18
+ "peft_type": "LORA",
19
+ "r": 32,
20
+ "rank_pattern": {},
21
+ "revision": null,
22
+ "target_modules": [
23
+ "gate_proj",
24
+ "up_proj",
25
+ "down_proj",
26
+ "v_proj",
27
+ "o_proj",
28
+ "q_proj",
29
+ "k_proj"
30
+ ],
31
+ "task_type": "CAUSAL_LM",
32
+ "use_dora": false,
33
+ "use_rslora": false
34
+ }
checkpoint-116/adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:8668d68993cbafac06993db9d1c83bc618398531aab80fdab4198b3b92fe2ccb
3
+ size 50503848
checkpoint-116/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:13f111d1655d1c0c5f946c1a507f8decff70637f9ab53f6b1f6bc534501332c0
3
+ size 202035450
checkpoint-116/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:edbf3bea1d4f693a677a0bf33551e5e3063cbd96faf6ce6d46f9d3b4ff62532d
3
+ size 14244
checkpoint-116/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:68424c30349bda77a4657d1902f52c5a348248ab657a58e4aee2ebfc370c25ab
3
+ size 1064
checkpoint-116/special_tokens_map.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": "</s>",
17
+ "unk_token": {
18
+ "content": "<unk>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ }
24
+ }
checkpoint-116/tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
3
+ size 499723
checkpoint-116/tokenizer_config.json ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": true,
3
+ "add_eos_token": false,
4
+ "add_prefix_space": true,
5
+ "added_tokens_decoder": {
6
+ "0": {
7
+ "content": "<unk>",
8
+ "lstrip": false,
9
+ "normalized": false,
10
+ "rstrip": false,
11
+ "single_word": false,
12
+ "special": true
13
+ },
14
+ "1": {
15
+ "content": "<s>",
16
+ "lstrip": false,
17
+ "normalized": false,
18
+ "rstrip": false,
19
+ "single_word": false,
20
+ "special": true
21
+ },
22
+ "2": {
23
+ "content": "</s>",
24
+ "lstrip": false,
25
+ "normalized": false,
26
+ "rstrip": false,
27
+ "single_word": false,
28
+ "special": true
29
+ }
30
+ },
31
+ "bos_token": "<s>",
32
+ "clean_up_tokenization_spaces": false,
33
+ "eos_token": "</s>",
34
+ "legacy": false,
35
+ "model_max_length": 1000000000000000019884624838656,
36
+ "pad_token": "</s>",
37
+ "padding_side": "right",
38
+ "sp_model_kwargs": {},
39
+ "spaces_between_special_tokens": false,
40
+ "tokenizer_class": "LlamaTokenizer",
41
+ "unk_token": "<unk>",
42
+ "use_default_system_prompt": false,
43
+ "use_fast": true
44
+ }
checkpoint-116/trainer_state.json ADDED
@@ -0,0 +1,965 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 3.899159663865546,
5
+ "eval_steps": 8,
6
+ "global_step": 116,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.03361344537815126,
13
+ "grad_norm": 0.115234375,
14
+ "learning_rate": 2e-05,
15
+ "loss": 1.768,
16
+ "step": 1
17
+ },
18
+ {
19
+ "epoch": 0.03361344537815126,
20
+ "eval_loss": 1.8648816347122192,
21
+ "eval_runtime": 18.2501,
22
+ "eval_samples_per_second": 2.74,
23
+ "eval_steps_per_second": 0.384,
24
+ "step": 1
25
+ },
26
+ {
27
+ "epoch": 0.06722689075630252,
28
+ "grad_norm": 0.10888671875,
29
+ "learning_rate": 4e-05,
30
+ "loss": 1.7838,
31
+ "step": 2
32
+ },
33
+ {
34
+ "epoch": 0.10084033613445378,
35
+ "grad_norm": 0.126953125,
36
+ "learning_rate": 6e-05,
37
+ "loss": 1.9413,
38
+ "step": 3
39
+ },
40
+ {
41
+ "epoch": 0.13445378151260504,
42
+ "grad_norm": 0.0986328125,
43
+ "learning_rate": 8e-05,
44
+ "loss": 1.7757,
45
+ "step": 4
46
+ },
47
+ {
48
+ "epoch": 0.16806722689075632,
49
+ "grad_norm": 0.12255859375,
50
+ "learning_rate": 0.0001,
51
+ "loss": 1.735,
52
+ "step": 5
53
+ },
54
+ {
55
+ "epoch": 0.20168067226890757,
56
+ "grad_norm": 0.10791015625,
57
+ "learning_rate": 0.00012,
58
+ "loss": 1.8269,
59
+ "step": 6
60
+ },
61
+ {
62
+ "epoch": 0.23529411764705882,
63
+ "grad_norm": 0.11669921875,
64
+ "learning_rate": 0.00014,
65
+ "loss": 1.8552,
66
+ "step": 7
67
+ },
68
+ {
69
+ "epoch": 0.2689075630252101,
70
+ "grad_norm": 0.119140625,
71
+ "learning_rate": 0.00016,
72
+ "loss": 1.8084,
73
+ "step": 8
74
+ },
75
+ {
76
+ "epoch": 0.2689075630252101,
77
+ "eval_loss": 1.8317129611968994,
78
+ "eval_runtime": 19.6984,
79
+ "eval_samples_per_second": 2.538,
80
+ "eval_steps_per_second": 0.355,
81
+ "step": 8
82
+ },
83
+ {
84
+ "epoch": 0.3025210084033613,
85
+ "grad_norm": 0.12255859375,
86
+ "learning_rate": 0.00018,
87
+ "loss": 1.7158,
88
+ "step": 9
89
+ },
90
+ {
91
+ "epoch": 0.33613445378151263,
92
+ "grad_norm": 0.134765625,
93
+ "learning_rate": 0.0002,
94
+ "loss": 1.8702,
95
+ "step": 10
96
+ },
97
+ {
98
+ "epoch": 0.3697478991596639,
99
+ "grad_norm": 0.12890625,
100
+ "learning_rate": 0.00019995608365087946,
101
+ "loss": 1.8307,
102
+ "step": 11
103
+ },
104
+ {
105
+ "epoch": 0.40336134453781514,
106
+ "grad_norm": 0.11474609375,
107
+ "learning_rate": 0.00019982437317643217,
108
+ "loss": 1.6583,
109
+ "step": 12
110
+ },
111
+ {
112
+ "epoch": 0.4369747899159664,
113
+ "grad_norm": 0.134765625,
114
+ "learning_rate": 0.0001996049842615217,
115
+ "loss": 1.6663,
116
+ "step": 13
117
+ },
118
+ {
119
+ "epoch": 0.47058823529411764,
120
+ "grad_norm": 0.12060546875,
121
+ "learning_rate": 0.00019929810960135172,
122
+ "loss": 1.7388,
123
+ "step": 14
124
+ },
125
+ {
126
+ "epoch": 0.5042016806722689,
127
+ "grad_norm": 0.126953125,
128
+ "learning_rate": 0.0001989040187322164,
129
+ "loss": 1.7485,
130
+ "step": 15
131
+ },
132
+ {
133
+ "epoch": 0.5378151260504201,
134
+ "grad_norm": 0.1259765625,
135
+ "learning_rate": 0.00019842305779475968,
136
+ "loss": 1.633,
137
+ "step": 16
138
+ },
139
+ {
140
+ "epoch": 0.5378151260504201,
141
+ "eval_loss": 1.7832777500152588,
142
+ "eval_runtime": 19.6833,
143
+ "eval_samples_per_second": 2.54,
144
+ "eval_steps_per_second": 0.356,
145
+ "step": 16
146
+ },
147
+ {
148
+ "epoch": 0.5714285714285714,
149
+ "grad_norm": 0.12109375,
150
+ "learning_rate": 0.0001978556492299504,
151
+ "loss": 1.8373,
152
+ "step": 17
153
+ },
154
+ {
155
+ "epoch": 0.6050420168067226,
156
+ "grad_norm": 0.1337890625,
157
+ "learning_rate": 0.0001972022914080411,
158
+ "loss": 1.6552,
159
+ "step": 18
160
+ },
161
+ {
162
+ "epoch": 0.6386554621848739,
163
+ "grad_norm": 0.126953125,
164
+ "learning_rate": 0.00019646355819083589,
165
+ "loss": 1.8113,
166
+ "step": 19
167
+ },
168
+ {
169
+ "epoch": 0.6722689075630253,
170
+ "grad_norm": 0.1220703125,
171
+ "learning_rate": 0.00019564009842765225,
172
+ "loss": 1.6544,
173
+ "step": 20
174
+ },
175
+ {
176
+ "epoch": 0.7058823529411765,
177
+ "grad_norm": 0.11669921875,
178
+ "learning_rate": 0.00019473263538541914,
179
+ "loss": 1.6649,
180
+ "step": 21
181
+ },
182
+ {
183
+ "epoch": 0.7394957983193278,
184
+ "grad_norm": 0.0986328125,
185
+ "learning_rate": 0.0001937419661134121,
186
+ "loss": 1.6868,
187
+ "step": 22
188
+ },
189
+ {
190
+ "epoch": 0.773109243697479,
191
+ "grad_norm": 0.126953125,
192
+ "learning_rate": 0.00019266896074318334,
193
+ "loss": 1.7762,
194
+ "step": 23
195
+ },
196
+ {
197
+ "epoch": 0.8067226890756303,
198
+ "grad_norm": 0.11279296875,
199
+ "learning_rate": 0.00019151456172430183,
200
+ "loss": 1.6737,
201
+ "step": 24
202
+ },
203
+ {
204
+ "epoch": 0.8067226890756303,
205
+ "eval_loss": 1.7643933296203613,
206
+ "eval_runtime": 19.6308,
207
+ "eval_samples_per_second": 2.547,
208
+ "eval_steps_per_second": 0.357,
209
+ "step": 24
210
+ },
211
+ {
212
+ "epoch": 0.8403361344537815,
213
+ "grad_norm": 0.1298828125,
214
+ "learning_rate": 0.00019027978299657436,
215
+ "loss": 1.6401,
216
+ "step": 25
217
+ },
218
+ {
219
+ "epoch": 0.8739495798319328,
220
+ "grad_norm": 0.099609375,
221
+ "learning_rate": 0.00018896570909947475,
222
+ "loss": 1.7068,
223
+ "step": 26
224
+ },
225
+ {
226
+ "epoch": 0.907563025210084,
227
+ "grad_norm": 0.12060546875,
228
+ "learning_rate": 0.0001875734942195637,
229
+ "loss": 1.8112,
230
+ "step": 27
231
+ },
232
+ {
233
+ "epoch": 0.9411764705882353,
234
+ "grad_norm": 0.1162109375,
235
+ "learning_rate": 0.00018610436117673555,
236
+ "loss": 1.6596,
237
+ "step": 28
238
+ },
239
+ {
240
+ "epoch": 0.9747899159663865,
241
+ "grad_norm": 0.12890625,
242
+ "learning_rate": 0.0001845596003501826,
243
+ "loss": 1.7936,
244
+ "step": 29
245
+ },
246
+ {
247
+ "epoch": 1.0084033613445378,
248
+ "grad_norm": 0.1240234375,
249
+ "learning_rate": 0.0001829405685450202,
250
+ "loss": 1.7947,
251
+ "step": 30
252
+ },
253
+ {
254
+ "epoch": 1.0420168067226891,
255
+ "grad_norm": 0.20703125,
256
+ "learning_rate": 0.00018124868780056814,
257
+ "loss": 1.6887,
258
+ "step": 31
259
+ },
260
+ {
261
+ "epoch": 1.0756302521008403,
262
+ "grad_norm": 0.1455078125,
263
+ "learning_rate": 0.00017948544414133534,
264
+ "loss": 1.6722,
265
+ "step": 32
266
+ },
267
+ {
268
+ "epoch": 1.0756302521008403,
269
+ "eval_loss": 1.7600828409194946,
270
+ "eval_runtime": 19.7105,
271
+ "eval_samples_per_second": 2.537,
272
+ "eval_steps_per_second": 0.355,
273
+ "step": 32
274
+ },
275
+ {
276
+ "epoch": 1.1092436974789917,
277
+ "grad_norm": 0.09814453125,
278
+ "learning_rate": 0.00017765238627180424,
279
+ "loss": 1.7145,
280
+ "step": 33
281
+ },
282
+ {
283
+ "epoch": 1.1428571428571428,
284
+ "grad_norm": 0.10693359375,
285
+ "learning_rate": 0.00017575112421616202,
286
+ "loss": 1.6609,
287
+ "step": 34
288
+ },
289
+ {
290
+ "epoch": 1.1764705882352942,
291
+ "grad_norm": 0.11572265625,
292
+ "learning_rate": 0.00017378332790417273,
293
+ "loss": 1.6681,
294
+ "step": 35
295
+ },
296
+ {
297
+ "epoch": 1.2100840336134453,
298
+ "grad_norm": 0.11767578125,
299
+ "learning_rate": 0.00017175072570443312,
300
+ "loss": 1.6641,
301
+ "step": 36
302
+ },
303
+ {
304
+ "epoch": 1.2436974789915967,
305
+ "grad_norm": 0.11376953125,
306
+ "learning_rate": 0.00016965510290629972,
307
+ "loss": 1.7011,
308
+ "step": 37
309
+ },
310
+ {
311
+ "epoch": 1.2773109243697478,
312
+ "grad_norm": 0.11572265625,
313
+ "learning_rate": 0.00016749830015182107,
314
+ "loss": 1.7171,
315
+ "step": 38
316
+ },
317
+ {
318
+ "epoch": 1.3109243697478992,
319
+ "grad_norm": 0.103515625,
320
+ "learning_rate": 0.00016528221181905217,
321
+ "loss": 1.6333,
322
+ "step": 39
323
+ },
324
+ {
325
+ "epoch": 1.3445378151260505,
326
+ "grad_norm": 0.111328125,
327
+ "learning_rate": 0.00016300878435817113,
328
+ "loss": 1.7162,
329
+ "step": 40
330
+ },
331
+ {
332
+ "epoch": 1.3445378151260505,
333
+ "eval_loss": 1.757140040397644,
334
+ "eval_runtime": 19.6485,
335
+ "eval_samples_per_second": 2.545,
336
+ "eval_steps_per_second": 0.356,
337
+ "step": 40
338
+ },
339
+ {
340
+ "epoch": 1.3781512605042017,
341
+ "grad_norm": 0.1484375,
342
+ "learning_rate": 0.00016068001458185936,
343
+ "loss": 1.6501,
344
+ "step": 41
345
+ },
346
+ {
347
+ "epoch": 1.4117647058823528,
348
+ "grad_norm": 0.1240234375,
349
+ "learning_rate": 0.0001582979479114472,
350
+ "loss": 1.6446,
351
+ "step": 42
352
+ },
353
+ {
354
+ "epoch": 1.4453781512605042,
355
+ "grad_norm": 0.119140625,
356
+ "learning_rate": 0.00015586467658036524,
357
+ "loss": 1.7104,
358
+ "step": 43
359
+ },
360
+ {
361
+ "epoch": 1.4789915966386555,
362
+ "grad_norm": 0.109375,
363
+ "learning_rate": 0.0001533823377964791,
364
+ "loss": 1.6146,
365
+ "step": 44
366
+ },
367
+ {
368
+ "epoch": 1.5126050420168067,
369
+ "grad_norm": 0.11572265625,
370
+ "learning_rate": 0.00015085311186492206,
371
+ "loss": 1.6448,
372
+ "step": 45
373
+ },
374
+ {
375
+ "epoch": 1.5462184873949578,
376
+ "grad_norm": 0.11572265625,
377
+ "learning_rate": 0.00014827922027307451,
378
+ "loss": 1.6735,
379
+ "step": 46
380
+ },
381
+ {
382
+ "epoch": 1.5798319327731094,
383
+ "grad_norm": 0.1181640625,
384
+ "learning_rate": 0.0001456629237393713,
385
+ "loss": 1.6604,
386
+ "step": 47
387
+ },
388
+ {
389
+ "epoch": 1.6134453781512605,
390
+ "grad_norm": 0.12060546875,
391
+ "learning_rate": 0.00014300652022765207,
392
+ "loss": 1.7046,
393
+ "step": 48
394
+ },
395
+ {
396
+ "epoch": 1.6134453781512605,
397
+ "eval_loss": 1.7558497190475464,
398
+ "eval_runtime": 19.7723,
399
+ "eval_samples_per_second": 2.529,
400
+ "eval_steps_per_second": 0.354,
401
+ "step": 48
402
+ },
403
+ {
404
+ "epoch": 1.6470588235294117,
405
+ "grad_norm": 0.1259765625,
406
+ "learning_rate": 0.00014031234292879725,
407
+ "loss": 1.694,
408
+ "step": 49
409
+ },
410
+ {
411
+ "epoch": 1.680672268907563,
412
+ "grad_norm": 0.1220703125,
413
+ "learning_rate": 0.00013758275821142382,
414
+ "loss": 1.625,
415
+ "step": 50
416
+ },
417
+ {
418
+ "epoch": 1.7142857142857144,
419
+ "grad_norm": 0.123046875,
420
+ "learning_rate": 0.0001348201635434399,
421
+ "loss": 1.6919,
422
+ "step": 51
423
+ },
424
+ {
425
+ "epoch": 1.7478991596638656,
426
+ "grad_norm": 0.1201171875,
427
+ "learning_rate": 0.00013202698538628376,
428
+ "loss": 1.6777,
429
+ "step": 52
430
+ },
431
+ {
432
+ "epoch": 1.7815126050420167,
433
+ "grad_norm": 0.12890625,
434
+ "learning_rate": 0.00012920567706369758,
435
+ "loss": 1.7762,
436
+ "step": 53
437
+ },
438
+ {
439
+ "epoch": 1.815126050420168,
440
+ "grad_norm": 0.126953125,
441
+ "learning_rate": 0.00012635871660690676,
442
+ "loss": 1.6668,
443
+ "step": 54
444
+ },
445
+ {
446
+ "epoch": 1.8487394957983194,
447
+ "grad_norm": 0.125,
448
+ "learning_rate": 0.00012348860457809838,
449
+ "loss": 1.7061,
450
+ "step": 55
451
+ },
452
+ {
453
+ "epoch": 1.8823529411764706,
454
+ "grad_norm": 0.1328125,
455
+ "learning_rate": 0.00012059786187410984,
456
+ "loss": 1.6714,
457
+ "step": 56
458
+ },
459
+ {
460
+ "epoch": 1.8823529411764706,
461
+ "eval_loss": 1.7563551664352417,
462
+ "eval_runtime": 19.6588,
463
+ "eval_samples_per_second": 2.543,
464
+ "eval_steps_per_second": 0.356,
465
+ "step": 56
466
+ },
467
+ {
468
+ "epoch": 1.9159663865546217,
469
+ "grad_norm": 0.1279296875,
470
+ "learning_rate": 0.0001176890275122573,
471
+ "loss": 1.6318,
472
+ "step": 57
473
+ },
474
+ {
475
+ "epoch": 1.949579831932773,
476
+ "grad_norm": 0.1318359375,
477
+ "learning_rate": 0.00011476465640024814,
478
+ "loss": 1.6693,
479
+ "step": 58
480
+ },
481
+ {
482
+ "epoch": 1.9831932773109244,
483
+ "grad_norm": 0.12353515625,
484
+ "learning_rate": 0.00011182731709213659,
485
+ "loss": 1.5927,
486
+ "step": 59
487
+ },
488
+ {
489
+ "epoch": 2.0168067226890756,
490
+ "grad_norm": 0.138671875,
491
+ "learning_rate": 0.00010887958953229349,
492
+ "loss": 1.6558,
493
+ "step": 60
494
+ },
495
+ {
496
+ "epoch": 2.0504201680672267,
497
+ "grad_norm": 0.134765625,
498
+ "learning_rate": 0.00010592406278937144,
499
+ "loss": 1.7352,
500
+ "step": 61
501
+ },
502
+ {
503
+ "epoch": 2.0840336134453783,
504
+ "grad_norm": 0.1328125,
505
+ "learning_rate": 0.00010296333278225599,
506
+ "loss": 1.6216,
507
+ "step": 62
508
+ },
509
+ {
510
+ "epoch": 2.1176470588235294,
511
+ "grad_norm": 0.1201171875,
512
+ "learning_rate": 0.0001,
513
+ "loss": 1.6365,
514
+ "step": 63
515
+ },
516
+ {
517
+ "epoch": 2.1512605042016806,
518
+ "grad_norm": 0.1298828125,
519
+ "learning_rate": 9.703666721774402e-05,
520
+ "loss": 1.6249,
521
+ "step": 64
522
+ },
523
+ {
524
+ "epoch": 2.1512605042016806,
525
+ "eval_loss": 1.756639838218689,
526
+ "eval_runtime": 19.5951,
527
+ "eval_samples_per_second": 2.552,
528
+ "eval_steps_per_second": 0.357,
529
+ "step": 64
530
+ },
531
+ {
532
+ "epoch": 2.184873949579832,
533
+ "grad_norm": 0.1357421875,
534
+ "learning_rate": 9.407593721062859e-05,
535
+ "loss": 1.653,
536
+ "step": 65
537
+ },
538
+ {
539
+ "epoch": 2.2184873949579833,
540
+ "grad_norm": 0.146484375,
541
+ "learning_rate": 9.112041046770653e-05,
542
+ "loss": 1.6545,
543
+ "step": 66
544
+ },
545
+ {
546
+ "epoch": 2.2521008403361344,
547
+ "grad_norm": 0.13671875,
548
+ "learning_rate": 8.817268290786343e-05,
549
+ "loss": 1.5787,
550
+ "step": 67
551
+ },
552
+ {
553
+ "epoch": 2.2857142857142856,
554
+ "grad_norm": 0.1328125,
555
+ "learning_rate": 8.523534359975189e-05,
556
+ "loss": 1.6532,
557
+ "step": 68
558
+ },
559
+ {
560
+ "epoch": 2.3193277310924367,
561
+ "grad_norm": 0.1396484375,
562
+ "learning_rate": 8.231097248774274e-05,
563
+ "loss": 1.6784,
564
+ "step": 69
565
+ },
566
+ {
567
+ "epoch": 2.3529411764705883,
568
+ "grad_norm": 0.1337890625,
569
+ "learning_rate": 7.940213812589018e-05,
570
+ "loss": 1.5721,
571
+ "step": 70
572
+ },
573
+ {
574
+ "epoch": 2.3865546218487395,
575
+ "grad_norm": 0.1416015625,
576
+ "learning_rate": 7.651139542190164e-05,
577
+ "loss": 1.5836,
578
+ "step": 71
579
+ },
580
+ {
581
+ "epoch": 2.4201680672268906,
582
+ "grad_norm": 0.146484375,
583
+ "learning_rate": 7.364128339309326e-05,
584
+ "loss": 1.5604,
585
+ "step": 72
586
+ },
587
+ {
588
+ "epoch": 2.4201680672268906,
589
+ "eval_loss": 1.7598735094070435,
590
+ "eval_runtime": 19.7508,
591
+ "eval_samples_per_second": 2.532,
592
+ "eval_steps_per_second": 0.354,
593
+ "step": 72
594
+ },
595
+ {
596
+ "epoch": 2.453781512605042,
597
+ "grad_norm": 0.1455078125,
598
+ "learning_rate": 7.079432293630244e-05,
599
+ "loss": 1.6259,
600
+ "step": 73
601
+ },
602
+ {
603
+ "epoch": 2.4873949579831933,
604
+ "grad_norm": 0.1484375,
605
+ "learning_rate": 6.797301461371625e-05,
606
+ "loss": 1.5811,
607
+ "step": 74
608
+ },
609
+ {
610
+ "epoch": 2.5210084033613445,
611
+ "grad_norm": 0.14453125,
612
+ "learning_rate": 6.517983645656014e-05,
613
+ "loss": 1.4929,
614
+ "step": 75
615
+ },
616
+ {
617
+ "epoch": 2.5546218487394956,
618
+ "grad_norm": 0.1572265625,
619
+ "learning_rate": 6.24172417885762e-05,
620
+ "loss": 1.7014,
621
+ "step": 76
622
+ },
623
+ {
624
+ "epoch": 2.588235294117647,
625
+ "grad_norm": 0.1484375,
626
+ "learning_rate": 5.96876570712028e-05,
627
+ "loss": 1.5623,
628
+ "step": 77
629
+ },
630
+ {
631
+ "epoch": 2.6218487394957983,
632
+ "grad_norm": 0.1474609375,
633
+ "learning_rate": 5.699347977234799e-05,
634
+ "loss": 1.6006,
635
+ "step": 78
636
+ },
637
+ {
638
+ "epoch": 2.6554621848739495,
639
+ "grad_norm": 0.150390625,
640
+ "learning_rate": 5.43370762606287e-05,
641
+ "loss": 1.6641,
642
+ "step": 79
643
+ },
644
+ {
645
+ "epoch": 2.689075630252101,
646
+ "grad_norm": 0.15234375,
647
+ "learning_rate": 5.172077972692553e-05,
648
+ "loss": 1.7003,
649
+ "step": 80
650
+ },
651
+ {
652
+ "epoch": 2.689075630252101,
653
+ "eval_loss": 1.761399269104004,
654
+ "eval_runtime": 19.6692,
655
+ "eval_samples_per_second": 2.542,
656
+ "eval_steps_per_second": 0.356,
657
+ "step": 80
658
+ },
659
+ {
660
+ "epoch": 2.722689075630252,
661
+ "grad_norm": 0.154296875,
662
+ "learning_rate": 4.914688813507797e-05,
663
+ "loss": 1.6923,
664
+ "step": 81
665
+ },
666
+ {
667
+ "epoch": 2.7563025210084033,
668
+ "grad_norm": 0.158203125,
669
+ "learning_rate": 4.661766220352097e-05,
670
+ "loss": 1.6819,
671
+ "step": 82
672
+ },
673
+ {
674
+ "epoch": 2.7899159663865545,
675
+ "grad_norm": 0.1513671875,
676
+ "learning_rate": 4.4135323419634766e-05,
677
+ "loss": 1.5649,
678
+ "step": 83
679
+ },
680
+ {
681
+ "epoch": 2.8235294117647056,
682
+ "grad_norm": 0.1494140625,
683
+ "learning_rate": 4.170205208855281e-05,
684
+ "loss": 1.608,
685
+ "step": 84
686
+ },
687
+ {
688
+ "epoch": 2.857142857142857,
689
+ "grad_norm": 0.1611328125,
690
+ "learning_rate": 3.931998541814069e-05,
691
+ "loss": 1.5474,
692
+ "step": 85
693
+ },
694
+ {
695
+ "epoch": 2.8907563025210083,
696
+ "grad_norm": 0.1552734375,
697
+ "learning_rate": 3.69912156418289e-05,
698
+ "loss": 1.6484,
699
+ "step": 86
700
+ },
701
+ {
702
+ "epoch": 2.92436974789916,
703
+ "grad_norm": 0.1484375,
704
+ "learning_rate": 3.471778818094785e-05,
705
+ "loss": 1.6145,
706
+ "step": 87
707
+ },
708
+ {
709
+ "epoch": 2.957983193277311,
710
+ "grad_norm": 0.158203125,
711
+ "learning_rate": 3.250169984817897e-05,
712
+ "loss": 1.7115,
713
+ "step": 88
714
+ },
715
+ {
716
+ "epoch": 2.957983193277311,
717
+ "eval_loss": 1.7605273723602295,
718
+ "eval_runtime": 19.7632,
719
+ "eval_samples_per_second": 2.53,
720
+ "eval_steps_per_second": 0.354,
721
+ "step": 88
722
+ },
723
+ {
724
+ "epoch": 2.991596638655462,
725
+ "grad_norm": 0.1591796875,
726
+ "learning_rate": 3.034489709370033e-05,
727
+ "loss": 1.6485,
728
+ "step": 89
729
+ },
730
+ {
731
+ "epoch": 3.0252100840336134,
732
+ "grad_norm": 0.16015625,
733
+ "learning_rate": 2.8249274295566864e-05,
734
+ "loss": 1.5097,
735
+ "step": 90
736
+ },
737
+ {
738
+ "epoch": 3.0588235294117645,
739
+ "grad_norm": 0.1513671875,
740
+ "learning_rate": 2.6216672095827266e-05,
741
+ "loss": 1.5918,
742
+ "step": 91
743
+ },
744
+ {
745
+ "epoch": 3.092436974789916,
746
+ "grad_norm": 0.1484375,
747
+ "learning_rate": 2.4248875783837987e-05,
748
+ "loss": 1.6116,
749
+ "step": 92
750
+ },
751
+ {
752
+ "epoch": 3.1260504201680672,
753
+ "grad_norm": 0.1552734375,
754
+ "learning_rate": 2.234761372819577e-05,
755
+ "loss": 1.5989,
756
+ "step": 93
757
+ },
758
+ {
759
+ "epoch": 3.1596638655462184,
760
+ "grad_norm": 0.154296875,
761
+ "learning_rate": 2.0514555858664663e-05,
762
+ "loss": 1.6061,
763
+ "step": 94
764
+ },
765
+ {
766
+ "epoch": 3.19327731092437,
767
+ "grad_norm": 0.16015625,
768
+ "learning_rate": 1.875131219943187e-05,
769
+ "loss": 1.549,
770
+ "step": 95
771
+ },
772
+ {
773
+ "epoch": 3.226890756302521,
774
+ "grad_norm": 0.15625,
775
+ "learning_rate": 1.7059431454979824e-05,
776
+ "loss": 1.5937,
777
+ "step": 96
778
+ },
779
+ {
780
+ "epoch": 3.226890756302521,
781
+ "eval_loss": 1.7609126567840576,
782
+ "eval_runtime": 19.7273,
783
+ "eval_samples_per_second": 2.535,
784
+ "eval_steps_per_second": 0.355,
785
+ "step": 96
786
+ },
787
+ {
788
+ "epoch": 3.2605042016806722,
789
+ "grad_norm": 0.1611328125,
790
+ "learning_rate": 1.5440399649817385e-05,
791
+ "loss": 1.6636,
792
+ "step": 97
793
+ },
794
+ {
795
+ "epoch": 3.2941176470588234,
796
+ "grad_norm": 0.15625,
797
+ "learning_rate": 1.3895638823264446e-05,
798
+ "loss": 1.6452,
799
+ "step": 98
800
+ },
801
+ {
802
+ "epoch": 3.327731092436975,
803
+ "grad_norm": 0.146484375,
804
+ "learning_rate": 1.2426505780436326e-05,
805
+ "loss": 1.6122,
806
+ "step": 99
807
+ },
808
+ {
809
+ "epoch": 3.361344537815126,
810
+ "grad_norm": 0.154296875,
811
+ "learning_rate": 1.103429090052528e-05,
812
+ "loss": 1.7153,
813
+ "step": 100
814
+ },
815
+ {
816
+ "epoch": 3.3949579831932772,
817
+ "grad_norm": 0.154296875,
818
+ "learning_rate": 9.720217003425647e-06,
819
+ "loss": 1.5094,
820
+ "step": 101
821
+ },
822
+ {
823
+ "epoch": 3.4285714285714284,
824
+ "grad_norm": 0.146484375,
825
+ "learning_rate": 8.485438275698154e-06,
826
+ "loss": 1.5703,
827
+ "step": 102
828
+ },
829
+ {
830
+ "epoch": 3.46218487394958,
831
+ "grad_norm": 0.1572265625,
832
+ "learning_rate": 7.331039256816663e-06,
833
+ "loss": 1.5971,
834
+ "step": 103
835
+ },
836
+ {
837
+ "epoch": 3.495798319327731,
838
+ "grad_norm": 0.158203125,
839
+ "learning_rate": 6.258033886587911e-06,
840
+ "loss": 1.655,
841
+ "step": 104
842
+ },
843
+ {
844
+ "epoch": 3.495798319327731,
845
+ "eval_loss": 1.7612451314926147,
846
+ "eval_runtime": 19.7385,
847
+ "eval_samples_per_second": 2.533,
848
+ "eval_steps_per_second": 0.355,
849
+ "step": 104
850
+ },
851
+ {
852
+ "epoch": 3.5294117647058822,
853
+ "grad_norm": 0.158203125,
854
+ "learning_rate": 5.267364614580861e-06,
855
+ "loss": 1.6928,
856
+ "step": 105
857
+ },
858
+ {
859
+ "epoch": 3.5630252100840334,
860
+ "grad_norm": 0.1474609375,
861
+ "learning_rate": 4.359901572347758e-06,
862
+ "loss": 1.6083,
863
+ "step": 106
864
+ },
865
+ {
866
+ "epoch": 3.596638655462185,
867
+ "grad_norm": 0.1591796875,
868
+ "learning_rate": 3.5364418091641373e-06,
869
+ "loss": 1.6547,
870
+ "step": 107
871
+ },
872
+ {
873
+ "epoch": 3.630252100840336,
874
+ "grad_norm": 0.1513671875,
875
+ "learning_rate": 2.7977085919589254e-06,
876
+ "loss": 1.5425,
877
+ "step": 108
878
+ },
879
+ {
880
+ "epoch": 3.6638655462184873,
881
+ "grad_norm": 0.154296875,
882
+ "learning_rate": 2.144350770049597e-06,
883
+ "loss": 1.5763,
884
+ "step": 109
885
+ },
886
+ {
887
+ "epoch": 3.697478991596639,
888
+ "grad_norm": 0.1513671875,
889
+ "learning_rate": 1.576942205240317e-06,
890
+ "loss": 1.513,
891
+ "step": 110
892
+ },
893
+ {
894
+ "epoch": 3.73109243697479,
895
+ "grad_norm": 0.15234375,
896
+ "learning_rate": 1.0959812677835968e-06,
897
+ "loss": 1.5338,
898
+ "step": 111
899
+ },
900
+ {
901
+ "epoch": 3.764705882352941,
902
+ "grad_norm": 0.150390625,
903
+ "learning_rate": 7.018903986483083e-07,
904
+ "loss": 1.5829,
905
+ "step": 112
906
+ },
907
+ {
908
+ "epoch": 3.764705882352941,
909
+ "eval_loss": 1.7612619400024414,
910
+ "eval_runtime": 19.83,
911
+ "eval_samples_per_second": 2.521,
912
+ "eval_steps_per_second": 0.353,
913
+ "step": 112
914
+ },
915
+ {
916
+ "epoch": 3.7983193277310923,
917
+ "grad_norm": 0.1572265625,
918
+ "learning_rate": 3.950157384783104e-07,
919
+ "loss": 1.5591,
920
+ "step": 113
921
+ },
922
+ {
923
+ "epoch": 3.831932773109244,
924
+ "grad_norm": 0.1533203125,
925
+ "learning_rate": 1.7562682356786487e-07,
926
+ "loss": 1.6213,
927
+ "step": 114
928
+ },
929
+ {
930
+ "epoch": 3.865546218487395,
931
+ "grad_norm": 0.1572265625,
932
+ "learning_rate": 4.391634912056519e-08,
933
+ "loss": 1.6246,
934
+ "step": 115
935
+ },
936
+ {
937
+ "epoch": 3.899159663865546,
938
+ "grad_norm": 0.1591796875,
939
+ "learning_rate": 0.0,
940
+ "loss": 1.5533,
941
+ "step": 116
942
+ }
943
+ ],
944
+ "logging_steps": 1,
945
+ "max_steps": 116,
946
+ "num_input_tokens_seen": 0,
947
+ "num_train_epochs": 4,
948
+ "save_steps": 29,
949
+ "stateful_callbacks": {
950
+ "TrainerControl": {
951
+ "args": {
952
+ "should_epoch_stop": false,
953
+ "should_evaluate": false,
954
+ "should_log": false,
955
+ "should_save": true,
956
+ "should_training_stop": true
957
+ },
958
+ "attributes": {}
959
+ }
960
+ },
961
+ "total_flos": 1.3243100413820928e+16,
962
+ "train_batch_size": 8,
963
+ "trial_name": null,
964
+ "trial_params": null
965
+ }
checkpoint-116/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6f22338230e6fb5499b3cb737d6ee991fc1b36ba21747e343dbc66d770479d2b
3
+ size 5944
checkpoint-29/README.md ADDED
@@ -0,0 +1,202 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ base_model: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
4
+ ---
5
+
6
+ # Model Card for Model ID
7
+
8
+ <!-- Provide a quick summary of what the model is/does. -->
9
+
10
+
11
+
12
+ ## Model Details
13
+
14
+ ### Model Description
15
+
16
+ <!-- Provide a longer summary of what this model is. -->
17
+
18
+
19
+
20
+ - **Developed by:** [More Information Needed]
21
+ - **Funded by [optional]:** [More Information Needed]
22
+ - **Shared by [optional]:** [More Information Needed]
23
+ - **Model type:** [More Information Needed]
24
+ - **Language(s) (NLP):** [More Information Needed]
25
+ - **License:** [More Information Needed]
26
+ - **Finetuned from model [optional]:** [More Information Needed]
27
+
28
+ ### Model Sources [optional]
29
+
30
+ <!-- Provide the basic links for the model. -->
31
+
32
+ - **Repository:** [More Information Needed]
33
+ - **Paper [optional]:** [More Information Needed]
34
+ - **Demo [optional]:** [More Information Needed]
35
+
36
+ ## Uses
37
+
38
+ <!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
39
+
40
+ ### Direct Use
41
+
42
+ <!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
43
+
44
+ [More Information Needed]
45
+
46
+ ### Downstream Use [optional]
47
+
48
+ <!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
49
+
50
+ [More Information Needed]
51
+
52
+ ### Out-of-Scope Use
53
+
54
+ <!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
55
+
56
+ [More Information Needed]
57
+
58
+ ## Bias, Risks, and Limitations
59
+
60
+ <!-- This section is meant to convey both technical and sociotechnical limitations. -->
61
+
62
+ [More Information Needed]
63
+
64
+ ### Recommendations
65
+
66
+ <!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
67
+
68
+ Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
69
+
70
+ ## How to Get Started with the Model
71
+
72
+ Use the code below to get started with the model.
73
+
74
+ [More Information Needed]
75
+
76
+ ## Training Details
77
+
78
+ ### Training Data
79
+
80
+ <!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
81
+
82
+ [More Information Needed]
83
+
84
+ ### Training Procedure
85
+
86
+ <!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
87
+
88
+ #### Preprocessing [optional]
89
+
90
+ [More Information Needed]
91
+
92
+
93
+ #### Training Hyperparameters
94
+
95
+ - **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
96
+
97
+ #### Speeds, Sizes, Times [optional]
98
+
99
+ <!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
100
+
101
+ [More Information Needed]
102
+
103
+ ## Evaluation
104
+
105
+ <!-- This section describes the evaluation protocols and provides the results. -->
106
+
107
+ ### Testing Data, Factors & Metrics
108
+
109
+ #### Testing Data
110
+
111
+ <!-- This should link to a Dataset Card if possible. -->
112
+
113
+ [More Information Needed]
114
+
115
+ #### Factors
116
+
117
+ <!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
118
+
119
+ [More Information Needed]
120
+
121
+ #### Metrics
122
+
123
+ <!-- These are the evaluation metrics being used, ideally with a description of why. -->
124
+
125
+ [More Information Needed]
126
+
127
+ ### Results
128
+
129
+ [More Information Needed]
130
+
131
+ #### Summary
132
+
133
+
134
+
135
+ ## Model Examination [optional]
136
+
137
+ <!-- Relevant interpretability work for the model goes here -->
138
+
139
+ [More Information Needed]
140
+
141
+ ## Environmental Impact
142
+
143
+ <!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
144
+
145
+ Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
146
+
147
+ - **Hardware Type:** [More Information Needed]
148
+ - **Hours used:** [More Information Needed]
149
+ - **Cloud Provider:** [More Information Needed]
150
+ - **Compute Region:** [More Information Needed]
151
+ - **Carbon Emitted:** [More Information Needed]
152
+
153
+ ## Technical Specifications [optional]
154
+
155
+ ### Model Architecture and Objective
156
+
157
+ [More Information Needed]
158
+
159
+ ### Compute Infrastructure
160
+
161
+ [More Information Needed]
162
+
163
+ #### Hardware
164
+
165
+ [More Information Needed]
166
+
167
+ #### Software
168
+
169
+ [More Information Needed]
170
+
171
+ ## Citation [optional]
172
+
173
+ <!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
174
+
175
+ **BibTeX:**
176
+
177
+ [More Information Needed]
178
+
179
+ **APA:**
180
+
181
+ [More Information Needed]
182
+
183
+ ## Glossary [optional]
184
+
185
+ <!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
186
+
187
+ [More Information Needed]
188
+
189
+ ## More Information [optional]
190
+
191
+ [More Information Needed]
192
+
193
+ ## Model Card Authors [optional]
194
+
195
+ [More Information Needed]
196
+
197
+ ## Model Card Contact
198
+
199
+ [More Information Needed]
200
+ ### Framework versions
201
+
202
+ - PEFT 0.11.1
checkpoint-29/adapter_config.json ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T",
5
+ "bias": "none",
6
+ "fan_in_fan_out": null,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layer_replication": null,
10
+ "layers_pattern": null,
11
+ "layers_to_transform": null,
12
+ "loftq_config": {},
13
+ "lora_alpha": 16,
14
+ "lora_dropout": 0.05,
15
+ "megatron_config": null,
16
+ "megatron_core": "megatron.core",
17
+ "modules_to_save": null,
18
+ "peft_type": "LORA",
19
+ "r": 32,
20
+ "rank_pattern": {},
21
+ "revision": null,
22
+ "target_modules": [
23
+ "gate_proj",
24
+ "up_proj",
25
+ "down_proj",
26
+ "v_proj",
27
+ "o_proj",
28
+ "q_proj",
29
+ "k_proj"
30
+ ],
31
+ "task_type": "CAUSAL_LM",
32
+ "use_dora": false,
33
+ "use_rslora": false
34
+ }
checkpoint-29/adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7f3306c7c3271b6df472652a3e758371296cdd9260f28eae96c6d5c93e8f261b
3
+ size 50503848
checkpoint-29/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:60450a89e284ab0412512635b6713a0507d3a55dc20cb18cd142230fe92e18c8
3
+ size 202035450
checkpoint-29/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f5dff5ca7ee6d9737c0fb532125f4108aa3bd942be0c6c415c0eee299436cfee
3
+ size 14244
checkpoint-29/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:82e0ea69a0f2d46a8611802e20de4ba9ab4c81307121d85f58745fed7e6bfae6
3
+ size 1064
checkpoint-29/special_tokens_map.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": "</s>",
17
+ "unk_token": {
18
+ "content": "<unk>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ }
24
+ }
checkpoint-29/tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
3
+ size 499723
checkpoint-29/tokenizer_config.json ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": true,
3
+ "add_eos_token": false,
4
+ "add_prefix_space": true,
5
+ "added_tokens_decoder": {
6
+ "0": {
7
+ "content": "<unk>",
8
+ "lstrip": false,
9
+ "normalized": false,
10
+ "rstrip": false,
11
+ "single_word": false,
12
+ "special": true
13
+ },
14
+ "1": {
15
+ "content": "<s>",
16
+ "lstrip": false,
17
+ "normalized": false,
18
+ "rstrip": false,
19
+ "single_word": false,
20
+ "special": true
21
+ },
22
+ "2": {
23
+ "content": "</s>",
24
+ "lstrip": false,
25
+ "normalized": false,
26
+ "rstrip": false,
27
+ "single_word": false,
28
+ "special": true
29
+ }
30
+ },
31
+ "bos_token": "<s>",
32
+ "clean_up_tokenization_spaces": false,
33
+ "eos_token": "</s>",
34
+ "legacy": false,
35
+ "model_max_length": 1000000000000000019884624838656,
36
+ "pad_token": "</s>",
37
+ "padding_side": "right",
38
+ "sp_model_kwargs": {},
39
+ "spaces_between_special_tokens": false,
40
+ "tokenizer_class": "LlamaTokenizer",
41
+ "unk_token": "<unk>",
42
+ "use_default_system_prompt": false,
43
+ "use_fast": true
44
+ }
checkpoint-29/trainer_state.json ADDED
@@ -0,0 +1,268 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 0.9747899159663865,
5
+ "eval_steps": 8,
6
+ "global_step": 29,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.03361344537815126,
13
+ "grad_norm": 0.115234375,
14
+ "learning_rate": 2e-05,
15
+ "loss": 1.768,
16
+ "step": 1
17
+ },
18
+ {
19
+ "epoch": 0.03361344537815126,
20
+ "eval_loss": 1.8648816347122192,
21
+ "eval_runtime": 18.2501,
22
+ "eval_samples_per_second": 2.74,
23
+ "eval_steps_per_second": 0.384,
24
+ "step": 1
25
+ },
26
+ {
27
+ "epoch": 0.06722689075630252,
28
+ "grad_norm": 0.10888671875,
29
+ "learning_rate": 4e-05,
30
+ "loss": 1.7838,
31
+ "step": 2
32
+ },
33
+ {
34
+ "epoch": 0.10084033613445378,
35
+ "grad_norm": 0.126953125,
36
+ "learning_rate": 6e-05,
37
+ "loss": 1.9413,
38
+ "step": 3
39
+ },
40
+ {
41
+ "epoch": 0.13445378151260504,
42
+ "grad_norm": 0.0986328125,
43
+ "learning_rate": 8e-05,
44
+ "loss": 1.7757,
45
+ "step": 4
46
+ },
47
+ {
48
+ "epoch": 0.16806722689075632,
49
+ "grad_norm": 0.12255859375,
50
+ "learning_rate": 0.0001,
51
+ "loss": 1.735,
52
+ "step": 5
53
+ },
54
+ {
55
+ "epoch": 0.20168067226890757,
56
+ "grad_norm": 0.10791015625,
57
+ "learning_rate": 0.00012,
58
+ "loss": 1.8269,
59
+ "step": 6
60
+ },
61
+ {
62
+ "epoch": 0.23529411764705882,
63
+ "grad_norm": 0.11669921875,
64
+ "learning_rate": 0.00014,
65
+ "loss": 1.8552,
66
+ "step": 7
67
+ },
68
+ {
69
+ "epoch": 0.2689075630252101,
70
+ "grad_norm": 0.119140625,
71
+ "learning_rate": 0.00016,
72
+ "loss": 1.8084,
73
+ "step": 8
74
+ },
75
+ {
76
+ "epoch": 0.2689075630252101,
77
+ "eval_loss": 1.8317129611968994,
78
+ "eval_runtime": 19.6984,
79
+ "eval_samples_per_second": 2.538,
80
+ "eval_steps_per_second": 0.355,
81
+ "step": 8
82
+ },
83
+ {
84
+ "epoch": 0.3025210084033613,
85
+ "grad_norm": 0.12255859375,
86
+ "learning_rate": 0.00018,
87
+ "loss": 1.7158,
88
+ "step": 9
89
+ },
90
+ {
91
+ "epoch": 0.33613445378151263,
92
+ "grad_norm": 0.134765625,
93
+ "learning_rate": 0.0002,
94
+ "loss": 1.8702,
95
+ "step": 10
96
+ },
97
+ {
98
+ "epoch": 0.3697478991596639,
99
+ "grad_norm": 0.12890625,
100
+ "learning_rate": 0.00019995608365087946,
101
+ "loss": 1.8307,
102
+ "step": 11
103
+ },
104
+ {
105
+ "epoch": 0.40336134453781514,
106
+ "grad_norm": 0.11474609375,
107
+ "learning_rate": 0.00019982437317643217,
108
+ "loss": 1.6583,
109
+ "step": 12
110
+ },
111
+ {
112
+ "epoch": 0.4369747899159664,
113
+ "grad_norm": 0.134765625,
114
+ "learning_rate": 0.0001996049842615217,
115
+ "loss": 1.6663,
116
+ "step": 13
117
+ },
118
+ {
119
+ "epoch": 0.47058823529411764,
120
+ "grad_norm": 0.12060546875,
121
+ "learning_rate": 0.00019929810960135172,
122
+ "loss": 1.7388,
123
+ "step": 14
124
+ },
125
+ {
126
+ "epoch": 0.5042016806722689,
127
+ "grad_norm": 0.126953125,
128
+ "learning_rate": 0.0001989040187322164,
129
+ "loss": 1.7485,
130
+ "step": 15
131
+ },
132
+ {
133
+ "epoch": 0.5378151260504201,
134
+ "grad_norm": 0.1259765625,
135
+ "learning_rate": 0.00019842305779475968,
136
+ "loss": 1.633,
137
+ "step": 16
138
+ },
139
+ {
140
+ "epoch": 0.5378151260504201,
141
+ "eval_loss": 1.7832777500152588,
142
+ "eval_runtime": 19.6833,
143
+ "eval_samples_per_second": 2.54,
144
+ "eval_steps_per_second": 0.356,
145
+ "step": 16
146
+ },
147
+ {
148
+ "epoch": 0.5714285714285714,
149
+ "grad_norm": 0.12109375,
150
+ "learning_rate": 0.0001978556492299504,
151
+ "loss": 1.8373,
152
+ "step": 17
153
+ },
154
+ {
155
+ "epoch": 0.6050420168067226,
156
+ "grad_norm": 0.1337890625,
157
+ "learning_rate": 0.0001972022914080411,
158
+ "loss": 1.6552,
159
+ "step": 18
160
+ },
161
+ {
162
+ "epoch": 0.6386554621848739,
163
+ "grad_norm": 0.126953125,
164
+ "learning_rate": 0.00019646355819083589,
165
+ "loss": 1.8113,
166
+ "step": 19
167
+ },
168
+ {
169
+ "epoch": 0.6722689075630253,
170
+ "grad_norm": 0.1220703125,
171
+ "learning_rate": 0.00019564009842765225,
172
+ "loss": 1.6544,
173
+ "step": 20
174
+ },
175
+ {
176
+ "epoch": 0.7058823529411765,
177
+ "grad_norm": 0.11669921875,
178
+ "learning_rate": 0.00019473263538541914,
179
+ "loss": 1.6649,
180
+ "step": 21
181
+ },
182
+ {
183
+ "epoch": 0.7394957983193278,
184
+ "grad_norm": 0.0986328125,
185
+ "learning_rate": 0.0001937419661134121,
186
+ "loss": 1.6868,
187
+ "step": 22
188
+ },
189
+ {
190
+ "epoch": 0.773109243697479,
191
+ "grad_norm": 0.126953125,
192
+ "learning_rate": 0.00019266896074318334,
193
+ "loss": 1.7762,
194
+ "step": 23
195
+ },
196
+ {
197
+ "epoch": 0.8067226890756303,
198
+ "grad_norm": 0.11279296875,
199
+ "learning_rate": 0.00019151456172430183,
200
+ "loss": 1.6737,
201
+ "step": 24
202
+ },
203
+ {
204
+ "epoch": 0.8067226890756303,
205
+ "eval_loss": 1.7643933296203613,
206
+ "eval_runtime": 19.6308,
207
+ "eval_samples_per_second": 2.547,
208
+ "eval_steps_per_second": 0.357,
209
+ "step": 24
210
+ },
211
+ {
212
+ "epoch": 0.8403361344537815,
213
+ "grad_norm": 0.1298828125,
214
+ "learning_rate": 0.00019027978299657436,
215
+ "loss": 1.6401,
216
+ "step": 25
217
+ },
218
+ {
219
+ "epoch": 0.8739495798319328,
220
+ "grad_norm": 0.099609375,
221
+ "learning_rate": 0.00018896570909947475,
222
+ "loss": 1.7068,
223
+ "step": 26
224
+ },
225
+ {
226
+ "epoch": 0.907563025210084,
227
+ "grad_norm": 0.12060546875,
228
+ "learning_rate": 0.0001875734942195637,
229
+ "loss": 1.8112,
230
+ "step": 27
231
+ },
232
+ {
233
+ "epoch": 0.9411764705882353,
234
+ "grad_norm": 0.1162109375,
235
+ "learning_rate": 0.00018610436117673555,
236
+ "loss": 1.6596,
237
+ "step": 28
238
+ },
239
+ {
240
+ "epoch": 0.9747899159663865,
241
+ "grad_norm": 0.12890625,
242
+ "learning_rate": 0.0001845596003501826,
243
+ "loss": 1.7936,
244
+ "step": 29
245
+ }
246
+ ],
247
+ "logging_steps": 1,
248
+ "max_steps": 116,
249
+ "num_input_tokens_seen": 0,
250
+ "num_train_epochs": 4,
251
+ "save_steps": 29,
252
+ "stateful_callbacks": {
253
+ "TrainerControl": {
254
+ "args": {
255
+ "should_epoch_stop": false,
256
+ "should_evaluate": false,
257
+ "should_log": false,
258
+ "should_save": true,
259
+ "should_training_stop": false
260
+ },
261
+ "attributes": {}
262
+ }
263
+ },
264
+ "total_flos": 3366220896141312.0,
265
+ "train_batch_size": 8,
266
+ "trial_name": null,
267
+ "trial_params": null
268
+ }
checkpoint-29/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6f22338230e6fb5499b3cb737d6ee991fc1b36ba21747e343dbc66d770479d2b
3
+ size 5944
checkpoint-58/README.md ADDED
@@ -0,0 +1,202 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ base_model: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
4
+ ---
5
+
6
+ # Model Card for Model ID
7
+
8
+ <!-- Provide a quick summary of what the model is/does. -->
9
+
10
+
11
+
12
+ ## Model Details
13
+
14
+ ### Model Description
15
+
16
+ <!-- Provide a longer summary of what this model is. -->
17
+
18
+
19
+
20
+ - **Developed by:** [More Information Needed]
21
+ - **Funded by [optional]:** [More Information Needed]
22
+ - **Shared by [optional]:** [More Information Needed]
23
+ - **Model type:** [More Information Needed]
24
+ - **Language(s) (NLP):** [More Information Needed]
25
+ - **License:** [More Information Needed]
26
+ - **Finetuned from model [optional]:** [More Information Needed]
27
+
28
+ ### Model Sources [optional]
29
+
30
+ <!-- Provide the basic links for the model. -->
31
+
32
+ - **Repository:** [More Information Needed]
33
+ - **Paper [optional]:** [More Information Needed]
34
+ - **Demo [optional]:** [More Information Needed]
35
+
36
+ ## Uses
37
+
38
+ <!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
39
+
40
+ ### Direct Use
41
+
42
+ <!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
43
+
44
+ [More Information Needed]
45
+
46
+ ### Downstream Use [optional]
47
+
48
+ <!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
49
+
50
+ [More Information Needed]
51
+
52
+ ### Out-of-Scope Use
53
+
54
+ <!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
55
+
56
+ [More Information Needed]
57
+
58
+ ## Bias, Risks, and Limitations
59
+
60
+ <!-- This section is meant to convey both technical and sociotechnical limitations. -->
61
+
62
+ [More Information Needed]
63
+
64
+ ### Recommendations
65
+
66
+ <!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
67
+
68
+ Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
69
+
70
+ ## How to Get Started with the Model
71
+
72
+ Use the code below to get started with the model.
73
+
74
+ [More Information Needed]
75
+
76
+ ## Training Details
77
+
78
+ ### Training Data
79
+
80
+ <!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
81
+
82
+ [More Information Needed]
83
+
84
+ ### Training Procedure
85
+
86
+ <!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
87
+
88
+ #### Preprocessing [optional]
89
+
90
+ [More Information Needed]
91
+
92
+
93
+ #### Training Hyperparameters
94
+
95
+ - **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
96
+
97
+ #### Speeds, Sizes, Times [optional]
98
+
99
+ <!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
100
+
101
+ [More Information Needed]
102
+
103
+ ## Evaluation
104
+
105
+ <!-- This section describes the evaluation protocols and provides the results. -->
106
+
107
+ ### Testing Data, Factors & Metrics
108
+
109
+ #### Testing Data
110
+
111
+ <!-- This should link to a Dataset Card if possible. -->
112
+
113
+ [More Information Needed]
114
+
115
+ #### Factors
116
+
117
+ <!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
118
+
119
+ [More Information Needed]
120
+
121
+ #### Metrics
122
+
123
+ <!-- These are the evaluation metrics being used, ideally with a description of why. -->
124
+
125
+ [More Information Needed]
126
+
127
+ ### Results
128
+
129
+ [More Information Needed]
130
+
131
+ #### Summary
132
+
133
+
134
+
135
+ ## Model Examination [optional]
136
+
137
+ <!-- Relevant interpretability work for the model goes here -->
138
+
139
+ [More Information Needed]
140
+
141
+ ## Environmental Impact
142
+
143
+ <!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
144
+
145
+ Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
146
+
147
+ - **Hardware Type:** [More Information Needed]
148
+ - **Hours used:** [More Information Needed]
149
+ - **Cloud Provider:** [More Information Needed]
150
+ - **Compute Region:** [More Information Needed]
151
+ - **Carbon Emitted:** [More Information Needed]
152
+
153
+ ## Technical Specifications [optional]
154
+
155
+ ### Model Architecture and Objective
156
+
157
+ [More Information Needed]
158
+
159
+ ### Compute Infrastructure
160
+
161
+ [More Information Needed]
162
+
163
+ #### Hardware
164
+
165
+ [More Information Needed]
166
+
167
+ #### Software
168
+
169
+ [More Information Needed]
170
+
171
+ ## Citation [optional]
172
+
173
+ <!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
174
+
175
+ **BibTeX:**
176
+
177
+ [More Information Needed]
178
+
179
+ **APA:**
180
+
181
+ [More Information Needed]
182
+
183
+ ## Glossary [optional]
184
+
185
+ <!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
186
+
187
+ [More Information Needed]
188
+
189
+ ## More Information [optional]
190
+
191
+ [More Information Needed]
192
+
193
+ ## Model Card Authors [optional]
194
+
195
+ [More Information Needed]
196
+
197
+ ## Model Card Contact
198
+
199
+ [More Information Needed]
200
+ ### Framework versions
201
+
202
+ - PEFT 0.11.1
checkpoint-58/adapter_config.json ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T",
5
+ "bias": "none",
6
+ "fan_in_fan_out": null,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layer_replication": null,
10
+ "layers_pattern": null,
11
+ "layers_to_transform": null,
12
+ "loftq_config": {},
13
+ "lora_alpha": 16,
14
+ "lora_dropout": 0.05,
15
+ "megatron_config": null,
16
+ "megatron_core": "megatron.core",
17
+ "modules_to_save": null,
18
+ "peft_type": "LORA",
19
+ "r": 32,
20
+ "rank_pattern": {},
21
+ "revision": null,
22
+ "target_modules": [
23
+ "gate_proj",
24
+ "up_proj",
25
+ "down_proj",
26
+ "v_proj",
27
+ "o_proj",
28
+ "q_proj",
29
+ "k_proj"
30
+ ],
31
+ "task_type": "CAUSAL_LM",
32
+ "use_dora": false,
33
+ "use_rslora": false
34
+ }
checkpoint-58/adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f07ef1ad6cfdd6061f1cad1529524d822d6c21409b40d6ca2c11e7a2ebd9b03f
3
+ size 50503848
checkpoint-58/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:712ca45af3a346ebbc26929883fe200d0d4fa0625f5f02944a41b2721a0dac4f
3
+ size 202035450
checkpoint-58/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7e2714773be96a258d965b3d5be5d9e51b91e01c435077d016f8a6cad6f5455b
3
+ size 14244
checkpoint-58/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:2ffd843f86241f649ae5523a8aa7c9d13157ef6bc9ba4fe819ec88f7f7923587
3
+ size 1064
checkpoint-58/special_tokens_map.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": "</s>",
17
+ "unk_token": {
18
+ "content": "<unk>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ }
24
+ }
checkpoint-58/tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
3
+ size 499723
checkpoint-58/tokenizer_config.json ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": true,
3
+ "add_eos_token": false,
4
+ "add_prefix_space": true,
5
+ "added_tokens_decoder": {
6
+ "0": {
7
+ "content": "<unk>",
8
+ "lstrip": false,
9
+ "normalized": false,
10
+ "rstrip": false,
11
+ "single_word": false,
12
+ "special": true
13
+ },
14
+ "1": {
15
+ "content": "<s>",
16
+ "lstrip": false,
17
+ "normalized": false,
18
+ "rstrip": false,
19
+ "single_word": false,
20
+ "special": true
21
+ },
22
+ "2": {
23
+ "content": "</s>",
24
+ "lstrip": false,
25
+ "normalized": false,
26
+ "rstrip": false,
27
+ "single_word": false,
28
+ "special": true
29
+ }
30
+ },
31
+ "bos_token": "<s>",
32
+ "clean_up_tokenization_spaces": false,
33
+ "eos_token": "</s>",
34
+ "legacy": false,
35
+ "model_max_length": 1000000000000000019884624838656,
36
+ "pad_token": "</s>",
37
+ "padding_side": "right",
38
+ "sp_model_kwargs": {},
39
+ "spaces_between_special_tokens": false,
40
+ "tokenizer_class": "LlamaTokenizer",
41
+ "unk_token": "<unk>",
42
+ "use_default_system_prompt": false,
43
+ "use_fast": true
44
+ }
checkpoint-58/trainer_state.json ADDED
@@ -0,0 +1,503 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 1.949579831932773,
5
+ "eval_steps": 8,
6
+ "global_step": 58,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.03361344537815126,
13
+ "grad_norm": 0.115234375,
14
+ "learning_rate": 2e-05,
15
+ "loss": 1.768,
16
+ "step": 1
17
+ },
18
+ {
19
+ "epoch": 0.03361344537815126,
20
+ "eval_loss": 1.8648816347122192,
21
+ "eval_runtime": 18.2501,
22
+ "eval_samples_per_second": 2.74,
23
+ "eval_steps_per_second": 0.384,
24
+ "step": 1
25
+ },
26
+ {
27
+ "epoch": 0.06722689075630252,
28
+ "grad_norm": 0.10888671875,
29
+ "learning_rate": 4e-05,
30
+ "loss": 1.7838,
31
+ "step": 2
32
+ },
33
+ {
34
+ "epoch": 0.10084033613445378,
35
+ "grad_norm": 0.126953125,
36
+ "learning_rate": 6e-05,
37
+ "loss": 1.9413,
38
+ "step": 3
39
+ },
40
+ {
41
+ "epoch": 0.13445378151260504,
42
+ "grad_norm": 0.0986328125,
43
+ "learning_rate": 8e-05,
44
+ "loss": 1.7757,
45
+ "step": 4
46
+ },
47
+ {
48
+ "epoch": 0.16806722689075632,
49
+ "grad_norm": 0.12255859375,
50
+ "learning_rate": 0.0001,
51
+ "loss": 1.735,
52
+ "step": 5
53
+ },
54
+ {
55
+ "epoch": 0.20168067226890757,
56
+ "grad_norm": 0.10791015625,
57
+ "learning_rate": 0.00012,
58
+ "loss": 1.8269,
59
+ "step": 6
60
+ },
61
+ {
62
+ "epoch": 0.23529411764705882,
63
+ "grad_norm": 0.11669921875,
64
+ "learning_rate": 0.00014,
65
+ "loss": 1.8552,
66
+ "step": 7
67
+ },
68
+ {
69
+ "epoch": 0.2689075630252101,
70
+ "grad_norm": 0.119140625,
71
+ "learning_rate": 0.00016,
72
+ "loss": 1.8084,
73
+ "step": 8
74
+ },
75
+ {
76
+ "epoch": 0.2689075630252101,
77
+ "eval_loss": 1.8317129611968994,
78
+ "eval_runtime": 19.6984,
79
+ "eval_samples_per_second": 2.538,
80
+ "eval_steps_per_second": 0.355,
81
+ "step": 8
82
+ },
83
+ {
84
+ "epoch": 0.3025210084033613,
85
+ "grad_norm": 0.12255859375,
86
+ "learning_rate": 0.00018,
87
+ "loss": 1.7158,
88
+ "step": 9
89
+ },
90
+ {
91
+ "epoch": 0.33613445378151263,
92
+ "grad_norm": 0.134765625,
93
+ "learning_rate": 0.0002,
94
+ "loss": 1.8702,
95
+ "step": 10
96
+ },
97
+ {
98
+ "epoch": 0.3697478991596639,
99
+ "grad_norm": 0.12890625,
100
+ "learning_rate": 0.00019995608365087946,
101
+ "loss": 1.8307,
102
+ "step": 11
103
+ },
104
+ {
105
+ "epoch": 0.40336134453781514,
106
+ "grad_norm": 0.11474609375,
107
+ "learning_rate": 0.00019982437317643217,
108
+ "loss": 1.6583,
109
+ "step": 12
110
+ },
111
+ {
112
+ "epoch": 0.4369747899159664,
113
+ "grad_norm": 0.134765625,
114
+ "learning_rate": 0.0001996049842615217,
115
+ "loss": 1.6663,
116
+ "step": 13
117
+ },
118
+ {
119
+ "epoch": 0.47058823529411764,
120
+ "grad_norm": 0.12060546875,
121
+ "learning_rate": 0.00019929810960135172,
122
+ "loss": 1.7388,
123
+ "step": 14
124
+ },
125
+ {
126
+ "epoch": 0.5042016806722689,
127
+ "grad_norm": 0.126953125,
128
+ "learning_rate": 0.0001989040187322164,
129
+ "loss": 1.7485,
130
+ "step": 15
131
+ },
132
+ {
133
+ "epoch": 0.5378151260504201,
134
+ "grad_norm": 0.1259765625,
135
+ "learning_rate": 0.00019842305779475968,
136
+ "loss": 1.633,
137
+ "step": 16
138
+ },
139
+ {
140
+ "epoch": 0.5378151260504201,
141
+ "eval_loss": 1.7832777500152588,
142
+ "eval_runtime": 19.6833,
143
+ "eval_samples_per_second": 2.54,
144
+ "eval_steps_per_second": 0.356,
145
+ "step": 16
146
+ },
147
+ {
148
+ "epoch": 0.5714285714285714,
149
+ "grad_norm": 0.12109375,
150
+ "learning_rate": 0.0001978556492299504,
151
+ "loss": 1.8373,
152
+ "step": 17
153
+ },
154
+ {
155
+ "epoch": 0.6050420168067226,
156
+ "grad_norm": 0.1337890625,
157
+ "learning_rate": 0.0001972022914080411,
158
+ "loss": 1.6552,
159
+ "step": 18
160
+ },
161
+ {
162
+ "epoch": 0.6386554621848739,
163
+ "grad_norm": 0.126953125,
164
+ "learning_rate": 0.00019646355819083589,
165
+ "loss": 1.8113,
166
+ "step": 19
167
+ },
168
+ {
169
+ "epoch": 0.6722689075630253,
170
+ "grad_norm": 0.1220703125,
171
+ "learning_rate": 0.00019564009842765225,
172
+ "loss": 1.6544,
173
+ "step": 20
174
+ },
175
+ {
176
+ "epoch": 0.7058823529411765,
177
+ "grad_norm": 0.11669921875,
178
+ "learning_rate": 0.00019473263538541914,
179
+ "loss": 1.6649,
180
+ "step": 21
181
+ },
182
+ {
183
+ "epoch": 0.7394957983193278,
184
+ "grad_norm": 0.0986328125,
185
+ "learning_rate": 0.0001937419661134121,
186
+ "loss": 1.6868,
187
+ "step": 22
188
+ },
189
+ {
190
+ "epoch": 0.773109243697479,
191
+ "grad_norm": 0.126953125,
192
+ "learning_rate": 0.00019266896074318334,
193
+ "loss": 1.7762,
194
+ "step": 23
195
+ },
196
+ {
197
+ "epoch": 0.8067226890756303,
198
+ "grad_norm": 0.11279296875,
199
+ "learning_rate": 0.00019151456172430183,
200
+ "loss": 1.6737,
201
+ "step": 24
202
+ },
203
+ {
204
+ "epoch": 0.8067226890756303,
205
+ "eval_loss": 1.7643933296203613,
206
+ "eval_runtime": 19.6308,
207
+ "eval_samples_per_second": 2.547,
208
+ "eval_steps_per_second": 0.357,
209
+ "step": 24
210
+ },
211
+ {
212
+ "epoch": 0.8403361344537815,
213
+ "grad_norm": 0.1298828125,
214
+ "learning_rate": 0.00019027978299657436,
215
+ "loss": 1.6401,
216
+ "step": 25
217
+ },
218
+ {
219
+ "epoch": 0.8739495798319328,
220
+ "grad_norm": 0.099609375,
221
+ "learning_rate": 0.00018896570909947475,
222
+ "loss": 1.7068,
223
+ "step": 26
224
+ },
225
+ {
226
+ "epoch": 0.907563025210084,
227
+ "grad_norm": 0.12060546875,
228
+ "learning_rate": 0.0001875734942195637,
229
+ "loss": 1.8112,
230
+ "step": 27
231
+ },
232
+ {
233
+ "epoch": 0.9411764705882353,
234
+ "grad_norm": 0.1162109375,
235
+ "learning_rate": 0.00018610436117673555,
236
+ "loss": 1.6596,
237
+ "step": 28
238
+ },
239
+ {
240
+ "epoch": 0.9747899159663865,
241
+ "grad_norm": 0.12890625,
242
+ "learning_rate": 0.0001845596003501826,
243
+ "loss": 1.7936,
244
+ "step": 29
245
+ },
246
+ {
247
+ "epoch": 1.0084033613445378,
248
+ "grad_norm": 0.1240234375,
249
+ "learning_rate": 0.0001829405685450202,
250
+ "loss": 1.7947,
251
+ "step": 30
252
+ },
253
+ {
254
+ "epoch": 1.0420168067226891,
255
+ "grad_norm": 0.20703125,
256
+ "learning_rate": 0.00018124868780056814,
257
+ "loss": 1.6887,
258
+ "step": 31
259
+ },
260
+ {
261
+ "epoch": 1.0756302521008403,
262
+ "grad_norm": 0.1455078125,
263
+ "learning_rate": 0.00017948544414133534,
264
+ "loss": 1.6722,
265
+ "step": 32
266
+ },
267
+ {
268
+ "epoch": 1.0756302521008403,
269
+ "eval_loss": 1.7600828409194946,
270
+ "eval_runtime": 19.7105,
271
+ "eval_samples_per_second": 2.537,
272
+ "eval_steps_per_second": 0.355,
273
+ "step": 32
274
+ },
275
+ {
276
+ "epoch": 1.1092436974789917,
277
+ "grad_norm": 0.09814453125,
278
+ "learning_rate": 0.00017765238627180424,
279
+ "loss": 1.7145,
280
+ "step": 33
281
+ },
282
+ {
283
+ "epoch": 1.1428571428571428,
284
+ "grad_norm": 0.10693359375,
285
+ "learning_rate": 0.00017575112421616202,
286
+ "loss": 1.6609,
287
+ "step": 34
288
+ },
289
+ {
290
+ "epoch": 1.1764705882352942,
291
+ "grad_norm": 0.11572265625,
292
+ "learning_rate": 0.00017378332790417273,
293
+ "loss": 1.6681,
294
+ "step": 35
295
+ },
296
+ {
297
+ "epoch": 1.2100840336134453,
298
+ "grad_norm": 0.11767578125,
299
+ "learning_rate": 0.00017175072570443312,
300
+ "loss": 1.6641,
301
+ "step": 36
302
+ },
303
+ {
304
+ "epoch": 1.2436974789915967,
305
+ "grad_norm": 0.11376953125,
306
+ "learning_rate": 0.00016965510290629972,
307
+ "loss": 1.7011,
308
+ "step": 37
309
+ },
310
+ {
311
+ "epoch": 1.2773109243697478,
312
+ "grad_norm": 0.11572265625,
313
+ "learning_rate": 0.00016749830015182107,
314
+ "loss": 1.7171,
315
+ "step": 38
316
+ },
317
+ {
318
+ "epoch": 1.3109243697478992,
319
+ "grad_norm": 0.103515625,
320
+ "learning_rate": 0.00016528221181905217,
321
+ "loss": 1.6333,
322
+ "step": 39
323
+ },
324
+ {
325
+ "epoch": 1.3445378151260505,
326
+ "grad_norm": 0.111328125,
327
+ "learning_rate": 0.00016300878435817113,
328
+ "loss": 1.7162,
329
+ "step": 40
330
+ },
331
+ {
332
+ "epoch": 1.3445378151260505,
333
+ "eval_loss": 1.757140040397644,
334
+ "eval_runtime": 19.6485,
335
+ "eval_samples_per_second": 2.545,
336
+ "eval_steps_per_second": 0.356,
337
+ "step": 40
338
+ },
339
+ {
340
+ "epoch": 1.3781512605042017,
341
+ "grad_norm": 0.1484375,
342
+ "learning_rate": 0.00016068001458185936,
343
+ "loss": 1.6501,
344
+ "step": 41
345
+ },
346
+ {
347
+ "epoch": 1.4117647058823528,
348
+ "grad_norm": 0.1240234375,
349
+ "learning_rate": 0.0001582979479114472,
350
+ "loss": 1.6446,
351
+ "step": 42
352
+ },
353
+ {
354
+ "epoch": 1.4453781512605042,
355
+ "grad_norm": 0.119140625,
356
+ "learning_rate": 0.00015586467658036524,
357
+ "loss": 1.7104,
358
+ "step": 43
359
+ },
360
+ {
361
+ "epoch": 1.4789915966386555,
362
+ "grad_norm": 0.109375,
363
+ "learning_rate": 0.0001533823377964791,
364
+ "loss": 1.6146,
365
+ "step": 44
366
+ },
367
+ {
368
+ "epoch": 1.5126050420168067,
369
+ "grad_norm": 0.11572265625,
370
+ "learning_rate": 0.00015085311186492206,
371
+ "loss": 1.6448,
372
+ "step": 45
373
+ },
374
+ {
375
+ "epoch": 1.5462184873949578,
376
+ "grad_norm": 0.11572265625,
377
+ "learning_rate": 0.00014827922027307451,
378
+ "loss": 1.6735,
379
+ "step": 46
380
+ },
381
+ {
382
+ "epoch": 1.5798319327731094,
383
+ "grad_norm": 0.1181640625,
384
+ "learning_rate": 0.0001456629237393713,
385
+ "loss": 1.6604,
386
+ "step": 47
387
+ },
388
+ {
389
+ "epoch": 1.6134453781512605,
390
+ "grad_norm": 0.12060546875,
391
+ "learning_rate": 0.00014300652022765207,
392
+ "loss": 1.7046,
393
+ "step": 48
394
+ },
395
+ {
396
+ "epoch": 1.6134453781512605,
397
+ "eval_loss": 1.7558497190475464,
398
+ "eval_runtime": 19.7723,
399
+ "eval_samples_per_second": 2.529,
400
+ "eval_steps_per_second": 0.354,
401
+ "step": 48
402
+ },
403
+ {
404
+ "epoch": 1.6470588235294117,
405
+ "grad_norm": 0.1259765625,
406
+ "learning_rate": 0.00014031234292879725,
407
+ "loss": 1.694,
408
+ "step": 49
409
+ },
410
+ {
411
+ "epoch": 1.680672268907563,
412
+ "grad_norm": 0.1220703125,
413
+ "learning_rate": 0.00013758275821142382,
414
+ "loss": 1.625,
415
+ "step": 50
416
+ },
417
+ {
418
+ "epoch": 1.7142857142857144,
419
+ "grad_norm": 0.123046875,
420
+ "learning_rate": 0.0001348201635434399,
421
+ "loss": 1.6919,
422
+ "step": 51
423
+ },
424
+ {
425
+ "epoch": 1.7478991596638656,
426
+ "grad_norm": 0.1201171875,
427
+ "learning_rate": 0.00013202698538628376,
428
+ "loss": 1.6777,
429
+ "step": 52
430
+ },
431
+ {
432
+ "epoch": 1.7815126050420167,
433
+ "grad_norm": 0.12890625,
434
+ "learning_rate": 0.00012920567706369758,
435
+ "loss": 1.7762,
436
+ "step": 53
437
+ },
438
+ {
439
+ "epoch": 1.815126050420168,
440
+ "grad_norm": 0.126953125,
441
+ "learning_rate": 0.00012635871660690676,
442
+ "loss": 1.6668,
443
+ "step": 54
444
+ },
445
+ {
446
+ "epoch": 1.8487394957983194,
447
+ "grad_norm": 0.125,
448
+ "learning_rate": 0.00012348860457809838,
449
+ "loss": 1.7061,
450
+ "step": 55
451
+ },
452
+ {
453
+ "epoch": 1.8823529411764706,
454
+ "grad_norm": 0.1328125,
455
+ "learning_rate": 0.00012059786187410984,
456
+ "loss": 1.6714,
457
+ "step": 56
458
+ },
459
+ {
460
+ "epoch": 1.8823529411764706,
461
+ "eval_loss": 1.7563551664352417,
462
+ "eval_runtime": 19.6588,
463
+ "eval_samples_per_second": 2.543,
464
+ "eval_steps_per_second": 0.356,
465
+ "step": 56
466
+ },
467
+ {
468
+ "epoch": 1.9159663865546217,
469
+ "grad_norm": 0.1279296875,
470
+ "learning_rate": 0.0001176890275122573,
471
+ "loss": 1.6318,
472
+ "step": 57
473
+ },
474
+ {
475
+ "epoch": 1.949579831932773,
476
+ "grad_norm": 0.1318359375,
477
+ "learning_rate": 0.00011476465640024814,
478
+ "loss": 1.6693,
479
+ "step": 58
480
+ }
481
+ ],
482
+ "logging_steps": 1,
483
+ "max_steps": 116,
484
+ "num_input_tokens_seen": 0,
485
+ "num_train_epochs": 4,
486
+ "save_steps": 29,
487
+ "stateful_callbacks": {
488
+ "TrainerControl": {
489
+ "args": {
490
+ "should_epoch_stop": false,
491
+ "should_evaluate": false,
492
+ "should_log": false,
493
+ "should_save": true,
494
+ "should_training_stop": false
495
+ },
496
+ "attributes": {}
497
+ }
498
+ },
499
+ "total_flos": 6643321582387200.0,
500
+ "train_batch_size": 8,
501
+ "trial_name": null,
502
+ "trial_params": null
503
+ }
checkpoint-58/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6f22338230e6fb5499b3cb737d6ee991fc1b36ba21747e343dbc66d770479d2b
3
+ size 5944
checkpoint-87/README.md ADDED
@@ -0,0 +1,202 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ library_name: peft
3
+ base_model: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
4
+ ---
5
+
6
+ # Model Card for Model ID
7
+
8
+ <!-- Provide a quick summary of what the model is/does. -->
9
+
10
+
11
+
12
+ ## Model Details
13
+
14
+ ### Model Description
15
+
16
+ <!-- Provide a longer summary of what this model is. -->
17
+
18
+
19
+
20
+ - **Developed by:** [More Information Needed]
21
+ - **Funded by [optional]:** [More Information Needed]
22
+ - **Shared by [optional]:** [More Information Needed]
23
+ - **Model type:** [More Information Needed]
24
+ - **Language(s) (NLP):** [More Information Needed]
25
+ - **License:** [More Information Needed]
26
+ - **Finetuned from model [optional]:** [More Information Needed]
27
+
28
+ ### Model Sources [optional]
29
+
30
+ <!-- Provide the basic links for the model. -->
31
+
32
+ - **Repository:** [More Information Needed]
33
+ - **Paper [optional]:** [More Information Needed]
34
+ - **Demo [optional]:** [More Information Needed]
35
+
36
+ ## Uses
37
+
38
+ <!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
39
+
40
+ ### Direct Use
41
+
42
+ <!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
43
+
44
+ [More Information Needed]
45
+
46
+ ### Downstream Use [optional]
47
+
48
+ <!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
49
+
50
+ [More Information Needed]
51
+
52
+ ### Out-of-Scope Use
53
+
54
+ <!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
55
+
56
+ [More Information Needed]
57
+
58
+ ## Bias, Risks, and Limitations
59
+
60
+ <!-- This section is meant to convey both technical and sociotechnical limitations. -->
61
+
62
+ [More Information Needed]
63
+
64
+ ### Recommendations
65
+
66
+ <!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
67
+
68
+ Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
69
+
70
+ ## How to Get Started with the Model
71
+
72
+ Use the code below to get started with the model.
73
+
74
+ [More Information Needed]
75
+
76
+ ## Training Details
77
+
78
+ ### Training Data
79
+
80
+ <!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
81
+
82
+ [More Information Needed]
83
+
84
+ ### Training Procedure
85
+
86
+ <!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
87
+
88
+ #### Preprocessing [optional]
89
+
90
+ [More Information Needed]
91
+
92
+
93
+ #### Training Hyperparameters
94
+
95
+ - **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
96
+
97
+ #### Speeds, Sizes, Times [optional]
98
+
99
+ <!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
100
+
101
+ [More Information Needed]
102
+
103
+ ## Evaluation
104
+
105
+ <!-- This section describes the evaluation protocols and provides the results. -->
106
+
107
+ ### Testing Data, Factors & Metrics
108
+
109
+ #### Testing Data
110
+
111
+ <!-- This should link to a Dataset Card if possible. -->
112
+
113
+ [More Information Needed]
114
+
115
+ #### Factors
116
+
117
+ <!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
118
+
119
+ [More Information Needed]
120
+
121
+ #### Metrics
122
+
123
+ <!-- These are the evaluation metrics being used, ideally with a description of why. -->
124
+
125
+ [More Information Needed]
126
+
127
+ ### Results
128
+
129
+ [More Information Needed]
130
+
131
+ #### Summary
132
+
133
+
134
+
135
+ ## Model Examination [optional]
136
+
137
+ <!-- Relevant interpretability work for the model goes here -->
138
+
139
+ [More Information Needed]
140
+
141
+ ## Environmental Impact
142
+
143
+ <!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
144
+
145
+ Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
146
+
147
+ - **Hardware Type:** [More Information Needed]
148
+ - **Hours used:** [More Information Needed]
149
+ - **Cloud Provider:** [More Information Needed]
150
+ - **Compute Region:** [More Information Needed]
151
+ - **Carbon Emitted:** [More Information Needed]
152
+
153
+ ## Technical Specifications [optional]
154
+
155
+ ### Model Architecture and Objective
156
+
157
+ [More Information Needed]
158
+
159
+ ### Compute Infrastructure
160
+
161
+ [More Information Needed]
162
+
163
+ #### Hardware
164
+
165
+ [More Information Needed]
166
+
167
+ #### Software
168
+
169
+ [More Information Needed]
170
+
171
+ ## Citation [optional]
172
+
173
+ <!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
174
+
175
+ **BibTeX:**
176
+
177
+ [More Information Needed]
178
+
179
+ **APA:**
180
+
181
+ [More Information Needed]
182
+
183
+ ## Glossary [optional]
184
+
185
+ <!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
186
+
187
+ [More Information Needed]
188
+
189
+ ## More Information [optional]
190
+
191
+ [More Information Needed]
192
+
193
+ ## Model Card Authors [optional]
194
+
195
+ [More Information Needed]
196
+
197
+ ## Model Card Contact
198
+
199
+ [More Information Needed]
200
+ ### Framework versions
201
+
202
+ - PEFT 0.11.1
checkpoint-87/adapter_config.json ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T",
5
+ "bias": "none",
6
+ "fan_in_fan_out": null,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layer_replication": null,
10
+ "layers_pattern": null,
11
+ "layers_to_transform": null,
12
+ "loftq_config": {},
13
+ "lora_alpha": 16,
14
+ "lora_dropout": 0.05,
15
+ "megatron_config": null,
16
+ "megatron_core": "megatron.core",
17
+ "modules_to_save": null,
18
+ "peft_type": "LORA",
19
+ "r": 32,
20
+ "rank_pattern": {},
21
+ "revision": null,
22
+ "target_modules": [
23
+ "gate_proj",
24
+ "up_proj",
25
+ "down_proj",
26
+ "v_proj",
27
+ "o_proj",
28
+ "q_proj",
29
+ "k_proj"
30
+ ],
31
+ "task_type": "CAUSAL_LM",
32
+ "use_dora": false,
33
+ "use_rslora": false
34
+ }
checkpoint-87/adapter_model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4c119a868a2f168125d890295ff7e7d1e8709c364ae22393fd56670ef283bd84
3
+ size 50503848
checkpoint-87/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:be530a8fa8568da1149a6eb3fa1e0d3e6641ab7b50fc6a724480c58020d8db51
3
+ size 202035450
checkpoint-87/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:abee6b45a92cc12ab71fa5ef10750e86110e096fada3acf2955d86fe41f121d3
3
+ size 14244
checkpoint-87/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:96dfdc4ac2eb14a7a5283672324c02dd21283974614857cbf29d0d55e24e2b3f
3
+ size 1064
checkpoint-87/special_tokens_map.json ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "bos_token": {
3
+ "content": "<s>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "</s>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "pad_token": "</s>",
17
+ "unk_token": {
18
+ "content": "<unk>",
19
+ "lstrip": false,
20
+ "normalized": false,
21
+ "rstrip": false,
22
+ "single_word": false
23
+ }
24
+ }
checkpoint-87/tokenizer.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
3
+ size 499723
checkpoint-87/tokenizer_config.json ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_bos_token": true,
3
+ "add_eos_token": false,
4
+ "add_prefix_space": true,
5
+ "added_tokens_decoder": {
6
+ "0": {
7
+ "content": "<unk>",
8
+ "lstrip": false,
9
+ "normalized": false,
10
+ "rstrip": false,
11
+ "single_word": false,
12
+ "special": true
13
+ },
14
+ "1": {
15
+ "content": "<s>",
16
+ "lstrip": false,
17
+ "normalized": false,
18
+ "rstrip": false,
19
+ "single_word": false,
20
+ "special": true
21
+ },
22
+ "2": {
23
+ "content": "</s>",
24
+ "lstrip": false,
25
+ "normalized": false,
26
+ "rstrip": false,
27
+ "single_word": false,
28
+ "special": true
29
+ }
30
+ },
31
+ "bos_token": "<s>",
32
+ "clean_up_tokenization_spaces": false,
33
+ "eos_token": "</s>",
34
+ "legacy": false,
35
+ "model_max_length": 1000000000000000019884624838656,
36
+ "pad_token": "</s>",
37
+ "padding_side": "right",
38
+ "sp_model_kwargs": {},
39
+ "spaces_between_special_tokens": false,
40
+ "tokenizer_class": "LlamaTokenizer",
41
+ "unk_token": "<unk>",
42
+ "use_default_system_prompt": false,
43
+ "use_fast": true
44
+ }
checkpoint-87/trainer_state.json ADDED
@@ -0,0 +1,730 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": null,
3
+ "best_model_checkpoint": null,
4
+ "epoch": 2.92436974789916,
5
+ "eval_steps": 8,
6
+ "global_step": 87,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.03361344537815126,
13
+ "grad_norm": 0.115234375,
14
+ "learning_rate": 2e-05,
15
+ "loss": 1.768,
16
+ "step": 1
17
+ },
18
+ {
19
+ "epoch": 0.03361344537815126,
20
+ "eval_loss": 1.8648816347122192,
21
+ "eval_runtime": 18.2501,
22
+ "eval_samples_per_second": 2.74,
23
+ "eval_steps_per_second": 0.384,
24
+ "step": 1
25
+ },
26
+ {
27
+ "epoch": 0.06722689075630252,
28
+ "grad_norm": 0.10888671875,
29
+ "learning_rate": 4e-05,
30
+ "loss": 1.7838,
31
+ "step": 2
32
+ },
33
+ {
34
+ "epoch": 0.10084033613445378,
35
+ "grad_norm": 0.126953125,
36
+ "learning_rate": 6e-05,
37
+ "loss": 1.9413,
38
+ "step": 3
39
+ },
40
+ {
41
+ "epoch": 0.13445378151260504,
42
+ "grad_norm": 0.0986328125,
43
+ "learning_rate": 8e-05,
44
+ "loss": 1.7757,
45
+ "step": 4
46
+ },
47
+ {
48
+ "epoch": 0.16806722689075632,
49
+ "grad_norm": 0.12255859375,
50
+ "learning_rate": 0.0001,
51
+ "loss": 1.735,
52
+ "step": 5
53
+ },
54
+ {
55
+ "epoch": 0.20168067226890757,
56
+ "grad_norm": 0.10791015625,
57
+ "learning_rate": 0.00012,
58
+ "loss": 1.8269,
59
+ "step": 6
60
+ },
61
+ {
62
+ "epoch": 0.23529411764705882,
63
+ "grad_norm": 0.11669921875,
64
+ "learning_rate": 0.00014,
65
+ "loss": 1.8552,
66
+ "step": 7
67
+ },
68
+ {
69
+ "epoch": 0.2689075630252101,
70
+ "grad_norm": 0.119140625,
71
+ "learning_rate": 0.00016,
72
+ "loss": 1.8084,
73
+ "step": 8
74
+ },
75
+ {
76
+ "epoch": 0.2689075630252101,
77
+ "eval_loss": 1.8317129611968994,
78
+ "eval_runtime": 19.6984,
79
+ "eval_samples_per_second": 2.538,
80
+ "eval_steps_per_second": 0.355,
81
+ "step": 8
82
+ },
83
+ {
84
+ "epoch": 0.3025210084033613,
85
+ "grad_norm": 0.12255859375,
86
+ "learning_rate": 0.00018,
87
+ "loss": 1.7158,
88
+ "step": 9
89
+ },
90
+ {
91
+ "epoch": 0.33613445378151263,
92
+ "grad_norm": 0.134765625,
93
+ "learning_rate": 0.0002,
94
+ "loss": 1.8702,
95
+ "step": 10
96
+ },
97
+ {
98
+ "epoch": 0.3697478991596639,
99
+ "grad_norm": 0.12890625,
100
+ "learning_rate": 0.00019995608365087946,
101
+ "loss": 1.8307,
102
+ "step": 11
103
+ },
104
+ {
105
+ "epoch": 0.40336134453781514,
106
+ "grad_norm": 0.11474609375,
107
+ "learning_rate": 0.00019982437317643217,
108
+ "loss": 1.6583,
109
+ "step": 12
110
+ },
111
+ {
112
+ "epoch": 0.4369747899159664,
113
+ "grad_norm": 0.134765625,
114
+ "learning_rate": 0.0001996049842615217,
115
+ "loss": 1.6663,
116
+ "step": 13
117
+ },
118
+ {
119
+ "epoch": 0.47058823529411764,
120
+ "grad_norm": 0.12060546875,
121
+ "learning_rate": 0.00019929810960135172,
122
+ "loss": 1.7388,
123
+ "step": 14
124
+ },
125
+ {
126
+ "epoch": 0.5042016806722689,
127
+ "grad_norm": 0.126953125,
128
+ "learning_rate": 0.0001989040187322164,
129
+ "loss": 1.7485,
130
+ "step": 15
131
+ },
132
+ {
133
+ "epoch": 0.5378151260504201,
134
+ "grad_norm": 0.1259765625,
135
+ "learning_rate": 0.00019842305779475968,
136
+ "loss": 1.633,
137
+ "step": 16
138
+ },
139
+ {
140
+ "epoch": 0.5378151260504201,
141
+ "eval_loss": 1.7832777500152588,
142
+ "eval_runtime": 19.6833,
143
+ "eval_samples_per_second": 2.54,
144
+ "eval_steps_per_second": 0.356,
145
+ "step": 16
146
+ },
147
+ {
148
+ "epoch": 0.5714285714285714,
149
+ "grad_norm": 0.12109375,
150
+ "learning_rate": 0.0001978556492299504,
151
+ "loss": 1.8373,
152
+ "step": 17
153
+ },
154
+ {
155
+ "epoch": 0.6050420168067226,
156
+ "grad_norm": 0.1337890625,
157
+ "learning_rate": 0.0001972022914080411,
158
+ "loss": 1.6552,
159
+ "step": 18
160
+ },
161
+ {
162
+ "epoch": 0.6386554621848739,
163
+ "grad_norm": 0.126953125,
164
+ "learning_rate": 0.00019646355819083589,
165
+ "loss": 1.8113,
166
+ "step": 19
167
+ },
168
+ {
169
+ "epoch": 0.6722689075630253,
170
+ "grad_norm": 0.1220703125,
171
+ "learning_rate": 0.00019564009842765225,
172
+ "loss": 1.6544,
173
+ "step": 20
174
+ },
175
+ {
176
+ "epoch": 0.7058823529411765,
177
+ "grad_norm": 0.11669921875,
178
+ "learning_rate": 0.00019473263538541914,
179
+ "loss": 1.6649,
180
+ "step": 21
181
+ },
182
+ {
183
+ "epoch": 0.7394957983193278,
184
+ "grad_norm": 0.0986328125,
185
+ "learning_rate": 0.0001937419661134121,
186
+ "loss": 1.6868,
187
+ "step": 22
188
+ },
189
+ {
190
+ "epoch": 0.773109243697479,
191
+ "grad_norm": 0.126953125,
192
+ "learning_rate": 0.00019266896074318334,
193
+ "loss": 1.7762,
194
+ "step": 23
195
+ },
196
+ {
197
+ "epoch": 0.8067226890756303,
198
+ "grad_norm": 0.11279296875,
199
+ "learning_rate": 0.00019151456172430183,
200
+ "loss": 1.6737,
201
+ "step": 24
202
+ },
203
+ {
204
+ "epoch": 0.8067226890756303,
205
+ "eval_loss": 1.7643933296203613,
206
+ "eval_runtime": 19.6308,
207
+ "eval_samples_per_second": 2.547,
208
+ "eval_steps_per_second": 0.357,
209
+ "step": 24
210
+ },
211
+ {
212
+ "epoch": 0.8403361344537815,
213
+ "grad_norm": 0.1298828125,
214
+ "learning_rate": 0.00019027978299657436,
215
+ "loss": 1.6401,
216
+ "step": 25
217
+ },
218
+ {
219
+ "epoch": 0.8739495798319328,
220
+ "grad_norm": 0.099609375,
221
+ "learning_rate": 0.00018896570909947475,
222
+ "loss": 1.7068,
223
+ "step": 26
224
+ },
225
+ {
226
+ "epoch": 0.907563025210084,
227
+ "grad_norm": 0.12060546875,
228
+ "learning_rate": 0.0001875734942195637,
229
+ "loss": 1.8112,
230
+ "step": 27
231
+ },
232
+ {
233
+ "epoch": 0.9411764705882353,
234
+ "grad_norm": 0.1162109375,
235
+ "learning_rate": 0.00018610436117673555,
236
+ "loss": 1.6596,
237
+ "step": 28
238
+ },
239
+ {
240
+ "epoch": 0.9747899159663865,
241
+ "grad_norm": 0.12890625,
242
+ "learning_rate": 0.0001845596003501826,
243
+ "loss": 1.7936,
244
+ "step": 29
245
+ },
246
+ {
247
+ "epoch": 1.0084033613445378,
248
+ "grad_norm": 0.1240234375,
249
+ "learning_rate": 0.0001829405685450202,
250
+ "loss": 1.7947,
251
+ "step": 30
252
+ },
253
+ {
254
+ "epoch": 1.0420168067226891,
255
+ "grad_norm": 0.20703125,
256
+ "learning_rate": 0.00018124868780056814,
257
+ "loss": 1.6887,
258
+ "step": 31
259
+ },
260
+ {
261
+ "epoch": 1.0756302521008403,
262
+ "grad_norm": 0.1455078125,
263
+ "learning_rate": 0.00017948544414133534,
264
+ "loss": 1.6722,
265
+ "step": 32
266
+ },
267
+ {
268
+ "epoch": 1.0756302521008403,
269
+ "eval_loss": 1.7600828409194946,
270
+ "eval_runtime": 19.7105,
271
+ "eval_samples_per_second": 2.537,
272
+ "eval_steps_per_second": 0.355,
273
+ "step": 32
274
+ },
275
+ {
276
+ "epoch": 1.1092436974789917,
277
+ "grad_norm": 0.09814453125,
278
+ "learning_rate": 0.00017765238627180424,
279
+ "loss": 1.7145,
280
+ "step": 33
281
+ },
282
+ {
283
+ "epoch": 1.1428571428571428,
284
+ "grad_norm": 0.10693359375,
285
+ "learning_rate": 0.00017575112421616202,
286
+ "loss": 1.6609,
287
+ "step": 34
288
+ },
289
+ {
290
+ "epoch": 1.1764705882352942,
291
+ "grad_norm": 0.11572265625,
292
+ "learning_rate": 0.00017378332790417273,
293
+ "loss": 1.6681,
294
+ "step": 35
295
+ },
296
+ {
297
+ "epoch": 1.2100840336134453,
298
+ "grad_norm": 0.11767578125,
299
+ "learning_rate": 0.00017175072570443312,
300
+ "loss": 1.6641,
301
+ "step": 36
302
+ },
303
+ {
304
+ "epoch": 1.2436974789915967,
305
+ "grad_norm": 0.11376953125,
306
+ "learning_rate": 0.00016965510290629972,
307
+ "loss": 1.7011,
308
+ "step": 37
309
+ },
310
+ {
311
+ "epoch": 1.2773109243697478,
312
+ "grad_norm": 0.11572265625,
313
+ "learning_rate": 0.00016749830015182107,
314
+ "loss": 1.7171,
315
+ "step": 38
316
+ },
317
+ {
318
+ "epoch": 1.3109243697478992,
319
+ "grad_norm": 0.103515625,
320
+ "learning_rate": 0.00016528221181905217,
321
+ "loss": 1.6333,
322
+ "step": 39
323
+ },
324
+ {
325
+ "epoch": 1.3445378151260505,
326
+ "grad_norm": 0.111328125,
327
+ "learning_rate": 0.00016300878435817113,
328
+ "loss": 1.7162,
329
+ "step": 40
330
+ },
331
+ {
332
+ "epoch": 1.3445378151260505,
333
+ "eval_loss": 1.757140040397644,
334
+ "eval_runtime": 19.6485,
335
+ "eval_samples_per_second": 2.545,
336
+ "eval_steps_per_second": 0.356,
337
+ "step": 40
338
+ },
339
+ {
340
+ "epoch": 1.3781512605042017,
341
+ "grad_norm": 0.1484375,
342
+ "learning_rate": 0.00016068001458185936,
343
+ "loss": 1.6501,
344
+ "step": 41
345
+ },
346
+ {
347
+ "epoch": 1.4117647058823528,
348
+ "grad_norm": 0.1240234375,
349
+ "learning_rate": 0.0001582979479114472,
350
+ "loss": 1.6446,
351
+ "step": 42
352
+ },
353
+ {
354
+ "epoch": 1.4453781512605042,
355
+ "grad_norm": 0.119140625,
356
+ "learning_rate": 0.00015586467658036524,
357
+ "loss": 1.7104,
358
+ "step": 43
359
+ },
360
+ {
361
+ "epoch": 1.4789915966386555,
362
+ "grad_norm": 0.109375,
363
+ "learning_rate": 0.0001533823377964791,
364
+ "loss": 1.6146,
365
+ "step": 44
366
+ },
367
+ {
368
+ "epoch": 1.5126050420168067,
369
+ "grad_norm": 0.11572265625,
370
+ "learning_rate": 0.00015085311186492206,
371
+ "loss": 1.6448,
372
+ "step": 45
373
+ },
374
+ {
375
+ "epoch": 1.5462184873949578,
376
+ "grad_norm": 0.11572265625,
377
+ "learning_rate": 0.00014827922027307451,
378
+ "loss": 1.6735,
379
+ "step": 46
380
+ },
381
+ {
382
+ "epoch": 1.5798319327731094,
383
+ "grad_norm": 0.1181640625,
384
+ "learning_rate": 0.0001456629237393713,
385
+ "loss": 1.6604,
386
+ "step": 47
387
+ },
388
+ {
389
+ "epoch": 1.6134453781512605,
390
+ "grad_norm": 0.12060546875,
391
+ "learning_rate": 0.00014300652022765207,
392
+ "loss": 1.7046,
393
+ "step": 48
394
+ },
395
+ {
396
+ "epoch": 1.6134453781512605,
397
+ "eval_loss": 1.7558497190475464,
398
+ "eval_runtime": 19.7723,
399
+ "eval_samples_per_second": 2.529,
400
+ "eval_steps_per_second": 0.354,
401
+ "step": 48
402
+ },
403
+ {
404
+ "epoch": 1.6470588235294117,
405
+ "grad_norm": 0.1259765625,
406
+ "learning_rate": 0.00014031234292879725,
407
+ "loss": 1.694,
408
+ "step": 49
409
+ },
410
+ {
411
+ "epoch": 1.680672268907563,
412
+ "grad_norm": 0.1220703125,
413
+ "learning_rate": 0.00013758275821142382,
414
+ "loss": 1.625,
415
+ "step": 50
416
+ },
417
+ {
418
+ "epoch": 1.7142857142857144,
419
+ "grad_norm": 0.123046875,
420
+ "learning_rate": 0.0001348201635434399,
421
+ "loss": 1.6919,
422
+ "step": 51
423
+ },
424
+ {
425
+ "epoch": 1.7478991596638656,
426
+ "grad_norm": 0.1201171875,
427
+ "learning_rate": 0.00013202698538628376,
428
+ "loss": 1.6777,
429
+ "step": 52
430
+ },
431
+ {
432
+ "epoch": 1.7815126050420167,
433
+ "grad_norm": 0.12890625,
434
+ "learning_rate": 0.00012920567706369758,
435
+ "loss": 1.7762,
436
+ "step": 53
437
+ },
438
+ {
439
+ "epoch": 1.815126050420168,
440
+ "grad_norm": 0.126953125,
441
+ "learning_rate": 0.00012635871660690676,
442
+ "loss": 1.6668,
443
+ "step": 54
444
+ },
445
+ {
446
+ "epoch": 1.8487394957983194,
447
+ "grad_norm": 0.125,
448
+ "learning_rate": 0.00012348860457809838,
449
+ "loss": 1.7061,
450
+ "step": 55
451
+ },
452
+ {
453
+ "epoch": 1.8823529411764706,
454
+ "grad_norm": 0.1328125,
455
+ "learning_rate": 0.00012059786187410984,
456
+ "loss": 1.6714,
457
+ "step": 56
458
+ },
459
+ {
460
+ "epoch": 1.8823529411764706,
461
+ "eval_loss": 1.7563551664352417,
462
+ "eval_runtime": 19.6588,
463
+ "eval_samples_per_second": 2.543,
464
+ "eval_steps_per_second": 0.356,
465
+ "step": 56
466
+ },
467
+ {
468
+ "epoch": 1.9159663865546217,
469
+ "grad_norm": 0.1279296875,
470
+ "learning_rate": 0.0001176890275122573,
471
+ "loss": 1.6318,
472
+ "step": 57
473
+ },
474
+ {
475
+ "epoch": 1.949579831932773,
476
+ "grad_norm": 0.1318359375,
477
+ "learning_rate": 0.00011476465640024814,
478
+ "loss": 1.6693,
479
+ "step": 58
480
+ },
481
+ {
482
+ "epoch": 1.9831932773109244,
483
+ "grad_norm": 0.12353515625,
484
+ "learning_rate": 0.00011182731709213659,
485
+ "loss": 1.5927,
486
+ "step": 59
487
+ },
488
+ {
489
+ "epoch": 2.0168067226890756,
490
+ "grad_norm": 0.138671875,
491
+ "learning_rate": 0.00010887958953229349,
492
+ "loss": 1.6558,
493
+ "step": 60
494
+ },
495
+ {
496
+ "epoch": 2.0504201680672267,
497
+ "grad_norm": 0.134765625,
498
+ "learning_rate": 0.00010592406278937144,
499
+ "loss": 1.7352,
500
+ "step": 61
501
+ },
502
+ {
503
+ "epoch": 2.0840336134453783,
504
+ "grad_norm": 0.1328125,
505
+ "learning_rate": 0.00010296333278225599,
506
+ "loss": 1.6216,
507
+ "step": 62
508
+ },
509
+ {
510
+ "epoch": 2.1176470588235294,
511
+ "grad_norm": 0.1201171875,
512
+ "learning_rate": 0.0001,
513
+ "loss": 1.6365,
514
+ "step": 63
515
+ },
516
+ {
517
+ "epoch": 2.1512605042016806,
518
+ "grad_norm": 0.1298828125,
519
+ "learning_rate": 9.703666721774402e-05,
520
+ "loss": 1.6249,
521
+ "step": 64
522
+ },
523
+ {
524
+ "epoch": 2.1512605042016806,
525
+ "eval_loss": 1.756639838218689,
526
+ "eval_runtime": 19.5951,
527
+ "eval_samples_per_second": 2.552,
528
+ "eval_steps_per_second": 0.357,
529
+ "step": 64
530
+ },
531
+ {
532
+ "epoch": 2.184873949579832,
533
+ "grad_norm": 0.1357421875,
534
+ "learning_rate": 9.407593721062859e-05,
535
+ "loss": 1.653,
536
+ "step": 65
537
+ },
538
+ {
539
+ "epoch": 2.2184873949579833,
540
+ "grad_norm": 0.146484375,
541
+ "learning_rate": 9.112041046770653e-05,
542
+ "loss": 1.6545,
543
+ "step": 66
544
+ },
545
+ {
546
+ "epoch": 2.2521008403361344,
547
+ "grad_norm": 0.13671875,
548
+ "learning_rate": 8.817268290786343e-05,
549
+ "loss": 1.5787,
550
+ "step": 67
551
+ },
552
+ {
553
+ "epoch": 2.2857142857142856,
554
+ "grad_norm": 0.1328125,
555
+ "learning_rate": 8.523534359975189e-05,
556
+ "loss": 1.6532,
557
+ "step": 68
558
+ },
559
+ {
560
+ "epoch": 2.3193277310924367,
561
+ "grad_norm": 0.1396484375,
562
+ "learning_rate": 8.231097248774274e-05,
563
+ "loss": 1.6784,
564
+ "step": 69
565
+ },
566
+ {
567
+ "epoch": 2.3529411764705883,
568
+ "grad_norm": 0.1337890625,
569
+ "learning_rate": 7.940213812589018e-05,
570
+ "loss": 1.5721,
571
+ "step": 70
572
+ },
573
+ {
574
+ "epoch": 2.3865546218487395,
575
+ "grad_norm": 0.1416015625,
576
+ "learning_rate": 7.651139542190164e-05,
577
+ "loss": 1.5836,
578
+ "step": 71
579
+ },
580
+ {
581
+ "epoch": 2.4201680672268906,
582
+ "grad_norm": 0.146484375,
583
+ "learning_rate": 7.364128339309326e-05,
584
+ "loss": 1.5604,
585
+ "step": 72
586
+ },
587
+ {
588
+ "epoch": 2.4201680672268906,
589
+ "eval_loss": 1.7598735094070435,
590
+ "eval_runtime": 19.7508,
591
+ "eval_samples_per_second": 2.532,
592
+ "eval_steps_per_second": 0.354,
593
+ "step": 72
594
+ },
595
+ {
596
+ "epoch": 2.453781512605042,
597
+ "grad_norm": 0.1455078125,
598
+ "learning_rate": 7.079432293630244e-05,
599
+ "loss": 1.6259,
600
+ "step": 73
601
+ },
602
+ {
603
+ "epoch": 2.4873949579831933,
604
+ "grad_norm": 0.1484375,
605
+ "learning_rate": 6.797301461371625e-05,
606
+ "loss": 1.5811,
607
+ "step": 74
608
+ },
609
+ {
610
+ "epoch": 2.5210084033613445,
611
+ "grad_norm": 0.14453125,
612
+ "learning_rate": 6.517983645656014e-05,
613
+ "loss": 1.4929,
614
+ "step": 75
615
+ },
616
+ {
617
+ "epoch": 2.5546218487394956,
618
+ "grad_norm": 0.1572265625,
619
+ "learning_rate": 6.24172417885762e-05,
620
+ "loss": 1.7014,
621
+ "step": 76
622
+ },
623
+ {
624
+ "epoch": 2.588235294117647,
625
+ "grad_norm": 0.1484375,
626
+ "learning_rate": 5.96876570712028e-05,
627
+ "loss": 1.5623,
628
+ "step": 77
629
+ },
630
+ {
631
+ "epoch": 2.6218487394957983,
632
+ "grad_norm": 0.1474609375,
633
+ "learning_rate": 5.699347977234799e-05,
634
+ "loss": 1.6006,
635
+ "step": 78
636
+ },
637
+ {
638
+ "epoch": 2.6554621848739495,
639
+ "grad_norm": 0.150390625,
640
+ "learning_rate": 5.43370762606287e-05,
641
+ "loss": 1.6641,
642
+ "step": 79
643
+ },
644
+ {
645
+ "epoch": 2.689075630252101,
646
+ "grad_norm": 0.15234375,
647
+ "learning_rate": 5.172077972692553e-05,
648
+ "loss": 1.7003,
649
+ "step": 80
650
+ },
651
+ {
652
+ "epoch": 2.689075630252101,
653
+ "eval_loss": 1.761399269104004,
654
+ "eval_runtime": 19.6692,
655
+ "eval_samples_per_second": 2.542,
656
+ "eval_steps_per_second": 0.356,
657
+ "step": 80
658
+ },
659
+ {
660
+ "epoch": 2.722689075630252,
661
+ "grad_norm": 0.154296875,
662
+ "learning_rate": 4.914688813507797e-05,
663
+ "loss": 1.6923,
664
+ "step": 81
665
+ },
666
+ {
667
+ "epoch": 2.7563025210084033,
668
+ "grad_norm": 0.158203125,
669
+ "learning_rate": 4.661766220352097e-05,
670
+ "loss": 1.6819,
671
+ "step": 82
672
+ },
673
+ {
674
+ "epoch": 2.7899159663865545,
675
+ "grad_norm": 0.1513671875,
676
+ "learning_rate": 4.4135323419634766e-05,
677
+ "loss": 1.5649,
678
+ "step": 83
679
+ },
680
+ {
681
+ "epoch": 2.8235294117647056,
682
+ "grad_norm": 0.1494140625,
683
+ "learning_rate": 4.170205208855281e-05,
684
+ "loss": 1.608,
685
+ "step": 84
686
+ },
687
+ {
688
+ "epoch": 2.857142857142857,
689
+ "grad_norm": 0.1611328125,
690
+ "learning_rate": 3.931998541814069e-05,
691
+ "loss": 1.5474,
692
+ "step": 85
693
+ },
694
+ {
695
+ "epoch": 2.8907563025210083,
696
+ "grad_norm": 0.1552734375,
697
+ "learning_rate": 3.69912156418289e-05,
698
+ "loss": 1.6484,
699
+ "step": 86
700
+ },
701
+ {
702
+ "epoch": 2.92436974789916,
703
+ "grad_norm": 0.1484375,
704
+ "learning_rate": 3.471778818094785e-05,
705
+ "loss": 1.6145,
706
+ "step": 87
707
+ }
708
+ ],
709
+ "logging_steps": 1,
710
+ "max_steps": 116,
711
+ "num_input_tokens_seen": 0,
712
+ "num_train_epochs": 4,
713
+ "save_steps": 29,
714
+ "stateful_callbacks": {
715
+ "TrainerControl": {
716
+ "args": {
717
+ "should_epoch_stop": false,
718
+ "should_evaluate": false,
719
+ "should_log": false,
720
+ "should_save": true,
721
+ "should_training_stop": false
722
+ },
723
+ "attributes": {}
724
+ }
725
+ },
726
+ "total_flos": 9933851341357056.0,
727
+ "train_batch_size": 8,
728
+ "trial_name": null,
729
+ "trial_params": null
730
+ }
checkpoint-87/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6f22338230e6fb5499b3cb737d6ee991fc1b36ba21747e343dbc66d770479d2b
3
+ size 5944
config.json ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T",
3
+ "architectures": [
4
+ "LlamaForCausalLM"
5
+ ],
6
+ "attention_bias": false,
7
+ "attention_dropout": 0.0,
8
+ "bos_token_id": 1,
9
+ "eos_token_id": 2,
10
+ "hidden_act": "silu",
11
+ "hidden_size": 2048,
12
+ "initializer_range": 0.02,
13
+ "intermediate_size": 5632,
14
+ "max_position_embeddings": 4096,
15
+ "mlp_bias": false,
16
+ "model_type": "llama",
17
+ "num_attention_heads": 32,
18
+ "num_hidden_layers": 22,
19
+ "num_key_value_heads": 4,
20
+ "pretraining_tp": 1,
21
+ "quantization_config": {
22
+ "_load_in_4bit": true,
23
+ "_load_in_8bit": false,
24
+ "bnb_4bit_compute_dtype": "float32",
25
+ "bnb_4bit_quant_storage": "bfloat16",
26
+ "bnb_4bit_quant_type": "nf4",
27
+ "bnb_4bit_use_double_quant": true,
28
+ "llm_int8_enable_fp32_cpu_offload": false,
29
+ "llm_int8_has_fp16_weight": false,
30
+ "llm_int8_skip_modules": null,
31
+ "llm_int8_threshold": 6.0,
32
+ "load_in_4bit": true,
33
+ "load_in_8bit": false,
34
+ "quant_method": "bitsandbytes"
35
+ },
36
+ "rms_norm_eps": 1e-05,
37
+ "rope_scaling": null,
38
+ "rope_theta": 10000.0,
39
+ "tie_word_embeddings": false,
40
+ "torch_dtype": "float32",
41
+ "transformers_version": "4.41.1",
42
+ "use_cache": false,
43
+ "vocab_size": 32000
44
+ }
content/outputs/qlora-out/README.md ADDED
@@ -0,0 +1,143 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: apache-2.0
3
+ library_name: peft
4
+ tags:
5
+ - generated_from_trainer
6
+ base_model: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
7
+ model-index:
8
+ - name: outputs/qlora-out
9
+ results: []
10
+ ---
11
+
12
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
13
+ should probably proofread and complete it, then remove this comment. -->
14
+
15
+ [<img src="https://raw.githubusercontent.com/OpenAccess-AI-Collective/axolotl/main/image/axolotl-badge-web.png" alt="Built with Axolotl" width="200" height="32"/>](https://github.com/OpenAccess-AI-Collective/axolotl)
16
+ <details><summary>See axolotl config</summary>
17
+
18
+ axolotl version: `0.4.1`
19
+ ```yaml
20
+ adapter: qlora
21
+ base_model: TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T
22
+ bf16: false
23
+ dataset_prepared_path: null
24
+ datasets:
25
+ - ds_tipe: json
26
+ path: pubmed_continual_pretraning_dataset.jsonl
27
+ type: completion
28
+ debug: null
29
+ deepspeed: null
30
+ early_stopping_patience: null
31
+ eval_sample_packing: false
32
+ evals_per_epoch: 4
33
+ flash_attention: false
34
+ fp16: null
35
+ fsdp: null
36
+ fsdp_config: null
37
+ gradient_accumulation_steps: 4
38
+ gradient_checkpointing: true
39
+ group_by_length: false
40
+ learning_rate: 0.0002
41
+ load_in_4bit: true
42
+ load_in_8bit: false
43
+ local_rank: null
44
+ logging_steps: 1
45
+ lora_alpha: 16
46
+ lora_dropout: 0.05
47
+ lora_fan_in_fan_out: null
48
+ lora_model_dir: null
49
+ lora_r: 32
50
+ lora_target_linear: true
51
+ lora_target_modules: null
52
+ lr_scheduler: cosine
53
+ micro_batch_size: 8
54
+ model_type: LlamaForCausalLM
55
+ num_epochs: 4
56
+ optimizer: paged_adamw_32bit
57
+ output_dir: ./outputs/qlora-out
58
+ pad_to_sequence_len: false
59
+ resume_from_checkpoint: null
60
+ sample_packing: false
61
+ saves_per_epoch: 1
62
+ sequence_len: 4096
63
+ special_tokens: null
64
+ strict: false
65
+ tf32: false
66
+ tokenizer_type: LlamaTokenizer
67
+ train_on_inputs: false
68
+ val_set_size: 0.05
69
+ wandb_entity: null
70
+ wandb_log_model: null
71
+ wandb_name: null
72
+ wandb_project: null
73
+ wandb_watch: null
74
+ warmup_steps: 10
75
+ weight_decay: 0.0
76
+ xformers_attention: null
77
+
78
+ ```
79
+
80
+ </details><br>
81
+
82
+ # outputs/qlora-out
83
+
84
+ This model is a fine-tuned version of [TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T](https://huggingface.co/TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T) on the None dataset.
85
+ It achieves the following results on the evaluation set:
86
+ - Loss: 1.7613
87
+
88
+ ## Model description
89
+
90
+ More information needed
91
+
92
+ ## Intended uses & limitations
93
+
94
+ More information needed
95
+
96
+ ## Training and evaluation data
97
+
98
+ More information needed
99
+
100
+ ## Training procedure
101
+
102
+ ### Training hyperparameters
103
+
104
+ The following hyperparameters were used during training:
105
+ - learning_rate: 0.0002
106
+ - train_batch_size: 8
107
+ - eval_batch_size: 8
108
+ - seed: 42
109
+ - gradient_accumulation_steps: 4
110
+ - total_train_batch_size: 32
111
+ - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
112
+ - lr_scheduler_type: cosine
113
+ - lr_scheduler_warmup_steps: 10
114
+ - num_epochs: 4
115
+
116
+ ### Training results
117
+
118
+ | Training Loss | Epoch | Step | Validation Loss |
119
+ |:-------------:|:------:|:----:|:---------------:|
120
+ | 1.768 | 0.0336 | 1 | 1.8649 |
121
+ | 1.8084 | 0.2689 | 8 | 1.8317 |
122
+ | 1.633 | 0.5378 | 16 | 1.7833 |
123
+ | 1.6737 | 0.8067 | 24 | 1.7644 |
124
+ | 1.6722 | 1.0756 | 32 | 1.7601 |
125
+ | 1.7162 | 1.3445 | 40 | 1.7571 |
126
+ | 1.7046 | 1.6134 | 48 | 1.7558 |
127
+ | 1.6714 | 1.8824 | 56 | 1.7564 |
128
+ | 1.6249 | 2.1513 | 64 | 1.7566 |
129
+ | 1.5604 | 2.4202 | 72 | 1.7599 |
130
+ | 1.7003 | 2.6891 | 80 | 1.7614 |
131
+ | 1.7115 | 2.9580 | 88 | 1.7605 |
132
+ | 1.5937 | 3.2269 | 96 | 1.7609 |
133
+ | 1.655 | 3.4958 | 104 | 1.7612 |
134
+ | 1.5829 | 3.7647 | 112 | 1.7613 |
135
+
136
+
137
+ ### Framework versions
138
+
139
+ - PEFT 0.11.1
140
+ - Transformers 4.41.1
141
+ - Pytorch 2.1.2+cu121
142
+ - Datasets 2.19.1
143
+ - Tokenizers 0.19.1
content/outputs/qlora-out/adapter_config.json ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "alpha_pattern": {},
3
+ "auto_mapping": null,
4
+ "base_model_name_or_path": "TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T",
5
+ "bias": "none",
6
+ "fan_in_fan_out": null,
7
+ "inference_mode": true,
8
+ "init_lora_weights": true,
9
+ "layer_replication": null,
10
+ "layers_pattern": null,
11
+ "layers_to_transform": null,
12
+ "loftq_config": {},
13
+ "lora_alpha": 16,
14
+ "lora_dropout": 0.05,
15
+ "megatron_config": null,
16
+ "megatron_core": "megatron.core",
17
+ "modules_to_save": null,
18
+ "peft_type": "LORA",
19
+ "r": 32,
20
+ "rank_pattern": {},
21
+ "revision": null,
22
+ "target_modules": [
23
+ "gate_proj",
24
+ "up_proj",
25
+ "down_proj",
26
+ "v_proj",
27
+ "o_proj",
28
+ "q_proj",
29
+ "k_proj"
30
+ ],
31
+ "task_type": "CAUSAL_LM",
32
+ "use_dora": false,
33
+ "use_rslora": false
34
+ }