End of training

README.md

@@ -1,11 +1,12 @@
 ---
 library_name: transformers
-base_model: trl-internal-testing/tiny-random-LlamaForCausalLM
+license: llama3.2
+base_model: NousResearch/Llama-3.2-1B
 tags:
 - axolotl
 - generated_from_trainer
 model-index:
-- name: dab16ec4-4ddf-4ee5-8888-3dc2a83f0f86
+- name: 0c2649cc-2fe7-4e88-b672-6da1fee4001f
   results: []
 ---
 
@@ -17,20 +18,21 @@ should probably proofread and complete it, then remove this comment. -->
 
 axolotl version: `0.4.1`
 ```yaml
-base_model: trl-internal-testing/tiny-random-LlamaForCausalLM
+base_model: NousResearch/Llama-3.2-1B
 batch_size: 32
 bf16: true
 chat_template: tokenizer_default_fallback_alpaca
 datasets:
 - data_files:
-  - f4a61305a746447c_train_data.json
+  - f51beb4c568b9128_train_data.json
   ds_type: json
   format: custom
-  path: /workspace/input_data/f4a61305a746447c_train_data.json
+  path: /workspace/input_data/f51beb4c568b9128_train_data.json
   type:
-    field_instruction: sentence1
-    field_output: sentence2
-    format: '{instruction}'
+    field_input: keywords
+    field_instruction: idea
+    field_output: full_response
+    format: '{instruction} {input}'
     no_input_format: '{instruction}'
     system_format: '{system}'
     system_prompt: ''
@@ -39,7 +41,7 @@ flash_attention: true
 gpu_memory_limit: 80GiB
 gradient_checkpointing: true
 group_by_length: true
-hub_model_id: willtensora/dab16ec4-4ddf-4ee5-8888-3dc2a83f0f86
+hub_model_id: willtensora/0c2649cc-2fe7-4e88-b672-6da1fee4001f
 hub_strategy: checkpoint
 learning_rate: 0.0002
 logging_steps: 10
@@ -55,13 +57,15 @@ sample_packing: false
 save_steps: 40
 save_total_limit: 1
 sequence_len: 2048
-tokenizer_type: LlamaTokenizerFast
+special_tokens:
+  pad_token: <|end_of_text|>
+tokenizer_type: PreTrainedTokenizerFast
 train_on_inputs: false
 trust_remote_code: true
 val_set_size: 0.1
 wandb_entity: ''
 wandb_mode: online
-wandb_name: trl-internal-testing/tiny-random-LlamaForCausalLM-/workspace/input_data/f4a61305a746447c_train_data.json
+wandb_name: NousResearch/Llama-3.2-1B-/workspace/input_data/f51beb4c568b9128_train_data.json
 wandb_project: Gradients-On-Demand
 wandb_run: your_name
 wandb_runid: default
@@ -72,9 +76,11 @@ xformers_attention: true
 
 </details><br>
 
-# dab16ec4-4ddf-4ee5-8888-3dc2a83f0f86
+# 0c2649cc-2fe7-4e88-b672-6da1fee4001f
 
-This model is a fine-tuned version of [trl-internal-testing/tiny-random-LlamaForCausalLM](https://huggingface.co/trl-internal-testing/tiny-random-LlamaForCausalLM) on the None dataset.
+This model is a fine-tuned version of [NousResearch/Llama-3.2-1B](https://huggingface.co/NousResearch/Llama-3.2-1B) on the None dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.0849
 
 ## Model description
 
@@ -103,13 +109,26 @@ The following hyperparameters were used during training:
 - total_eval_batch_size: 32
 - optimizer: Use OptimizerNames.ADAMW_BNB with betas=(0.9,0.999) and epsilon=1e-08 and optimizer_args=No additional optimizer arguments
 - lr_scheduler_type: cosine
-- training_steps: 13
+- lr_scheduler_warmup_steps: 12
+- training_steps: 258
 
 ### Training results
 
-| Training Loss | Epoch | Step | Validation Loss |
-|:-------------:|:-----:|:----:|:---------------:|
-| No log        | 0.01  | 1    | 10.3686         |
+| Training Loss | Epoch  | Step | Validation Loss |
+|:-------------:|:------:|:----:|:---------------:|
+| No log        | 0.0005 | 1    | 0.2074          |
+| 0.5472        | 0.0097 | 20   | 0.1746          |
+| 0.3199        | 0.0194 | 40   | 0.2036          |
+| 0.2013        | 0.0291 | 60   | 0.1772          |
+| 0.0903        | 0.0388 | 80   | 0.1702          |
+| 0.0875        | 0.0485 | 100  | 0.2040          |
+| 0.1425        | 0.0582 | 120  | 0.1392          |
+| 0.1982        | 0.0679 | 140  | 0.1194          |
+| 0.1372        | 0.0776 | 160  | 0.1014          |
+| 0.0278        | 0.0873 | 180  | 0.0952          |
+| 0.0248        | 0.0970 | 200  | 0.0893          |
+| 0.1051        | 0.1067 | 220  | 0.0875          |
+| 0.0649        | 0.1164 | 240  | 0.0849          |
 
 
 ### Framework versions

generation_config.json

@@ -1,8 +1,7 @@
 {
   "_from_model_config": true,
-  "bos_token_id": 0,
+  "bos_token_id": 128000,
   "do_sample": true,
-  "eos_token_id": 1,
-  "pad_token_id": 2,
+  "eos_token_id": 128001,
   "transformers_version": "4.46.0"
 }

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:70872e8af35c48abf8dc8a0f41f28f7673e23a762cd5f5a4707b0788bf617ebf
-size 2071661
+oid sha256:4a3488e39325dea60c21ab0cf3a2715d26192702fde06183582341380d5a328b
+size 2471678226