Model save

README.md

@@ -0,0 +1,69 @@
+---
+base_model: meta-llama/Meta-Llama-3-8B
+datasets:
+- generator
+library_name: peft
+license: llama3
+tags:
+- trl
+- sft
+- generated_from_trainer
+model-index:
+- name: llama3.1-8b-gpt4o_100k_summarize-lora
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# llama3.1-8b-gpt4o_100k_summarize-lora
+
+This model is a fine-tuned version of [meta-llama/Meta-Llama-3-8B](https://huggingface.co/meta-llama/Meta-Llama-3-8B) on the generator dataset.
+It achieves the following results on the evaluation set:
+- Loss: 2.0161
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0003
+- train_batch_size: 4
+- eval_batch_size: 4
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 2
+- total_train_batch_size: 64
+- total_eval_batch_size: 32
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 1
+
+### Training results
+
+| Training Loss | Epoch | Step | Validation Loss |
+|:-------------:|:-----:|:----:|:---------------:|
+| 0.8469        | 1.0   | 195  | 2.0161          |
+
+
+### Framework versions
+
+- PEFT 0.13.0
+- Transformers 4.45.1
+- Pytorch 2.4.1+cu121
+- Datasets 3.0.1
+- Tokenizers 0.20.0

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 5.7571792321406566e+17,
+    "train_loss": 0.9914888846568573,
+    "train_runtime": 2084.3097,
+    "train_samples": 115376,
+    "train_samples_per_second": 5.986,
+    "train_steps_per_second": 0.094
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 1.0,
+    "total_flos": 5.7571792321406566e+17,
+    "train_loss": 0.9914888846568573,
+    "train_runtime": 2084.3097,
+    "train_samples": 115376,
+    "train_samples_per_second": 5.986,
+    "train_steps_per_second": 0.094
+}

trainer_state.json

@@ -0,0 +1,330 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 500,
+  "global_step": 195,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.005128205128205128,
+      "grad_norm": 0.7734681367874146,
+      "learning_rate": 1.4999999999999999e-05,
+      "loss": 2.2537,
+      "step": 1
+    },
+    {
+      "epoch": 0.02564102564102564,
+      "grad_norm": 0.9921783804893494,
+      "learning_rate": 7.5e-05,
+      "loss": 2.2441,
+      "step": 5
+    },
+    {
+      "epoch": 0.05128205128205128,
+      "grad_norm": 1.209192156791687,
+      "learning_rate": 0.00015,
+      "loss": 2.1391,
+      "step": 10
+    },
+    {
+      "epoch": 0.07692307692307693,
+      "grad_norm": 1.4021319150924683,
+      "learning_rate": 0.000225,
+      "loss": 1.6962,
+      "step": 15
+    },
+    {
+      "epoch": 0.10256410256410256,
+      "grad_norm": 2.129197359085083,
+      "learning_rate": 0.0003,
+      "loss": 1.2642,
+      "step": 20
+    },
+    {
+      "epoch": 0.1282051282051282,
+      "grad_norm": 0.6822463870048523,
+      "learning_rate": 0.00029939614409928584,
+      "loss": 1.15,
+      "step": 25
+    },
+    {
+      "epoch": 0.15384615384615385,
+      "grad_norm": 0.5067035555839539,
+      "learning_rate": 0.00029758943828979444,
+      "loss": 1.078,
+      "step": 30
+    },
+    {
+      "epoch": 0.1794871794871795,
+      "grad_norm": 0.40783199667930603,
+      "learning_rate": 0.00029459442910437797,
+      "loss": 1.0186,
+      "step": 35
+    },
+    {
+      "epoch": 0.20512820512820512,
+      "grad_norm": 0.274424284696579,
+      "learning_rate": 0.00029043523059596053,
+      "loss": 0.9822,
+      "step": 40
+    },
+    {
+      "epoch": 0.23076923076923078,
+      "grad_norm": 0.241612508893013,
+      "learning_rate": 0.0002851453301853628,
+      "loss": 0.9603,
+      "step": 45
+    },
+    {
+      "epoch": 0.2564102564102564,
+      "grad_norm": 0.18045982718467712,
+      "learning_rate": 0.0002787673190402799,
+      "loss": 0.9451,
+      "step": 50
+    },
+    {
+      "epoch": 0.28205128205128205,
+      "grad_norm": 0.24599388241767883,
+      "learning_rate": 0.0002713525491562421,
+      "loss": 0.9247,
+      "step": 55
+    },
+    {
+      "epoch": 0.3076923076923077,
+      "grad_norm": 0.23441565036773682,
+      "learning_rate": 0.00026296071990054165,
+      "loss": 0.9203,
+      "step": 60
+    },
+    {
+      "epoch": 0.3333333333333333,
+      "grad_norm": 0.20591339468955994,
+      "learning_rate": 0.0002536593973480297,
+      "loss": 0.9067,
+      "step": 65
+    },
+    {
+      "epoch": 0.358974358974359,
+      "grad_norm": 0.46566951274871826,
+      "learning_rate": 0.00024352347027881003,
+      "loss": 0.9017,
+      "step": 70
+    },
+    {
+      "epoch": 0.38461538461538464,
+      "grad_norm": 0.21229182183742523,
+      "learning_rate": 0.00023263454721781537,
+      "loss": 0.9016,
+      "step": 75
+    },
+    {
+      "epoch": 0.41025641025641024,
+      "grad_norm": 0.22548721730709076,
+      "learning_rate": 0.0002210802993709498,
+      "loss": 0.8903,
+      "step": 80
+    },
+    {
+      "epoch": 0.4358974358974359,
+      "grad_norm": 0.18637174367904663,
+      "learning_rate": 0.00020895375474808852,
+      "loss": 0.8883,
+      "step": 85
+    },
+    {
+      "epoch": 0.46153846153846156,
+      "grad_norm": 0.1939140409231186,
+      "learning_rate": 0.0001963525491562421,
+      "loss": 0.8799,
+      "step": 90
+    },
+    {
+      "epoch": 0.48717948717948717,
+      "grad_norm": 0.2150397151708603,
+      "learning_rate": 0.00018337814009344714,
+      "loss": 0.8776,
+      "step": 95
+    },
+    {
+      "epoch": 0.5128205128205128,
+      "grad_norm": 0.21106576919555664,
+      "learning_rate": 0.00017013498987264832,
+      "loss": 0.8775,
+      "step": 100
+    },
+    {
+      "epoch": 0.5384615384615384,
+      "grad_norm": 0.21716269850730896,
+      "learning_rate": 0.00015672972455257723,
+      "loss": 0.8734,
+      "step": 105
+    },
+    {
+      "epoch": 0.5641025641025641,
+      "grad_norm": 0.22086042165756226,
+      "learning_rate": 0.0001432702754474228,
+      "loss": 0.8512,
+      "step": 110
+    },
+    {
+      "epoch": 0.5897435897435898,
+      "grad_norm": 0.232639878988266,
+      "learning_rate": 0.0001298650101273517,
+      "loss": 0.8718,
+      "step": 115
+    },
+    {
+      "epoch": 0.6153846153846154,
+      "grad_norm": 0.21026825904846191,
+      "learning_rate": 0.00011662185990655284,
+      "loss": 0.8579,
+      "step": 120
+    },
+    {
+      "epoch": 0.6410256410256411,
+      "grad_norm": 0.19612908363342285,
+      "learning_rate": 0.0001036474508437579,
+      "loss": 0.859,
+      "step": 125
+    },
+    {
+      "epoch": 0.6666666666666666,
+      "grad_norm": 0.21048156917095184,
+      "learning_rate": 9.104624525191145e-05,
+      "loss": 0.8543,
+      "step": 130
+    },
+    {
+      "epoch": 0.6923076923076923,
+      "grad_norm": 0.18239416182041168,
+      "learning_rate": 7.89197006290502e-05,
+      "loss": 0.8531,
+      "step": 135
+    },
+    {
+      "epoch": 0.717948717948718,
+      "grad_norm": 0.18775755167007446,
+      "learning_rate": 6.736545278218463e-05,
+      "loss": 0.858,
+      "step": 140
+    },
+    {
+      "epoch": 0.7435897435897436,
+      "grad_norm": 0.20988330245018005,
+      "learning_rate": 5.6476529721189974e-05,
+      "loss": 0.8532,
+      "step": 145
+    },
+    {
+      "epoch": 0.7692307692307693,
+      "grad_norm": 0.20845285058021545,
+      "learning_rate": 4.63406026519703e-05,
+      "loss": 0.8562,
+      "step": 150
+    },
+    {
+      "epoch": 0.7948717948717948,
+      "grad_norm": 0.1928585022687912,
+      "learning_rate": 3.7039280099458366e-05,
+      "loss": 0.8567,
+      "step": 155
+    },
+    {
+      "epoch": 0.8205128205128205,
+      "grad_norm": 0.19923464953899384,
+      "learning_rate": 2.8647450843757897e-05,
+      "loss": 0.8492,
+      "step": 160
+    },
+    {
+      "epoch": 0.8461538461538461,
+      "grad_norm": 0.17737220227718353,
+      "learning_rate": 2.1232680959720082e-05,
+      "loss": 0.8404,
+      "step": 165
+    },
+    {
+      "epoch": 0.8717948717948718,
+      "grad_norm": 0.18061356246471405,
+      "learning_rate": 1.4854669814637143e-05,
+      "loss": 0.8493,
+      "step": 170
+    },
+    {
+      "epoch": 0.8974358974358975,
+      "grad_norm": 0.17233729362487793,
+      "learning_rate": 9.564769404039419e-06,
+      "loss": 0.849,
+      "step": 175
+    },
+    {
+      "epoch": 0.9230769230769231,
+      "grad_norm": 0.20439976453781128,
+      "learning_rate": 5.405570895622013e-06,
+      "loss": 0.8434,
+      "step": 180
+    },
+    {
+      "epoch": 0.9487179487179487,
+      "grad_norm": 0.1979411542415619,
+      "learning_rate": 2.4105617102055496e-06,
+      "loss": 0.8441,
+      "step": 185
+    },
+    {
+      "epoch": 0.9743589743589743,
+      "grad_norm": 0.18543799221515656,
+      "learning_rate": 6.038559007141397e-07,
+      "loss": 0.8526,
+      "step": 190
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 0.225219264626503,
+      "learning_rate": 0.0,
+      "loss": 0.8469,
+      "step": 195
+    },
+    {
+      "epoch": 1.0,
+      "eval_loss": 2.0160739421844482,
+      "eval_runtime": 1.1142,
+      "eval_samples_per_second": 8.975,
+      "eval_steps_per_second": 0.897,
+      "step": 195
+    },
+    {
+      "epoch": 1.0,
+      "step": 195,
+      "total_flos": 5.7571792321406566e+17,
+      "train_loss": 0.9914888846568573,
+      "train_runtime": 2084.3097,
+      "train_samples_per_second": 5.986,
+      "train_steps_per_second": 0.094
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 195,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 5.7571792321406566e+17,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}