Model save

README.md

@@ -2,13 +2,12 @@
 license: gemma
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - sft
 - generated_from_trainer
 base_model: google/gemma-2b
 datasets:
-- llama-duo/synth_summarize_dataset_dedup
+- generator
 model-index:
 - name: gemma2b-summarize-gemini1_5flash-256k
   results: []
@@ -19,9 +18,9 @@ should probably proofread and complete it, then remove this comment. -->
 
 # gemma2b-summarize-gemini1_5flash-256k
 
-This model is a fine-tuned version of [google/gemma-2b](https://huggingface.co/google/gemma-2b) on the llama-duo/synth_summarize_dataset_dedup dataset.
+This model is a fine-tuned version of [google/gemma-2b](https://huggingface.co/google/gemma-2b) on the generator dataset.
 It achieves the following results on the evaluation set:
-- Loss: 2.5038
+- Loss: 2.5669
 
 ## Model description
 
@@ -45,40 +44,35 @@ The following hyperparameters were used during training:
 - eval_batch_size: 8
 - seed: 42
 - distributed_type: multi-GPU
-- num_devices: 4
+- num_devices: 8
 - gradient_accumulation_steps: 2
-- total_train_batch_size: 64
-- total_eval_batch_size: 32
+- total_train_batch_size: 128
+- total_eval_batch_size: 64
 - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
 - lr_scheduler_type: cosine
 - lr_scheduler_warmup_ratio: 0.1
-- num_epochs: 15
+- num_epochs: 10
 
 ### Training results
 
-| Training Loss | Epoch   | Step | Validation Loss |
-|:-------------:|:-------:|:----:|:---------------:|
-| 1.0614        | 0.9988  | 414  | 2.4760          |
-| 1.0004        | 2.0     | 829  | 2.4481          |
-| 0.9586        | 2.9988  | 1243 | 2.4426          |
-| 0.9412        | 4.0     | 1658 | 2.4496          |
-| 0.9325        | 4.9988  | 2072 | 2.4600          |
-| 0.9129        | 6.0     | 2487 | 2.4629          |
-| 0.8995        | 6.9988  | 2901 | 2.4703          |
-| 0.8999        | 8.0     | 3316 | 2.4830          |
-| 0.8762        | 8.9988  | 3730 | 2.4934          |
-| 0.8821        | 10.0    | 4145 | 2.4974          |
-| 0.8697        | 10.9988 | 4559 | 2.5013          |
-| 0.8729        | 12.0    | 4974 | 2.5031          |
-| 0.8779        | 12.9988 | 5388 | 2.5023          |
-| 0.8743        | 14.0    | 5803 | 2.5033          |
-| 0.8746        | 14.9819 | 6210 | 2.5038          |
+| Training Loss | Epoch  | Step | Validation Loss |
+|:-------------:|:------:|:----:|:---------------:|
+| 1.0246        | 0.9976 | 207  | 2.4550          |
+| 0.9556        | 2.0    | 415  | 2.4530          |
+| 0.9114        | 2.9976 | 622  | 2.4641          |
+| 0.8927        | 4.0    | 830  | 2.4882          |
+| 0.8752        | 4.9976 | 1037 | 2.5081          |
+| 0.8602        | 6.0    | 1245 | 2.5277          |
+| 0.8464        | 6.9976 | 1452 | 2.5513          |
+| 0.8353        | 8.0    | 1660 | 2.5615          |
+| 0.8267        | 8.9976 | 1867 | 2.5674          |
+| 0.827         | 9.9759 | 2070 | 2.5669          |
 
 
 ### Framework versions
 
 - PEFT 0.11.1
-- Transformers 4.40.1
-- Pytorch 2.2.0+cu121
+- Transformers 4.41.2
+- Pytorch 2.3.1+cu121
 - Datasets 2.19.2
 - Tokenizers 0.19.1

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b9bf333514cb70e73dda09561ebd8fba4d5658337e542a559555c3ad57cf41ab
+oid sha256:465c1b3666bad23b46105166da89ba228655adfb0e551b2809da6f6c1f2df5f3
 size 78480320

all_results.json

@@ -1,14 +1,9 @@
 {
-    "epoch": 14.981905910735826,
-    "eval_loss": 2.503845453262329,
-    "eval_runtime": 0.5015,
-    "eval_samples": 25,
-    "eval_samples_per_second": 19.941,
-    "eval_steps_per_second": 1.994,
-    "total_flos": 4.863451355047526e+18,
-    "train_loss": 0.9534509487582098,
-    "train_runtime": 21285.3674,
+    "epoch": 9.975903614457831,
+    "total_flos": 3.290190024938619e+18,
+    "train_loss": 0.9333097650233099,
+    "train_runtime": 14306.303,
     "train_samples": 253412,
-    "train_samples_per_second": 18.683,
-    "train_steps_per_second": 0.292
+    "train_samples_per_second": 18.532,
+    "train_steps_per_second": 0.145
 }

runs/Jun10_09-38-09_48ddfe8e991f/events.out.tfevents.1718012562.48ddfe8e991f.334310.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:314c1368e9577920bf9375f6e653416745d5241d15d260dd90f822bb1a8ed8e8
-size 92554
+oid sha256:35b82431d49a0c3e5f8201a73d7045835a5bd2c881a40f073d0db0fae3d9a474
+size 96133

train_results.json

@@ -1,9 +1,9 @@
 {
-    "epoch": 14.981905910735826,
-    "total_flos": 4.863451355047526e+18,
-    "train_loss": 0.9534509487582098,
-    "train_runtime": 21285.3674,
+    "epoch": 9.975903614457831,
+    "total_flos": 3.290190024938619e+18,
+    "train_loss": 0.9333097650233099,
+    "train_runtime": 14306.303,
     "train_samples": 253412,
-    "train_samples_per_second": 18.683,
-    "train_steps_per_second": 0.292
+    "train_samples_per_second": 18.532,
+    "train_steps_per_second": 0.145
 }