Model save

all_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.0,
     "total_flos": 0.0,
-    "train_loss": 0.7470545699963203,
-    "train_runtime": 930.2413,
+    "train_loss": 0.7437324760816036,
+    "train_runtime": 1103.4698,
     "train_samples": 2484,
-    "train_samples_per_second": 2.67,
-    "train_steps_per_second": 0.084
+    "train_samples_per_second": 2.251,
+    "train_steps_per_second": 0.071
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:684bdb27c40770cf3cddf81911175143ce9a1b30623967ed1f37a205e01501cb
+oid sha256:92113357029c867f634c8a73dd82a58a5de916fc2ad25a88d2f242e94fab8678
 size 4877660776

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:16a83d750d00de4c43911d14230c7cabcdef1d11399abe8950f5fe6eec28bfd8
+oid sha256:91ab47564d3b43114c1c7d1733184beb0817feaea0ab9c7eadc2e28c2a57b94d
 size 4932751008

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8111c6fab11befea669f8bfbb233fdc1a15e0ece5ff7a61267b5b8a957ff5330
+oid sha256:82e6c9c65d0bd2d0fd8aa827bd906ab44e7047f59ff3efed7e515b46ce049d99
 size 4330865200

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e7afdab785183d4921848c4d75fadb9aa7f29ebb31a374667b269420fd23bfd6
+oid sha256:f461197be3ca75b615cc2a1b55bee0eb4ee17bff23dc1bf01a3a24a0903c4e65
 size 1089994880

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 1.0,
     "total_flos": 0.0,
-    "train_loss": 0.7470545699963203,
-    "train_runtime": 930.2413,
+    "train_loss": 0.7437324760816036,
+    "train_runtime": 1103.4698,
     "train_samples": 2484,
-    "train_samples_per_second": 2.67,
-    "train_steps_per_second": 0.084
+    "train_samples_per_second": 2.251,
+    "train_steps_per_second": 0.071
 }

trainer_state.json

@@ -10,7 +10,7 @@
   "log_history": [
     {
       "epoch": 0.01282051282051282,
-      "grad_norm": 101.89968076534468,
+      "grad_norm": 101.98915917164486,
       "learning_rate": 6.25e-08,
       "logits/generated": -1.0679720640182495,
       "logits/real": -0.826486349105835,
@@ -25,117 +25,117 @@
     },
     {
       "epoch": 0.1282051282051282,
-      "grad_norm": 143.51099887453523,
+      "grad_norm": 80.52500100713164,
       "learning_rate": 4.857142857142857e-07,
-      "logits/generated": -0.8684251308441162,
-      "logits/real": -0.6457947492599487,
-      "logps/generated": -273.63232421875,
-      "logps/real": -274.0804748535156,
-      "loss": 0.9133,
-      "rewards/accuracies": 0.625,
-      "rewards/generated": 0.16648532450199127,
-      "rewards/margins": 0.10824514925479889,
-      "rewards/real": 0.27473047375679016,
+      "logits/generated": -0.8668110966682434,
+      "logits/real": -0.644612193107605,
+      "logps/generated": -273.2930908203125,
+      "logps/real": -273.6876220703125,
+      "loss": 0.907,
+      "rewards/accuracies": 0.5277777910232544,
+      "rewards/generated": 0.20040562748908997,
+      "rewards/margins": 0.11361115425825119,
+      "rewards/real": 0.31401681900024414,
       "step": 10
     },
     {
       "epoch": 0.2564102564102564,
-      "grad_norm": 61.070639184089565,
+      "grad_norm": 61.621972969126524,
       "learning_rate": 4.142857142857143e-07,
-      "logits/generated": -1.0700271129608154,
-      "logits/real": -0.7057520747184753,
-      "logps/generated": -261.4720153808594,
-      "logps/real": -257.3920593261719,
-      "loss": 0.7717,
-      "rewards/accuracies": 0.75,
-      "rewards/generated": 0.8906627893447876,
-      "rewards/margins": 0.7612454295158386,
-      "rewards/real": 1.6519081592559814,
+      "logits/generated": -1.0656638145446777,
+      "logits/real": -0.7032974362373352,
+      "logps/generated": -260.37921142578125,
+      "logps/real": -255.3711395263672,
+      "loss": 0.7653,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/generated": 0.9999414682388306,
+      "rewards/margins": 0.8540604710578918,
+      "rewards/real": 1.8540016412734985,
       "step": 20
     },
     {
       "epoch": 0.38461538461538464,
-      "grad_norm": 64.32897890365741,
+      "grad_norm": 59.21144986605617,
       "learning_rate": 3.4285714285714286e-07,
-      "logits/generated": -0.9067907333374023,
-      "logits/real": -0.659667432308197,
-      "logps/generated": -261.84893798828125,
-      "logps/real": -250.3210906982422,
-      "loss": 0.7144,
+      "logits/generated": -0.9283930063247681,
+      "logits/real": -0.6714473962783813,
+      "logps/generated": -261.3749084472656,
+      "logps/real": -249.65054321289062,
+      "loss": 0.7105,
       "rewards/accuracies": 0.75,
-      "rewards/generated": 1.7948137521743774,
-      "rewards/margins": 0.8462657928466797,
-      "rewards/real": 2.6410794258117676,
+      "rewards/generated": 1.842218041419983,
+      "rewards/margins": 0.8659162521362305,
+      "rewards/real": 2.708134412765503,
       "step": 30
     },
     {
       "epoch": 0.5128205128205128,
-      "grad_norm": 164.54242317315587,
+      "grad_norm": 62.53453913007679,
       "learning_rate": 2.714285714285714e-07,
-      "logits/generated": -0.9878751039505005,
-      "logits/real": -0.6697134375572205,
-      "logps/generated": -254.94430541992188,
-      "logps/real": -251.97024536132812,
-      "loss": 0.7095,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/generated": 1.6531692743301392,
-      "rewards/margins": 1.1178901195526123,
-      "rewards/real": 2.771059513092041,
+      "logits/generated": -0.9731477499008179,
+      "logits/real": -0.6548992395401001,
+      "logps/generated": -253.58157348632812,
+      "logps/real": -250.48837280273438,
+      "loss": 0.7065,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/generated": 1.789442777633667,
+      "rewards/margins": 1.1298000812530518,
+      "rewards/real": 2.919243097305298,
       "step": 40
     },
     {
       "epoch": 0.6410256410256411,
-      "grad_norm": 65.89825293961819,
+      "grad_norm": 63.255829698141724,
       "learning_rate": 2e-07,
-      "logits/generated": -0.9043434262275696,
-      "logits/real": -0.767203152179718,
-      "logps/generated": -254.6318817138672,
-      "logps/real": -240.71481323242188,
-      "loss": 0.6938,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/generated": 2.046715497970581,
-      "rewards/margins": 0.7744835615158081,
-      "rewards/real": 2.8211989402770996,
+      "logits/generated": -0.8815720677375793,
+      "logits/real": -0.7497758269309998,
+      "logps/generated": -253.22402954101562,
+      "logps/real": -238.9423370361328,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/generated": 2.1875,
+      "rewards/margins": 0.8109487295150757,
+      "rewards/real": 2.998448610305786,
       "step": 50
     },
     {
       "epoch": 0.7692307692307693,
-      "grad_norm": 124.30731613732516,
+      "grad_norm": 77.61305253400987,
       "learning_rate": 1.2857142857142855e-07,
-      "logits/generated": -1.0582646131515503,
-      "logits/real": -0.6999293565750122,
-      "logps/generated": -245.7989044189453,
-      "logps/real": -248.5264129638672,
-      "loss": 0.7132,
+      "logits/generated": -1.0364863872528076,
+      "logits/real": -0.6802612543106079,
+      "logps/generated": -244.5989227294922,
+      "logps/real": -247.1646270751953,
+      "loss": 0.7152,
       "rewards/accuracies": 0.8500000238418579,
-      "rewards/generated": 1.8123178482055664,
-      "rewards/margins": 1.1895701885223389,
-      "rewards/real": 3.0018882751464844,
+      "rewards/generated": 1.9323114156723022,
+      "rewards/margins": 1.2057563066482544,
+      "rewards/real": 3.1380679607391357,
       "step": 60
     },
     {
       "epoch": 0.8974358974358975,
-      "grad_norm": 65.42190072046155,
+      "grad_norm": 53.32977807319664,
       "learning_rate": 5.714285714285714e-08,
-      "logits/generated": -1.1064507961273193,
-      "logits/real": -0.7838689684867859,
-      "logps/generated": -245.507568359375,
-      "logps/real": -237.945068359375,
-      "loss": 0.7122,
-      "rewards/accuracies": 0.75,
-      "rewards/generated": 1.7316808700561523,
-      "rewards/margins": 1.1938471794128418,
-      "rewards/real": 2.925528049468994,
+      "logits/generated": -1.086891531944275,
+      "logits/real": -0.7703801989555359,
+      "logps/generated": -244.48086547851562,
+      "logps/real": -236.3184814453125,
+      "loss": 0.7159,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/generated": 1.834351897239685,
+      "rewards/margins": 1.2538334131240845,
+      "rewards/real": 3.0881857872009277,
       "step": 70
     },
     {
       "epoch": 1.0,
       "step": 78,
       "total_flos": 0.0,
-      "train_loss": 0.7470545699963203,
-      "train_runtime": 930.2413,
-      "train_samples_per_second": 2.67,
-      "train_steps_per_second": 0.084
+      "train_loss": 0.7437324760816036,
+      "train_runtime": 1103.4698,
+      "train_samples_per_second": 2.251,
+      "train_steps_per_second": 0.071
     }
   ],
   "logging_steps": 10,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dbda660047aab9616bdbc58c392057e52f11f679c812a40158afebcfae515e2e
+oid sha256:21ea6eb09e073776a07bcf698fec99f405955d2f050aac5a20b0120d2d119aaf
 size 6392