Model save

Browse files

Files changed (8) hide show

README.md +16 -21
adapter_config.json +4 -4
adapter_model.safetensors +1 -1
all_results.json +6 -19
runs/Jul31_17-29-05_ip-172-16-2-184.us-west-2.compute.internal/events.out.tfevents.1722447034.ip-172-16-2-184.us-west-2.compute.internal.23039.0 +3 -0
train_results.json +6 -6
trainer_state.json +882 -1048
training_args.bin +1 -1

README.md CHANGED Viewed

@@ -1,12 +1,8 @@
 ---
 base_model: HuggingFaceH4/zephyr-7b-gemma-sft-v0.1
-datasets:
-- argilla/dpo-mix-7k
-- RedaAlami/PKU-SafeRLHF-Processed
 library_name: peft
 license: other
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
@@ -20,17 +16,17 @@ should probably proofread and complete it, then remove this comment. -->
 # zephyr-7b-gemma-dpo
-This model is a fine-tuned version of [HuggingFaceH4/zephyr-7b-gemma-sft-v0.1](https://huggingface.co/HuggingFaceH4/zephyr-7b-gemma-sft-v0.1) on the argilla/dpo-mix-7k and the RedaAlami/PKU-SafeRLHF-Processed datasets.
 It achieves the following results on the evaluation set:
-- Loss: 0.6478
-- Rewards/chosen: -0.3452
-- Rewards/rejected: -0.5788
-- Rewards/accuracies: 0.6169
-- Rewards/margins: 0.2336
-- Logps/rejected: -334.5554
-- Logps/chosen: -295.9647
-- Logits/rejected: 436.0139
-- Logits/chosen: 452.6414
 ## Model description
@@ -67,13 +63,12 @@ The following hyperparameters were used during training:
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 0.7052        | 0.2602 | 100  | 0.7032          | 0.1828         | 0.1374           | 0.5148             | 0.0454          | -320.2303      | -285.4035    | 437.8931        | 454.4476      |
-| 0.6851        | 0.5205 | 200  | 0.6794          | 0.1534         | 0.0240           | 0.5991             | 0.1294          | -322.4987      | -285.9917    | 436.5674        | 453.2177      |
-| 0.6545        | 0.7807 | 300  | 0.6632          | -0.0335        | -0.2290          | 0.5962             | 0.1955          | -327.5587      | -289.7299    | 435.9517        | 452.6035      |
-| 0.6428        | 1.0410 | 400  | 0.6532          | -0.3583        | -0.5844          | 0.6154             | 0.2261          | -334.6671      | -296.2265    | 436.0371        | 452.6768      |
-| 0.6366        | 1.3012 | 500  | 0.6521          | -0.3063        | -0.5602          | 0.6124             | 0.2539          | -334.1831      | -295.1856    | 436.1843        | 452.8112      |
-| 0.6058        | 1.5615 | 600  | 0.6497          | -0.3389        | -0.5751          | 0.6139             | 0.2362          | -334.4804      | -295.8380    | 436.0276        | 452.6521      |
-| 0.6368        | 1.8217 | 700  | 0.6449          | -0.3403        | -0.5854          | 0.6065             | 0.2451          | -334.6864      | -295.8665    | 436.0117        | 452.6395      |
 ### Framework versions

 ---
 base_model: HuggingFaceH4/zephyr-7b-gemma-sft-v0.1
 library_name: peft
 license: other
 tags:
 - trl
 - dpo
 - generated_from_trainer
 # zephyr-7b-gemma-dpo
+This model is a fine-tuned version of [HuggingFaceH4/zephyr-7b-gemma-sft-v0.1](https://huggingface.co/HuggingFaceH4/zephyr-7b-gemma-sft-v0.1) on an unknown dataset.
 It achieves the following results on the evaluation set:
+- Loss: 97.2543
+- Rewards/chosen: 0.0424
+- Rewards/rejected: 0.0341
+- Rewards/accuracies: 0.5976
+- Rewards/margins: 0.0083
+- Logps/rejected: -2.3888
+- Logps/chosen: -2.3300
+- Logits/rejected: 384.5274
+- Logits/chosen: 412.5387
 ## Model description
 | Training Loss | Epoch  | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:------:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
+| 99.2543       | 0.3017 | 100  | 98.5109         | 0.0407         | 0.0354           | 0.5822             | 0.0053          | -2.3624        | -2.3625      | 390.8526        | 418.0560      |
+| 98.8709       | 0.6033 | 200  | 98.0235         | 0.0431         | 0.0367           | 0.5788             | 0.0063          | -2.3359        | -2.3153      | 388.3781        | 415.9555      |
+| 97.9389       | 0.9050 | 300  | 97.6159         | 0.0460         | 0.0381           | 0.5959             | 0.0078          | -2.3082        | -2.2581      | 386.4085        | 414.2633      |
+| 96.4776       | 1.2066 | 400  | 97.3138         | 0.0431         | 0.0347           | 0.5908             | 0.0083          | -2.3763        | -2.3158      | 385.0537        | 413.0242      |
+| 97.3613       | 1.5083 | 500  | 97.2518         | 0.0430         | 0.0346           | 0.5908             | 0.0083          | -2.3781        | -2.3180      | 384.5959        | 412.6117      |
+| 97.5077       | 1.8100 | 600  | 97.2543         | 0.0424         | 0.0341           | 0.5976             | 0.0083          | -2.3888        | -2.3300      | 384.5274        | 412.5387      |
 ### Framework versions

adapter_config.json CHANGED Viewed

@@ -21,12 +21,12 @@
   "revision": null,
   "target_modules": [
     "q_proj",
     "o_proj",
-    "up_proj",
-    "down_proj",
     "gate_proj",
-    "v_proj",
-    "k_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

   "revision": null,
   "target_modules": [
     "q_proj",
+    "v_proj",
     "o_proj",
     "gate_proj",
+    "k_proj",
+    "up_proj",
+    "down_proj"
   ],
   "task_type": "CAUSAL_LM",
   "use_dora": false,

adapter_model.safetensors CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9b468c405e70c66881f16bd3329bd7a94f9692e53fde6d171c6cef241df7b86b
 size 800117240

 version https://git-lfs.github.com/spec/v1
+oid sha256:f025b6f9d30b18456154dbd2150f8ec520d8c69f8064271ab7873aec7f04a36d
 size 800117240

all_results.json CHANGED Viewed

@@ -1,22 +1,9 @@
 {
-    "epoch": 1.9986987638256344,
-    "eval_logits/chosen": 452.641357421875,
-    "eval_logits/rejected": 436.01385498046875,
-    "eval_logps/chosen": -295.9647216796875,
-    "eval_logps/rejected": -334.555419921875,
-    "eval_loss": 0.6477869153022766,
-    "eval_rewards/accuracies": 0.6168639063835144,
-    "eval_rewards/chosen": -0.34524381160736084,
-    "eval_rewards/margins": 0.23360556364059448,
-    "eval_rewards/rejected": -0.5788493752479553,
-    "eval_runtime": 243.7753,
-    "eval_samples": 5406,
-    "eval_samples_per_second": 22.176,
-    "eval_steps_per_second": 0.693,
     "total_flos": 0.0,
-    "train_loss": 0.6547494133313497,
-    "train_runtime": 8032.2795,
-    "train_samples": 49171,
-    "train_samples_per_second": 12.243,
-    "train_steps_per_second": 0.096
 }

 {
+    "epoch": 1.9969834087481146,
     "total_flos": 0.0,
+    "train_loss": 98.05829228519313,
+    "train_runtime": 4353.733,
+    "train_samples": 42421,
+    "train_samples_per_second": 19.487,
+    "train_steps_per_second": 0.152
 }

runs/Jul31_17-29-05_ip-172-16-2-184.us-west-2.compute.internal/events.out.tfevents.1722447034.ip-172-16-2-184.us-west-2.compute.internal.23039.0 ADDED Viewed

	@@ -0,0 +1,3 @@

+version https://git-lfs.github.com/spec/v1
+oid sha256:98ce2a38ed58a0f59f46acdeb64bad41c8f006bbc47eab5d75dd1a664187bf9f
+size 56440

train_results.json CHANGED Viewed

@@ -1,9 +1,9 @@
 {
-    "epoch": 1.9986987638256344,
     "total_flos": 0.0,
-    "train_loss": 0.6547494133313497,
-    "train_runtime": 8032.2795,
-    "train_samples": 49171,
-    "train_samples_per_second": 12.243,
-    "train_steps_per_second": 0.096
 }

 {
+    "epoch": 1.9969834087481146,
     "total_flos": 0.0,
+    "train_loss": 98.05829228519313,
+    "train_runtime": 4353.733,
+    "train_samples": 42421,
+    "train_samples_per_second": 19.487,
+    "train_steps_per_second": 0.152
 }

trainer_state.json CHANGED Viewed

@@ -1,22 +1,22 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 1.9986987638256344,
   "eval_steps": 100,
-  "global_step": 768,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.002602472348731295,
-      "grad_norm": 9.0,
-      "learning_rate": 6.493506493506494e-09,
-      "logits/chosen": 411.1029357910156,
-      "logits/rejected": 362.02178955078125,
-      "logps/chosen": -352.47296142578125,
-      "logps/rejected": -387.4255065917969,
-      "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -24,1269 +24,1103 @@
       "step": 1
     },
     {
-      "epoch": 0.026024723487312947,
-      "grad_norm": 8.5625,
-      "learning_rate": 6.493506493506492e-08,
-      "logits/chosen": 378.90606689453125,
-      "logits/rejected": 347.66900634765625,
-      "logps/chosen": -288.4970397949219,
-      "logps/rejected": -333.1199645996094,
-      "loss": 0.7101,
-      "rewards/accuracies": 0.4513888955116272,
-      "rewards/chosen": 0.0017441289965063334,
-      "rewards/margins": 0.01993246003985405,
-      "rewards/rejected": -0.01818833500146866,
       "step": 10
     },
     {
-      "epoch": 0.05204944697462589,
-      "grad_norm": 8.75,
-      "learning_rate": 1.2987012987012984e-07,
-      "logits/chosen": 381.0403137207031,
-      "logits/rejected": 360.0636291503906,
-      "logps/chosen": -274.6163024902344,
-      "logps/rejected": -299.0308532714844,
-      "loss": 0.7185,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -0.016971342265605927,
-      "rewards/margins": -0.047712840139865875,
-      "rewards/rejected": 0.030741501599550247,
       "step": 20
     },
     {
-      "epoch": 0.07807417046193885,
-      "grad_norm": 9.0625,
-      "learning_rate": 1.948051948051948e-07,
-      "logits/chosen": 382.80517578125,
-      "logits/rejected": 375.8597412109375,
-      "logps/chosen": -305.6173095703125,
-      "logps/rejected": -332.44134521484375,
-      "loss": 0.709,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -0.005826466716825962,
-      "rewards/margins": -0.014216383919119835,
-      "rewards/rejected": 0.00838992465287447,
       "step": 30
     },
     {
-      "epoch": 0.10409889394925179,
-      "grad_norm": 8.125,
-      "learning_rate": 2.597402597402597e-07,
-      "logits/chosen": 368.2964172363281,
-      "logits/rejected": 368.75994873046875,
-      "logps/chosen": -303.63433837890625,
-      "logps/rejected": -310.3212890625,
-      "loss": 0.7122,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -0.0020552538335323334,
-      "rewards/margins": 0.017374467104673386,
-      "rewards/rejected": -0.019429724663496017,
       "step": 40
     },
     {
-      "epoch": 0.13012361743656473,
-      "grad_norm": 10.0,
-      "learning_rate": 3.2467532467532465e-07,
-      "logits/chosen": 374.7029724121094,
-      "logits/rejected": 367.4124450683594,
-      "logps/chosen": -314.57452392578125,
-      "logps/rejected": -332.7318420410156,
-      "loss": 0.7084,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.022785179316997528,
-      "rewards/margins": 0.039472438395023346,
-      "rewards/rejected": -0.01668725535273552,
       "step": 50
     },
     {
-      "epoch": 0.1561483409238777,
-      "grad_norm": 9.0,
-      "learning_rate": 3.896103896103896e-07,
-      "logits/chosen": 370.33184814453125,
-      "logits/rejected": 370.1338806152344,
-      "logps/chosen": -295.30523681640625,
-      "logps/rejected": -298.3065185546875,
-      "loss": 0.6999,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.04296935349702835,
-      "rewards/margins": 0.012664027512073517,
-      "rewards/rejected": 0.03030533157289028,
       "step": 60
     },
     {
-      "epoch": 0.18217306441119063,
-      "grad_norm": 9.3125,
-      "learning_rate": 4.545454545454545e-07,
-      "logits/chosen": 411.93780517578125,
-      "logits/rejected": 376.9971923828125,
-      "logps/chosen": -285.2339172363281,
-      "logps/rejected": -330.7354431152344,
-      "loss": 0.7152,
-      "rewards/accuracies": 0.518750011920929,
-      "rewards/chosen": 0.061416469514369965,
-      "rewards/margins": 0.01710355281829834,
-      "rewards/rejected": 0.044312912970781326,
       "step": 70
     },
     {
-      "epoch": 0.20819778789850357,
-      "grad_norm": 9.6875,
-      "learning_rate": 4.999767464405451e-07,
-      "logits/chosen": 388.18426513671875,
-      "logits/rejected": 372.40509033203125,
-      "logps/chosen": -304.78607177734375,
-      "logps/rejected": -350.2904357910156,
-      "loss": 0.7012,
-      "rewards/accuracies": 0.518750011920929,
-      "rewards/chosen": 0.0943768322467804,
-      "rewards/margins": 0.008914275094866753,
-      "rewards/rejected": 0.0854625552892685,
       "step": 80
     },
     {
-      "epoch": 0.2342225113858165,
-      "grad_norm": 9.875,
-      "learning_rate": 4.995634701567891e-07,
-      "logits/chosen": 383.4454040527344,
-      "logits/rejected": 348.91796875,
-      "logps/chosen": -281.39263916015625,
-      "logps/rejected": -323.1919860839844,
-      "loss": 0.704,
-      "rewards/accuracies": 0.543749988079071,
-      "rewards/chosen": 0.15481603145599365,
-      "rewards/margins": 0.03042496182024479,
-      "rewards/rejected": 0.12439107894897461,
       "step": 90
     },
     {
-      "epoch": 0.26024723487312945,
-      "grad_norm": 10.0625,
-      "learning_rate": 4.986344312601082e-07,
-      "logits/chosen": 387.20343017578125,
-      "logits/rejected": 395.9942932128906,
-      "logps/chosen": -293.97088623046875,
-      "logps/rejected": -312.58624267578125,
-      "loss": 0.7052,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.19738170504570007,
-      "rewards/margins": 0.052183426916599274,
-      "rewards/rejected": 0.14519831538200378,
       "step": 100
     },
     {
-      "epoch": 0.26024723487312945,
-      "eval_logits/chosen": 454.44757080078125,
-      "eval_logits/rejected": 437.89306640625,
-      "eval_logps/chosen": -285.4035339355469,
-      "eval_logps/rejected": -320.2303161621094,
-      "eval_loss": 0.7031933069229126,
-      "eval_rewards/accuracies": 0.5147929191589355,
-      "eval_rewards/chosen": 0.18281590938568115,
-      "eval_rewards/margins": 0.04541104659438133,
-      "eval_rewards/rejected": 0.13740485906600952,
-      "eval_runtime": 253.4391,
-      "eval_samples_per_second": 21.331,
-      "eval_steps_per_second": 0.667,
       "step": 100
     },
     {
-      "epoch": 0.28627195836044245,
-      "grad_norm": 9.5625,
-      "learning_rate": 4.971915497571788e-07,
-      "logits/chosen": 383.96282958984375,
-      "logits/rejected": 361.9601135253906,
-      "logps/chosen": -300.56646728515625,
-      "logps/rejected": -338.2516174316406,
-      "loss": 0.6997,
-      "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": 0.2231082022190094,
-      "rewards/margins": 0.06056695431470871,
-      "rewards/rejected": 0.1625412404537201,
       "step": 110
     },
     {
-      "epoch": 0.3122966818477554,
-      "grad_norm": 9.6875,
-      "learning_rate": 4.952378075921676e-07,
-      "logits/chosen": 384.23895263671875,
-      "logits/rejected": 363.35589599609375,
-      "logps/chosen": -306.78302001953125,
-      "logps/rejected": -331.3634033203125,
-      "loss": 0.6849,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": 0.18709978461265564,
-      "rewards/margins": 0.051194410771131516,
-      "rewards/rejected": 0.13590538501739502,
       "step": 120
     },
     {
-      "epoch": 0.3383214053350683,
-      "grad_norm": 8.625,
-      "learning_rate": 4.927772424840701e-07,
-      "logits/chosen": 397.5968017578125,
-      "logits/rejected": 365.3117980957031,
-      "logps/chosen": -282.23553466796875,
-      "logps/rejected": -322.7259216308594,
-      "loss": 0.6841,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.23038265109062195,
-      "rewards/margins": 0.12032196670770645,
-      "rewards/rejected": 0.1100606918334961,
       "step": 130
     },
     {
-      "epoch": 0.36434612882238127,
-      "grad_norm": 9.6875,
-      "learning_rate": 4.898149395821217e-07,
-      "logits/chosen": 390.6437072753906,
-      "logits/rejected": 364.450439453125,
-      "logps/chosen": -276.3985900878906,
-      "logps/rejected": -327.2912292480469,
-      "loss": 0.6807,
-      "rewards/accuracies": 0.543749988079071,
-      "rewards/chosen": 0.2373248040676117,
-      "rewards/margins": 0.10833799839019775,
-      "rewards/rejected": 0.12898679077625275,
       "step": 140
     },
     {
-      "epoch": 0.3903708523096942,
-      "grad_norm": 8.25,
-      "learning_rate": 4.863570209565277e-07,
-      "logits/chosen": 382.4102478027344,
-      "logits/rejected": 376.72918701171875,
-      "logps/chosen": -300.83026123046875,
-      "logps/rejected": -310.814697265625,
-      "loss": 0.6788,
-      "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": 0.2639835476875305,
-      "rewards/margins": 0.0948825255036354,
-      "rewards/rejected": 0.1691010296344757,
       "step": 150
     },
     {
-      "epoch": 0.41639557579700714,
-      "grad_norm": 9.5,
-      "learning_rate": 4.824106329462312e-07,
-      "logits/chosen": 358.6318664550781,
-      "logits/rejected": 373.10601806640625,
-      "logps/chosen": -314.7348327636719,
-      "logps/rejected": -300.9188232421875,
-      "loss": 0.6969,
-      "rewards/accuracies": 0.4124999940395355,
-      "rewards/chosen": 0.17297904193401337,
-      "rewards/margins": -0.0854780301451683,
-      "rewards/rejected": 0.25845709443092346,
       "step": 160
     },
     {
-      "epoch": 0.4424202992843201,
-      "grad_norm": 9.0625,
-      "learning_rate": 4.779839313898674e-07,
-      "logits/chosen": 383.4333190917969,
-      "logits/rejected": 393.3439025878906,
-      "logps/chosen": -330.3350524902344,
-      "logps/rejected": -342.2207946777344,
-      "loss": 0.6778,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.19442406296730042,
-      "rewards/margins": 0.1012936383485794,
-      "rewards/rejected": 0.09313042461872101,
       "step": 170
     },
     {
-      "epoch": 0.468445022771633,
-      "grad_norm": 9.75,
-      "learning_rate": 4.730860647704252e-07,
-      "logits/chosen": 382.30145263671875,
-      "logits/rejected": 371.52410888671875,
-      "logps/chosen": -282.8478088378906,
-      "logps/rejected": -313.7384948730469,
-      "loss": 0.6791,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.27071475982666016,
-      "rewards/margins": 0.12174586206674576,
-      "rewards/rejected": 0.14896893501281738,
       "step": 180
     },
     {
-      "epoch": 0.494469746258946,
-      "grad_norm": 8.25,
-      "learning_rate": 4.677271553084514e-07,
-      "logits/chosen": 369.3918151855469,
-      "logits/rejected": 352.20196533203125,
-      "logps/chosen": -303.9125671386719,
-      "logps/rejected": -319.8887939453125,
-      "loss": 0.6857,
       "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.14780566096305847,
-      "rewards/margins": 0.09987500309944153,
-      "rewards/rejected": 0.04793066531419754,
       "step": 190
     },
     {
-      "epoch": 0.5204944697462589,
-      "grad_norm": 9.1875,
-      "learning_rate": 4.619182780428723e-07,
-      "logits/chosen": 383.06085205078125,
-      "logits/rejected": 361.99041748046875,
-      "logps/chosen": -280.6048278808594,
-      "logps/rejected": -300.1059875488281,
-      "loss": 0.6851,
       "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.1767120063304901,
-      "rewards/margins": 0.10154225677251816,
-      "rewards/rejected": 0.07516975700855255,
       "step": 200
     },
     {
-      "epoch": 0.5204944697462589,
-      "eval_logits/chosen": 453.2177429199219,
-      "eval_logits/rejected": 436.5673522949219,
-      "eval_logps/chosen": -285.99169921875,
-      "eval_logps/rejected": -322.4987487792969,
-      "eval_loss": 0.6793849468231201,
-      "eval_rewards/accuracies": 0.5991124510765076,
-      "eval_rewards/chosen": 0.15340714156627655,
-      "eval_rewards/margins": 0.12942208349704742,
-      "eval_rewards/rejected": 0.02398504503071308,
-      "eval_runtime": 244.8119,
-      "eval_samples_per_second": 22.082,
-      "eval_steps_per_second": 0.69,
       "step": 200
     },
     {
-      "epoch": 0.5465191932335719,
-      "grad_norm": 9.375,
-      "learning_rate": 4.5567143794266337e-07,
-      "logits/chosen": 400.5397033691406,
-      "logits/rejected": 364.52850341796875,
-      "logps/chosen": -275.1554870605469,
-      "logps/rejected": -342.13067626953125,
-      "loss": 0.6697,
       "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.12358863651752472,
-      "rewards/margins": 0.12654754519462585,
-      "rewards/rejected": -0.002958917524665594,
       "step": 210
     },
     {
-      "epoch": 0.5725439167208849,
-      "grad_norm": 8.375,
-      "learning_rate": 4.4899954509667134e-07,
-      "logits/chosen": 376.6017150878906,
-      "logits/rejected": 354.7825622558594,
-      "logps/chosen": -288.1871337890625,
-      "logps/rejected": -336.48321533203125,
-      "loss": 0.6807,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": 0.1387133151292801,
-      "rewards/margins": 0.1342148333787918,
-      "rewards/rejected": 0.004498471971601248,
       "step": 220
     },
     {
-      "epoch": 0.5985686402081978,
-      "grad_norm": 10.1875,
-      "learning_rate": 4.4191638803286144e-07,
-      "logits/chosen": 372.1169128417969,
-      "logits/rejected": 363.5325012207031,
-      "logps/chosen": -299.6369934082031,
-      "logps/rejected": -315.469970703125,
-      "loss": 0.6743,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": 0.11066161096096039,
-      "rewards/margins": 0.11400572210550308,
-      "rewards/rejected": -0.0033441067207604647,
       "step": 230
     },
     {
-      "epoch": 0.6245933636955108,
-      "grad_norm": 8.4375,
-      "learning_rate": 4.3443660522213157e-07,
-      "logits/chosen": 385.5279235839844,
-      "logits/rejected": 365.88372802734375,
-      "logps/chosen": -293.209228515625,
-      "logps/rejected": -317.05718994140625,
-      "loss": 0.6522,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.16928274929523468,
-      "rewards/margins": 0.16216634213924408,
-      "rewards/rejected": 0.007116401102393866,
       "step": 240
     },
     {
-      "epoch": 0.6506180871828237,
-      "grad_norm": 9.375,
-      "learning_rate": 4.265756548255822e-07,
-      "logits/chosen": 362.12689208984375,
-      "logits/rejected": 352.41986083984375,
-      "logps/chosen": -302.4978332519531,
-      "logps/rejected": -311.34228515625,
-      "loss": 0.6726,
-      "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": 0.0975576788187027,
-      "rewards/margins": 0.1262621432542801,
-      "rewards/rejected": -0.0287044458091259,
       "step": 250
     },
     {
-      "epoch": 0.6766428106701367,
-      "grad_norm": 9.125,
-      "learning_rate": 4.1834978274776867e-07,
-      "logits/chosen": 379.18072509765625,
-      "logits/rejected": 365.29168701171875,
-      "logps/chosen": -290.32928466796875,
-      "logps/rejected": -310.4072265625,
-      "loss": 0.6705,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.08413325250148773,
-      "rewards/margins": 0.1175273060798645,
-      "rewards/rejected": -0.033394038677215576,
       "step": 260
     },
     {
-      "epoch": 0.7026675341574495,
-      "grad_norm": 11.3125,
-      "learning_rate": 4.0977598906195386e-07,
-      "logits/chosen": 375.8214416503906,
-      "logits/rejected": 360.27264404296875,
-      "logps/chosen": -273.41180419921875,
-      "logps/rejected": -320.2526550292969,
-      "loss": 0.6468,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": 0.08752383291721344,
-      "rewards/margins": 0.22164182364940643,
-      "rewards/rejected": -0.134117990732193,
       "step": 270
     },
     {
-      "epoch": 0.7286922576447625,
-      "grad_norm": 11.1875,
-      "learning_rate": 4.00871992876753e-07,
-      "logits/chosen": 373.965087890625,
-      "logits/rejected": 366.2101135253906,
-      "logps/chosen": -300.20367431640625,
-      "logps/rejected": -311.20953369140625,
-      "loss": 0.6494,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": 0.033880796283483505,
-      "rewards/margins": 0.18614129722118378,
-      "rewards/rejected": -0.15226049721240997,
       "step": 280
     },
     {
-      "epoch": 0.7547169811320755,
-      "grad_norm": 7.90625,
-      "learning_rate": 3.9165619571677645e-07,
-      "logits/chosen": 363.5823059082031,
-      "logits/rejected": 354.50970458984375,
-      "logps/chosen": -297.46087646484375,
-      "logps/rejected": -325.23760986328125,
-      "loss": 0.6671,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": -0.09576047956943512,
-      "rewards/margins": 0.17094558477401733,
-      "rewards/rejected": -0.26670604944229126,
       "step": 290
     },
     {
-      "epoch": 0.7807417046193884,
-      "grad_norm": 10.5,
-      "learning_rate": 3.8214764349295194e-07,
-      "logits/chosen": 377.7972412109375,
-      "logits/rejected": 362.15045166015625,
-      "logps/chosen": -326.63104248046875,
-      "logps/rejected": -344.16009521484375,
-      "loss": 0.6545,
-      "rewards/accuracies": 0.5687500238418579,
-      "rewards/chosen": -0.101107656955719,
-      "rewards/margins": 0.14511139690876007,
-      "rewards/rejected": -0.24621903896331787,
       "step": 300
     },
     {
-      "epoch": 0.7807417046193884,
-      "eval_logits/chosen": 452.603515625,
-      "eval_logits/rejected": 435.95166015625,
-      "eval_logps/chosen": -289.7298583984375,
-      "eval_logps/rejected": -327.5587463378906,
-      "eval_loss": 0.6631770133972168,
-      "eval_rewards/accuracies": 0.5961538553237915,
-      "eval_rewards/chosen": -0.03350303694605827,
-      "eval_rewards/margins": 0.19551357626914978,
-      "eval_rewards/rejected": -0.22901661694049835,
-      "eval_runtime": 244.6727,
-      "eval_samples_per_second": 22.095,
-      "eval_steps_per_second": 0.691,
       "step": 300
     },
     {
-      "epoch": 0.8067664281067014,
-      "grad_norm": 7.53125,
-      "learning_rate": 3.7236598714111955e-07,
-      "logits/chosen": 378.12774658203125,
-      "logits/rejected": 344.31512451171875,
-      "logps/chosen": -293.63519287109375,
-      "logps/rejected": -344.9774169921875,
-      "loss": 0.6417,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.05039479583501816,
-      "rewards/margins": 0.2865242063999176,
-      "rewards/rejected": -0.33691897988319397,
       "step": 310
     },
     {
-      "epoch": 0.8327911515940143,
-      "grad_norm": 9.5625,
-      "learning_rate": 3.623314420102467e-07,
-      "logits/chosen": 377.9627685546875,
-      "logits/rejected": 364.48223876953125,
-      "logps/chosen": -284.373779296875,
-      "logps/rejected": -311.5481262207031,
-      "loss": 0.6498,
       "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -0.042569078505039215,
-      "rewards/margins": 0.20671768486499786,
-      "rewards/rejected": -0.2492867410182953,
       "step": 320
     },
     {
-      "epoch": 0.8588158750813273,
-      "grad_norm": 8.6875,
-      "learning_rate": 3.520647460841938e-07,
-      "logits/chosen": 371.0107727050781,
-      "logits/rejected": 363.4261169433594,
-      "logps/chosen": -309.06732177734375,
-      "logps/rejected": -317.2771911621094,
-      "loss": 0.6432,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.14327314496040344,
-      "rewards/margins": 0.23333874344825745,
-      "rewards/rejected": -0.3766118884086609,
       "step": 330
     },
     {
-      "epoch": 0.8848405985686402,
-      "grad_norm": 8.75,
-      "learning_rate": 3.415871171233708e-07,
-      "logits/chosen": 382.25439453125,
-      "logits/rejected": 356.82818603515625,
-      "logps/chosen": -283.73760986328125,
-      "logps/rejected": -317.1445617675781,
-      "loss": 0.6552,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -0.18387103080749512,
-      "rewards/margins": 0.21161150932312012,
-      "rewards/rejected": -0.39548248052597046,
       "step": 340
     },
     {
-      "epoch": 0.9108653220559532,
-      "grad_norm": 9.0625,
-      "learning_rate": 3.309202088148608e-07,
-      "logits/chosen": 386.0477600097656,
-      "logits/rejected": 353.7298278808594,
-      "logps/chosen": -316.064453125,
-      "logps/rejected": -360.25201416015625,
-      "loss": 0.6564,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -0.45992183685302734,
-      "rewards/margins": 0.1660740077495575,
-      "rewards/rejected": -0.6259958148002625,
       "step": 350
     },
     {
-      "epoch": 0.936890045543266,
-      "grad_norm": 11.625,
-      "learning_rate": 3.200860660216302e-07,
-      "logits/chosen": 394.1874084472656,
-      "logits/rejected": 357.0087585449219,
-      "logps/chosen": -284.3465270996094,
-      "logps/rejected": -341.0668640136719,
-      "loss": 0.6376,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.26841384172439575,
-      "rewards/margins": 0.28497201204299927,
-      "rewards/rejected": -0.5533859133720398,
       "step": 360
     },
     {
-      "epoch": 0.962914769030579,
-      "grad_norm": 9.1875,
-      "learning_rate": 3.091070792233124e-07,
-      "logits/chosen": 379.9867248535156,
-      "logits/rejected": 363.60565185546875,
-      "logps/chosen": -330.55035400390625,
-      "logps/rejected": -357.5587158203125,
-      "loss": 0.6562,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": -0.36982443928718567,
-      "rewards/margins": 0.1465008407831192,
-      "rewards/rejected": -0.5163252949714661,
       "step": 370
     },
     {
-      "epoch": 0.988939492517892,
-      "grad_norm": 112.0,
-      "learning_rate": 2.9800593824272024e-07,
-      "logits/chosen": 377.43341064453125,
-      "logits/rejected": 351.53948974609375,
-      "logps/chosen": -288.1562194824219,
-      "logps/rejected": -326.09930419921875,
-      "loss": 0.6717,
-      "rewards/accuracies": 0.543749988079071,
-      "rewards/chosen": -0.4318203032016754,
-      "rewards/margins": 0.1724158674478531,
-      "rewards/rejected": -0.6042361259460449,
       "step": 380
     },
     {
-      "epoch": 1.014964216005205,
-      "grad_norm": 11.4375,
-      "learning_rate": 2.8680558535371687e-07,
-      "logits/chosen": 367.515380859375,
-      "logits/rejected": 362.1560974121094,
-      "logps/chosen": -295.6492614746094,
-      "logps/rejected": -316.13189697265625,
-      "loss": 0.6406,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -0.38629141449928284,
-      "rewards/margins": 0.227385476231575,
-      "rewards/rejected": -0.613676905632019,
       "step": 390
     },
     {
-      "epoch": 1.0409889394925178,
-      "grad_norm": 8.75,
-      "learning_rate": 2.755291678673574e-07,
-      "logits/chosen": 382.78900146484375,
-      "logits/rejected": 349.6816101074219,
-      "logps/chosen": -312.42962646484375,
-      "logps/rejected": -368.1940002441406,
-      "loss": 0.6428,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.32089877128601074,
-      "rewards/margins": 0.36698684096336365,
-      "rewards/rejected": -0.6878856420516968,
       "step": 400
     },
     {
-      "epoch": 1.0409889394925178,
-      "eval_logits/chosen": 452.67681884765625,
-      "eval_logits/rejected": 436.037109375,
-      "eval_logps/chosen": -296.22650146484375,
-      "eval_logps/rejected": -334.6671142578125,
-      "eval_loss": 0.6531640291213989,
-      "eval_rewards/accuracies": 0.6153846383094788,
-      "eval_rewards/chosen": -0.3583340048789978,
-      "eval_rewards/margins": 0.2260989397764206,
-      "eval_rewards/rejected": -0.5844328999519348,
-      "eval_runtime": 244.5749,
-      "eval_samples_per_second": 22.104,
-      "eval_steps_per_second": 0.691,
       "step": 400
     },
     {
-      "epoch": 1.0670136629798308,
-      "grad_norm": 9.8125,
-      "learning_rate": 2.6419999029428816e-07,
-      "logits/chosen": 393.82427978515625,
-      "logits/rejected": 353.9914855957031,
-      "logps/chosen": -282.713623046875,
-      "logps/rejected": -346.3770751953125,
-      "loss": 0.6338,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.3410654664039612,
-      "rewards/margins": 0.20948953926563263,
-      "rewards/rejected": -0.5505550503730774,
       "step": 410
     },
     {
-      "epoch": 1.0930383864671438,
-      "grad_norm": 8.9375,
-      "learning_rate": 2.5284146618226805e-07,
-      "logits/chosen": 399.9950256347656,
-      "logits/rejected": 378.6581115722656,
-      "logps/chosen": -306.4803466796875,
-      "logps/rejected": -339.96636962890625,
-      "loss": 0.6206,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -0.2285558432340622,
-      "rewards/margins": 0.35167884826660156,
-      "rewards/rejected": -0.580234706401825,
       "step": 420
     },
     {
-      "epoch": 1.1190631099544568,
-      "grad_norm": 8.375,
-      "learning_rate": 2.414770697283471e-07,
-      "logits/chosen": 388.2931213378906,
-      "logits/rejected": 368.3885498046875,
-      "logps/chosen": -279.95123291015625,
-      "logps/rejected": -318.58660888671875,
-      "loss": 0.6254,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.31028804183006287,
-      "rewards/margins": 0.23735752701759338,
-      "rewards/rejected": -0.5476455092430115,
       "step": 430
     },
     {
-      "epoch": 1.1450878334417696,
-      "grad_norm": 11.1875,
-      "learning_rate": 2.3013028726570433e-07,
-      "logits/chosen": 384.466552734375,
-      "logits/rejected": 378.22186279296875,
-      "logps/chosen": -324.28790283203125,
-      "logps/rejected": -317.89752197265625,
-      "loss": 0.6483,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -0.37500661611557007,
-      "rewards/margins": 0.15624721348285675,
-      "rewards/rejected": -0.5312538743019104,
       "step": 440
     },
     {
-      "epoch": 1.1711125569290826,
-      "grad_norm": 10.8125,
-      "learning_rate": 2.1882456872540343e-07,
-      "logits/chosen": 389.3873291015625,
-      "logits/rejected": 368.3778381347656,
-      "logps/chosen": -296.072998046875,
-      "logps/rejected": -338.9192810058594,
-      "loss": 0.6395,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.3294864594936371,
-      "rewards/margins": 0.21306844055652618,
-      "rewards/rejected": -0.5425547957420349,
       "step": 450
     },
     {
-      "epoch": 1.1971372804163956,
-      "grad_norm": 9.875,
-      "learning_rate": 2.075832791733802e-07,
-      "logits/chosen": 373.2823791503906,
-      "logits/rejected": 373.5628967285156,
-      "logps/chosen": -311.630126953125,
-      "logps/rejected": -316.8753356933594,
-      "loss": 0.6239,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.2192085236310959,
-      "rewards/margins": 0.2490987330675125,
-      "rewards/rejected": -0.4683072566986084,
       "step": 460
     },
     {
-      "epoch": 1.2231620039037086,
-      "grad_norm": 8.4375,
-      "learning_rate": 1.9642965052281615e-07,
-      "logits/chosen": 375.5801696777344,
-      "logits/rejected": 357.92327880859375,
-      "logps/chosen": -288.80816650390625,
-      "logps/rejected": -314.3056335449219,
-      "loss": 0.627,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.2664826512336731,
-      "rewards/margins": 0.22547881305217743,
-      "rewards/rejected": -0.4919614791870117,
       "step": 470
     },
     {
-      "epoch": 1.2491867273910215,
-      "grad_norm": 11.8125,
-      "learning_rate": 1.8538673352169466e-07,
-      "logits/chosen": 392.02789306640625,
-      "logits/rejected": 378.76702880859375,
-      "logps/chosen": -311.27508544921875,
-      "logps/rejected": -346.3468322753906,
-      "loss": 0.6254,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -0.3370845913887024,
-      "rewards/margins": 0.30168086290359497,
-      "rewards/rejected": -0.6387654542922974,
       "step": 480
     },
     {
-      "epoch": 1.2752114508783343,
-      "grad_norm": 7.84375,
-      "learning_rate": 1.7447735011476267e-07,
-      "logits/chosen": 372.53924560546875,
-      "logits/rejected": 353.3471374511719,
-      "logps/chosen": -287.72393798828125,
-      "logps/rejected": -329.53619384765625,
-      "loss": 0.6328,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -0.3344346880912781,
-      "rewards/margins": 0.40995296835899353,
-      "rewards/rejected": -0.7443875670433044,
       "step": 490
     },
     {
-      "epoch": 1.3012361743656473,
-      "grad_norm": 8.375,
-      "learning_rate": 1.6372404627835178e-07,
-      "logits/chosen": 376.4175109863281,
-      "logits/rejected": 357.60369873046875,
-      "logps/chosen": -305.36932373046875,
-      "logps/rejected": -334.09979248046875,
-      "loss": 0.6366,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -0.2382354736328125,
-      "rewards/margins": 0.29306212067604065,
-      "rewards/rejected": -0.5312975645065308,
       "step": 500
     },
     {
-      "epoch": 1.3012361743656473,
-      "eval_logits/chosen": 452.81121826171875,
-      "eval_logits/rejected": 436.1842956542969,
-      "eval_logps/chosen": -295.1855773925781,
-      "eval_logps/rejected": -334.1830749511719,
-      "eval_loss": 0.6521105170249939,
-      "eval_rewards/accuracies": 0.6124260425567627,
-      "eval_rewards/chosen": -0.30628812313079834,
-      "eval_rewards/margins": 0.2539446949958801,
-      "eval_rewards/rejected": -0.5602327585220337,
-      "eval_runtime": 244.6483,
-      "eval_samples_per_second": 22.097,
-      "eval_steps_per_second": 0.691,
       "step": 500
     },
     {
-      "epoch": 1.3272608978529603,
-      "grad_norm": 27.375,
-      "learning_rate": 1.5314904542553098e-07,
-      "logits/chosen": 381.5070495605469,
-      "logits/rejected": 362.86529541015625,
-      "logps/chosen": -311.13287353515625,
-      "logps/rejected": -328.11505126953125,
-      "loss": 0.6295,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -0.2357090413570404,
-      "rewards/margins": 0.3728547990322113,
-      "rewards/rejected": -0.6085638999938965,
       "step": 510
     },
     {
-      "epoch": 1.3532856213402733,
-      "grad_norm": 10.75,
-      "learning_rate": 1.4277420247788842e-07,
-      "logits/chosen": 373.0950622558594,
-      "logits/rejected": 353.08367919921875,
-      "logps/chosen": -291.31494140625,
-      "logps/rejected": -334.69415283203125,
-      "loss": 0.6207,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -0.318630576133728,
-      "rewards/margins": 0.3065961003303528,
-      "rewards/rejected": -0.6252266764640808,
       "step": 520
     },
     {
-      "epoch": 1.3793103448275863,
-      "grad_norm": 8.8125,
-      "learning_rate": 1.3262095869885905e-07,
-      "logits/chosen": 385.84002685546875,
-      "logits/rejected": 390.05853271484375,
-      "logps/chosen": -330.57440185546875,
-      "logps/rejected": -347.67303466796875,
-      "loss": 0.624,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.3374934792518616,
-      "rewards/margins": 0.22613167762756348,
-      "rewards/rejected": -0.563625156879425,
       "step": 530
     },
     {
-      "epoch": 1.405335068314899,
-      "grad_norm": 9.75,
-      "learning_rate": 1.2271029738194257e-07,
-      "logits/chosen": 392.96490478515625,
-      "logits/rejected": 369.8482360839844,
-      "logps/chosen": -286.5850524902344,
-      "logps/rejected": -321.9965515136719,
-      "loss": 0.6266,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -0.23913364112377167,
-      "rewards/margins": 0.265684574842453,
-      "rewards/rejected": -0.5048182606697083,
       "step": 540
     },
     {
-      "epoch": 1.431359791802212,
-      "grad_norm": 9.5,
-      "learning_rate": 1.1306270048538966e-07,
-      "logits/chosen": 375.2906799316406,
-      "logits/rejected": 369.0203552246094,
-      "logps/chosen": -302.0591735839844,
-      "logps/rejected": -318.1397399902344,
-      "loss": 0.6149,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -0.330008327960968,
-      "rewards/margins": 0.2058134377002716,
-      "rewards/rejected": -0.535821795463562,
       "step": 550
     },
     {
-      "epoch": 1.457384515289525,
-      "grad_norm": 11.25,
-      "learning_rate": 1.0369810630297657e-07,
-      "logits/chosen": 370.2327575683594,
-      "logits/rejected": 373.281005859375,
-      "logps/chosen": -332.6309814453125,
-      "logps/rejected": -339.60247802734375,
-      "loss": 0.6251,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -0.3642995357513428,
-      "rewards/margins": 0.20028725266456604,
-      "rewards/rejected": -0.5645867586135864,
       "step": 560
     },
     {
-      "epoch": 1.483409238776838,
-      "grad_norm": 9.75,
-      "learning_rate": 9.463586825834938e-08,
-      "logits/chosen": 371.750244140625,
-      "logits/rejected": 374.09259033203125,
-      "logps/chosen": -320.38555908203125,
-      "logps/rejected": -332.42120361328125,
-      "loss": 0.6486,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.2637823820114136,
-      "rewards/margins": 0.28345996141433716,
-      "rewards/rejected": -0.5472423434257507,
       "step": 570
     },
     {
-      "epoch": 1.509433962264151,
-      "grad_norm": 9.9375,
-      "learning_rate": 8.589471490809472e-08,
-      "logits/chosen": 388.10174560546875,
-      "logits/rejected": 363.66436767578125,
-      "logps/chosen": -293.32745361328125,
-      "logps/rejected": -341.19866943359375,
-      "loss": 0.6028,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.26182278990745544,
-      "rewards/margins": 0.36554405093193054,
-      "rewards/rejected": -0.6273669004440308,
       "step": 580
     },
     {
-      "epoch": 1.5354586857514638,
-      "grad_norm": 12.375,
-      "learning_rate": 7.749271123619888e-08,
-      "logits/chosen": 380.56036376953125,
-      "logits/rejected": 354.895263671875,
-      "logps/chosen": -273.80010986328125,
-      "logps/rejected": -324.5169372558594,
-      "loss": 0.631,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -0.1536741703748703,
-      "rewards/margins": 0.3123035430908203,
-      "rewards/rejected": -0.46597766876220703,
       "step": 590
     },
     {
-      "epoch": 1.5614834092387768,
-      "grad_norm": 9.5625,
-      "learning_rate": 6.944722131988392e-08,
-      "logits/chosen": 388.1387023925781,
-      "logits/rejected": 355.89056396484375,
-      "logps/chosen": -300.9095153808594,
-      "logps/rejected": -333.97088623046875,
-      "loss": 0.6058,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -0.23438140749931335,
-      "rewards/margins": 0.3118259310722351,
-      "rewards/rejected": -0.5462073087692261,
       "step": 600
     },
     {
-      "epoch": 1.5614834092387768,
-      "eval_logits/chosen": 452.652099609375,
-      "eval_logits/rejected": 436.0275573730469,
-      "eval_logps/chosen": -295.8379821777344,
-      "eval_logps/rejected": -334.4803771972656,
-      "eval_loss": 0.6497182250022888,
-      "eval_rewards/accuracies": 0.6139053106307983,
-      "eval_rewards/chosen": -0.3389085829257965,
-      "eval_rewards/margins": 0.23618672788143158,
-      "eval_rewards/rejected": -0.5750953555107117,
-      "eval_runtime": 244.5623,
-      "eval_samples_per_second": 22.105,
-      "eval_steps_per_second": 0.691,
       "step": 600
     },
     {
-      "epoch": 1.5875081327260898,
-      "grad_norm": 9.25,
-      "learning_rate": 6.177487244398008e-08,
-      "logits/chosen": 386.1551513671875,
-      "logits/rejected": 376.6158142089844,
-      "logps/chosen": -306.791259765625,
-      "logps/rejected": -328.063232421875,
-      "loss": 0.6288,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -0.24376244843006134,
-      "rewards/margins": 0.39431554079055786,
-      "rewards/rejected": -0.638077974319458,
       "step": 610
     },
     {
-      "epoch": 1.6135328562134026,
-      "grad_norm": 8.875,
-      "learning_rate": 5.449152073799615e-08,
-      "logits/chosen": 390.0867614746094,
-      "logits/rejected": 356.75006103515625,
-      "logps/chosen": -306.01861572265625,
-      "logps/rejected": -342.045166015625,
-      "loss": 0.6306,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.274763286113739,
-      "rewards/margins": 0.3244563043117523,
-      "rewards/rejected": -0.599219560623169,
       "step": 620
     },
     {
-      "epoch": 1.6395575797007158,
-      "grad_norm": 10.3125,
-      "learning_rate": 4.761221840690585e-08,
-      "logits/chosen": 384.375244140625,
-      "logits/rejected": 363.5393981933594,
-      "logps/chosen": -304.85076904296875,
-      "logps/rejected": -343.6796875,
-      "loss": 0.6297,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -0.2900192439556122,
-      "rewards/margins": 0.30680161714553833,
-      "rewards/rejected": -0.5968209505081177,
       "step": 630
     },
     {
-      "epoch": 1.6655823031880286,
-      "grad_norm": 9.1875,
-      "learning_rate": 4.115118262337128e-08,
-      "logits/chosen": 381.7267150878906,
-      "logits/rejected": 368.2362060546875,
-      "logps/chosen": -302.25628662109375,
-      "logps/rejected": -344.24676513671875,
-      "loss": 0.6238,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.2637556195259094,
-      "rewards/margins": 0.33825674653053284,
-      "rewards/rejected": -0.6020123362541199,
       "step": 640
     },
     {
-      "epoch": 1.6916070266753416,
-      "grad_norm": 9.4375,
-      "learning_rate": 3.5121766145694173e-08,
-      "logits/chosen": 382.5013732910156,
-      "logits/rejected": 357.2362976074219,
-      "logps/chosen": -290.78717041015625,
-      "logps/rejected": -344.69677734375,
-      "loss": 0.628,
       "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -0.31657862663269043,
-      "rewards/margins": 0.3126802146434784,
-      "rewards/rejected": -0.6292588710784912,
       "step": 650
     },
     {
-      "epoch": 1.7176317501626546,
-      "grad_norm": 9.4375,
-      "learning_rate": 2.9536429722216205e-08,
-      "logits/chosen": 376.56842041015625,
-      "logits/rejected": 344.4474182128906,
-      "logps/chosen": -279.5611267089844,
-      "logps/rejected": -347.4361572265625,
-      "loss": 0.6492,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -0.393665611743927,
-      "rewards/margins": 0.2809843122959137,
-      "rewards/rejected": -0.6746498942375183,
       "step": 660
     },
     {
-      "epoch": 1.7436564736499673,
-      "grad_norm": 9.0625,
-      "learning_rate": 2.4406716339200743e-08,
-      "logits/chosen": 373.3739013671875,
-      "logits/rejected": 362.2355041503906,
-      "logps/chosen": -303.25,
-      "logps/rejected": -325.88995361328125,
-      "loss": 0.6358,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -0.233059361577034,
-      "rewards/margins": 0.3218488097190857,
-      "rewards/rejected": -0.5549081563949585,
-      "step": 670
-    },
-    {
-      "epoch": 1.7696811971372806,
-      "grad_norm": 8.0625,
-      "learning_rate": 1.974322736541509e-08,
-      "logits/chosen": 396.20916748046875,
-      "logits/rejected": 368.4210510253906,
-      "logps/chosen": -294.27386474609375,
-      "logps/rejected": -334.74005126953125,
-      "loss": 0.6414,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -0.34432071447372437,
-      "rewards/margins": 0.23387746512889862,
-      "rewards/rejected": -0.5781981945037842,
-      "step": 680
-    },
-    {
-      "epoch": 1.7957059206245933,
-      "grad_norm": 9.0,
-      "learning_rate": 1.5555600642715437e-08,
-      "logits/chosen": 397.7704162597656,
-      "logits/rejected": 368.53717041015625,
-      "logps/chosen": -307.35321044921875,
-      "logps/rejected": -370.9716796875,
-      "loss": 0.6298,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -0.25926095247268677,
-      "rewards/margins": 0.26611703634262085,
-      "rewards/rejected": -0.5253779888153076,
-      "step": 690
-    },
-    {
-      "epoch": 1.8217306441119063,
-      "grad_norm": 9.5625,
-      "learning_rate": 1.1852490567913653e-08,
-      "logits/chosen": 377.498291015625,
-      "logits/rejected": 364.7190856933594,
-      "logps/chosen": -297.59246826171875,
-      "logps/rejected": -319.68499755859375,
-      "loss": 0.6368,
-      "rewards/accuracies": 0.6875,
-      "rewards/chosen": -0.3016144633293152,
-      "rewards/margins": 0.32235556840896606,
-      "rewards/rejected": -0.6239700317382812,
-      "step": 700
-    },
-    {
-      "epoch": 1.8217306441119063,
-      "eval_logits/chosen": 452.6395263671875,
-      "eval_logits/rejected": 436.0116882324219,
-      "eval_logps/chosen": -295.8665466308594,
-      "eval_logps/rejected": -334.6864013671875,
-      "eval_loss": 0.6448772549629211,
-      "eval_rewards/accuracies": 0.6065088510513306,
-      "eval_rewards/chosen": -0.3403345048427582,
-      "eval_rewards/margins": 0.24506251513957977,
-      "eval_rewards/rejected": -0.5853970646858215,
-      "eval_runtime": 244.6363,
-      "eval_samples_per_second": 22.098,
-      "eval_steps_per_second": 0.691,
-      "step": 700
-    },
-    {
-      "epoch": 1.8477553675992193,
-      "grad_norm": 11.625,
-      "learning_rate": 8.641550207089038e-09,
-      "logits/chosen": 388.95428466796875,
-      "logits/rejected": 358.6634826660156,
-      "logps/chosen": -323.3162841796875,
-      "logps/rejected": -371.1590270996094,
-      "loss": 0.6193,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -0.3497847318649292,
-      "rewards/margins": 0.32699352502822876,
-      "rewards/rejected": -0.676778256893158,
-      "step": 710
-    },
-    {
-      "epoch": 1.873780091086532,
-      "grad_norm": 10.4375,
-      "learning_rate": 5.929415479310279e-09,
-      "logits/chosen": 371.3935241699219,
-      "logits/rejected": 376.6529235839844,
-      "logps/chosen": -321.6485290527344,
-      "logps/rejected": -330.58721923828125,
-      "loss": 0.6461,
-      "rewards/accuracies": 0.53125,
-      "rewards/chosen": -0.29033127427101135,
-      "rewards/margins": 0.1948806494474411,
-      "rewards/rejected": -0.48521193861961365,
-      "step": 720
-    },
-    {
-      "epoch": 1.8998048145738453,
-      "grad_norm": 9.3125,
-      "learning_rate": 3.721691442452768e-09,
-      "logits/chosen": 370.8646240234375,
-      "logits/rejected": 361.8319396972656,
-      "logps/chosen": -303.0124816894531,
-      "logps/rejected": -321.4366760253906,
-      "loss": 0.6354,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -0.2958618998527527,
-      "rewards/margins": 0.2960815727710724,
-      "rewards/rejected": -0.5919433832168579,
-      "step": 730
-    },
-    {
-      "epoch": 1.925829538061158,
-      "grad_norm": 9.9375,
-      "learning_rate": 2.0229407094547735e-09,
-      "logits/chosen": 377.24005126953125,
-      "logits/rejected": 357.33294677734375,
-      "logps/chosen": -277.88214111328125,
-      "logps/rejected": -321.13739013671875,
-      "loss": 0.6277,
-      "rewards/accuracies": 0.699999988079071,
-      "rewards/chosen": -0.2274915874004364,
-      "rewards/margins": 0.35707369446754456,
-      "rewards/rejected": -0.584565281867981,
-      "step": 740
-    },
-    {
-      "epoch": 1.951854261548471,
-      "grad_norm": 10.0625,
-      "learning_rate": 8.366740189520715e-10,
-      "logits/chosen": 353.9988098144531,
-      "logits/rejected": 346.21380615234375,
-      "logps/chosen": -301.86627197265625,
-      "logps/rejected": -304.49468994140625,
-      "loss": 0.6407,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -0.36653071641921997,
-      "rewards/margins": 0.22654423117637634,
-      "rewards/rejected": -0.5930749773979187,
-      "step": 750
-    },
-    {
-      "epoch": 1.977878985035784,
-      "grad_norm": 9.375,
-      "learning_rate": 1.6534297977804923e-10,
-      "logits/chosen": 381.37811279296875,
-      "logits/rejected": 356.3533020019531,
-      "logps/chosen": -299.4158935546875,
-      "logps/rejected": -340.39569091796875,
-      "loss": 0.6361,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -0.2855472266674042,
-      "rewards/margins": 0.29285091161727905,
-      "rewards/rejected": -0.5783981084823608,
-      "step": 760
-    },
-    {
-      "epoch": 1.9986987638256344,
-      "step": 768,
       "total_flos": 0.0,
-      "train_loss": 0.6547494133313497,
-      "train_runtime": 8032.2795,
-      "train_samples_per_second": 12.243,
-      "train_steps_per_second": 0.096
     }
   ],
   "logging_steps": 10,
-  "max_steps": 768,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 2,
   "save_steps": 500,

 {
   "best_metric": null,
   "best_model_checkpoint": null,
+  "epoch": 1.9969834087481146,
   "eval_steps": 100,
+  "global_step": 662,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
+      "epoch": 0.0030165912518853697,
+      "grad_norm": 237.0,
+      "learning_rate": 7.462686567164179e-09,
+      "logits/chosen": 373.7113952636719,
+      "logits/rejected": 314.0401306152344,
+      "logps/chosen": -3.7142391204833984,
+      "logps/rejected": -3.6281590461730957,
+      "loss": 100.0,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
       "step": 1
     },
     {
+      "epoch": 0.030165912518853696,
+      "grad_norm": 113.5,
+      "learning_rate": 7.462686567164178e-08,
+      "logits/chosen": 357.4179992675781,
+      "logits/rejected": 339.513916015625,
+      "logps/chosen": -3.1980221271514893,
+      "logps/rejected": -3.2019872665405273,
+      "loss": 99.9521,
+      "rewards/accuracies": 0.5069444179534912,
+      "rewards/chosen": 0.0002816318301483989,
+      "rewards/margins": 0.000606835528742522,
+      "rewards/rejected": -0.00032520375680178404,
       "step": 10
     },
     {
+      "epoch": 0.06033182503770739,
+      "grad_norm": 142.0,
+      "learning_rate": 1.4925373134328355e-07,
+      "logits/chosen": 375.2931823730469,
+      "logits/rejected": 341.9668273925781,
+      "logps/chosen": -3.4338252544403076,
+      "logps/rejected": -3.202441453933716,
+      "loss": 100.1094,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -0.0006929364753887057,
+      "rewards/margins": -0.0008935144869610667,
+      "rewards/rejected": 0.00020057809888385236,
       "step": 20
     },
     {
+      "epoch": 0.09049773755656108,
+      "grad_norm": 105.5,
+      "learning_rate": 2.2388059701492537e-07,
+      "logits/chosen": 367.0047302246094,
+      "logits/rejected": 345.61358642578125,
+      "logps/chosen": -3.1884350776672363,
+      "logps/rejected": -3.089536666870117,
+      "loss": 99.8614,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.0009225490503013134,
+      "rewards/margins": 0.0005482577835209668,
+      "rewards/rejected": 0.0003742911503650248,
       "step": 30
     },
     {
+      "epoch": 0.12066365007541478,
+      "grad_norm": 212.0,
+      "learning_rate": 2.985074626865671e-07,
+      "logits/chosen": 375.55523681640625,
+      "logits/rejected": 343.5054931640625,
+      "logps/chosen": -3.3171133995056152,
+      "logps/rejected": -3.01216459274292,
+      "loss": 99.8979,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.0019296016544103622,
+      "rewards/margins": 0.00038303257315419614,
+      "rewards/rejected": 0.0015465689357370138,
       "step": 40
     },
     {
+      "epoch": 0.15082956259426847,
+      "grad_norm": 171.0,
+      "learning_rate": 3.7313432835820895e-07,
+      "logits/chosen": 362.37359619140625,
+      "logits/rejected": 326.4159240722656,
+      "logps/chosen": -3.03434681892395,
+      "logps/rejected": -3.188572645187378,
+      "loss": 99.8304,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.004757150541990995,
+      "rewards/margins": -0.00031133147422224283,
+      "rewards/rejected": 0.005068481899797916,
       "step": 50
     },
     {
+      "epoch": 0.18099547511312217,
+      "grad_norm": 92.0,
+      "learning_rate": 4.4776119402985074e-07,
+      "logits/chosen": 352.8628845214844,
+      "logits/rejected": 327.94586181640625,
+      "logps/chosen": -2.9519481658935547,
+      "logps/rejected": -3.0041961669921875,
+      "loss": 99.6399,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": 0.009530991315841675,
+      "rewards/margins": 0.001031916355714202,
+      "rewards/rejected": 0.008499075658619404,
       "step": 60
     },
     {
+      "epoch": 0.21116138763197587,
+      "grad_norm": 103.0,
+      "learning_rate": 4.999686376024363e-07,
+      "logits/chosen": 357.00048828125,
+      "logits/rejected": 333.3026123046875,
+      "logps/chosen": -2.858609676361084,
+      "logps/rejected": -2.8480465412139893,
+      "loss": 99.3579,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.01719777286052704,
+      "rewards/margins": 0.0022748487535864115,
+      "rewards/rejected": 0.01492292433977127,
       "step": 70
     },
     {
+      "epoch": 0.24132730015082957,
+      "grad_norm": 106.5,
+      "learning_rate": 4.994113027425108e-07,
+      "logits/chosen": 379.38946533203125,
+      "logits/rejected": 339.73944091796875,
+      "logps/chosen": -2.7288854122161865,
+      "logps/rejected": -2.6808323860168457,
+      "loss": 98.8914,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": 0.025090601295232773,
+      "rewards/margins": 0.005764222703874111,
+      "rewards/rejected": 0.019326379522681236,
       "step": 80
     },
     {
+      "epoch": 0.27149321266968324,
+      "grad_norm": 112.0,
+      "learning_rate": 4.981588138481958e-07,
+      "logits/chosen": 376.42950439453125,
+      "logits/rejected": 345.72589111328125,
+      "logps/chosen": -2.639547824859619,
+      "logps/rejected": -2.540241241455078,
+      "loss": 98.8622,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": 0.03617015480995178,
+      "rewards/margins": 0.011906561441719532,
+      "rewards/rejected": 0.024263592436909676,
       "step": 90
     },
     {
+      "epoch": 0.30165912518853694,
+      "grad_norm": 109.0,
+      "learning_rate": 4.962146618306821e-07,
+      "logits/chosen": 381.8599853515625,
+      "logits/rejected": 347.7571716308594,
+      "logps/chosen": -2.4746594429016113,
+      "logps/rejected": -2.393070697784424,
+      "loss": 99.2543,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.04532874748110771,
+      "rewards/margins": 0.010249687358736992,
+      "rewards/rejected": 0.03507906198501587,
       "step": 100
     },
     {
+      "epoch": 0.30165912518853694,
+      "eval_logits/chosen": 418.0560302734375,
+      "eval_logits/rejected": 390.85260009765625,
+      "eval_logps/chosen": -2.36246657371521,
+      "eval_logps/rejected": -2.362391233444214,
+      "eval_loss": 98.51091003417969,
+      "eval_rewards/accuracies": 0.5821917653083801,
+      "eval_rewards/chosen": 0.040737785398960114,
+      "eval_rewards/margins": 0.005313507281243801,
+      "eval_rewards/rejected": 0.03542427718639374,
+      "eval_runtime": 99.1497,
+      "eval_samples_per_second": 46.959,
+      "eval_steps_per_second": 1.473,
       "step": 100
     },
     {
+      "epoch": 0.33182503770739064,
+      "grad_norm": 94.5,
+      "learning_rate": 4.935842653903397e-07,
+      "logits/chosen": 363.9966735839844,
+      "logits/rejected": 347.78033447265625,
+      "logps/chosen": -2.4134836196899414,
+      "logps/rejected": -2.3732988834381104,
+      "loss": 98.8696,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.04237458482384682,
+      "rewards/margins": 0.005867284722626209,
+      "rewards/rejected": 0.03650730103254318,
       "step": 110
     },
     {
+      "epoch": 0.36199095022624433,
+      "grad_norm": 109.0,
+      "learning_rate": 4.902749559138277e-07,
+      "logits/chosen": 381.50604248046875,
+      "logits/rejected": 344.4244384765625,
+      "logps/chosen": -2.260192394256592,
+      "logps/rejected": -2.3423144817352295,
+      "loss": 99.2348,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.042268361896276474,
+      "rewards/margins": 0.004213915206491947,
+      "rewards/rejected": 0.0380544476211071,
       "step": 120
     },
     {
+      "epoch": 0.39215686274509803,
+      "grad_norm": 111.5,
+      "learning_rate": 4.862959570402049e-07,
+      "logits/chosen": 355.4747314453125,
+      "logits/rejected": 347.9737854003906,
+      "logps/chosen": -2.242130756378174,
+      "logps/rejected": -2.236783981323242,
+      "loss": 99.0442,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": 0.03943895921111107,
+      "rewards/margins": 0.00034976023016497493,
+      "rewards/rejected": 0.039089202880859375,
       "step": 130
     },
     {
+      "epoch": 0.42232277526395173,
+      "grad_norm": 100.5,
+      "learning_rate": 4.816583589529929e-07,
+      "logits/chosen": 377.9671630859375,
+      "logits/rejected": 330.95989990234375,
+      "logps/chosen": -2.2830498218536377,
+      "logps/rejected": -2.2647957801818848,
+      "loss": 98.1403,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.055894386023283005,
+      "rewards/margins": 0.010847574099898338,
+      "rewards/rejected": 0.045046813786029816,
       "step": 140
     },
     {
+      "epoch": 0.45248868778280543,
+      "grad_norm": 93.0,
+      "learning_rate": 4.7637508746984616e-07,
+      "logits/chosen": 348.13671875,
+      "logits/rejected": 340.65460205078125,
+      "logps/chosen": -2.119765281677246,
+      "logps/rejected": -2.2865583896636963,
+      "loss": 98.2056,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.05023769661784172,
+      "rewards/margins": 0.0025197002105414867,
+      "rewards/rejected": 0.04771799594163895,
       "step": 150
     },
     {
+      "epoch": 0.48265460030165913,
+      "grad_norm": 94.5,
+      "learning_rate": 4.7046086801597966e-07,
+      "logits/chosen": 354.6838684082031,
+      "logits/rejected": 344.14971923828125,
+      "logps/chosen": -2.1116466522216797,
+      "logps/rejected": -2.1644375324249268,
+      "loss": 98.8372,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": 0.047597456723451614,
+      "rewards/margins": -0.0014352608704939485,
+      "rewards/rejected": 0.04903271794319153,
       "step": 160
     },
     {
+      "epoch": 0.5128205128205128,
+      "grad_norm": 84.0,
+      "learning_rate": 4.639321845817675e-07,
+      "logits/chosen": 367.41046142578125,
+      "logits/rejected": 350.5602111816406,
+      "logps/chosen": -2.1424355506896973,
+      "logps/rejected": -2.1355576515197754,
+      "loss": 99.3001,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": 0.048554591834545135,
+      "rewards/margins": 0.00010763984028017148,
+      "rewards/rejected": 0.048446957021951675,
       "step": 170
     },
     {
+      "epoch": 0.5429864253393665,
+      "grad_norm": 103.5,
+      "learning_rate": 4.568072337789056e-07,
+      "logits/chosen": 355.62255859375,
+      "logits/rejected": 353.52374267578125,
+      "logps/chosen": -2.1198911666870117,
+      "logps/rejected": -2.168549060821533,
+      "loss": 99.3479,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": 0.05148975923657417,
+      "rewards/margins": 0.001858557341620326,
+      "rewards/rejected": 0.04963120073080063,
       "step": 180
     },
     {
+      "epoch": 0.5731523378582202,
+      "grad_norm": 120.0,
+      "learning_rate": 4.4910587412319077e-07,
+      "logits/chosen": 360.10418701171875,
+      "logits/rejected": 338.6171875,
+      "logps/chosen": -2.1945066452026367,
+      "logps/rejected": -2.278280735015869,
+      "loss": 98.6843,
       "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.050669699907302856,
+      "rewards/margins": 0.004689137917011976,
+      "rewards/rejected": 0.04598056524991989,
       "step": 190
     },
     {
+      "epoch": 0.6033182503770739,
+      "grad_norm": 150.0,
+      "learning_rate": 4.408495706852757e-07,
+      "logits/chosen": 367.1837158203125,
+      "logits/rejected": 335.16845703125,
+      "logps/chosen": -2.285163402557373,
+      "logps/rejected": -2.4447309970855713,
+      "loss": 98.8709,
       "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.04992702230811119,
+      "rewards/margins": 0.005152065306901932,
+      "rewards/rejected": 0.04477496072649956,
       "step": 200
     },
     {
+      "epoch": 0.6033182503770739,
+      "eval_logits/chosen": 415.9554748535156,
+      "eval_logits/rejected": 388.3780822753906,
+      "eval_logps/chosen": -2.315298318862915,
+      "eval_logps/rejected": -2.3358802795410156,
+      "eval_loss": 98.0234603881836,
+      "eval_rewards/accuracies": 0.5787671208381653,
+      "eval_rewards/chosen": 0.04309620335698128,
+      "eval_rewards/margins": 0.006346376612782478,
+      "eval_rewards/rejected": 0.03674982488155365,
+      "eval_runtime": 96.1345,
+      "eval_samples_per_second": 48.432,
+      "eval_steps_per_second": 1.519,
       "step": 200
     },
     {
+      "epoch": 0.6334841628959276,
+      "grad_norm": 162.0,
+      "learning_rate": 4.3206133526366716e-07,
+      "logits/chosen": 381.5917053222656,
+      "logits/rejected": 350.63568115234375,
+      "logps/chosen": -2.362278699874878,
+      "logps/rejected": -2.354440927505493,
+      "loss": 97.5361,
       "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.04696507379412651,
+      "rewards/margins": 0.01031709648668766,
+      "rewards/rejected": 0.0366479754447937,
       "step": 210
     },
     {
+      "epoch": 0.6636500754147813,
+      "grad_norm": 88.0,
+      "learning_rate": 4.227656622467162e-07,
+      "logits/chosen": 363.15777587890625,
+      "logits/rejected": 331.46624755859375,
+      "logps/chosen": -2.2479934692382812,
+      "logps/rejected": -2.419908046722412,
+      "loss": 98.4741,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": 0.04176827520132065,
+      "rewards/margins": 0.0058479527942836285,
+      "rewards/rejected": 0.03592032194137573,
       "step": 220
     },
     {
+      "epoch": 0.693815987933635,
+      "grad_norm": 114.0,
+      "learning_rate": 4.129884603423642e-07,
+      "logits/chosen": 359.25927734375,
+      "logits/rejected": 348.3480529785156,
+      "logps/chosen": -2.296457529067993,
+      "logps/rejected": -2.342721462249756,
+      "loss": 98.0148,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": 0.04244069382548332,
+      "rewards/margins": 0.0055845663882792,
+      "rewards/rejected": 0.03685613349080086,
       "step": 230
     },
     {
+      "epoch": 0.7239819004524887,
+      "grad_norm": 166.0,
+      "learning_rate": 4.0275698036592786e-07,
+      "logits/chosen": 349.7359619140625,
+      "logits/rejected": 340.6862487792969,
+      "logps/chosen": -2.2498726844787598,
+      "logps/rejected": -2.308311939239502,
+      "loss": 98.0968,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": 0.0436752513051033,
+      "rewards/margins": 0.0003684187831822783,
+      "rewards/rejected": 0.04330682009458542,
       "step": 240
     },
     {
+      "epoch": 0.7541478129713424,
+      "grad_norm": 115.5,
+      "learning_rate": 3.920997392871882e-07,
+      "logits/chosen": 366.28228759765625,
+      "logits/rejected": 329.5648193359375,
+      "logps/chosen": -2.187605619430542,
+      "logps/rejected": -2.240809679031372,
+      "loss": 98.3289,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": 0.05280689522624016,
+      "rewards/margins": 0.007444704882800579,
+      "rewards/rejected": 0.045362185686826706,
       "step": 250
     },
     {
+      "epoch": 0.7843137254901961,
+      "grad_norm": 161.0,
+      "learning_rate": 3.8104644074848177e-07,
+      "logits/chosen": 367.5448303222656,
+      "logits/rejected": 359.0502014160156,
+      "logps/chosen": -2.094212293624878,
+      "logps/rejected": -2.2239794731140137,
+      "loss": 97.8614,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.058206796646118164,
+      "rewards/margins": 0.00784910749644041,
+      "rewards/rejected": 0.05035768076777458,
       "step": 260
     },
     {
+      "epoch": 0.8144796380090498,
+      "grad_norm": 132.0,
+      "learning_rate": 3.696278922753216e-07,
+      "logits/chosen": 354.0914611816406,
+      "logits/rejected": 350.6158447265625,
+      "logps/chosen": -2.169219493865967,
+      "logps/rejected": -2.2195897102355957,
+      "loss": 98.4471,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": 0.046092383563518524,
+      "rewards/margins": -0.0004284932219889015,
+      "rewards/rejected": 0.04652087762951851,
       "step": 270
     },
     {
+      "epoch": 0.8446455505279035,
+      "grad_norm": 113.5,
+      "learning_rate": 3.5787591941029836e-07,
+      "logits/chosen": 353.6844482421875,
+      "logits/rejected": 332.7743835449219,
+      "logps/chosen": -2.208927869796753,
+      "logps/rejected": -2.241152286529541,
+      "loss": 98.0145,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.04912891611456871,
+      "rewards/margins": 0.007012406829744577,
+      "rewards/rejected": 0.04211651161313057,
       "step": 280
     },
     {
+      "epoch": 0.8748114630467572,
+      "grad_norm": 99.0,
+      "learning_rate": 3.4582327700958453e-07,
+      "logits/chosen": 358.00921630859375,
+      "logits/rejected": 338.9405822753906,
+      "logps/chosen": -2.2120420932769775,
+      "logps/rejected": -2.275679111480713,
+      "loss": 98.1666,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.048364873975515366,
+      "rewards/margins": 0.007225348148494959,
+      "rewards/rejected": 0.041139524430036545,
       "step": 290
     },
     {
+      "epoch": 0.9049773755656109,
+      "grad_norm": 177.0,
+      "learning_rate": 3.3350355794927597e-07,
+      "logits/chosen": 355.8531494140625,
+      "logits/rejected": 348.1767578125,
+      "logps/chosen": -2.248812198638916,
+      "logps/rejected": -2.373812198638916,
+      "loss": 97.9389,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.04782190918922424,
+      "rewards/margins": 0.0014587871264666319,
+      "rewards/rejected": 0.04636312276124954,
       "step": 300
     },
     {
+      "epoch": 0.9049773755656109,
+      "eval_logits/chosen": 414.2632751464844,
+      "eval_logits/rejected": 386.40850830078125,
+      "eval_logps/chosen": -2.2581100463867188,
+      "eval_logps/rejected": -2.3082242012023926,
+      "eval_loss": 97.61585998535156,
+      "eval_rewards/accuracies": 0.5958904027938843,
+      "eval_rewards/chosen": 0.0459556020796299,
+      "eval_rewards/margins": 0.007822984829545021,
+      "eval_rewards/rejected": 0.038132619112730026,
+      "eval_runtime": 96.7019,
+      "eval_samples_per_second": 48.148,
+      "eval_steps_per_second": 1.51,
       "step": 300
     },
     {
+      "epoch": 0.9351432880844646,
+      "grad_norm": 175.0,
+      "learning_rate": 3.209510994960208e-07,
+      "logits/chosen": 356.83721923828125,
+      "logits/rejected": 340.94195556640625,
+      "logps/chosen": -2.3349146842956543,
+      "logps/rejected": -2.351058006286621,
+      "loss": 97.2842,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": 0.04642525687813759,
+      "rewards/margins": 0.008910289034247398,
+      "rewards/rejected": 0.03751496225595474,
       "step": 310
     },
     {
+      "epoch": 0.9653092006033183,
+      "grad_norm": 159.0,
+      "learning_rate": 3.082008876028986e-07,
+      "logits/chosen": 363.7893981933594,
+      "logits/rejected": 325.7989501953125,
+      "logps/chosen": -2.269832134246826,
+      "logps/rejected": -2.3510982990264893,
+      "loss": 97.3576,
       "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": 0.04809769243001938,
+      "rewards/margins": 0.013297428376972675,
+      "rewards/rejected": 0.03480026498436928,
       "step": 320
     },
     {
+      "epoch": 0.995475113122172,
+      "grad_norm": 191.0,
+      "learning_rate": 2.952884593972944e-07,
+      "logits/chosen": 365.5209045410156,
+      "logits/rejected": 338.1622009277344,
+      "logps/chosen": -2.251650094985962,
+      "logps/rejected": -2.3790125846862793,
+      "loss": 97.8277,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": 0.053557444363832474,
+      "rewards/margins": 0.00977501180022955,
+      "rewards/rejected": 0.04378242418169975,
       "step": 330
     },
     {
+      "epoch": 1.0256410256410255,
+      "grad_norm": 228.0,
+      "learning_rate": 2.822498041325508e-07,
+      "logits/chosen": 361.9426574707031,
+      "logits/rejected": 331.2440490722656,
+      "logps/chosen": -2.251028537750244,
+      "logps/rejected": -2.407641887664795,
+      "loss": 97.2834,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.05633380264043808,
+      "rewards/margins": 0.014148990623652935,
+      "rewards/rejected": 0.04218481108546257,
       "step": 340
     },
     {
+      "epoch": 1.0558069381598794,
+      "grad_norm": 113.0,
+      "learning_rate": 2.6912126287946387e-07,
+      "logits/chosen": 360.66326904296875,
+      "logits/rejected": 353.2472839355469,
+      "logps/chosen": -2.278568744659424,
+      "logps/rejected": -2.3496108055114746,
+      "loss": 97.7777,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": 0.04154600575566292,
+      "rewards/margins": 0.00018900888971984386,
+      "rewards/rejected": 0.04135699197649956,
       "step": 350
     },
     {
+      "epoch": 1.085972850678733,
+      "grad_norm": 87.0,
+      "learning_rate": 2.5593942723720076e-07,
+      "logits/chosen": 353.33563232421875,
+      "logits/rejected": 335.6075134277344,
+      "logps/chosen": -2.326427936553955,
+      "logps/rejected": -2.2644736766815186,
+      "loss": 98.5909,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": 0.050872981548309326,
+      "rewards/margins": 0.008244507014751434,
+      "rewards/rejected": 0.04262847453355789,
       "step": 360
     },
     {
+      "epoch": 1.1161387631975868,
+      "grad_norm": 150.0,
+      "learning_rate": 2.427410373459502e-07,
+      "logits/chosen": 360.3174133300781,
+      "logits/rejected": 337.7137451171875,
+      "logps/chosen": -2.3095028400421143,
+      "logps/rejected": -2.3610920906066895,
+      "loss": 97.0909,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": 0.045982178300619125,
+      "rewards/margins": 0.01000190433114767,
+      "rewards/rejected": 0.03598027303814888,
       "step": 370
     },
     {
+      "epoch": 1.1463046757164403,
+      "grad_norm": 174.0,
+      "learning_rate": 2.2956287948556162e-07,
+      "logits/chosen": 354.63238525390625,
+      "logits/rejected": 343.02880859375,
+      "logps/chosen": -2.2987804412841797,
+      "logps/rejected": -2.3496735095977783,
+      "loss": 97.7205,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": 0.041914135217666626,
+      "rewards/margins": 0.004762929864227772,
+      "rewards/rejected": 0.03715119883418083,
       "step": 380
     },
     {
+      "epoch": 1.1764705882352942,
+      "grad_norm": 158.0,
+      "learning_rate": 2.164416835455862e-07,
+      "logits/chosen": 356.2020568847656,
+      "logits/rejected": 331.1781921386719,
+      "logps/chosen": -2.3598244190216064,
+      "logps/rejected": -2.406748056411743,
+      "loss": 97.8897,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.05264422297477722,
+      "rewards/margins": 0.006141997873783112,
+      "rewards/rejected": 0.04650222510099411,
       "step": 390
     },
     {
+      "epoch": 1.2066365007541477,
+      "grad_norm": 122.5,
+      "learning_rate": 2.0341402065248575e-07,
+      "logits/chosen": 365.3482360839844,
+      "logits/rejected": 355.1469421386719,
+      "logps/chosen": -2.242918014526367,
+      "logps/rejected": -2.4804000854492188,
+      "loss": 96.4776,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": 0.04187587648630142,
+      "rewards/margins": 0.00307876686565578,
+      "rewards/rejected": 0.03879711031913757,
       "step": 400
     },
     {
+      "epoch": 1.2066365007541477,
+      "eval_logits/chosen": 413.0242004394531,
+      "eval_logits/rejected": 385.0536804199219,
+      "eval_logps/chosen": -2.3157808780670166,
+      "eval_logps/rejected": -2.376335859298706,
+      "eval_loss": 97.31377410888672,
+      "eval_rewards/accuracies": 0.590753436088562,
+      "eval_rewards/chosen": 0.04307207837700844,
+      "eval_rewards/margins": 0.008345033042132854,
+      "eval_rewards/rejected": 0.03472704440355301,
+      "eval_runtime": 96.7791,
+      "eval_samples_per_second": 48.11,
+      "eval_steps_per_second": 1.509,
       "step": 400
     },
     {
+      "epoch": 1.2368024132730016,
+      "grad_norm": 121.0,
+      "learning_rate": 1.9051620123934536e-07,
+      "logits/chosen": 372.9312438964844,
+      "logits/rejected": 332.6072692871094,
+      "logps/chosen": -2.3628134727478027,
+      "logps/rejected": -2.4130213260650635,
+      "loss": 97.2639,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.04606416076421738,
+      "rewards/margins": 0.011483157984912395,
+      "rewards/rejected": 0.034581005573272705,
       "step": 410
     },
     {
+      "epoch": 1.2669683257918551,
+      "grad_norm": 141.0,
+      "learning_rate": 1.7778417384218248e-07,
+      "logits/chosen": 365.7309875488281,
+      "logits/rejected": 330.0118408203125,
+      "logps/chosen": -2.3192696571350098,
+      "logps/rejected": -2.434166431427002,
+      "loss": 97.2612,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.04589037224650383,
+      "rewards/margins": 0.010156502947211266,
+      "rewards/rejected": 0.03573386371135712,
       "step": 420
     },
     {
+      "epoch": 1.297134238310709,
+      "grad_norm": 175.0,
+      "learning_rate": 1.652534249049305e-07,
+      "logits/chosen": 364.436279296875,
+      "logits/rejected": 337.96990966796875,
+      "logps/chosen": -2.2914252281188965,
+      "logps/rejected": -2.368446111679077,
+      "loss": 96.7924,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": 0.05218750238418579,
+      "rewards/margins": 0.009751560166478157,
+      "rewards/rejected": 0.042435940355062485,
       "step": 430
     },
     {
+      "epoch": 1.3273001508295625,
+      "grad_norm": 119.0,
+      "learning_rate": 1.5295887987235433e-07,
+      "logits/chosen": 360.49114990234375,
+      "logits/rejected": 341.19036865234375,
+      "logps/chosen": -2.2844796180725098,
+      "logps/rejected": -2.5002331733703613,
+      "loss": 97.8876,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.04997866973280907,
+      "rewards/margins": 0.00823974795639515,
+      "rewards/rejected": 0.04173891991376877,
       "step": 440
     },
     {
+      "epoch": 1.3574660633484164,
+      "grad_norm": 169.0,
+      "learning_rate": 1.4093480584657152e-07,
+      "logits/chosen": 338.6693115234375,
+      "logits/rejected": 338.9823303222656,
+      "logps/chosen": -2.340331792831421,
+      "logps/rejected": -2.3622262477874756,
+      "loss": 97.1683,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": 0.045739155262708664,
+      "rewards/margins": 0.00735636567696929,
+      "rewards/rejected": 0.038382794708013535,
       "step": 450
     },
     {
+      "epoch": 1.38763197586727,
+      "grad_norm": 149.0,
+      "learning_rate": 1.292147160784931e-07,
+      "logits/chosen": 356.4623107910156,
+      "logits/rejected": 323.6207580566406,
+      "logps/chosen": -2.2748360633850098,
+      "logps/rejected": -2.365034580230713,
+      "loss": 98.4686,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": 0.04644192382693291,
+      "rewards/margins": 0.010739983059465885,
+      "rewards/rejected": 0.0357019416987896,
       "step": 460
     },
     {
+      "epoch": 1.4177978883861238,
+      "grad_norm": 125.5,
+      "learning_rate": 1.1783127656038188e-07,
+      "logits/chosen": 349.2054443359375,
+      "logits/rejected": 347.08544921875,
+      "logps/chosen": -2.277270793914795,
+      "logps/rejected": -2.3475308418273926,
+      "loss": 97.2242,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.04547736421227455,
+      "rewards/margins": 0.006285218056291342,
+      "rewards/rejected": 0.03919214755296707,
       "step": 470
     },
     {
+      "epoch": 1.4479638009049773,
+      "grad_norm": 169.0,
+      "learning_rate": 1.068162149798737e-07,
+      "logits/chosen": 349.33013916015625,
+      "logits/rejected": 320.25604248046875,
+      "logps/chosen": -2.3431038856506348,
+      "logps/rejected": -2.404095411300659,
+      "loss": 97.0285,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": 0.04836113005876541,
+      "rewards/margins": 0.011659981682896614,
+      "rewards/rejected": 0.036701153963804245,
       "step": 480
     },
     {
+      "epoch": 1.4781297134238311,
+      "grad_norm": 131.0,
+      "learning_rate": 9.620023228922112e-08,
+      "logits/chosen": 344.032958984375,
+      "logits/rejected": 335.2795715332031,
+      "logps/chosen": -2.3322200775146484,
+      "logps/rejected": -2.4139535427093506,
+      "loss": 97.409,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.04861091822385788,
+      "rewards/margins": 0.00725546944886446,
+      "rewards/rejected": 0.041355449706315994,
       "step": 490
     },
     {
+      "epoch": 1.5082956259426847,
+      "grad_norm": 135.0,
+      "learning_rate": 8.601291713623316e-08,
+      "logits/chosen": 352.27392578125,
+      "logits/rejected": 346.0445861816406,
+      "logps/chosen": -2.2179293632507324,
+      "logps/rejected": -2.3674466609954834,
+      "loss": 97.3613,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": 0.04390779882669449,
+      "rewards/margins": 0.003659948008134961,
+      "rewards/rejected": 0.0402478463947773,
       "step": 500
     },
     {
+      "epoch": 1.5082956259426847,
+      "eval_logits/chosen": 412.6116638183594,
+      "eval_logits/rejected": 384.59588623046875,
+      "eval_logps/chosen": -2.3179564476013184,
+      "eval_logps/rejected": -2.378140687942505,
+      "eval_loss": 97.25182342529297,
+      "eval_rewards/accuracies": 0.590753436088562,
+      "eval_rewards/chosen": 0.04296330735087395,
+      "eval_rewards/margins": 0.008326511830091476,
+      "eval_rewards/rejected": 0.03463679552078247,
+      "eval_runtime": 96.7441,
+      "eval_samples_per_second": 48.127,
+      "eval_steps_per_second": 1.509,
       "step": 500
     },
     {
+      "epoch": 1.5384615384615383,
+      "grad_norm": 118.5,
+      "learning_rate": 7.628266339540659e-08,
+      "logits/chosen": 348.74603271484375,
+      "logits/rejected": 347.08380126953125,
+      "logps/chosen": -2.342663288116455,
+      "logps/rejected": -2.448847532272339,
+      "loss": 97.8001,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.04741663485765457,
+      "rewards/margins": 0.0025564138777554035,
+      "rewards/rejected": 0.04486021772027016,
       "step": 510
     },
     {
+      "epoch": 1.5686274509803921,
+      "grad_norm": 215.0,
+      "learning_rate": 6.70365910291057e-08,
+      "logits/chosen": 357.7579040527344,
+      "logits/rejected": 342.58221435546875,
+      "logps/chosen": -2.3019728660583496,
+      "logps/rejected": -2.442549228668213,
+      "loss": 96.9522,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.04647397994995117,
+      "rewards/margins": 0.005314859561622143,
+      "rewards/rejected": 0.041159119457006454,
       "step": 520
     },
     {
+      "epoch": 1.598793363499246,
+      "grad_norm": 121.0,
+      "learning_rate": 5.830047049936254e-08,
+      "logits/chosen": 358.22900390625,
+      "logits/rejected": 328.1488342285156,
+      "logps/chosen": -2.2621231079101562,
+      "logps/rejected": -2.453052282333374,
+      "loss": 96.6704,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": 0.045031510293483734,
+      "rewards/margins": 0.008471885696053505,
+      "rewards/rejected": 0.03655962646007538,
       "step": 530
     },
     {
+      "epoch": 1.6289592760180995,
+      "grad_norm": 168.0,
+      "learning_rate": 5.009865094097732e-08,
+      "logits/chosen": 343.6116638183594,
+      "logits/rejected": 343.15179443359375,
+      "logps/chosen": -2.414348602294922,
+      "logps/rejected": -2.4327187538146973,
+      "loss": 97.8265,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": 0.044759172946214676,
+      "rewards/margins": 0.0038251220248639584,
+      "rewards/rejected": 0.04093404486775398,
       "step": 540
     },
     {
+      "epoch": 1.6591251885369531,
+      "grad_norm": 378.0,
+      "learning_rate": 4.245399229611238e-08,
+      "logits/chosen": 353.45245361328125,
+      "logits/rejected": 337.67340087890625,
+      "logps/chosen": -2.2361724376678467,
+      "logps/rejected": -2.310511589050293,
+      "loss": 97.9185,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.04042404517531395,
+      "rewards/margins": 0.003903269302099943,
+      "rewards/rejected": 0.03652077168226242,
       "step": 550
     },
     {
+      "epoch": 1.689291101055807,
+      "grad_norm": 147.0,
+      "learning_rate": 3.538780159953347e-08,
+      "logits/chosen": 348.5847473144531,
+      "logits/rejected": 323.38995361328125,
+      "logps/chosen": -2.261584997177124,
+      "logps/rejected": -2.404850482940674,
+      "loss": 97.2381,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.04695659875869751,
+      "rewards/margins": 0.00860162265598774,
+      "rewards/rejected": 0.03835497424006462,
       "step": 560
     },
     {
+      "epoch": 1.7194570135746607,
+      "grad_norm": 213.0,
+      "learning_rate": 2.8919773592082337e-08,
+      "logits/chosen": 356.572021484375,
+      "logits/rejected": 327.9063720703125,
+      "logps/chosen": -2.3302974700927734,
+      "logps/rejected": -2.465226173400879,
+      "loss": 97.1971,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.046825435012578964,
+      "rewards/margins": 0.011095492169260979,
+      "rewards/rejected": 0.035729944705963135,
       "step": 570
     },
     {
+      "epoch": 1.7496229260935143,
+      "grad_norm": 155.0,
+      "learning_rate": 2.3067935827901343e-08,
+      "logits/chosen": 363.5878601074219,
+      "logits/rejected": 354.7928161621094,
+      "logps/chosen": -2.347442150115967,
+      "logps/rejected": -2.424699068069458,
+      "loss": 97.9048,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": 0.042837705463171005,
+      "rewards/margins": 0.0025561857037246227,
+      "rewards/rejected": 0.04028152674436569,
       "step": 580
     },
     {
+      "epoch": 1.779788838612368,
+      "grad_norm": 152.0,
+      "learning_rate": 1.7848598428407024e-08,
+      "logits/chosen": 358.1939697265625,
+      "logits/rejected": 329.384765625,
+      "logps/chosen": -2.302311897277832,
+      "logps/rejected": -2.4380781650543213,
+      "loss": 97.3207,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.04442282021045685,
+      "rewards/margins": 0.011918460950255394,
+      "rewards/rejected": 0.032504357397556305,
       "step": 590
     },
     {
+      "epoch": 1.8099547511312217,
+      "grad_norm": 215.0,
+      "learning_rate": 1.3276308623055982e-08,
+      "logits/chosen": 346.2601623535156,
+      "logits/rejected": 337.3775329589844,
+      "logps/chosen": -2.2827913761138916,
+      "logps/rejected": -2.39813494682312,
+      "loss": 97.5077,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.040391139686107635,
+      "rewards/margins": 0.005558500997722149,
+      "rewards/rejected": 0.03483264148235321,
       "step": 600
     },
     {
+      "epoch": 1.8099547511312217,
+      "eval_logits/chosen": 412.5386962890625,
+      "eval_logits/rejected": 384.52740478515625,
+      "eval_logps/chosen": -2.330043315887451,
+      "eval_logps/rejected": -2.3888137340545654,
+      "eval_loss": 97.25433349609375,
+      "eval_rewards/accuracies": 0.5976027250289917,
+      "eval_rewards/chosen": 0.042358946055173874,
+      "eval_rewards/margins": 0.0082557899877429,
+      "eval_rewards/rejected": 0.0341031588613987,
+      "eval_runtime": 96.6091,
+      "eval_samples_per_second": 48.194,
+      "eval_steps_per_second": 1.511,
       "step": 600
     },
     {
+      "epoch": 1.8401206636500755,
+      "grad_norm": 199.0,
+      "learning_rate": 9.363810203607686e-09,
+      "logits/chosen": 346.93450927734375,
+      "logits/rejected": 334.490966796875,
+      "logps/chosen": -2.3141541481018066,
+      "logps/rejected": -2.3323051929473877,
+      "loss": 97.5854,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": 0.0445205494761467,
+      "rewards/margins": 0.007130137179046869,
+      "rewards/rejected": 0.03739041090011597,
       "step": 610
     },
     {
+      "epoch": 1.8702865761689291,
+      "grad_norm": 212.0,
+      "learning_rate": 6.1220080048908495e-09,
+      "logits/chosen": 351.2411193847656,
+      "logits/rejected": 340.6080017089844,
+      "logps/chosen": -2.309572696685791,
+      "logps/rejected": -2.4471845626831055,
+      "loss": 97.3323,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.04287445545196533,
+      "rewards/margins": 0.006148182787001133,
+      "rewards/rejected": 0.036726273596286774,
       "step": 620
     },
     {
+      "epoch": 1.9004524886877827,
+      "grad_norm": 173.0,
+      "learning_rate": 3.5599375110729747e-09,
+      "logits/chosen": 373.3907165527344,
+      "logits/rejected": 343.541259765625,
+      "logps/chosen": -2.3878846168518066,
+      "logps/rejected": -2.472883939743042,
+      "loss": 97.3339,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": 0.05199785903096199,
+      "rewards/margins": 0.012289008125662804,
+      "rewards/rejected": 0.039708852767944336,
       "step": 630
     },
     {
+      "epoch": 1.9306184012066365,
+      "grad_norm": 144.0,
+      "learning_rate": 1.6847396721454688e-09,
+      "logits/chosen": 359.8321228027344,
+      "logits/rejected": 329.95562744140625,
+      "logps/chosen": -2.36277437210083,
+      "logps/rejected": -2.395962953567505,
+      "loss": 96.6934,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": 0.04414898157119751,
+      "rewards/margins": 0.011453721672296524,
+      "rewards/rejected": 0.03269525617361069,
       "step": 640
     },
     {
+      "epoch": 1.9607843137254903,
+      "grad_norm": 211.0,
+      "learning_rate": 5.016410008151228e-10,
+      "logits/chosen": 355.52593994140625,
+      "logits/rejected": 320.21697998046875,
+      "logps/chosen": -2.4310154914855957,
+      "logps/rejected": -2.3878276348114014,
+      "loss": 97.3498,
       "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": 0.043542712926864624,
+      "rewards/margins": 0.00873391143977642,
+      "rewards/rejected": 0.034808795899152756,
       "step": 650
     },
     {
+      "epoch": 1.990950226244344,
+      "grad_norm": 177.0,
+      "learning_rate": 1.3939005275909189e-11,
+      "logits/chosen": 360.26409912109375,
+      "logits/rejected": 332.65447998046875,
+      "logps/chosen": -2.311514377593994,
+      "logps/rejected": -2.424405574798584,
+      "loss": 97.3456,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.04267590120434761,
+      "rewards/margins": 0.012712600640952587,
+      "rewards/rejected": 0.02996329963207245,
       "step": 660
     },
     {
+      "epoch": 1.9969834087481146,
+      "step": 662,
       "total_flos": 0.0,
+      "train_loss": 98.05829228519313,
+      "train_runtime": 4353.733,
+      "train_samples_per_second": 19.487,
+      "train_steps_per_second": 0.152
     }
   ],
   "logging_steps": 10,
+  "max_steps": 662,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 2,
   "save_steps": 500,

training_args.bin CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ee67c3729e4228cc628ede6c8eff9eefccbd7f9aed587e64f3767287592ace92
 size 5432

 version https://git-lfs.github.com/spec/v1
+oid sha256:75a9c23580e1086440d267260beb97e075785f866324343c758faa8d04ff6c0c
 size 5432