diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,10544 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.8999400119976004,
+  "eval_steps": 500,
+  "global_step": 7501,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.995203836930456e-09,
+      "logits/chosen": -2.424614667892456,
+      "logits/rejected": -1.9891018867492676,
+      "logps/chosen": -441.5737609863281,
+      "logps/rejected": -473.3967590332031,
+      "loss": 0.1361,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 5.995203836930456e-08,
+      "logits/chosen": -2.110785484313965,
+      "logits/rejected": -1.7659941911697388,
+      "logps/chosen": -209.29656982421875,
+      "logps/rejected": -153.5054931640625,
+      "loss": 0.207,
+      "rewards/accuracies": 0.3333333432674408,
+      "rewards/chosen": 2.990659231727477e-05,
+      "rewards/margins": -3.354326690896414e-05,
+      "rewards/rejected": 6.344986468320712e-05,
+      "step": 10
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.1990407673860913e-07,
+      "logits/chosen": -1.9711062908172607,
+      "logits/rejected": -1.6703836917877197,
+      "logps/chosen": -187.1942138671875,
+      "logps/rejected": -146.91098022460938,
+      "loss": 0.1869,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 1.7819638742366806e-05,
+      "rewards/margins": -4.986067142453976e-05,
+      "rewards/rejected": 6.768031744286418e-05,
+      "step": 20
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.7985611510791368e-07,
+      "logits/chosen": -2.0937609672546387,
+      "logits/rejected": -1.779779076576233,
+      "logps/chosen": -271.91497802734375,
+      "logps/rejected": -197.81089782714844,
+      "loss": 0.1615,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.3488960576069076e-05,
+      "rewards/margins": 5.5867491028038785e-05,
+      "rewards/rejected": -6.935644341865554e-05,
+      "step": 30
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 2.3980815347721825e-07,
+      "logits/chosen": -1.813472032546997,
+      "logits/rejected": -1.6542946100234985,
+      "logps/chosen": -180.65975952148438,
+      "logps/rejected": -205.70913696289062,
+      "loss": 0.1741,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 4.0745311707723886e-05,
+      "rewards/margins": -5.638422953779809e-05,
+      "rewards/rejected": 9.712954488350078e-05,
+      "step": 40
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 2.997601918465228e-07,
+      "logits/chosen": -2.049846649169922,
+      "logits/rejected": -1.7906068563461304,
+      "logps/chosen": -215.6183624267578,
+      "logps/rejected": -221.0026092529297,
+      "loss": 0.2307,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.00011617661220952868,
+      "rewards/margins": 0.00026756798615679145,
+      "rewards/rejected": -0.00015139135939534754,
+      "step": 50
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 3.5971223021582736e-07,
+      "logits/chosen": -2.021110773086548,
+      "logits/rejected": -1.5324536561965942,
+      "logps/chosen": -217.2840118408203,
+      "logps/rejected": -155.36181640625,
+      "loss": 0.1582,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -3.3291718864347786e-05,
+      "rewards/margins": 0.0003940206079278141,
+      "rewards/rejected": -0.00042731230496428907,
+      "step": 60
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 4.1966426858513196e-07,
+      "logits/chosen": -2.0625147819519043,
+      "logits/rejected": -1.635331392288208,
+      "logps/chosen": -219.1893768310547,
+      "logps/rejected": -171.37765502929688,
+      "loss": 0.197,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.0001982348767342046,
+      "rewards/margins": 0.00021397133241407573,
+      "rewards/rejected": -0.00041220622370019555,
+      "step": 70
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 4.796163069544365e-07,
+      "logits/chosen": -2.066410541534424,
+      "logits/rejected": -1.633291244506836,
+      "logps/chosen": -288.09136962890625,
+      "logps/rejected": -254.0281982421875,
+      "loss": 0.1874,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.0007181699620559812,
+      "rewards/margins": 0.0003048782527912408,
+      "rewards/rejected": -0.0010230483021587133,
+      "step": 80
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.39568345323741e-07,
+      "logits/chosen": -2.0009377002716064,
+      "logits/rejected": -1.5648455619812012,
+      "logps/chosen": -227.7441864013672,
+      "logps/rejected": -174.01083374023438,
+      "loss": 0.1822,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.0005296675371937454,
+      "rewards/margins": 0.001410541357472539,
+      "rewards/rejected": -0.0019402087200433016,
+      "step": 90
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.995203836930456e-07,
+      "logits/chosen": -1.837164282798767,
+      "logits/rejected": -1.9389890432357788,
+      "logps/chosen": -163.2045135498047,
+      "logps/rejected": -230.808837890625,
+      "loss": 0.1929,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.0020309616811573505,
+      "rewards/margins": 0.0006099173915572464,
+      "rewards/rejected": -0.0026408787816762924,
+      "step": 100
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 6.594724220623502e-07,
+      "logits/chosen": -1.9973289966583252,
+      "logits/rejected": -1.5120570659637451,
+      "logps/chosen": -170.47203063964844,
+      "logps/rejected": -135.51675415039062,
+      "loss": 0.2767,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.004789027385413647,
+      "rewards/margins": 0.0034963604994118214,
+      "rewards/rejected": -0.008285388350486755,
+      "step": 110
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 7.194244604316547e-07,
+      "logits/chosen": -2.063746213912964,
+      "logits/rejected": -1.807268500328064,
+      "logps/chosen": -243.6378936767578,
+      "logps/rejected": -241.0843963623047,
+      "loss": 0.254,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.008300217799842358,
+      "rewards/margins": 0.007192461285740137,
+      "rewards/rejected": -0.015492679551243782,
+      "step": 120
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 7.793764988009593e-07,
+      "logits/chosen": -1.722473382949829,
+      "logits/rejected": -1.4418509006500244,
+      "logps/chosen": -228.91159057617188,
+      "logps/rejected": -233.1204376220703,
+      "loss": 0.258,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.018205298110842705,
+      "rewards/margins": 0.01084174681454897,
+      "rewards/rejected": -0.02904704213142395,
+      "step": 130
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 8.393285371702639e-07,
+      "logits/chosen": -1.773154616355896,
+      "logits/rejected": -1.7045183181762695,
+      "logps/chosen": -188.4173126220703,
+      "logps/rejected": -255.24850463867188,
+      "loss": 0.1761,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.032684169709682465,
+      "rewards/margins": 0.013167713768780231,
+      "rewards/rejected": -0.04585187882184982,
+      "step": 140
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 8.992805755395684e-07,
+      "logits/chosen": -1.9206039905548096,
+      "logits/rejected": -1.482641577720642,
+      "logps/chosen": -267.39373779296875,
+      "logps/rejected": -268.87335205078125,
+      "loss": 0.2179,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.05459556728601456,
+      "rewards/margins": 0.020206941291689873,
+      "rewards/rejected": -0.07480250298976898,
+      "step": 150
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.59232613908873e-07,
+      "logits/chosen": -1.8588768243789673,
+      "logits/rejected": -1.4004710912704468,
+      "logps/chosen": -277.7038269042969,
+      "logps/rejected": -238.27685546875,
+      "loss": 0.2108,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.08276136219501495,
+      "rewards/margins": 0.02755787968635559,
+      "rewards/rejected": -0.11031924188137054,
+      "step": 160
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.0191846522781776e-06,
+      "logits/chosen": -1.8110958337783813,
+      "logits/rejected": -1.617334008216858,
+      "logps/chosen": -311.5564880371094,
+      "logps/rejected": -312.31256103515625,
+      "loss": 0.1871,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.11200787872076035,
+      "rewards/margins": 0.011968320235610008,
+      "rewards/rejected": -0.1239762082695961,
+      "step": 170
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.079136690647482e-06,
+      "logits/chosen": -1.981802225112915,
+      "logits/rejected": -1.722188949584961,
+      "logps/chosen": -272.49395751953125,
+      "logps/rejected": -308.3351745605469,
+      "loss": 0.1623,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.08306683599948883,
+      "rewards/margins": 0.03644068166613579,
+      "rewards/rejected": -0.11950752884149551,
+      "step": 180
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.1390887290167866e-06,
+      "logits/chosen": -1.9837507009506226,
+      "logits/rejected": -1.5758998394012451,
+      "logps/chosen": -399.0294189453125,
+      "logps/rejected": -415.584228515625,
+      "loss": 0.0927,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.11670782417058945,
+      "rewards/margins": 0.06923219561576843,
+      "rewards/rejected": -0.18594001233577728,
+      "step": 190
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.1990407673860912e-06,
+      "logits/chosen": -1.993973970413208,
+      "logits/rejected": -1.7477480173110962,
+      "logps/chosen": -434.3821716308594,
+      "logps/rejected": -445.232666015625,
+      "loss": 0.1626,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.16982071101665497,
+      "rewards/margins": 0.021237283945083618,
+      "rewards/rejected": -0.19105800986289978,
+      "step": 200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.2589928057553958e-06,
+      "logits/chosen": -1.9031627178192139,
+      "logits/rejected": -1.6432006359100342,
+      "logps/chosen": -375.82196044921875,
+      "logps/rejected": -430.86712646484375,
+      "loss": 0.1676,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.17942743003368378,
+      "rewards/margins": 0.056316059082746506,
+      "rewards/rejected": -0.2357434779405594,
+      "step": 210
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.3189448441247004e-06,
+      "logits/chosen": -1.8880250453948975,
+      "logits/rejected": -1.4063551425933838,
+      "logps/chosen": -490.26922607421875,
+      "logps/rejected": -506.9161682128906,
+      "loss": 0.1167,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.25285932421684265,
+      "rewards/margins": 0.06803429871797562,
+      "rewards/rejected": -0.3208935856819153,
+      "step": 220
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.378896882494005e-06,
+      "logits/chosen": -1.9972703456878662,
+      "logits/rejected": -1.3149311542510986,
+      "logps/chosen": -419.0880432128906,
+      "logps/rejected": -468.30572509765625,
+      "loss": 0.1956,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.1997191160917282,
+      "rewards/margins": 0.08645063638687134,
+      "rewards/rejected": -0.28616976737976074,
+      "step": 230
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.4388489208633094e-06,
+      "logits/chosen": -1.8106658458709717,
+      "logits/rejected": -1.567525029182434,
+      "logps/chosen": -407.79693603515625,
+      "logps/rejected": -482.77410888671875,
+      "loss": 0.1365,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.16676925122737885,
+      "rewards/margins": 0.06083636358380318,
+      "rewards/rejected": -0.22760562598705292,
+      "step": 240
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.4988009592326142e-06,
+      "logits/chosen": -1.8049312829971313,
+      "logits/rejected": -1.664839506149292,
+      "logps/chosen": -433.3682556152344,
+      "logps/rejected": -512.4707641601562,
+      "loss": 0.1664,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.21932223439216614,
+      "rewards/margins": 0.05535857751965523,
+      "rewards/rejected": -0.2746807932853699,
+      "step": 250
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.5587529976019186e-06,
+      "logits/chosen": -1.8094069957733154,
+      "logits/rejected": -1.5070183277130127,
+      "logps/chosen": -428.5262145996094,
+      "logps/rejected": -491.21026611328125,
+      "loss": 0.2173,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.2610072195529938,
+      "rewards/margins": 0.06348416954278946,
+      "rewards/rejected": -0.32449138164520264,
+      "step": 260
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.618705035971223e-06,
+      "logits/chosen": -1.5745047330856323,
+      "logits/rejected": -1.2507710456848145,
+      "logps/chosen": -367.3899841308594,
+      "logps/rejected": -457.46514892578125,
+      "loss": 0.1599,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.2029729187488556,
+      "rewards/margins": 0.09070710837841034,
+      "rewards/rejected": -0.29368001222610474,
+      "step": 270
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.6786570743405278e-06,
+      "logits/chosen": -1.9639122486114502,
+      "logits/rejected": -1.6996285915374756,
+      "logps/chosen": -426.679931640625,
+      "logps/rejected": -467.06201171875,
+      "loss": 0.1736,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.24603013694286346,
+      "rewards/margins": 0.05953250080347061,
+      "rewards/rejected": -0.30556267499923706,
+      "step": 280
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.7386091127098322e-06,
+      "logits/chosen": -2.002865791320801,
+      "logits/rejected": -1.5287902355194092,
+      "logps/chosen": -542.0328979492188,
+      "logps/rejected": -618.3682250976562,
+      "loss": 0.149,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.2986288368701935,
+      "rewards/margins": 0.09563031792640686,
+      "rewards/rejected": -0.39425915479660034,
+      "step": 290
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.7985611510791368e-06,
+      "logits/chosen": -1.7436326742172241,
+      "logits/rejected": -1.568961262702942,
+      "logps/chosen": -621.4896850585938,
+      "logps/rejected": -707.8096923828125,
+      "loss": 0.1426,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.3426869511604309,
+      "rewards/margins": 0.0701553151011467,
+      "rewards/rejected": -0.4128422737121582,
+      "step": 300
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.8585131894484414e-06,
+      "logits/chosen": -1.9436708688735962,
+      "logits/rejected": -1.7049095630645752,
+      "logps/chosen": -525.7359619140625,
+      "logps/rejected": -581.0355224609375,
+      "loss": 0.1895,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.30332881212234497,
+      "rewards/margins": 0.0772644504904747,
+      "rewards/rejected": -0.38059327006340027,
+      "step": 310
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.918465227817746e-06,
+      "logits/chosen": -1.8621517419815063,
+      "logits/rejected": -1.4305992126464844,
+      "logps/chosen": -411.10870361328125,
+      "logps/rejected": -479.657958984375,
+      "loss": 0.174,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.26565515995025635,
+      "rewards/margins": 0.08919227123260498,
+      "rewards/rejected": -0.35484743118286133,
+      "step": 320
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.9784172661870504e-06,
+      "logits/chosen": -2.023552894592285,
+      "logits/rejected": -1.889641523361206,
+      "logps/chosen": -465.8629455566406,
+      "logps/rejected": -552.2799072265625,
+      "loss": 0.1876,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.2822990417480469,
+      "rewards/margins": 0.06060437485575676,
+      "rewards/rejected": -0.34290340542793274,
+      "step": 330
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.0383693045563552e-06,
+      "logits/chosen": -1.855674386024475,
+      "logits/rejected": -1.6946580410003662,
+      "logps/chosen": -563.6864624023438,
+      "logps/rejected": -681.2032470703125,
+      "loss": 0.1688,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.39154255390167236,
+      "rewards/margins": 0.10727505385875702,
+      "rewards/rejected": -0.4988176226615906,
+      "step": 340
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.0983213429256596e-06,
+      "logits/chosen": -1.7664921283721924,
+      "logits/rejected": -1.5293089151382446,
+      "logps/chosen": -501.54376220703125,
+      "logps/rejected": -567.0556640625,
+      "loss": 0.2155,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.3383738398551941,
+      "rewards/margins": 0.07988253980875015,
+      "rewards/rejected": -0.41825634241104126,
+      "step": 350
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.158273381294964e-06,
+      "logits/chosen": -1.7843986749649048,
+      "logits/rejected": -1.5340216159820557,
+      "logps/chosen": -553.5965576171875,
+      "logps/rejected": -605.9610595703125,
+      "loss": 0.2004,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3518095016479492,
+      "rewards/margins": 0.07239037752151489,
+      "rewards/rejected": -0.4241998791694641,
+      "step": 360
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.218225419664269e-06,
+      "logits/chosen": -1.6985795497894287,
+      "logits/rejected": -1.4312691688537598,
+      "logps/chosen": -414.53564453125,
+      "logps/rejected": -476.00079345703125,
+      "loss": 0.0987,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.21437928080558777,
+      "rewards/margins": 0.05803290754556656,
+      "rewards/rejected": -0.2724122405052185,
+      "step": 370
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.2781774580335732e-06,
+      "logits/chosen": -2.03554105758667,
+      "logits/rejected": -1.405547022819519,
+      "logps/chosen": -376.60809326171875,
+      "logps/rejected": -366.76483154296875,
+      "loss": 0.187,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.14595827460289001,
+      "rewards/margins": 0.06921260058879852,
+      "rewards/rejected": -0.21517090499401093,
+      "step": 380
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.3381294964028776e-06,
+      "logits/chosen": -1.7824195623397827,
+      "logits/rejected": -1.6312448978424072,
+      "logps/chosen": -326.83660888671875,
+      "logps/rejected": -384.1517028808594,
+      "loss": 0.1209,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.1401316523551941,
+      "rewards/margins": 0.05587666481733322,
+      "rewards/rejected": -0.19600829482078552,
+      "step": 390
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.3980815347721824e-06,
+      "logits/chosen": -1.925445795059204,
+      "logits/rejected": -1.5636764764785767,
+      "logps/chosen": -343.93121337890625,
+      "logps/rejected": -391.94244384765625,
+      "loss": 0.1761,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.1491929143667221,
+      "rewards/margins": 0.07938452064990997,
+      "rewards/rejected": -0.2285774201154709,
+      "step": 400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.458033573141487e-06,
+      "logits/chosen": -1.8797461986541748,
+      "logits/rejected": -1.5031611919403076,
+      "logps/chosen": -480.9188537597656,
+      "logps/rejected": -600.8464965820312,
+      "loss": 0.1698,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.2560928165912628,
+      "rewards/margins": 0.12094642221927643,
+      "rewards/rejected": -0.37703922390937805,
+      "step": 410
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.5179856115107916e-06,
+      "logits/chosen": -1.9264554977416992,
+      "logits/rejected": -1.5078175067901611,
+      "logps/chosen": -451.627685546875,
+      "logps/rejected": -581.966064453125,
+      "loss": 0.1105,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.2343265265226364,
+      "rewards/margins": 0.1343805342912674,
+      "rewards/rejected": -0.3687070608139038,
+      "step": 420
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.577937649880096e-06,
+      "logits/chosen": -1.9310779571533203,
+      "logits/rejected": -1.819917917251587,
+      "logps/chosen": -499.9461364746094,
+      "logps/rejected": -614.4093017578125,
+      "loss": 0.2088,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.31922388076782227,
+      "rewards/margins": 0.0791390985250473,
+      "rewards/rejected": -0.3983629643917084,
+      "step": 430
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.637889688249401e-06,
+      "logits/chosen": -2.215282678604126,
+      "logits/rejected": -1.6334387063980103,
+      "logps/chosen": -722.10986328125,
+      "logps/rejected": -734.7113037109375,
+      "loss": 0.1113,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.35955536365509033,
+      "rewards/margins": 0.13125815987586975,
+      "rewards/rejected": -0.49081355333328247,
+      "step": 440
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.6978417266187052e-06,
+      "logits/chosen": -1.806089162826538,
+      "logits/rejected": -1.491436243057251,
+      "logps/chosen": -651.7131958007812,
+      "logps/rejected": -793.5899658203125,
+      "loss": 0.1312,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.4119444489479065,
+      "rewards/margins": 0.10372235625982285,
+      "rewards/rejected": -0.5156668424606323,
+      "step": 450
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 2.75779376498801e-06,
+      "logits/chosen": -1.9653629064559937,
+      "logits/rejected": -1.6969804763793945,
+      "logps/chosen": -711.3712158203125,
+      "logps/rejected": -791.8515014648438,
+      "loss": 0.1511,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.45856189727783203,
+      "rewards/margins": 0.08421288430690765,
+      "rewards/rejected": -0.5427747964859009,
+      "step": 460
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 2.8177458033573145e-06,
+      "logits/chosen": -1.813819169998169,
+      "logits/rejected": -1.4911987781524658,
+      "logps/chosen": -689.6634521484375,
+      "logps/rejected": -726.7325439453125,
+      "loss": 0.1855,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.5158717036247253,
+      "rewards/margins": 0.05104184150695801,
+      "rewards/rejected": -0.5669134855270386,
+      "step": 470
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 2.877697841726619e-06,
+      "logits/chosen": -1.9170444011688232,
+      "logits/rejected": -1.8240591287612915,
+      "logps/chosen": -734.8348388671875,
+      "logps/rejected": -727.4771118164062,
+      "loss": 0.1573,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.49010977149009705,
+      "rewards/margins": 0.026711275801062584,
+      "rewards/rejected": -0.5168210864067078,
+      "step": 480
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 2.9376498800959237e-06,
+      "logits/chosen": -1.9889415502548218,
+      "logits/rejected": -1.5214885473251343,
+      "logps/chosen": -682.8226318359375,
+      "logps/rejected": -780.7635498046875,
+      "loss": 0.1237,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.4781157970428467,
+      "rewards/margins": 0.12150125205516815,
+      "rewards/rejected": -0.5996171236038208,
+      "step": 490
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 2.9976019184652285e-06,
+      "logits/chosen": -1.9164119958877563,
+      "logits/rejected": -1.6286271810531616,
+      "logps/chosen": -480.3116149902344,
+      "logps/rejected": -545.7355346679688,
+      "loss": 0.1548,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.2581985592842102,
+      "rewards/margins": 0.08454164117574692,
+      "rewards/rejected": -0.3427402377128601,
+      "step": 500
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.0575539568345324e-06,
+      "logits/chosen": -2.170527935028076,
+      "logits/rejected": -1.9709542989730835,
+      "logps/chosen": -363.12579345703125,
+      "logps/rejected": -425.01226806640625,
+      "loss": 0.158,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.18245753645896912,
+      "rewards/margins": 0.06167198345065117,
+      "rewards/rejected": -0.2441294938325882,
+      "step": 510
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.1175059952038373e-06,
+      "logits/chosen": -1.945233941078186,
+      "logits/rejected": -1.428051233291626,
+      "logps/chosen": -505.4390563964844,
+      "logps/rejected": -523.0216064453125,
+      "loss": 0.1455,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.2762835621833801,
+      "rewards/margins": 0.07719194144010544,
+      "rewards/rejected": -0.35347551107406616,
+      "step": 520
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.177458033573142e-06,
+      "logits/chosen": -2.112267255783081,
+      "logits/rejected": -1.676775574684143,
+      "logps/chosen": -490.1148376464844,
+      "logps/rejected": -528.5180053710938,
+      "loss": 0.2065,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.29186129570007324,
+      "rewards/margins": 0.09129883348941803,
+      "rewards/rejected": -0.38316017389297485,
+      "step": 530
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.237410071942446e-06,
+      "logits/chosen": -1.9815177917480469,
+      "logits/rejected": -1.6378504037857056,
+      "logps/chosen": -520.5175170898438,
+      "logps/rejected": -590.74072265625,
+      "loss": 0.1532,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.30734989047050476,
+      "rewards/margins": 0.07567085325717926,
+      "rewards/rejected": -0.3830207586288452,
+      "step": 540
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.297362110311751e-06,
+      "logits/chosen": -2.0492682456970215,
+      "logits/rejected": -1.7660118341445923,
+      "logps/chosen": -530.2151489257812,
+      "logps/rejected": -621.3672485351562,
+      "loss": 0.0998,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.33154135942459106,
+      "rewards/margins": 0.10831041634082794,
+      "rewards/rejected": -0.439851850271225,
+      "step": 550
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.3573141486810557e-06,
+      "logits/chosen": -2.0890469551086426,
+      "logits/rejected": -1.8378829956054688,
+      "logps/chosen": -528.0261840820312,
+      "logps/rejected": -566.3738403320312,
+      "loss": 0.1213,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.2908535599708557,
+      "rewards/margins": 0.06711023300886154,
+      "rewards/rejected": -0.3579638600349426,
+      "step": 560
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.4172661870503596e-06,
+      "logits/chosen": -2.201676607131958,
+      "logits/rejected": -1.6598789691925049,
+      "logps/chosen": -511.14739990234375,
+      "logps/rejected": -609.4656982421875,
+      "loss": 0.1234,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.27405616641044617,
+      "rewards/margins": 0.11833932250738144,
+      "rewards/rejected": -0.3923954665660858,
+      "step": 570
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.4772182254196645e-06,
+      "logits/chosen": -2.3007311820983887,
+      "logits/rejected": -1.7538681030273438,
+      "logps/chosen": -518.4376220703125,
+      "logps/rejected": -539.1800537109375,
+      "loss": 0.185,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.24007895588874817,
+      "rewards/margins": 0.10247524082660675,
+      "rewards/rejected": -0.3425541818141937,
+      "step": 580
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.5371702637889693e-06,
+      "logits/chosen": -1.7686113119125366,
+      "logits/rejected": -1.4686288833618164,
+      "logps/chosen": -631.5836791992188,
+      "logps/rejected": -729.1126708984375,
+      "loss": 0.112,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.41966742277145386,
+      "rewards/margins": 0.14250221848487854,
+      "rewards/rejected": -0.5621696710586548,
+      "step": 590
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.5971223021582737e-06,
+      "logits/chosen": -1.930607557296753,
+      "logits/rejected": -1.5225746631622314,
+      "logps/chosen": -678.86572265625,
+      "logps/rejected": -724.9920043945312,
+      "loss": 0.2078,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.4797598719596863,
+      "rewards/margins": 0.0777256190776825,
+      "rewards/rejected": -0.5574854612350464,
+      "step": 600
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.657074340527578e-06,
+      "logits/chosen": -2.141531467437744,
+      "logits/rejected": -1.972782850265503,
+      "logps/chosen": -482.4869689941406,
+      "logps/rejected": -535.6817626953125,
+      "loss": 0.167,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.2537267804145813,
+      "rewards/margins": 0.08190487325191498,
+      "rewards/rejected": -0.33563166856765747,
+      "step": 610
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.717026378896883e-06,
+      "logits/chosen": -1.9434223175048828,
+      "logits/rejected": -1.6648263931274414,
+      "logps/chosen": -437.40679931640625,
+      "logps/rejected": -555.54296875,
+      "loss": 0.1658,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.2473764419555664,
+      "rewards/margins": 0.10345631837844849,
+      "rewards/rejected": -0.3508327603340149,
+      "step": 620
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.7769784172661873e-06,
+      "logits/chosen": -1.996471643447876,
+      "logits/rejected": -1.7018743753433228,
+      "logps/chosen": -401.1328125,
+      "logps/rejected": -464.3004455566406,
+      "loss": 0.0817,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.14634720981121063,
+      "rewards/margins": 0.08606003224849701,
+      "rewards/rejected": -0.23240725696086884,
+      "step": 630
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.836930455635492e-06,
+      "logits/chosen": -2.011030673980713,
+      "logits/rejected": -1.7729896306991577,
+      "logps/chosen": -449.0770568847656,
+      "logps/rejected": -510.6094665527344,
+      "loss": 0.1439,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.20279690623283386,
+      "rewards/margins": 0.0904121920466423,
+      "rewards/rejected": -0.29320910573005676,
+      "step": 640
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.896882494004797e-06,
+      "logits/chosen": -2.1106112003326416,
+      "logits/rejected": -1.7339891195297241,
+      "logps/chosen": -473.05426025390625,
+      "logps/rejected": -498.491455078125,
+      "loss": 0.1022,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.22257809340953827,
+      "rewards/margins": 0.07292576134204865,
+      "rewards/rejected": -0.2955038845539093,
+      "step": 650
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.956834532374101e-06,
+      "logits/chosen": -1.9875895977020264,
+      "logits/rejected": -1.7550048828125,
+      "logps/chosen": -426.18267822265625,
+      "logps/rejected": -565.2482299804688,
+      "loss": 0.1765,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.25893327593803406,
+      "rewards/margins": 0.09751396626234055,
+      "rewards/rejected": -0.3564472794532776,
+      "step": 660
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.016786570743406e-06,
+      "logits/chosen": -2.1742594242095947,
+      "logits/rejected": -1.7391729354858398,
+      "logps/chosen": -417.650390625,
+      "logps/rejected": -507.044677734375,
+      "loss": 0.0907,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.25809940695762634,
+      "rewards/margins": 0.1263275444507599,
+      "rewards/rejected": -0.38442692160606384,
+      "step": 670
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.0767386091127105e-06,
+      "logits/chosen": -2.1468255519866943,
+      "logits/rejected": -1.5991113185882568,
+      "logps/chosen": -518.39892578125,
+      "logps/rejected": -582.115478515625,
+      "loss": 0.1524,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.26489752531051636,
+      "rewards/margins": 0.13144561648368835,
+      "rewards/rejected": -0.3963431715965271,
+      "step": 680
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.1366906474820145e-06,
+      "logits/chosen": -2.1409378051757812,
+      "logits/rejected": -1.960959792137146,
+      "logps/chosen": -629.4721069335938,
+      "logps/rejected": -685.7987060546875,
+      "loss": 0.1611,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.40351057052612305,
+      "rewards/margins": 0.08392702043056488,
+      "rewards/rejected": -0.48743757605552673,
+      "step": 690
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.196642685851319e-06,
+      "logits/chosen": -2.0796456336975098,
+      "logits/rejected": -1.714124083518982,
+      "logps/chosen": -467.7308654785156,
+      "logps/rejected": -556.42919921875,
+      "loss": 0.1988,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.3264928162097931,
+      "rewards/margins": 0.08575908094644547,
+      "rewards/rejected": -0.41225185990333557,
+      "step": 700
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.256594724220624e-06,
+      "logits/chosen": -2.1705305576324463,
+      "logits/rejected": -1.7953819036483765,
+      "logps/chosen": -410.27691650390625,
+      "logps/rejected": -460.1272888183594,
+      "loss": 0.1767,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.18935605883598328,
+      "rewards/margins": 0.06809468567371368,
+      "rewards/rejected": -0.25745075941085815,
+      "step": 710
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.316546762589928e-06,
+      "logits/chosen": -2.1713924407958984,
+      "logits/rejected": -1.8006865978240967,
+      "logps/chosen": -366.3392028808594,
+      "logps/rejected": -428.9869689941406,
+      "loss": 0.1039,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.13448777794837952,
+      "rewards/margins": 0.0845385193824768,
+      "rewards/rejected": -0.21902629733085632,
+      "step": 720
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.376498800959233e-06,
+      "logits/chosen": -1.943730115890503,
+      "logits/rejected": -1.634590744972229,
+      "logps/chosen": -486.34857177734375,
+      "logps/rejected": -539.9793701171875,
+      "loss": 0.1916,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.2783839702606201,
+      "rewards/margins": 0.09620850533246994,
+      "rewards/rejected": -0.37459245324134827,
+      "step": 730
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.436450839328538e-06,
+      "logits/chosen": -1.8877426385879517,
+      "logits/rejected": -1.5067088603973389,
+      "logps/chosen": -515.3486328125,
+      "logps/rejected": -589.1819458007812,
+      "loss": 0.1318,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.32613372802734375,
+      "rewards/margins": 0.12069141864776611,
+      "rewards/rejected": -0.4468251168727875,
+      "step": 740
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.496402877697842e-06,
+      "logits/chosen": -2.281191825866699,
+      "logits/rejected": -1.8005530834197998,
+      "logps/chosen": -625.2354125976562,
+      "logps/rejected": -752.2579345703125,
+      "loss": 0.0832,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.3820364475250244,
+      "rewards/margins": 0.1880682408809662,
+      "rewards/rejected": -0.570104718208313,
+      "step": 750
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.5563549160671465e-06,
+      "logits/chosen": -1.9659464359283447,
+      "logits/rejected": -1.501859426498413,
+      "logps/chosen": -702.9942626953125,
+      "logps/rejected": -701.7636108398438,
+      "loss": 0.2488,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.4992641806602478,
+      "rewards/margins": 0.06576540321111679,
+      "rewards/rejected": -0.5650296807289124,
+      "step": 760
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.616306954436451e-06,
+      "logits/chosen": -2.0486137866973877,
+      "logits/rejected": -1.6541773080825806,
+      "logps/chosen": -585.8255615234375,
+      "logps/rejected": -702.7525024414062,
+      "loss": 0.1409,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.4078814387321472,
+      "rewards/margins": 0.1429028958082199,
+      "rewards/rejected": -0.5507842898368835,
+      "step": 770
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.676258992805755e-06,
+      "logits/chosen": -2.0564146041870117,
+      "logits/rejected": -1.7639620304107666,
+      "logps/chosen": -491.6483459472656,
+      "logps/rejected": -594.8165893554688,
+      "loss": 0.1234,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.308620810508728,
+      "rewards/margins": 0.12181546539068222,
+      "rewards/rejected": -0.43043628334999084,
+      "step": 780
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.73621103117506e-06,
+      "logits/chosen": -2.059473752975464,
+      "logits/rejected": -1.9862353801727295,
+      "logps/chosen": -501.5008850097656,
+      "logps/rejected": -590.1875610351562,
+      "loss": 0.1277,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3276519477367401,
+      "rewards/margins": 0.07991748303174973,
+      "rewards/rejected": -0.4075694680213928,
+      "step": 790
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.796163069544365e-06,
+      "logits/chosen": -2.139965295791626,
+      "logits/rejected": -1.6807670593261719,
+      "logps/chosen": -660.7980346679688,
+      "logps/rejected": -671.5537719726562,
+      "loss": 0.1609,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.415930837392807,
+      "rewards/margins": 0.0948338657617569,
+      "rewards/rejected": -0.5107647180557251,
+      "step": 800
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.856115107913669e-06,
+      "logits/chosen": -2.151519775390625,
+      "logits/rejected": -1.6908838748931885,
+      "logps/chosen": -526.8802490234375,
+      "logps/rejected": -620.55078125,
+      "loss": 0.1585,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.34467440843582153,
+      "rewards/margins": 0.09073910862207413,
+      "rewards/rejected": -0.43541350960731506,
+      "step": 810
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.916067146282974e-06,
+      "logits/chosen": -2.223308801651001,
+      "logits/rejected": -1.6892244815826416,
+      "logps/chosen": -552.6560668945312,
+      "logps/rejected": -626.216552734375,
+      "loss": 0.1396,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.35285186767578125,
+      "rewards/margins": 0.12594757974147797,
+      "rewards/rejected": -0.4787994921207428,
+      "step": 820
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.9760191846522785e-06,
+      "logits/chosen": -2.1306354999542236,
+      "logits/rejected": -1.9563677310943604,
+      "logps/chosen": -547.3248901367188,
+      "logps/rejected": -642.46533203125,
+      "loss": 0.1675,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.3764510750770569,
+      "rewards/margins": 0.08799637854099274,
+      "rewards/rejected": -0.4644474387168884,
+      "step": 830
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.9999921064257284e-06,
+      "logits/chosen": -2.000413179397583,
+      "logits/rejected": -1.5254223346710205,
+      "logps/chosen": -621.1843872070312,
+      "logps/rejected": -774.5809326171875,
+      "loss": 0.1628,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.3899049460887909,
+      "rewards/margins": 0.1542535126209259,
+      "rewards/rejected": -0.5441584587097168,
+      "step": 840
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.9999438680968e-06,
+      "logits/chosen": -1.8323099613189697,
+      "logits/rejected": -1.2995059490203857,
+      "logps/chosen": -750.2576904296875,
+      "logps/rejected": -869.3472900390625,
+      "loss": 0.1665,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.5629902482032776,
+      "rewards/margins": 0.15459483861923218,
+      "rewards/rejected": -0.7175850868225098,
+      "step": 850
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.999851777603122e-06,
+      "logits/chosen": -2.1188056468963623,
+      "logits/rejected": -1.8229395151138306,
+      "logps/chosen": -577.459228515625,
+      "logps/rejected": -690.314208984375,
+      "loss": 0.1175,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.34011557698249817,
+      "rewards/margins": 0.12348371744155884,
+      "rewards/rejected": -0.463599294424057,
+      "step": 860
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.999715836560074e-06,
+      "logits/chosen": -2.089865207672119,
+      "logits/rejected": -1.6631124019622803,
+      "logps/chosen": -498.9947204589844,
+      "logps/rejected": -581.304443359375,
+      "loss": 0.199,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.3167702555656433,
+      "rewards/margins": 0.09270444512367249,
+      "rewards/rejected": -0.4094746708869934,
+      "step": 870
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.999536047352236e-06,
+      "logits/chosen": -2.047680616378784,
+      "logits/rejected": -1.7472779750823975,
+      "logps/chosen": -504.05181884765625,
+      "logps/rejected": -548.29541015625,
+      "loss": 0.1703,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.33363524079322815,
+      "rewards/margins": 0.06062453240156174,
+      "rewards/rejected": -0.3942597508430481,
+      "step": 880
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.999312413133335e-06,
+      "logits/chosen": -2.267192840576172,
+      "logits/rejected": -1.705100417137146,
+      "logps/chosen": -584.5316162109375,
+      "logps/rejected": -662.5524291992188,
+      "loss": 0.1351,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.3158092796802521,
+      "rewards/margins": 0.12349255383014679,
+      "rewards/rejected": -0.4393017888069153,
+      "step": 890
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.999044937826198e-06,
+      "logits/chosen": -2.1212897300720215,
+      "logits/rejected": -1.4789259433746338,
+      "logps/chosen": -542.8187866210938,
+      "logps/rejected": -657.3876953125,
+      "loss": 0.1755,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.3331352472305298,
+      "rewards/margins": 0.13251715898513794,
+      "rewards/rejected": -0.4656524062156677,
+      "step": 900
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.998733626122679e-06,
+      "logits/chosen": -2.2352664470672607,
+      "logits/rejected": -2.049133777618408,
+      "logps/chosen": -371.13348388671875,
+      "logps/rejected": -442.18096923828125,
+      "loss": 0.1089,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.1648525893688202,
+      "rewards/margins": 0.09664817154407501,
+      "rewards/rejected": -0.2615007758140564,
+      "step": 910
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.998378483483577e-06,
+      "logits/chosen": -2.283607006072998,
+      "logits/rejected": -1.7711458206176758,
+      "logps/chosen": -349.93487548828125,
+      "logps/rejected": -355.8497619628906,
+      "loss": 0.1901,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.15371933579444885,
+      "rewards/margins": 0.09378610551357269,
+      "rewards/rejected": -0.24750542640686035,
+      "step": 920
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.997979516138542e-06,
+      "logits/chosen": -2.0300064086914062,
+      "logits/rejected": -1.6172802448272705,
+      "logps/chosen": -407.49078369140625,
+      "logps/rejected": -496.5247497558594,
+      "loss": 0.1487,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.2352338284254074,
+      "rewards/margins": 0.10448497533798218,
+      "rewards/rejected": -0.3397188186645508,
+      "step": 930
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.997536731085962e-06,
+      "logits/chosen": -2.2076451778411865,
+      "logits/rejected": -1.7691433429718018,
+      "logps/chosen": -545.2886962890625,
+      "logps/rejected": -630.4237060546875,
+      "loss": 0.1057,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.28509920835494995,
+      "rewards/margins": 0.11583169549703598,
+      "rewards/rejected": -0.4009309411048889,
+      "step": 940
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.997050136092847e-06,
+      "logits/chosen": -2.1567330360412598,
+      "logits/rejected": -1.6220133304595947,
+      "logps/chosen": -547.7337036132812,
+      "logps/rejected": -612.4246215820312,
+      "loss": 0.1358,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.32976025342941284,
+      "rewards/margins": 0.14104151725769043,
+      "rewards/rejected": -0.4708017408847809,
+      "step": 950
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.996519739694684e-06,
+      "logits/chosen": -2.0740866661071777,
+      "logits/rejected": -1.9312379360198975,
+      "logps/chosen": -579.1875,
+      "logps/rejected": -645.7134399414062,
+      "loss": 0.1225,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.34073591232299805,
+      "rewards/margins": 0.09059334546327591,
+      "rewards/rejected": -0.43132925033569336,
+      "step": 960
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.995945551195296e-06,
+      "logits/chosen": -2.064605236053467,
+      "logits/rejected": -1.6799293756484985,
+      "logps/chosen": -594.6480712890625,
+      "logps/rejected": -676.9332275390625,
+      "loss": 0.175,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.4062989354133606,
+      "rewards/margins": 0.1238042339682579,
+      "rewards/rejected": -0.5301030874252319,
+      "step": 970
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.995327580666672e-06,
+      "logits/chosen": -2.331757068634033,
+      "logits/rejected": -1.5995452404022217,
+      "logps/chosen": -653.6688232421875,
+      "logps/rejected": -793.6797485351562,
+      "loss": 0.0968,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.4400145411491394,
+      "rewards/margins": 0.2155471295118332,
+      "rewards/rejected": -0.655561625957489,
+      "step": 980
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.994665838948792e-06,
+      "logits/chosen": -2.1293423175811768,
+      "logits/rejected": -2.020312547683716,
+      "logps/chosen": -577.7098999023438,
+      "logps/rejected": -701.9240112304688,
+      "loss": 0.1479,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.3618104159832001,
+      "rewards/margins": 0.08071889728307724,
+      "rewards/rejected": -0.4425292909145355,
+      "step": 990
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.993960337649441e-06,
+      "logits/chosen": -2.156223773956299,
+      "logits/rejected": -1.5423715114593506,
+      "logps/chosen": -575.0502319335938,
+      "logps/rejected": -647.4953002929688,
+      "loss": 0.2001,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.3424583077430725,
+      "rewards/margins": 0.13414214551448822,
+      "rewards/rejected": -0.4766004681587219,
+      "step": 1000
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.993211089144e-06,
+      "logits/chosen": -2.215156078338623,
+      "logits/rejected": -1.6531171798706055,
+      "logps/chosen": -514.7477416992188,
+      "logps/rejected": -605.0260009765625,
+      "loss": 0.1432,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.27666550874710083,
+      "rewards/margins": 0.1487276554107666,
+      "rewards/rejected": -0.42539310455322266,
+      "step": 1010
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.992418106575232e-06,
+      "logits/chosen": -2.4343996047973633,
+      "logits/rejected": -1.8728506565093994,
+      "logps/chosen": -603.20458984375,
+      "logps/rejected": -604.4957275390625,
+      "loss": 0.1333,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.30504000186920166,
+      "rewards/margins": 0.08572148531675339,
+      "rewards/rejected": -0.39076146483421326,
+      "step": 1020
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.9915814038530505e-06,
+      "logits/chosen": -2.3881208896636963,
+      "logits/rejected": -2.0142359733581543,
+      "logps/chosen": -467.87493896484375,
+      "logps/rejected": -507.83367919921875,
+      "loss": 0.1771,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.27033811807632446,
+      "rewards/margins": 0.07256729900836945,
+      "rewards/rejected": -0.3429053723812103,
+      "step": 1030
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.990700995654274e-06,
+      "logits/chosen": -2.276477813720703,
+      "logits/rejected": -1.8213142156600952,
+      "logps/chosen": -627.15625,
+      "logps/rejected": -697.53173828125,
+      "loss": 0.1295,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.4485446512699127,
+      "rewards/margins": 0.09285398572683334,
+      "rewards/rejected": -0.5413986444473267,
+      "step": 1040
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.9897768974223726e-06,
+      "logits/chosen": -2.3381855487823486,
+      "logits/rejected": -1.9102046489715576,
+      "logps/chosen": -530.7308349609375,
+      "logps/rejected": -650.5681762695312,
+      "loss": 0.1563,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3197484314441681,
+      "rewards/margins": 0.1567644625902176,
+      "rewards/rejected": -0.4765128493309021,
+      "step": 1050
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.9888091253671925e-06,
+      "logits/chosen": -2.19991397857666,
+      "logits/rejected": -1.5455358028411865,
+      "logps/chosen": -541.7783203125,
+      "logps/rejected": -601.699951171875,
+      "loss": 0.119,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.3362812101840973,
+      "rewards/margins": 0.12328728288412094,
+      "rewards/rejected": -0.45956850051879883,
+      "step": 1060
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.9877976964646755e-06,
+      "logits/chosen": -2.207604169845581,
+      "logits/rejected": -1.9821321964263916,
+      "logps/chosen": -570.8699951171875,
+      "logps/rejected": -670.33935546875,
+      "loss": 0.1209,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.3377350866794586,
+      "rewards/margins": 0.13441041111946106,
+      "rewards/rejected": -0.4721454977989197,
+      "step": 1070
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.986742628456559e-06,
+      "logits/chosen": -2.212739944458008,
+      "logits/rejected": -1.5888116359710693,
+      "logps/chosen": -536.6373291015625,
+      "logps/rejected": -573.2736206054688,
+      "loss": 0.1903,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.29524531960487366,
+      "rewards/margins": 0.12274769693613052,
+      "rewards/rejected": -0.4179930090904236,
+      "step": 1080
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.985643939850063e-06,
+      "logits/chosen": -2.31105637550354,
+      "logits/rejected": -1.8029263019561768,
+      "logps/chosen": -518.2979736328125,
+      "logps/rejected": -548.7676391601562,
+      "loss": 0.1241,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.26929497718811035,
+      "rewards/margins": 0.094261035323143,
+      "rewards/rejected": -0.36355599761009216,
+      "step": 1090
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.984501649917573e-06,
+      "logits/chosen": -2.1341145038604736,
+      "logits/rejected": -1.6650168895721436,
+      "logps/chosen": -512.9805908203125,
+      "logps/rejected": -584.8742065429688,
+      "loss": 0.1607,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.33220353722572327,
+      "rewards/margins": 0.09670991450548172,
+      "rewards/rejected": -0.428913414478302,
+      "step": 1100
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.98331577869629e-06,
+      "logits/chosen": -2.1692538261413574,
+      "logits/rejected": -1.8654272556304932,
+      "logps/chosen": -629.990234375,
+      "logps/rejected": -680.2612915039062,
+      "loss": 0.1021,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3584403693675995,
+      "rewards/margins": 0.08045858889818192,
+      "rewards/rejected": -0.4388989508152008,
+      "step": 1110
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.982086346987891e-06,
+      "logits/chosen": -1.9992872476577759,
+      "logits/rejected": -1.7333484888076782,
+      "logps/chosen": -528.8548583984375,
+      "logps/rejected": -596.2742309570312,
+      "loss": 0.1817,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3123190999031067,
+      "rewards/margins": 0.0845862478017807,
+      "rewards/rejected": -0.3969053626060486,
+      "step": 1120
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.980813376358157e-06,
+      "logits/chosen": -1.9758859872817993,
+      "logits/rejected": -1.6551954746246338,
+      "logps/chosen": -515.8092651367188,
+      "logps/rejected": -627.9628295898438,
+      "loss": 0.1201,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3216765820980072,
+      "rewards/margins": 0.10043396800756454,
+      "rewards/rejected": -0.42211055755615234,
+      "step": 1130
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.9794968891365955e-06,
+      "logits/chosen": -2.1555192470550537,
+      "logits/rejected": -1.7460895776748657,
+      "logps/chosen": -550.7313842773438,
+      "logps/rejected": -608.6207275390625,
+      "loss": 0.1357,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.30990099906921387,
+      "rewards/margins": 0.09260927140712738,
+      "rewards/rejected": -0.40251025557518005,
+      "step": 1140
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.978136908416052e-06,
+      "logits/chosen": -2.2638537883758545,
+      "logits/rejected": -1.7636549472808838,
+      "logps/chosen": -482.6693420410156,
+      "logps/rejected": -637.0721435546875,
+      "loss": 0.1588,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.3305802643299103,
+      "rewards/margins": 0.15013296902179718,
+      "rewards/rejected": -0.48071321845054626,
+      "step": 1150
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.976733458052301e-06,
+      "logits/chosen": -2.1523945331573486,
+      "logits/rejected": -1.643943428993225,
+      "logps/chosen": -537.0808715820312,
+      "logps/rejected": -697.762939453125,
+      "loss": 0.1183,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3862455487251282,
+      "rewards/margins": 0.1877693235874176,
+      "rewards/rejected": -0.5740148425102234,
+      "step": 1160
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.975286562663629e-06,
+      "logits/chosen": -2.2173848152160645,
+      "logits/rejected": -2.006077289581299,
+      "logps/chosen": -576.416259765625,
+      "logps/rejected": -593.7449951171875,
+      "loss": 0.1409,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.34326648712158203,
+      "rewards/margins": 0.06652097404003143,
+      "rewards/rejected": -0.40978750586509705,
+      "step": 1170
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.9737962476304045e-06,
+      "logits/chosen": -2.2371926307678223,
+      "logits/rejected": -1.789444923400879,
+      "logps/chosen": -464.71990966796875,
+      "logps/rejected": -567.7537231445312,
+      "loss": 0.1375,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.29233497381210327,
+      "rewards/margins": 0.13133782148361206,
+      "rewards/rejected": -0.4236728250980377,
+      "step": 1180
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.972262539094633e-06,
+      "logits/chosen": -2.120291233062744,
+      "logits/rejected": -1.7952181100845337,
+      "logps/chosen": -475.3655700683594,
+      "logps/rejected": -480.8006286621094,
+      "loss": 0.1864,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.28726550936698914,
+      "rewards/margins": 0.03819563239812851,
+      "rewards/rejected": -0.32546114921569824,
+      "step": 1190
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.970685463959489e-06,
+      "logits/chosen": -2.294468879699707,
+      "logits/rejected": -1.779756784439087,
+      "logps/chosen": -437.14044189453125,
+      "logps/rejected": -509.7494201660156,
+      "loss": 0.1243,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.26150578260421753,
+      "rewards/margins": 0.11649823188781738,
+      "rewards/rejected": -0.3780039846897125,
+      "step": 1200
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.969065049888861e-06,
+      "logits/chosen": -2.3901519775390625,
+      "logits/rejected": -1.738590955734253,
+      "logps/chosen": -369.5718688964844,
+      "logps/rejected": -553.3377075195312,
+      "loss": 0.1188,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.20310676097869873,
+      "rewards/margins": 0.18624350428581238,
+      "rewards/rejected": -0.3893502652645111,
+      "step": 1210
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.9674013253068535e-06,
+      "logits/chosen": -2.280198097229004,
+      "logits/rejected": -2.058004856109619,
+      "logps/chosen": -547.2525634765625,
+      "logps/rejected": -635.6800537109375,
+      "loss": 0.1347,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3341754376888275,
+      "rewards/margins": 0.10133830457925797,
+      "rewards/rejected": -0.43551379442214966,
+      "step": 1220
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.96569431939729e-06,
+      "logits/chosen": -1.9538536071777344,
+      "logits/rejected": -1.7261974811553955,
+      "logps/chosen": -607.1433715820312,
+      "logps/rejected": -676.9395751953125,
+      "loss": 0.1334,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.4256073832511902,
+      "rewards/margins": 0.09686996042728424,
+      "rewards/rejected": -0.5224773287773132,
+      "step": 1230
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.963944062103205e-06,
+      "logits/chosen": -2.1461334228515625,
+      "logits/rejected": -1.8456847667694092,
+      "logps/chosen": -607.056884765625,
+      "logps/rejected": -701.8773193359375,
+      "loss": 0.1563,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.39632710814476013,
+      "rewards/margins": 0.11460691690444946,
+      "rewards/rejected": -0.5109339952468872,
+      "step": 1240
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.9621505841263155e-06,
+      "logits/chosen": -2.0759506225585938,
+      "logits/rejected": -1.7105655670166016,
+      "logps/chosen": -541.2208862304688,
+      "logps/rejected": -608.587646484375,
+      "loss": 0.1214,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.36031174659729004,
+      "rewards/margins": 0.08165458589792252,
+      "rewards/rejected": -0.44196629524230957,
+      "step": 1250
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.960313916926486e-06,
+      "logits/chosen": -2.1210339069366455,
+      "logits/rejected": -1.9969221353530884,
+      "logps/chosen": -502.31219482421875,
+      "logps/rejected": -580.4507446289062,
+      "loss": 0.1301,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3091687560081482,
+      "rewards/margins": 0.08267490565776825,
+      "rewards/rejected": -0.39184361696243286,
+      "step": 1260
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.958434092721172e-06,
+      "logits/chosen": -2.146009922027588,
+      "logits/rejected": -1.791215181350708,
+      "logps/chosen": -439.869140625,
+      "logps/rejected": -544.2640991210938,
+      "loss": 0.15,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.26890939474105835,
+      "rewards/margins": 0.1035056933760643,
+      "rewards/rejected": -0.37241509556770325,
+      "step": 1270
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.956511144484858e-06,
+      "logits/chosen": -2.2720885276794434,
+      "logits/rejected": -1.6973850727081299,
+      "logps/chosen": -580.974853515625,
+      "logps/rejected": -622.9429931640625,
+      "loss": 0.1407,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3187558948993683,
+      "rewards/margins": 0.11552183330059052,
+      "rewards/rejected": -0.4342777132987976,
+      "step": 1280
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.954545105948479e-06,
+      "logits/chosen": -2.400979995727539,
+      "logits/rejected": -2.2261736392974854,
+      "logps/chosen": -499.32208251953125,
+      "logps/rejected": -594.3624267578125,
+      "loss": 0.1611,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.26005902886390686,
+      "rewards/margins": 0.10248663276433945,
+      "rewards/rejected": -0.3625456690788269,
+      "step": 1290
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.952536011598828e-06,
+      "logits/chosen": -2.25364351272583,
+      "logits/rejected": -2.1474640369415283,
+      "logps/chosen": -393.96551513671875,
+      "logps/rejected": -510.79840087890625,
+      "loss": 0.234,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.2129390686750412,
+      "rewards/margins": 0.07593324780464172,
+      "rewards/rejected": -0.28887230157852173,
+      "step": 1300
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.950483896677949e-06,
+      "logits/chosen": -2.23642635345459,
+      "logits/rejected": -1.7574459314346313,
+      "logps/chosen": -526.2327270507812,
+      "logps/rejected": -678.3369140625,
+      "loss": 0.1103,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3006700575351715,
+      "rewards/margins": 0.19242578744888306,
+      "rewards/rejected": -0.4930958151817322,
+      "step": 1310
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.948388797182525e-06,
+      "logits/chosen": -2.0806617736816406,
+      "logits/rejected": -2.0248215198516846,
+      "logps/chosen": -527.6939697265625,
+      "logps/rejected": -661.6565551757812,
+      "loss": 0.1965,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3822014033794403,
+      "rewards/margins": 0.0886811688542366,
+      "rewards/rejected": -0.4708825647830963,
+      "step": 1320
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.9462507498632404e-06,
+      "logits/chosen": -1.9994096755981445,
+      "logits/rejected": -1.6206257343292236,
+      "logps/chosen": -432.0537109375,
+      "logps/rejected": -510.92047119140625,
+      "loss": 0.1232,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.2707695960998535,
+      "rewards/margins": 0.09937051683664322,
+      "rewards/rejected": -0.37014010548591614,
+      "step": 1330
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.944069792224138e-06,
+      "logits/chosen": -2.147658586502075,
+      "logits/rejected": -1.7564668655395508,
+      "logps/chosen": -534.7755126953125,
+      "logps/rejected": -547.2291259765625,
+      "loss": 0.1658,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.2872273921966553,
+      "rewards/margins": 0.0723971277475357,
+      "rewards/rejected": -0.3596245348453522,
+      "step": 1340
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.941845962521961e-06,
+      "logits/chosen": -2.402040958404541,
+      "logits/rejected": -1.9889097213745117,
+      "logps/chosen": -436.06390380859375,
+      "logps/rejected": -565.5316772460938,
+      "loss": 0.122,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.28458279371261597,
+      "rewards/margins": 0.15276896953582764,
+      "rewards/rejected": -0.4373517632484436,
+      "step": 1350
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.939579299765485e-06,
+      "logits/chosen": -2.296431303024292,
+      "logits/rejected": -1.9629325866699219,
+      "logps/chosen": -527.28173828125,
+      "logps/rejected": -682.8411254882812,
+      "loss": 0.1196,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.372751921415329,
+      "rewards/margins": 0.12490395456552505,
+      "rewards/rejected": -0.49765586853027344,
+      "step": 1360
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.937269843714831e-06,
+      "logits/chosen": -2.11098313331604,
+      "logits/rejected": -1.7684904336929321,
+      "logps/chosen": -561.1141357421875,
+      "logps/rejected": -687.3109741210938,
+      "loss": 0.1608,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.3640119433403015,
+      "rewards/margins": 0.12598463892936707,
+      "rewards/rejected": -0.4899965822696686,
+      "step": 1370
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.934917634880766e-06,
+      "logits/chosen": -2.0837790966033936,
+      "logits/rejected": -1.7617356777191162,
+      "logps/chosen": -551.3839721679688,
+      "logps/rejected": -679.1558837890625,
+      "loss": 0.1416,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3760112226009369,
+      "rewards/margins": 0.1157335489988327,
+      "rewards/rejected": -0.4917447566986084,
+      "step": 1380
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.932522714523996e-06,
+      "logits/chosen": -2.045963764190674,
+      "logits/rejected": -1.8890432119369507,
+      "logps/chosen": -541.0911865234375,
+      "logps/rejected": -626.9671630859375,
+      "loss": 0.1467,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.37426990270614624,
+      "rewards/margins": 0.09441323578357697,
+      "rewards/rejected": -0.4686830937862396,
+      "step": 1390
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.930085124654443e-06,
+      "logits/chosen": -2.1963815689086914,
+      "logits/rejected": -1.408853530883789,
+      "logps/chosen": -723.8082275390625,
+      "logps/rejected": -742.334716796875,
+      "loss": 0.1221,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3928244709968567,
+      "rewards/margins": 0.1440887153148651,
+      "rewards/rejected": -0.5369132161140442,
+      "step": 1400
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.927604908030503e-06,
+      "logits/chosen": -2.0483460426330566,
+      "logits/rejected": -1.8593266010284424,
+      "logps/chosen": -485.88262939453125,
+      "logps/rejected": -600.080078125,
+      "loss": 0.178,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3080288767814636,
+      "rewards/margins": 0.10219831764698029,
+      "rewards/rejected": -0.4102272391319275,
+      "step": 1410
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.9250821081583e-06,
+      "logits/chosen": -2.1222591400146484,
+      "logits/rejected": -1.7079252004623413,
+      "logps/chosen": -502.71453857421875,
+      "logps/rejected": -646.7982788085938,
+      "loss": 0.0943,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.2986127734184265,
+      "rewards/margins": 0.15742304921150208,
+      "rewards/rejected": -0.4560357928276062,
+      "step": 1420
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.922516769290921e-06,
+      "logits/chosen": -2.3326635360717773,
+      "logits/rejected": -1.712327241897583,
+      "logps/chosen": -646.2364501953125,
+      "logps/rejected": -754.2018432617188,
+      "loss": 0.0933,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.4021947383880615,
+      "rewards/margins": 0.1420992910861969,
+      "rewards/rejected": -0.544293999671936,
+      "step": 1430
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.919908936427643e-06,
+      "logits/chosen": -2.184134006500244,
+      "logits/rejected": -1.8271191120147705,
+      "logps/chosen": -512.01220703125,
+      "logps/rejected": -612.6547241210938,
+      "loss": 0.1374,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.3094896674156189,
+      "rewards/margins": 0.10854814946651459,
+      "rewards/rejected": -0.41803789138793945,
+      "step": 1440
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.917258655313137e-06,
+      "logits/chosen": -2.093576431274414,
+      "logits/rejected": -1.9420554637908936,
+      "logps/chosen": -548.6033325195312,
+      "logps/rejected": -686.27001953125,
+      "loss": 0.1507,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.4107818007469177,
+      "rewards/margins": 0.11964146047830582,
+      "rewards/rejected": -0.5304232835769653,
+      "step": 1450
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.914565972436677e-06,
+      "logits/chosen": -2.2028658390045166,
+      "logits/rejected": -1.7589342594146729,
+      "logps/chosen": -576.9755249023438,
+      "logps/rejected": -718.1185302734375,
+      "loss": 0.1207,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.36530739068984985,
+      "rewards/margins": 0.17530110478401184,
+      "rewards/rejected": -0.5406085252761841,
+      "step": 1460
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.911830935031308e-06,
+      "logits/chosen": -2.065781593322754,
+      "logits/rejected": -1.8709427118301392,
+      "logps/chosen": -566.509521484375,
+      "logps/rejected": -660.6765747070312,
+      "loss": 0.1777,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3564848303794861,
+      "rewards/margins": 0.10104473680257797,
+      "rewards/rejected": -0.45752960443496704,
+      "step": 1470
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.909053591073034e-06,
+      "logits/chosen": -2.340172290802002,
+      "logits/rejected": -1.900390386581421,
+      "logps/chosen": -415.19647216796875,
+      "logps/rejected": -483.5662536621094,
+      "loss": 0.1885,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.20236341655254364,
+      "rewards/margins": 0.1149483323097229,
+      "rewards/rejected": -0.31731173396110535,
+      "step": 1480
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.906233989279967e-06,
+      "logits/chosen": -2.532696008682251,
+      "logits/rejected": -2.0081982612609863,
+      "logps/chosen": -478.8548889160156,
+      "logps/rejected": -499.87506103515625,
+      "loss": 0.1287,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.24814286828041077,
+      "rewards/margins": 0.06408984959125519,
+      "rewards/rejected": -0.31223273277282715,
+      "step": 1490
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.903372179111473e-06,
+      "logits/chosen": -2.113211154937744,
+      "logits/rejected": -2.026672124862671,
+      "logps/chosen": -543.4927978515625,
+      "logps/rejected": -606.6187133789062,
+      "loss": 0.1258,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.3248176574707031,
+      "rewards/margins": 0.07738800346851349,
+      "rewards/rejected": -0.4022056460380554,
+      "step": 1500
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.900468210767309e-06,
+      "logits/chosen": -2.0895724296569824,
+      "logits/rejected": -1.6837047338485718,
+      "logps/chosen": -524.8826904296875,
+      "logps/rejected": -556.540771484375,
+      "loss": 0.197,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.32450398802757263,
+      "rewards/margins": 0.07545824348926544,
+      "rewards/rejected": -0.39996224641799927,
+      "step": 1510
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.897522135186737e-06,
+      "logits/chosen": -2.1736972332000732,
+      "logits/rejected": -1.8453476428985596,
+      "logps/chosen": -604.2501220703125,
+      "logps/rejected": -684.3479614257812,
+      "loss": 0.1477,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.365187406539917,
+      "rewards/margins": 0.10084176063537598,
+      "rewards/rejected": -0.46602916717529297,
+      "step": 1520
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.894534004047635e-06,
+      "logits/chosen": -2.2742676734924316,
+      "logits/rejected": -1.9172481298446655,
+      "logps/chosen": -627.14599609375,
+      "logps/rejected": -699.8477783203125,
+      "loss": 0.067,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.34163469076156616,
+      "rewards/margins": 0.10761366039514542,
+      "rewards/rejected": -0.44924837350845337,
+      "step": 1530
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.891503869765586e-06,
+      "logits/chosen": -2.259845495223999,
+      "logits/rejected": -1.9497085809707642,
+      "logps/chosen": -550.7775268554688,
+      "logps/rejected": -693.5928955078125,
+      "loss": 0.0696,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.31246715784072876,
+      "rewards/margins": 0.17972543835639954,
+      "rewards/rejected": -0.4921925961971283,
+      "step": 1540
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.888431785492964e-06,
+      "logits/chosen": -2.254445791244507,
+      "logits/rejected": -1.7232528924942017,
+      "logps/chosen": -507.88079833984375,
+      "logps/rejected": -612.9020385742188,
+      "loss": 0.1578,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.33192700147628784,
+      "rewards/margins": 0.12616361677646637,
+      "rewards/rejected": -0.4580906331539154,
+      "step": 1550
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.8853178051179965e-06,
+      "logits/chosen": -2.27195405960083,
+      "logits/rejected": -1.739133596420288,
+      "logps/chosen": -574.9097290039062,
+      "logps/rejected": -595.2188110351562,
+      "loss": 0.118,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.3243725001811981,
+      "rewards/margins": 0.09614486992359161,
+      "rewards/rejected": -0.4205173850059509,
+      "step": 1560
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.882161983263822e-06,
+      "logits/chosen": -2.2348380088806152,
+      "logits/rejected": -1.915328025817871,
+      "logps/chosen": -456.7100524902344,
+      "logps/rejected": -510.7151794433594,
+      "loss": 0.1315,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.26327812671661377,
+      "rewards/margins": 0.057186055928468704,
+      "rewards/rejected": -0.32046419382095337,
+      "step": 1570
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.8789643752875315e-06,
+      "logits/chosen": -2.503678798675537,
+      "logits/rejected": -1.697729468345642,
+      "logps/chosen": -496.685791015625,
+      "logps/rejected": -506.0799865722656,
+      "loss": 0.0846,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.21371059119701385,
+      "rewards/margins": 0.1099679097533226,
+      "rewards/rejected": -0.32367846369743347,
+      "step": 1580
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.875725037279197e-06,
+      "logits/chosen": -2.4171833992004395,
+      "logits/rejected": -1.9364573955535889,
+      "logps/chosen": -570.8368530273438,
+      "logps/rejected": -673.1084594726562,
+      "loss": 0.1197,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.3213617205619812,
+      "rewards/margins": 0.1403539478778839,
+      "rewards/rejected": -0.4617156982421875,
+      "step": 1590
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.8724440260608885e-06,
+      "logits/chosen": -2.3573861122131348,
+      "logits/rejected": -1.9183915853500366,
+      "logps/chosen": -515.6995849609375,
+      "logps/rejected": -658.369384765625,
+      "loss": 0.1243,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.33093100786209106,
+      "rewards/margins": 0.1580355167388916,
+      "rewards/rejected": -0.4889664649963379,
+      "step": 1600
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.8691213991856755e-06,
+      "logits/chosen": -2.4707210063934326,
+      "logits/rejected": -2.0764760971069336,
+      "logps/chosen": -498.86962890625,
+      "logps/rejected": -596.0845947265625,
+      "loss": 0.1338,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.30956193804740906,
+      "rewards/margins": 0.12143248319625854,
+      "rewards/rejected": -0.4309944212436676,
+      "step": 1610
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.8657572149366195e-06,
+      "logits/chosen": -2.3442182540893555,
+      "logits/rejected": -2.127903461456299,
+      "logps/chosen": -513.7960815429688,
+      "logps/rejected": -579.8582763671875,
+      "loss": 0.1339,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.29489654302597046,
+      "rewards/margins": 0.08306767046451569,
+      "rewards/rejected": -0.37796422839164734,
+      "step": 1620
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.8623515323257496e-06,
+      "logits/chosen": -2.1487810611724854,
+      "logits/rejected": -2.0269265174865723,
+      "logps/chosen": -594.8941650390625,
+      "logps/rejected": -682.0096435546875,
+      "loss": 0.1421,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3908698856830597,
+      "rewards/margins": 0.06920132786035538,
+      "rewards/rejected": -0.46007123589515686,
+      "step": 1630
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.85890441109303e-06,
+      "logits/chosen": -2.210763692855835,
+      "logits/rejected": -1.9094241857528687,
+      "logps/chosen": -625.5291748046875,
+      "logps/rejected": -732.5994873046875,
+      "loss": 0.1253,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.4071863293647766,
+      "rewards/margins": 0.13136863708496094,
+      "rewards/rejected": -0.5385549068450928,
+      "step": 1640
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.855415911705308e-06,
+      "logits/chosen": -2.2696292400360107,
+      "logits/rejected": -2.162616729736328,
+      "logps/chosen": -523.4466552734375,
+      "logps/rejected": -608.39453125,
+      "loss": 0.1232,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3096820116043091,
+      "rewards/margins": 0.08442376554012299,
+      "rewards/rejected": -0.3941057622432709,
+      "step": 1650
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.851886095355259e-06,
+      "logits/chosen": -2.3265531063079834,
+      "logits/rejected": -1.7164676189422607,
+      "logps/chosen": -618.269287109375,
+      "logps/rejected": -668.5130615234375,
+      "loss": 0.1077,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.3504432737827301,
+      "rewards/margins": 0.12268070131540298,
+      "rewards/rejected": -0.4731239676475525,
+      "step": 1660
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.848315023960308e-06,
+      "logits/chosen": -2.2978243827819824,
+      "logits/rejected": -1.7795026302337646,
+      "logps/chosen": -558.755859375,
+      "logps/rejected": -562.528076171875,
+      "loss": 0.1368,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.34597286581993103,
+      "rewards/margins": 0.08709348738193512,
+      "rewards/rejected": -0.43306636810302734,
+      "step": 1670
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.844702760161546e-06,
+      "logits/chosen": -2.1904449462890625,
+      "logits/rejected": -1.6820290088653564,
+      "logps/chosen": -514.9161987304688,
+      "logps/rejected": -607.1641235351562,
+      "loss": 0.1452,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.33464479446411133,
+      "rewards/margins": 0.12145761400461197,
+      "rewards/rejected": -0.4561024308204651,
+      "step": 1680
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.841049367322631e-06,
+      "logits/chosen": -2.1526894569396973,
+      "logits/rejected": -1.7920734882354736,
+      "logps/chosen": -543.8094482421875,
+      "logps/rejected": -632.2418212890625,
+      "loss": 0.077,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.31233692169189453,
+      "rewards/margins": 0.11530987173318863,
+      "rewards/rejected": -0.42764681577682495,
+      "step": 1690
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.837354909528675e-06,
+      "logits/chosen": -2.1050491333007812,
+      "logits/rejected": -2.1262621879577637,
+      "logps/chosen": -483.9656677246094,
+      "logps/rejected": -576.7960815429688,
+      "loss": 0.171,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.3369120955467224,
+      "rewards/margins": 0.08613753318786621,
+      "rewards/rejected": -0.42304959893226624,
+      "step": 1700
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.833619451585122e-06,
+      "logits/chosen": -2.2384800910949707,
+      "logits/rejected": -1.7839508056640625,
+      "logps/chosen": -474.2342224121094,
+      "logps/rejected": -574.675537109375,
+      "loss": 0.0992,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.26608026027679443,
+      "rewards/margins": 0.12795677781105042,
+      "rewards/rejected": -0.39403706789016724,
+      "step": 1710
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.829843059016611e-06,
+      "logits/chosen": -2.2431914806365967,
+      "logits/rejected": -1.808118462562561,
+      "logps/chosen": -444.66259765625,
+      "logps/rejected": -542.5850219726562,
+      "loss": 0.1987,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.29531484842300415,
+      "rewards/margins": 0.09632458537817001,
+      "rewards/rejected": -0.3916394114494324,
+      "step": 1720
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.826025798065823e-06,
+      "logits/chosen": -2.2982006072998047,
+      "logits/rejected": -2.1897151470184326,
+      "logps/chosen": -486.81219482421875,
+      "logps/rejected": -561.9449462890625,
+      "loss": 0.2002,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.27266815304756165,
+      "rewards/margins": 0.10728361457586288,
+      "rewards/rejected": -0.3799517750740051,
+      "step": 1730
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.8221677356923255e-06,
+      "logits/chosen": -2.184744358062744,
+      "logits/rejected": -1.9015491008758545,
+      "logps/chosen": -422.75006103515625,
+      "logps/rejected": -595.830810546875,
+      "loss": 0.1617,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.2709837853908539,
+      "rewards/margins": 0.15053145587444305,
+      "rewards/rejected": -0.42151522636413574,
+      "step": 1740
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.8182689395713925e-06,
+      "logits/chosen": -2.3053829669952393,
+      "logits/rejected": -1.7083690166473389,
+      "logps/chosen": -571.0980224609375,
+      "logps/rejected": -749.0482177734375,
+      "loss": 0.1525,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.3781247138977051,
+      "rewards/margins": 0.2000313699245453,
+      "rewards/rejected": -0.5781561136245728,
+      "step": 1750
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.814329478092818e-06,
+      "logits/chosen": -2.335254669189453,
+      "logits/rejected": -1.8961292505264282,
+      "logps/chosen": -563.0028076171875,
+      "logps/rejected": -639.2133178710938,
+      "loss": 0.1358,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.33126476407051086,
+      "rewards/margins": 0.0919918343424797,
+      "rewards/rejected": -0.4232565760612488,
+      "step": 1760
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.810349420359722e-06,
+      "logits/chosen": -2.175225019454956,
+      "logits/rejected": -1.623108148574829,
+      "logps/chosen": -554.420166015625,
+      "logps/rejected": -694.8674926757812,
+      "loss": 0.1494,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3656291365623474,
+      "rewards/margins": 0.14364008605480194,
+      "rewards/rejected": -0.5092692375183105,
+      "step": 1770
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.806328836187328e-06,
+      "logits/chosen": -2.2877261638641357,
+      "logits/rejected": -1.8044350147247314,
+      "logps/chosen": -564.4354248046875,
+      "logps/rejected": -648.5870971679688,
+      "loss": 0.1824,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.34948137402534485,
+      "rewards/margins": 0.1296634078025818,
+      "rewards/rejected": -0.47914475202560425,
+      "step": 1780
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.802267796101749e-06,
+      "logits/chosen": -2.080324411392212,
+      "logits/rejected": -1.6770923137664795,
+      "logps/chosen": -698.7471313476562,
+      "logps/rejected": -747.7000122070312,
+      "loss": 0.1323,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.44999605417251587,
+      "rewards/margins": 0.11367656290531158,
+      "rewards/rejected": -0.563672661781311,
+      "step": 1790
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.798166371338745e-06,
+      "logits/chosen": -2.2113351821899414,
+      "logits/rejected": -2.0211894512176514,
+      "logps/chosen": -616.6056518554688,
+      "logps/rejected": -728.697998046875,
+      "loss": 0.1892,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.39488235116004944,
+      "rewards/margins": 0.0976155549287796,
+      "rewards/rejected": -0.49249786138534546,
+      "step": 1800
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.79402463384247e-06,
+      "logits/chosen": -2.105198383331299,
+      "logits/rejected": -1.8947727680206299,
+      "logps/chosen": -591.4815063476562,
+      "logps/rejected": -676.2838134765625,
+      "loss": 0.1633,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.3773289918899536,
+      "rewards/margins": 0.07217184454202652,
+      "rewards/rejected": -0.44950079917907715,
+      "step": 1810
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.78984265626422e-06,
+      "logits/chosen": -2.050130844116211,
+      "logits/rejected": -1.8681789636611938,
+      "logps/chosen": -475.3292541503906,
+      "logps/rejected": -573.3819580078125,
+      "loss": 0.1183,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.3002835810184479,
+      "rewards/margins": 0.10083530098199844,
+      "rewards/rejected": -0.40111884474754333,
+      "step": 1820
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.785620511961148e-06,
+      "logits/chosen": -2.3349571228027344,
+      "logits/rejected": -1.9975881576538086,
+      "logps/chosen": -558.6568603515625,
+      "logps/rejected": -648.95361328125,
+      "loss": 0.1382,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3172159194946289,
+      "rewards/margins": 0.11887846142053604,
+      "rewards/rejected": -0.43609437346458435,
+      "step": 1830
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.781358274994985e-06,
+      "logits/chosen": -2.4314303398132324,
+      "logits/rejected": -2.033545732498169,
+      "logps/chosen": -447.5263671875,
+      "logps/rejected": -527.177734375,
+      "loss": 0.1656,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.2411189079284668,
+      "rewards/margins": 0.11009607464075089,
+      "rewards/rejected": -0.35121503472328186,
+      "step": 1840
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.777056020130737e-06,
+      "logits/chosen": -2.474180221557617,
+      "logits/rejected": -1.891824722290039,
+      "logps/chosen": -622.7772216796875,
+      "logps/rejected": -739.7864990234375,
+      "loss": 0.119,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3389725089073181,
+      "rewards/margins": 0.16335345804691315,
+      "rewards/rejected": -0.5023259520530701,
+      "step": 1850
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.772713822835374e-06,
+      "logits/chosen": -2.1120100021362305,
+      "logits/rejected": -1.643057107925415,
+      "logps/chosen": -609.477294921875,
+      "logps/rejected": -749.2017822265625,
+      "loss": 0.1599,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.42751917243003845,
+      "rewards/margins": 0.15787552297115326,
+      "rewards/rejected": -0.5853947401046753,
+      "step": 1860
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.768331759276506e-06,
+      "logits/chosen": -2.3515539169311523,
+      "logits/rejected": -2.054558515548706,
+      "logps/chosen": -555.1827392578125,
+      "logps/rejected": -633.0570068359375,
+      "loss": 0.0821,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.2991371154785156,
+      "rewards/margins": 0.09553690254688263,
+      "rewards/rejected": -0.39467400312423706,
+      "step": 1870
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.763909906321048e-06,
+      "logits/chosen": -2.3349084854125977,
+      "logits/rejected": -1.9230949878692627,
+      "logps/chosen": -465.7528381347656,
+      "logps/rejected": -548.5799560546875,
+      "loss": 0.1583,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.24164226651191711,
+      "rewards/margins": 0.11772511899471283,
+      "rewards/rejected": -0.35936737060546875,
+      "step": 1880
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.759448341533872e-06,
+      "logits/chosen": -2.1321113109588623,
+      "logits/rejected": -1.8927738666534424,
+      "logps/chosen": -506.82080078125,
+      "logps/rejected": -644.9132080078125,
+      "loss": 0.1572,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.29169219732284546,
+      "rewards/margins": 0.14229023456573486,
+      "rewards/rejected": -0.4339824616909027,
+      "step": 1890
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.754947143176445e-06,
+      "logits/chosen": -2.244537830352783,
+      "logits/rejected": -1.6140339374542236,
+      "logps/chosen": -424.0584411621094,
+      "logps/rejected": -502.7630920410156,
+      "loss": 0.1242,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.25831303000450134,
+      "rewards/margins": 0.12335425615310669,
+      "rewards/rejected": -0.3816673159599304,
+      "step": 1900
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.750406390205456e-06,
+      "logits/chosen": -2.331876754760742,
+      "logits/rejected": -2.2618374824523926,
+      "logps/chosen": -513.6726684570312,
+      "logps/rejected": -554.1386108398438,
+      "loss": 0.1692,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.25643008947372437,
+      "rewards/margins": 0.06760696321725845,
+      "rewards/rejected": -0.32403701543807983,
+      "step": 1910
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.745826162271433e-06,
+      "logits/chosen": -2.5929977893829346,
+      "logits/rejected": -2.0814719200134277,
+      "logps/chosen": -463.6415100097656,
+      "logps/rejected": -580.9536743164062,
+      "loss": 0.1207,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.2199324369430542,
+      "rewards/margins": 0.14355406165122986,
+      "rewards/rejected": -0.36348649859428406,
+      "step": 1920
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.741206539717343e-06,
+      "logits/chosen": -2.5063323974609375,
+      "logits/rejected": -1.8149082660675049,
+      "logps/chosen": -449.05548095703125,
+      "logps/rejected": -577.72216796875,
+      "loss": 0.1695,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.21354766190052032,
+      "rewards/margins": 0.17583154141902924,
+      "rewards/rejected": -0.3893791735172272,
+      "step": 1930
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.736547603577185e-06,
+      "logits/chosen": -2.0604281425476074,
+      "logits/rejected": -2.019392490386963,
+      "logps/chosen": -591.45849609375,
+      "logps/rejected": -649.7518310546875,
+      "loss": 0.2077,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.42575398087501526,
+      "rewards/margins": 0.07095947116613388,
+      "rewards/rejected": -0.49671345949172974,
+      "step": 1940
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.731849435574568e-06,
+      "logits/chosen": -2.3695755004882812,
+      "logits/rejected": -2.0847103595733643,
+      "logps/chosen": -555.1213989257812,
+      "logps/rejected": -688.3885498046875,
+      "loss": 0.1436,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.34852778911590576,
+      "rewards/margins": 0.15153101086616516,
+      "rewards/rejected": -0.5000588297843933,
+      "step": 1950
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.727112118121279e-06,
+      "logits/chosen": -2.2374966144561768,
+      "logits/rejected": -1.9865261316299438,
+      "logps/chosen": -546.40966796875,
+      "logps/rejected": -612.305908203125,
+      "loss": 0.1169,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.3476561903953552,
+      "rewards/margins": 0.086991086602211,
+      "rewards/rejected": -0.43464726209640503,
+      "step": 1960
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.722335734315833e-06,
+      "logits/chosen": -2.2488903999328613,
+      "logits/rejected": -1.7812178134918213,
+      "logps/chosen": -618.6195068359375,
+      "logps/rejected": -674.5559692382812,
+      "loss": 0.1072,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.3556477427482605,
+      "rewards/margins": 0.13487717509269714,
+      "rewards/rejected": -0.49052494764328003,
+      "step": 1970
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.7175203679420175e-06,
+      "logits/chosen": -2.1708850860595703,
+      "logits/rejected": -1.6388574838638306,
+      "logps/chosen": -602.9005126953125,
+      "logps/rejected": -758.0128173828125,
+      "loss": 0.1337,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.42405787110328674,
+      "rewards/margins": 0.16360315680503845,
+      "rewards/rejected": -0.5876610279083252,
+      "step": 1980
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.712666103467428e-06,
+      "logits/chosen": -2.3118343353271484,
+      "logits/rejected": -2.123844861984253,
+      "logps/chosen": -566.5917358398438,
+      "logps/rejected": -657.926513671875,
+      "loss": 0.1481,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3595176339149475,
+      "rewards/margins": 0.11952359974384308,
+      "rewards/rejected": -0.479041188955307,
+      "step": 1990
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.707773026041975e-06,
+      "logits/chosen": -2.334312915802002,
+      "logits/rejected": -2.110157012939453,
+      "logps/chosen": -622.9669189453125,
+      "logps/rejected": -696.2811889648438,
+      "loss": 0.1821,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.38160938024520874,
+      "rewards/margins": 0.1228628158569336,
+      "rewards/rejected": -0.5044721364974976,
+      "step": 2000
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.702841221496403e-06,
+      "logits/chosen": -2.297335624694824,
+      "logits/rejected": -1.8612556457519531,
+      "logps/chosen": -617.4207153320312,
+      "logps/rejected": -651.8953247070312,
+      "loss": 0.1728,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3849654197692871,
+      "rewards/margins": 0.08339057862758636,
+      "rewards/rejected": -0.4683559834957123,
+      "step": 2010
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.697870776340776e-06,
+      "logits/chosen": -2.5197081565856934,
+      "logits/rejected": -1.9300096035003662,
+      "logps/chosen": -492.34857177734375,
+      "logps/rejected": -604.9501953125,
+      "loss": 0.1583,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.2718012034893036,
+      "rewards/margins": 0.1724461019039154,
+      "rewards/rejected": -0.444247305393219,
+      "step": 2020
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.692861777762963e-06,
+      "logits/chosen": -2.263380527496338,
+      "logits/rejected": -1.7025277614593506,
+      "logps/chosen": -504.82379150390625,
+      "logps/rejected": -653.6024169921875,
+      "loss": 0.1046,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.2909550070762634,
+      "rewards/margins": 0.2033686339855194,
+      "rewards/rejected": -0.49432364106178284,
+      "step": 2030
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.68781431362711e-06,
+      "logits/chosen": -2.340829849243164,
+      "logits/rejected": -1.7469650506973267,
+      "logps/chosen": -675.9423828125,
+      "logps/rejected": -786.3250732421875,
+      "loss": 0.1799,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.4498385787010193,
+      "rewards/margins": 0.15624050796031952,
+      "rewards/rejected": -0.6060791015625,
+      "step": 2040
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.6827284724720955e-06,
+      "logits/chosen": -2.283242702484131,
+      "logits/rejected": -1.9447603225708008,
+      "logps/chosen": -577.5584106445312,
+      "logps/rejected": -636.4556884765625,
+      "loss": 0.1848,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.35255223512649536,
+      "rewards/margins": 0.10452032089233398,
+      "rewards/rejected": -0.45707258582115173,
+      "step": 2050
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.677604343509981e-06,
+      "logits/chosen": -2.372417449951172,
+      "logits/rejected": -1.864935278892517,
+      "logps/chosen": -469.20037841796875,
+      "logps/rejected": -560.1400146484375,
+      "loss": 0.1179,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.25479021668434143,
+      "rewards/margins": 0.1337851583957672,
+      "rewards/rejected": -0.38857537508010864,
+      "step": 2060
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.672442016624444e-06,
+      "logits/chosen": -2.5201709270477295,
+      "logits/rejected": -2.1217703819274902,
+      "logps/chosen": -538.6556396484375,
+      "logps/rejected": -617.9935302734375,
+      "loss": 0.1397,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.2754242420196533,
+      "rewards/margins": 0.14079084992408752,
+      "rewards/rejected": -0.41621509194374084,
+      "step": 2070
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.6672415823692e-06,
+      "logits/chosen": -2.1822547912597656,
+      "logits/rejected": -1.7586694955825806,
+      "logps/chosen": -590.7755737304688,
+      "logps/rejected": -682.9185791015625,
+      "loss": 0.1519,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3181314766407013,
+      "rewards/margins": 0.13592323660850525,
+      "rewards/rejected": -0.45405474305152893,
+      "step": 2080
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.662003131966418e-06,
+      "logits/chosen": -2.531625986099243,
+      "logits/rejected": -1.9204866886138916,
+      "logps/chosen": -497.391845703125,
+      "logps/rejected": -568.7432250976562,
+      "loss": 0.1685,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.2895485758781433,
+      "rewards/margins": 0.10035135596990585,
+      "rewards/rejected": -0.38989996910095215,
+      "step": 2090
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.6567267573051176e-06,
+      "logits/chosen": -2.2632105350494385,
+      "logits/rejected": -2.0153698921203613,
+      "logps/chosen": -499.95037841796875,
+      "logps/rejected": -559.6163330078125,
+      "loss": 0.1783,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.3090696930885315,
+      "rewards/margins": 0.05365429446101189,
+      "rewards/rejected": -0.3627239167690277,
+      "step": 2100
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.651412550939556e-06,
+      "logits/chosen": -2.3321595191955566,
+      "logits/rejected": -1.652076005935669,
+      "logps/chosen": -530.0022583007812,
+      "logps/rejected": -673.1708374023438,
+      "loss": 0.1112,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3329431116580963,
+      "rewards/margins": 0.19165830314159393,
+      "rewards/rejected": -0.524601399898529,
+      "step": 2110
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.646060606087608e-06,
+      "logits/chosen": -2.2596230506896973,
+      "logits/rejected": -1.9692060947418213,
+      "logps/chosen": -607.5302124023438,
+      "logps/rejected": -676.9356079101562,
+      "loss": 0.1672,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.3833482265472412,
+      "rewards/margins": 0.11330167204141617,
+      "rewards/rejected": -0.4966498911380768,
+      "step": 2120
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.640671016629129e-06,
+      "logits/chosen": -2.1524815559387207,
+      "logits/rejected": -2.0016331672668457,
+      "logps/chosen": -613.8076171875,
+      "logps/rejected": -692.9964599609375,
+      "loss": 0.1269,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3971349596977234,
+      "rewards/margins": 0.08355464041233063,
+      "rewards/rejected": -0.4806896150112152,
+      "step": 2130
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.635243877104307e-06,
+      "logits/chosen": -2.2989161014556885,
+      "logits/rejected": -2.0060343742370605,
+      "logps/chosen": -461.67169189453125,
+      "logps/rejected": -542.5751342773438,
+      "loss": 0.159,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.245099276304245,
+      "rewards/margins": 0.08059612661600113,
+      "rewards/rejected": -0.3256953954696655,
+      "step": 2140
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.629779282712007e-06,
+      "logits/chosen": -2.2033112049102783,
+      "logits/rejected": -1.780149221420288,
+      "logps/chosen": -494.9944763183594,
+      "logps/rejected": -584.2511596679688,
+      "loss": 0.1278,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.27719539403915405,
+      "rewards/margins": 0.11807969957590103,
+      "rewards/rejected": -0.3952751159667969,
+      "step": 2150
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.6242773293080965e-06,
+      "logits/chosen": -2.4133999347686768,
+      "logits/rejected": -1.7591224908828735,
+      "logps/chosen": -603.9238891601562,
+      "logps/rejected": -696.4195556640625,
+      "loss": 0.1128,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.32847195863723755,
+      "rewards/margins": 0.13869908452033997,
+      "rewards/rejected": -0.4671711027622223,
+      "step": 2160
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.618738113403772e-06,
+      "logits/chosen": -2.2003252506256104,
+      "logits/rejected": -1.5519673824310303,
+      "logps/chosen": -699.3399658203125,
+      "logps/rejected": -798.82373046875,
+      "loss": 0.0796,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.4113682806491852,
+      "rewards/margins": 0.15183985233306885,
+      "rewards/rejected": -0.5632081031799316,
+      "step": 2170
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.613161732163857e-06,
+      "logits/chosen": -2.265425443649292,
+      "logits/rejected": -1.9274418354034424,
+      "logps/chosen": -635.97802734375,
+      "logps/rejected": -723.0447387695312,
+      "loss": 0.196,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.4592297673225403,
+      "rewards/margins": 0.11523201316595078,
+      "rewards/rejected": -0.574461817741394,
+      "step": 2180
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.607548283405103e-06,
+      "logits/chosen": -2.4492762088775635,
+      "logits/rejected": -2.0186238288879395,
+      "logps/chosen": -664.1460571289062,
+      "logps/rejected": -792.5499267578125,
+      "loss": 0.1637,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.4228591024875641,
+      "rewards/margins": 0.18332651257514954,
+      "rewards/rejected": -0.6061855554580688,
+      "step": 2190
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.601897865594473e-06,
+      "logits/chosen": -2.2052628993988037,
+      "logits/rejected": -1.8468005657196045,
+      "logps/chosen": -788.9705810546875,
+      "logps/rejected": -918.4918212890625,
+      "loss": 0.1027,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.5626048445701599,
+      "rewards/margins": 0.1202419176697731,
+      "rewards/rejected": -0.6828467845916748,
+      "step": 2200
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.596210577847415e-06,
+      "logits/chosen": -2.061143398284912,
+      "logits/rejected": -1.593901515007019,
+      "logps/chosen": -631.2333984375,
+      "logps/rejected": -768.0970458984375,
+      "loss": 0.1546,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.4390382766723633,
+      "rewards/margins": 0.1623464673757553,
+      "rewards/rejected": -0.6013847589492798,
+      "step": 2210
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.590486519926118e-06,
+      "logits/chosen": -2.0461859703063965,
+      "logits/rejected": -1.9552223682403564,
+      "logps/chosen": -555.0554809570312,
+      "logps/rejected": -679.8436279296875,
+      "loss": 0.1053,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3374500870704651,
+      "rewards/margins": 0.13243895769119263,
+      "rewards/rejected": -0.4698890149593353,
+      "step": 2220
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.584725792237772e-06,
+      "logits/chosen": -2.1382579803466797,
+      "logits/rejected": -1.6836185455322266,
+      "logps/chosen": -599.0387573242188,
+      "logps/rejected": -736.6492919921875,
+      "loss": 0.1318,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3522716164588928,
+      "rewards/margins": 0.14818063378334045,
+      "rewards/rejected": -0.5004522204399109,
+      "step": 2230
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.578928495832795e-06,
+      "logits/chosen": -2.397468090057373,
+      "logits/rejected": -1.7665141820907593,
+      "logps/chosen": -537.0330810546875,
+      "logps/rejected": -578.9915771484375,
+      "loss": 0.136,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.29920458793640137,
+      "rewards/margins": 0.11508778482675552,
+      "rewards/rejected": -0.4142923355102539,
+      "step": 2240
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.57309473240307e-06,
+      "logits/chosen": -2.2755134105682373,
+      "logits/rejected": -1.7020740509033203,
+      "logps/chosen": -472.3487854003906,
+      "logps/rejected": -494.22222900390625,
+      "loss": 0.1255,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.23541271686553955,
+      "rewards/margins": 0.09914446622133255,
+      "rewards/rejected": -0.3345571756362915,
+      "step": 2250
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.567224604280157e-06,
+      "logits/chosen": -2.0108704566955566,
+      "logits/rejected": -1.8880360126495361,
+      "logps/chosen": -455.2273864746094,
+      "logps/rejected": -641.1407470703125,
+      "loss": 0.1208,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.30368906259536743,
+      "rewards/margins": 0.1440240442752838,
+      "rewards/rejected": -0.44771307706832886,
+      "step": 2260
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.561318214433499e-06,
+      "logits/chosen": -2.209198474884033,
+      "logits/rejected": -2.1322221755981445,
+      "logps/chosen": -467.77801513671875,
+      "logps/rejected": -574.4549560546875,
+      "loss": 0.162,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.2779673635959625,
+      "rewards/margins": 0.09591646492481232,
+      "rewards/rejected": -0.37388381361961365,
+      "step": 2270
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.555375666468613e-06,
+      "logits/chosen": -2.290503978729248,
+      "logits/rejected": -1.8968385457992554,
+      "logps/chosen": -508.3070373535156,
+      "logps/rejected": -587.0977172851562,
+      "loss": 0.1905,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.26000916957855225,
+      "rewards/margins": 0.11419705301523209,
+      "rewards/rejected": -0.37420621514320374,
+      "step": 2280
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.549397064625275e-06,
+      "logits/chosen": -2.1739487648010254,
+      "logits/rejected": -2.021857738494873,
+      "logps/chosen": -506.8651428222656,
+      "logps/rejected": -600.6451416015625,
+      "loss": 0.112,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.3014863431453705,
+      "rewards/margins": 0.08067801594734192,
+      "rewards/rejected": -0.3821643888950348,
+      "step": 2290
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.543382513775696e-06,
+      "logits/chosen": -2.289529323577881,
+      "logits/rejected": -1.876098871231079,
+      "logps/chosen": -377.9178466796875,
+      "logps/rejected": -465.83837890625,
+      "loss": 0.1334,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.1864103376865387,
+      "rewards/margins": 0.11499436944723129,
+      "rewards/rejected": -0.3014047145843506,
+      "step": 2300
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.5373321194226736e-06,
+      "logits/chosen": -2.244719982147217,
+      "logits/rejected": -1.8957116603851318,
+      "logps/chosen": -428.60089111328125,
+      "logps/rejected": -543.7210083007812,
+      "loss": 0.0971,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.20496077835559845,
+      "rewards/margins": 0.11617553234100342,
+      "rewards/rejected": -0.32113632559776306,
+      "step": 2310
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.531245987697747e-06,
+      "logits/chosen": -2.5063188076019287,
+      "logits/rejected": -2.1888763904571533,
+      "logps/chosen": -403.67169189453125,
+      "logps/rejected": -444.4043884277344,
+      "loss": 0.1387,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.15920114517211914,
+      "rewards/margins": 0.06529294699430466,
+      "rewards/rejected": -0.224494069814682,
+      "step": 2320
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.525124225359332e-06,
+      "logits/chosen": -2.3165335655212402,
+      "logits/rejected": -1.957654595375061,
+      "logps/chosen": -442.14727783203125,
+      "logps/rejected": -512.8746337890625,
+      "loss": 0.1721,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.2139579951763153,
+      "rewards/margins": 0.11900858581066132,
+      "rewards/rejected": -0.33296656608581543,
+      "step": 2330
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.518966939790854e-06,
+      "logits/chosen": -2.357377529144287,
+      "logits/rejected": -2.039689779281616,
+      "logps/chosen": -445.66912841796875,
+      "logps/rejected": -475.7320251464844,
+      "loss": 0.1792,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.18836535513401031,
+      "rewards/margins": 0.08137214928865433,
+      "rewards/rejected": -0.26973751187324524,
+      "step": 2340
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.512774238998858e-06,
+      "logits/chosen": -2.1866211891174316,
+      "logits/rejected": -1.884800672531128,
+      "logps/chosen": -478.59515380859375,
+      "logps/rejected": -567.1848754882812,
+      "loss": 0.1672,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.30711108446121216,
+      "rewards/margins": 0.10226906836032867,
+      "rewards/rejected": -0.40938013792037964,
+      "step": 2350
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.506546231611116e-06,
+      "logits/chosen": -2.0459065437316895,
+      "logits/rejected": -1.9586502313613892,
+      "logps/chosen": -624.876708984375,
+      "logps/rejected": -822.8359375,
+      "loss": 0.0622,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.41109973192214966,
+      "rewards/margins": 0.18004652857780457,
+      "rewards/rejected": -0.5911462306976318,
+      "step": 2360
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.500283026874724e-06,
+      "logits/chosen": -2.4889118671417236,
+      "logits/rejected": -1.9851830005645752,
+      "logps/chosen": -546.8642578125,
+      "logps/rejected": -661.696533203125,
+      "loss": 0.144,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3060414791107178,
+      "rewards/margins": 0.14917024970054626,
+      "rewards/rejected": -0.45521172881126404,
+      "step": 2370
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.493984734654184e-06,
+      "logits/chosen": -2.3801920413970947,
+      "logits/rejected": -2.1415188312530518,
+      "logps/chosen": -444.19732666015625,
+      "logps/rejected": -567.556396484375,
+      "loss": 0.1146,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.2435305416584015,
+      "rewards/margins": 0.1497594267129898,
+      "rewards/rejected": -0.3932898938655853,
+      "step": 2380
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.487651465429475e-06,
+      "logits/chosen": -2.4305508136749268,
+      "logits/rejected": -2.191523790359497,
+      "logps/chosen": -455.58941650390625,
+      "logps/rejected": -599.6484375,
+      "loss": 0.1809,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.2352205216884613,
+      "rewards/margins": 0.169716477394104,
+      "rewards/rejected": -0.4049369692802429,
+      "step": 2390
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.481283330294118e-06,
+      "logits/chosen": -2.083350658416748,
+      "logits/rejected": -1.5051873922348022,
+      "logps/chosen": -577.2568359375,
+      "logps/rejected": -683.6553955078125,
+      "loss": 0.185,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.39005208015441895,
+      "rewards/margins": 0.14638492465019226,
+      "rewards/rejected": -0.5364369750022888,
+      "step": 2400
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.474880440953224e-06,
+      "logits/chosen": -2.1407008171081543,
+      "logits/rejected": -1.9724195003509521,
+      "logps/chosen": -538.424560546875,
+      "logps/rejected": -696.5970458984375,
+      "loss": 0.0757,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.37489649653434753,
+      "rewards/margins": 0.13986638188362122,
+      "rewards/rejected": -0.5147628784179688,
+      "step": 2410
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.468442909721541e-06,
+      "logits/chosen": -2.1652956008911133,
+      "logits/rejected": -1.9329149723052979,
+      "logps/chosen": -600.6764526367188,
+      "logps/rejected": -741.7485961914062,
+      "loss": 0.1251,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.41478079557418823,
+      "rewards/margins": 0.13973450660705566,
+      "rewards/rejected": -0.5545153021812439,
+      "step": 2420
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.4619708495214735e-06,
+      "logits/chosen": -2.343940496444702,
+      "logits/rejected": -1.8537992238998413,
+      "logps/chosen": -679.3366088867188,
+      "logps/rejected": -690.858642578125,
+      "loss": 0.1689,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.4020530581474304,
+      "rewards/margins": 0.09724564850330353,
+      "rewards/rejected": -0.49929872155189514,
+      "step": 2430
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.455464373881112e-06,
+      "logits/chosen": -2.12073016166687,
+      "logits/rejected": -1.9531166553497314,
+      "logps/chosen": -544.91455078125,
+      "logps/rejected": -656.95849609375,
+      "loss": 0.1617,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.33384081721305847,
+      "rewards/margins": 0.14037808775901794,
+      "rewards/rejected": -0.4742189049720764,
+      "step": 2440
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.4489235969322355e-06,
+      "logits/chosen": -2.2637524604797363,
+      "logits/rejected": -1.9729868173599243,
+      "logps/chosen": -562.5989379882812,
+      "logps/rejected": -662.3556518554688,
+      "loss": 0.1635,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.4096534848213196,
+      "rewards/margins": 0.10285242646932602,
+      "rewards/rejected": -0.5125058889389038,
+      "step": 2450
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.442348633408312e-06,
+      "logits/chosen": -1.9970605373382568,
+      "logits/rejected": -1.5610849857330322,
+      "logps/chosen": -653.9733276367188,
+      "logps/rejected": -783.4227294921875,
+      "loss": 0.1265,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.49916648864746094,
+      "rewards/margins": 0.1468081921339035,
+      "rewards/rejected": -0.6459746956825256,
+      "step": 2460
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.435739598642484e-06,
+      "logits/chosen": -2.2038464546203613,
+      "logits/rejected": -1.9102834463119507,
+      "logps/chosen": -621.3877563476562,
+      "logps/rejected": -669.69775390625,
+      "loss": 0.0698,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3649560511112213,
+      "rewards/margins": 0.0884665995836258,
+      "rewards/rejected": -0.4534226357936859,
+      "step": 2470
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.429096608565547e-06,
+      "logits/chosen": -1.9743452072143555,
+      "logits/rejected": -1.4776114225387573,
+      "logps/chosen": -577.5357055664062,
+      "logps/rejected": -675.8516845703125,
+      "loss": 0.1426,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.36466556787490845,
+      "rewards/margins": 0.14127275347709656,
+      "rewards/rejected": -0.5059383511543274,
+      "step": 2480
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.422419779703916e-06,
+      "logits/chosen": -2.4348769187927246,
+      "logits/rejected": -2.0393269062042236,
+      "logps/chosen": -465.75579833984375,
+      "logps/rejected": -540.5274658203125,
+      "loss": 0.1457,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.2878037095069885,
+      "rewards/margins": 0.10460519790649414,
+      "rewards/rejected": -0.39240890741348267,
+      "step": 2490
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.415709229177579e-06,
+      "logits/chosen": -2.3408713340759277,
+      "logits/rejected": -1.9532690048217773,
+      "logps/chosen": -458.122314453125,
+      "logps/rejected": -622.7427368164062,
+      "loss": 0.1622,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.2399168759584427,
+      "rewards/margins": 0.1462162733078003,
+      "rewards/rejected": -0.3861331641674042,
+      "step": 2500
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.408965074698048e-06,
+      "logits/chosen": -2.3715767860412598,
+      "logits/rejected": -1.9510250091552734,
+      "logps/chosen": -458.63055419921875,
+      "logps/rejected": -553.1646728515625,
+      "loss": 0.1645,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.257379949092865,
+      "rewards/margins": 0.12068530172109604,
+      "rewards/rejected": -0.37806522846221924,
+      "step": 2510
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.402187434566286e-06,
+      "logits/chosen": -2.0662636756896973,
+      "logits/rejected": -2.135746479034424,
+      "logps/chosen": -542.5437622070312,
+      "logps/rejected": -650.4852905273438,
+      "loss": 0.1612,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3462911546230316,
+      "rewards/margins": 0.09853918850421906,
+      "rewards/rejected": -0.4448303282260895,
+      "step": 2520
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.395376427670641e-06,
+      "logits/chosen": -2.0513243675231934,
+      "logits/rejected": -1.9635378122329712,
+      "logps/chosen": -655.1318359375,
+      "logps/rejected": -836.2184448242188,
+      "loss": 0.1038,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.41789698600769043,
+      "rewards/margins": 0.1554383784532547,
+      "rewards/rejected": -0.5733353495597839,
+      "step": 2530
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.388532173484754e-06,
+      "logits/chosen": -2.3079288005828857,
+      "logits/rejected": -1.702330231666565,
+      "logps/chosen": -500.818603515625,
+      "logps/rejected": -609.0797729492188,
+      "loss": 0.1116,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.2638341188430786,
+      "rewards/margins": 0.15045282244682312,
+      "rewards/rejected": -0.41428691148757935,
+      "step": 2540
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.381654792065464e-06,
+      "logits/chosen": -2.4457876682281494,
+      "logits/rejected": -1.8283277750015259,
+      "logps/chosen": -548.6522827148438,
+      "logps/rejected": -576.9744262695312,
+      "loss": 0.1617,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.2628456950187683,
+      "rewards/margins": 0.14158931374549866,
+      "rewards/rejected": -0.4044349789619446,
+      "step": 2550
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.374744404050706e-06,
+      "logits/chosen": -2.486788511276245,
+      "logits/rejected": -1.8276809453964233,
+      "logps/chosen": -449.22491455078125,
+      "logps/rejected": -640.1329345703125,
+      "loss": 0.1291,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.21849986910820007,
+      "rewards/margins": 0.20482178032398224,
+      "rewards/rejected": -0.4233216345310211,
+      "step": 2560
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.367801130657391e-06,
+      "logits/chosen": -2.3643078804016113,
+      "logits/rejected": -1.8557987213134766,
+      "logps/chosen": -602.333984375,
+      "logps/rejected": -656.6599731445312,
+      "loss": 0.1349,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.32483401894569397,
+      "rewards/margins": 0.11426816135644913,
+      "rewards/rejected": -0.4391021728515625,
+      "step": 2570
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.3608250936792816e-06,
+      "logits/chosen": -2.4594244956970215,
+      "logits/rejected": -2.0317907333374023,
+      "logps/chosen": -568.8768310546875,
+      "logps/rejected": -663.7427368164062,
+      "loss": 0.1487,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.3338078558444977,
+      "rewards/margins": 0.13653399050235748,
+      "rewards/rejected": -0.47034183144569397,
+      "step": 2580
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.353816415484853e-06,
+      "logits/chosen": -2.4827380180358887,
+      "logits/rejected": -1.9951280355453491,
+      "logps/chosen": -597.8345947265625,
+      "logps/rejected": -661.3265380859375,
+      "loss": 0.105,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3690278232097626,
+      "rewards/margins": 0.10864318907260895,
+      "rewards/rejected": -0.4776710569858551,
+      "step": 2590
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.346775219015152e-06,
+      "logits/chosen": -2.2886929512023926,
+      "logits/rejected": -1.88126540184021,
+      "logps/chosen": -648.250244140625,
+      "logps/rejected": -734.9500122070312,
+      "loss": 0.1671,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.40541452169418335,
+      "rewards/margins": 0.10185621678829193,
+      "rewards/rejected": -0.5072706937789917,
+      "step": 2600
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.339701627781633e-06,
+      "logits/chosen": -2.2254672050476074,
+      "logits/rejected": -1.9947378635406494,
+      "logps/chosen": -542.703369140625,
+      "logps/rejected": -632.771484375,
+      "loss": 0.0978,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3216991126537323,
+      "rewards/margins": 0.10727803409099579,
+      "rewards/rejected": -0.4289771616458893,
+      "step": 2610
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.332595765863998e-06,
+      "logits/chosen": -2.1905245780944824,
+      "logits/rejected": -1.7435497045516968,
+      "logps/chosen": -473.333251953125,
+      "logps/rejected": -572.7846069335938,
+      "loss": 0.1025,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3315790295600891,
+      "rewards/margins": 0.11216048896312714,
+      "rewards/rejected": -0.44373950362205505,
+      "step": 2620
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.325457757908016e-06,
+      "logits/chosen": -2.332416296005249,
+      "logits/rejected": -1.8026313781738281,
+      "logps/chosen": -539.1818237304688,
+      "logps/rejected": -628.17236328125,
+      "loss": 0.1351,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3181731104850769,
+      "rewards/margins": 0.12451257556676865,
+      "rewards/rejected": -0.44268566370010376,
+      "step": 2630
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.3182877291233395e-06,
+      "logits/chosen": -2.2804439067840576,
+      "logits/rejected": -1.6812254190444946,
+      "logps/chosen": -516.1134643554688,
+      "logps/rejected": -646.3065185546875,
+      "loss": 0.1515,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.3417758047580719,
+      "rewards/margins": 0.15962377190589905,
+      "rewards/rejected": -0.5013996362686157,
+      "step": 2640
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.311085805281306e-06,
+      "logits/chosen": -2.2838168144226074,
+      "logits/rejected": -1.9283697605133057,
+      "logps/chosen": -618.880126953125,
+      "logps/rejected": -693.0870971679688,
+      "loss": 0.1441,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.33808597922325134,
+      "rewards/margins": 0.11199036985635757,
+      "rewards/rejected": -0.4500764012336731,
+      "step": 2650
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.303852112712731e-06,
+      "logits/chosen": -2.393462657928467,
+      "logits/rejected": -1.87359619140625,
+      "logps/chosen": -624.3165283203125,
+      "logps/rejected": -701.8876953125,
+      "loss": 0.1013,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.372732549905777,
+      "rewards/margins": 0.1467903107404709,
+      "rewards/rejected": -0.5195228457450867,
+      "step": 2660
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.2965867783056965e-06,
+      "logits/chosen": -2.429837942123413,
+      "logits/rejected": -1.668766975402832,
+      "logps/chosen": -484.904296875,
+      "logps/rejected": -643.1995849609375,
+      "loss": 0.0779,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.2645077705383301,
+      "rewards/margins": 0.2067512720823288,
+      "rewards/rejected": -0.4712590277194977,
+      "step": 2670
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.289289929503319e-06,
+      "logits/chosen": -1.9798911809921265,
+      "logits/rejected": -2.018995761871338,
+      "logps/chosen": -633.0222778320312,
+      "logps/rejected": -773.7440795898438,
+      "loss": 0.0891,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.39304956793785095,
+      "rewards/margins": 0.11399887502193451,
+      "rewards/rejected": -0.507048487663269,
+      "step": 2680
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.28196169430152e-06,
+      "logits/chosen": -2.292060375213623,
+      "logits/rejected": -1.9822555780410767,
+      "logps/chosen": -453.82196044921875,
+      "logps/rejected": -592.3687744140625,
+      "loss": 0.1392,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.2621261179447174,
+      "rewards/margins": 0.16394677758216858,
+      "rewards/rejected": -0.426072895526886,
+      "step": 2690
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.274602201246775e-06,
+      "logits/chosen": -2.3844289779663086,
+      "logits/rejected": -2.15895414352417,
+      "logps/chosen": -552.2619018554688,
+      "logps/rejected": -700.951904296875,
+      "loss": 0.1204,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.3467998504638672,
+      "rewards/margins": 0.1456369012594223,
+      "rewards/rejected": -0.4924367070198059,
+      "step": 2700
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.267211579433865e-06,
+      "logits/chosen": -2.4319164752960205,
+      "logits/rejected": -1.991904854774475,
+      "logps/chosen": -503.736328125,
+      "logps/rejected": -686.3350830078125,
+      "loss": 0.129,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.2772873640060425,
+      "rewards/margins": 0.19375869631767273,
+      "rewards/rejected": -0.47104611992836,
+      "step": 2710
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.259789958503606e-06,
+      "logits/chosen": -2.0806498527526855,
+      "logits/rejected": -1.6907981634140015,
+      "logps/chosen": -723.2483520507812,
+      "logps/rejected": -817.8671264648438,
+      "loss": 0.1057,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.48365211486816406,
+      "rewards/margins": 0.12595288455486298,
+      "rewards/rejected": -0.6096049547195435,
+      "step": 2720
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.252337468640578e-06,
+      "logits/chosen": -2.1475765705108643,
+      "logits/rejected": -1.615252137184143,
+      "logps/chosen": -500.94012451171875,
+      "logps/rejected": -643.6009521484375,
+      "loss": 0.1486,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.3485940098762512,
+      "rewards/margins": 0.1678835153579712,
+      "rewards/rejected": -0.5164775252342224,
+      "step": 2730
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.244854240570844e-06,
+      "logits/chosen": -2.2018234729766846,
+      "logits/rejected": -1.8951069116592407,
+      "logps/chosen": -687.0391845703125,
+      "logps/rejected": -774.2957153320312,
+      "loss": 0.1638,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.46429556608200073,
+      "rewards/margins": 0.08806496858596802,
+      "rewards/rejected": -0.5523605346679688,
+      "step": 2740
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.237340405559648e-06,
+      "logits/chosen": -2.339503288269043,
+      "logits/rejected": -2.0241925716400146,
+      "logps/chosen": -646.4700927734375,
+      "logps/rejected": -749.6617431640625,
+      "loss": 0.1386,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.41365042328834534,
+      "rewards/margins": 0.14192424714565277,
+      "rewards/rejected": -0.5555745959281921,
+      "step": 2750
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.229796095409124e-06,
+      "logits/chosen": -2.402667284011841,
+      "logits/rejected": -2.0132956504821777,
+      "logps/chosen": -507.3562927246094,
+      "logps/rejected": -603.4364624023438,
+      "loss": 0.1634,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.30583763122558594,
+      "rewards/margins": 0.14279913902282715,
+      "rewards/rejected": -0.4486367702484131,
+      "step": 2760
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.222221442455975e-06,
+      "logits/chosen": -2.1386983394622803,
+      "logits/rejected": -1.9748146533966064,
+      "logps/chosen": -591.2689208984375,
+      "logps/rejected": -662.3856201171875,
+      "loss": 0.1371,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.3731406331062317,
+      "rewards/margins": 0.08443150669336319,
+      "rewards/rejected": -0.4575721323490143,
+      "step": 2770
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.2146165795691565e-06,
+      "logits/chosen": -2.4209771156311035,
+      "logits/rejected": -1.9494943618774414,
+      "logps/chosen": -545.9030151367188,
+      "logps/rejected": -571.9857788085938,
+      "loss": 0.1755,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.3263731598854065,
+      "rewards/margins": 0.07779963314533234,
+      "rewards/rejected": -0.4041728079319,
+      "step": 2780
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.206981640147543e-06,
+      "logits/chosen": -2.3957254886627197,
+      "logits/rejected": -1.9466619491577148,
+      "logps/chosen": -435.2061462402344,
+      "logps/rejected": -548.5030517578125,
+      "loss": 0.1443,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.24618563055992126,
+      "rewards/margins": 0.15140727162361145,
+      "rewards/rejected": -0.3975929319858551,
+      "step": 2790
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.199316758117592e-06,
+      "logits/chosen": -2.166459560394287,
+      "logits/rejected": -1.6141250133514404,
+      "logps/chosen": -446.7137145996094,
+      "logps/rejected": -546.3388671875,
+      "loss": 0.1684,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.26606470346450806,
+      "rewards/margins": 0.1279333531856537,
+      "rewards/rejected": -0.39399799704551697,
+      "step": 2800
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.191622067930987e-06,
+      "logits/chosen": -2.2459752559661865,
+      "logits/rejected": -1.888383150100708,
+      "logps/chosen": -629.9856567382812,
+      "logps/rejected": -765.0233154296875,
+      "loss": 0.0971,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.3976503312587738,
+      "rewards/margins": 0.14301955699920654,
+      "rewards/rejected": -0.5406699180603027,
+      "step": 2810
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.1838977045622884e-06,
+      "logits/chosen": -2.3816401958465576,
+      "logits/rejected": -2.141226291656494,
+      "logps/chosen": -595.0026245117188,
+      "logps/rejected": -665.93798828125,
+      "loss": 0.1644,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.35240158438682556,
+      "rewards/margins": 0.08603041619062424,
+      "rewards/rejected": -0.4384320378303528,
+      "step": 2820
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.1761438035065624e-06,
+      "logits/chosen": -2.346287250518799,
+      "logits/rejected": -1.902046799659729,
+      "logps/chosen": -461.8497009277344,
+      "logps/rejected": -574.8619384765625,
+      "loss": 0.1602,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.24480530619621277,
+      "rewards/margins": 0.13314317166805267,
+      "rewards/rejected": -0.37794849276542664,
+      "step": 2830
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.168360500777e-06,
+      "logits/chosen": -2.4022815227508545,
+      "logits/rejected": -2.202157497406006,
+      "logps/chosen": -516.7845458984375,
+      "logps/rejected": -601.9164428710938,
+      "loss": 0.1249,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.307770699262619,
+      "rewards/margins": 0.10829315334558487,
+      "rewards/rejected": -0.4160638749599457,
+      "step": 2840
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.160547932902536e-06,
+      "logits/chosen": -2.46279239654541,
+      "logits/rejected": -1.8549070358276367,
+      "logps/chosen": -515.3394775390625,
+      "logps/rejected": -598.8408203125,
+      "loss": 0.1673,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.2791273891925812,
+      "rewards/margins": 0.13737359642982483,
+      "rewards/rejected": -0.4165009558200836,
+      "step": 2850
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.152706236925453e-06,
+      "logits/chosen": -2.3299038410186768,
+      "logits/rejected": -1.846632719039917,
+      "logps/chosen": -516.0804443359375,
+      "logps/rejected": -610.2926025390625,
+      "loss": 0.1276,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.32791393995285034,
+      "rewards/margins": 0.1283039152622223,
+      "rewards/rejected": -0.4562179148197174,
+      "step": 2860
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.144835550398977e-06,
+      "logits/chosen": -2.4411633014678955,
+      "logits/rejected": -2.0878303050994873,
+      "logps/chosen": -513.5189208984375,
+      "logps/rejected": -575.8346557617188,
+      "loss": 0.1411,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.2800028920173645,
+      "rewards/margins": 0.11319861561059952,
+      "rewards/rejected": -0.39320144057273865,
+      "step": 2870
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.136936011384864e-06,
+      "logits/chosen": -2.304033041000366,
+      "logits/rejected": -1.8952850103378296,
+      "logps/chosen": -537.4888916015625,
+      "logps/rejected": -581.5336303710938,
+      "loss": 0.1117,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.3097744584083557,
+      "rewards/margins": 0.08337071537971497,
+      "rewards/rejected": -0.39314520359039307,
+      "step": 2880
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.129007758450982e-06,
+      "logits/chosen": -2.2374026775360107,
+      "logits/rejected": -1.661385178565979,
+      "logps/chosen": -566.9647216796875,
+      "logps/rejected": -679.7523193359375,
+      "loss": 0.133,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.36568838357925415,
+      "rewards/margins": 0.15959905087947845,
+      "rewards/rejected": -0.525287389755249,
+      "step": 2890
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.121050930668871e-06,
+      "logits/chosen": -2.4135258197784424,
+      "logits/rejected": -2.161256790161133,
+      "logps/chosen": -491.43377685546875,
+      "logps/rejected": -549.2619018554688,
+      "loss": 0.1165,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.2968185544013977,
+      "rewards/margins": 0.07787463068962097,
+      "rewards/rejected": -0.37469321489334106,
+      "step": 2900
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.113065667611319e-06,
+      "logits/chosen": -2.4103140830993652,
+      "logits/rejected": -1.8133437633514404,
+      "logps/chosen": -599.7228393554688,
+      "logps/rejected": -698.2857666015625,
+      "loss": 0.1019,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.3744727075099945,
+      "rewards/margins": 0.15850642323493958,
+      "rewards/rejected": -0.5329791307449341,
+      "step": 2910
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.105052109349896e-06,
+      "logits/chosen": -2.2020657062530518,
+      "logits/rejected": -1.7776159048080444,
+      "logps/chosen": -602.0916748046875,
+      "logps/rejected": -672.6879272460938,
+      "loss": 0.1687,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.41790199279785156,
+      "rewards/margins": 0.10488717257976532,
+      "rewards/rejected": -0.5227892398834229,
+      "step": 2920
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.097010396452511e-06,
+      "logits/chosen": -1.9920505285263062,
+      "logits/rejected": -1.8179048299789429,
+      "logps/chosen": -642.3668823242188,
+      "logps/rejected": -779.47314453125,
+      "loss": 0.1083,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.47601833939552307,
+      "rewards/margins": 0.13789525628089905,
+      "rewards/rejected": -0.6139136552810669,
+      "step": 2930
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.088940669980936e-06,
+      "logits/chosen": -2.103344202041626,
+      "logits/rejected": -1.5821688175201416,
+      "logps/chosen": -658.018798828125,
+      "logps/rejected": -812.3699340820312,
+      "loss": 0.1833,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.483877569437027,
+      "rewards/margins": 0.17431114614009857,
+      "rewards/rejected": -0.6581886410713196,
+      "step": 2940
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.080843071488343e-06,
+      "logits/chosen": -2.0326786041259766,
+      "logits/rejected": -1.7171694040298462,
+      "logps/chosen": -767.70654296875,
+      "logps/rejected": -804.4965209960938,
+      "loss": 0.118,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.5029066205024719,
+      "rewards/margins": 0.07210102677345276,
+      "rewards/rejected": -0.5750076174736023,
+      "step": 2950
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.072717743016807e-06,
+      "logits/chosen": -2.176539421081543,
+      "logits/rejected": -2.028989315032959,
+      "logps/chosen": -619.164306640625,
+      "logps/rejected": -758.9608154296875,
+      "loss": 0.1328,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.4217703938484192,
+      "rewards/margins": 0.12057630717754364,
+      "rewards/rejected": -0.5423466563224792,
+      "step": 2960
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.064564827094827e-06,
+      "logits/chosen": -2.3864095211029053,
+      "logits/rejected": -2.0271217823028564,
+      "logps/chosen": -544.8698120117188,
+      "logps/rejected": -676.0510864257812,
+      "loss": 0.0999,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3405817151069641,
+      "rewards/margins": 0.1475837379693985,
+      "rewards/rejected": -0.4881654381752014,
+      "step": 2970
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.056384466734819e-06,
+      "logits/chosen": -1.9559199810028076,
+      "logits/rejected": -1.4649156332015991,
+      "logps/chosen": -629.2734985351562,
+      "logps/rejected": -724.0560302734375,
+      "loss": 0.153,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.4238099157810211,
+      "rewards/margins": 0.1426152139902115,
+      "rewards/rejected": -0.5664251446723938,
+      "step": 2980
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.048176805430608e-06,
+      "logits/chosen": -2.014777421951294,
+      "logits/rejected": -1.88828444480896,
+      "logps/chosen": -690.5521240234375,
+      "logps/rejected": -770.0137939453125,
+      "loss": 0.1625,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.47654733061790466,
+      "rewards/margins": 0.10130522400140762,
+      "rewards/rejected": -0.5778525471687317,
+      "step": 2990
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.039941987154913e-06,
+      "logits/chosen": -2.303227424621582,
+      "logits/rejected": -1.641969919204712,
+      "logps/chosen": -582.7745971679688,
+      "logps/rejected": -706.5143432617188,
+      "loss": 0.1341,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3689119517803192,
+      "rewards/margins": 0.18556420505046844,
+      "rewards/rejected": -0.5544760823249817,
+      "step": 3000
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.031680156356822e-06,
+      "logits/chosen": -2.340054988861084,
+      "logits/rejected": -1.765934944152832,
+      "logps/chosen": -652.1713256835938,
+      "logps/rejected": -786.2620849609375,
+      "loss": 0.0971,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.39940738677978516,
+      "rewards/margins": 0.170322984457016,
+      "rewards/rejected": -0.5697304010391235,
+      "step": 3010
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.023391457959253e-06,
+      "logits/chosen": -2.143876791000366,
+      "logits/rejected": -1.6205317974090576,
+      "logps/chosen": -543.5670776367188,
+      "logps/rejected": -658.7613525390625,
+      "loss": 0.1384,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.35222360491752625,
+      "rewards/margins": 0.14551807940006256,
+      "rewards/rejected": -0.49774169921875,
+      "step": 3020
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.015076037356419e-06,
+      "logits/chosen": -1.9576947689056396,
+      "logits/rejected": -1.616519570350647,
+      "logps/chosen": -648.9378051757812,
+      "logps/rejected": -696.1199951171875,
+      "loss": 0.1978,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.43091464042663574,
+      "rewards/margins": 0.07831801474094391,
+      "rewards/rejected": -0.5092326402664185,
+      "step": 3030
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.006734040411272e-06,
+      "logits/chosen": -2.0965278148651123,
+      "logits/rejected": -1.639369010925293,
+      "logps/chosen": -622.4537353515625,
+      "logps/rejected": -693.1015625,
+      "loss": 0.1978,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.43814611434936523,
+      "rewards/margins": 0.11496999114751816,
+      "rewards/rejected": -0.5531162023544312,
+      "step": 3040
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.998365613452947e-06,
+      "logits/chosen": -2.098613739013672,
+      "logits/rejected": -2.035844564437866,
+      "logps/chosen": -491.0465393066406,
+      "logps/rejected": -628.3836669921875,
+      "loss": 0.1286,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.3210960626602173,
+      "rewards/margins": 0.09204693138599396,
+      "rewards/rejected": -0.41314297914505005,
+      "step": 3050
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.9899709032741955e-06,
+      "logits/chosen": -2.441481113433838,
+      "logits/rejected": -1.9549480676651,
+      "logps/chosen": -504.31103515625,
+      "logps/rejected": -644.4075927734375,
+      "loss": 0.1419,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.3220273554325104,
+      "rewards/margins": 0.1593136340379715,
+      "rewards/rejected": -0.4813409745693207,
+      "step": 3060
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.981550057128809e-06,
+      "logits/chosen": -2.4221749305725098,
+      "logits/rejected": -1.8501135110855103,
+      "logps/chosen": -522.7767333984375,
+      "logps/rejected": -610.2904052734375,
+      "loss": 0.0917,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.30943506956100464,
+      "rewards/margins": 0.14679715037345886,
+      "rewards/rejected": -0.4562322497367859,
+      "step": 3070
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.973103222729037e-06,
+      "logits/chosen": -2.311864137649536,
+      "logits/rejected": -2.161663055419922,
+      "logps/chosen": -518.4806518554688,
+      "logps/rejected": -619.0294799804688,
+      "loss": 0.176,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.31779712438583374,
+      "rewards/margins": 0.10514382272958755,
+      "rewards/rejected": -0.4229409694671631,
+      "step": 3080
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.964630548242997e-06,
+      "logits/chosen": -2.0899147987365723,
+      "logits/rejected": -1.6714637279510498,
+      "logps/chosen": -476.2728576660156,
+      "logps/rejected": -585.7185668945312,
+      "loss": 0.1478,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.27700093388557434,
+      "rewards/margins": 0.15577425062656403,
+      "rewards/rejected": -0.43277519941329956,
+      "step": 3090
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.956132182292071e-06,
+      "logits/chosen": -2.242954969406128,
+      "logits/rejected": -1.90801203250885,
+      "logps/chosen": -664.0401611328125,
+      "logps/rejected": -780.6242065429688,
+      "loss": 0.0914,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.4059290289878845,
+      "rewards/margins": 0.1536424607038498,
+      "rewards/rejected": -0.5595714449882507,
+      "step": 3100
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.947608273948305e-06,
+      "logits/chosen": -2.2666306495666504,
+      "logits/rejected": -2.0380451679229736,
+      "logps/chosen": -501.05926513671875,
+      "logps/rejected": -597.31494140625,
+      "loss": 0.1327,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.3443632125854492,
+      "rewards/margins": 0.1188286542892456,
+      "rewards/rejected": -0.46319183707237244,
+      "step": 3110
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.939058972731788e-06,
+      "logits/chosen": -2.3747003078460693,
+      "logits/rejected": -2.1182050704956055,
+      "logps/chosen": -484.81732177734375,
+      "logps/rejected": -627.9736328125,
+      "loss": 0.16,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.337240070104599,
+      "rewards/margins": 0.1566658765077591,
+      "rewards/rejected": -0.4939059317111969,
+      "step": 3120
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.9304844286080356e-06,
+      "logits/chosen": -2.1962616443634033,
+      "logits/rejected": -1.771802544593811,
+      "logps/chosen": -595.945556640625,
+      "logps/rejected": -673.62744140625,
+      "loss": 0.101,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3727676570415497,
+      "rewards/margins": 0.11937081813812256,
+      "rewards/rejected": -0.49213847517967224,
+      "step": 3130
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.921884791985351e-06,
+      "logits/chosen": -2.307659864425659,
+      "logits/rejected": -1.8763517141342163,
+      "logps/chosen": -645.28369140625,
+      "logps/rejected": -755.6948852539062,
+      "loss": 0.1358,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.39711305499076843,
+      "rewards/margins": 0.1316022276878357,
+      "rewards/rejected": -0.5287152528762817,
+      "step": 3140
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.913260213712195e-06,
+      "logits/chosen": -2.1602864265441895,
+      "logits/rejected": -1.7082303762435913,
+      "logps/chosen": -691.4942626953125,
+      "logps/rejected": -803.3652954101562,
+      "loss": 0.1674,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.4596913456916809,
+      "rewards/margins": 0.12983541190624237,
+      "rewards/rejected": -0.5895268321037292,
+      "step": 3150
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.9046108450745365e-06,
+      "logits/chosen": -2.079827308654785,
+      "logits/rejected": -1.7302945852279663,
+      "logps/chosen": -688.3924560546875,
+      "logps/rejected": -738.8493041992188,
+      "loss": 0.1812,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.4927944242954254,
+      "rewards/margins": 0.08438103646039963,
+      "rewards/rejected": -0.5771754384040833,
+      "step": 3160
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.895936837793195e-06,
+      "logits/chosen": -2.296090602874756,
+      "logits/rejected": -2.1320481300354004,
+      "logps/chosen": -624.8693237304688,
+      "logps/rejected": -718.9822998046875,
+      "loss": 0.1027,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.3919276297092438,
+      "rewards/margins": 0.09653235226869583,
+      "rewards/rejected": -0.488459974527359,
+      "step": 3170
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.887238344021187e-06,
+      "logits/chosen": -2.236269474029541,
+      "logits/rejected": -1.7453107833862305,
+      "logps/chosen": -560.397705078125,
+      "logps/rejected": -756.7366943359375,
+      "loss": 0.1343,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3717997968196869,
+      "rewards/margins": 0.22282366454601288,
+      "rewards/rejected": -0.5946235060691833,
+      "step": 3180
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.878515516341051e-06,
+      "logits/chosen": -2.197040557861328,
+      "logits/rejected": -1.8816732168197632,
+      "logps/chosen": -639.8453369140625,
+      "logps/rejected": -794.7584228515625,
+      "loss": 0.1464,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.36621785163879395,
+      "rewards/margins": 0.1651550829410553,
+      "rewards/rejected": -0.5313729047775269,
+      "step": 3190
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.869768507762174e-06,
+      "logits/chosen": -2.1581335067749023,
+      "logits/rejected": -1.7568200826644897,
+      "logps/chosen": -650.58984375,
+      "logps/rejected": -715.04150390625,
+      "loss": 0.122,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.4995553493499756,
+      "rewards/margins": 0.09361995756626129,
+      "rewards/rejected": -0.5931754112243652,
+      "step": 3200
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.860997471718103e-06,
+      "logits/chosen": -2.468562126159668,
+      "logits/rejected": -1.6961658000946045,
+      "logps/chosen": -568.7840576171875,
+      "logps/rejected": -668.2062377929688,
+      "loss": 0.1735,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.35263791680336,
+      "rewards/margins": 0.17861270904541016,
+      "rewards/rejected": -0.5312505960464478,
+      "step": 3210
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.852202562063861e-06,
+      "logits/chosen": -2.2896485328674316,
+      "logits/rejected": -1.9326906204223633,
+      "logps/chosen": -612.377685546875,
+      "logps/rejected": -653.1135864257812,
+      "loss": 0.1128,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.3872022330760956,
+      "rewards/margins": 0.07518110424280167,
+      "rewards/rejected": -0.46238332986831665,
+      "step": 3220
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.843383933073243e-06,
+      "logits/chosen": -2.2557928562164307,
+      "logits/rejected": -1.7827682495117188,
+      "logps/chosen": -551.4180908203125,
+      "logps/rejected": -673.4695434570312,
+      "loss": 0.1063,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.3300485610961914,
+      "rewards/margins": 0.14839136600494385,
+      "rewards/rejected": -0.47843995690345764,
+      "step": 3230
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.834541739436111e-06,
+      "logits/chosen": -2.2872281074523926,
+      "logits/rejected": -1.9268043041229248,
+      "logps/chosen": -470.2852478027344,
+      "logps/rejected": -588.62841796875,
+      "loss": 0.2102,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.2869749367237091,
+      "rewards/margins": 0.13786783814430237,
+      "rewards/rejected": -0.4248427748680115,
+      "step": 3240
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.82567613625568e-06,
+      "logits/chosen": -2.439807176589966,
+      "logits/rejected": -2.346358060836792,
+      "logps/chosen": -529.2238159179688,
+      "logps/rejected": -608.7293090820312,
+      "loss": 0.1016,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.25156983733177185,
+      "rewards/margins": 0.08280692249536514,
+      "rewards/rejected": -0.33437681198120117,
+      "step": 3250
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.816787279045796e-06,
+      "logits/chosen": -2.176093816757202,
+      "logits/rejected": -1.813281774520874,
+      "logps/chosen": -390.67327880859375,
+      "logps/rejected": -536.8662109375,
+      "loss": 0.1261,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.23340579867362976,
+      "rewards/margins": 0.1577015221118927,
+      "rewards/rejected": -0.39110735058784485,
+      "step": 3260
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.807875323728216e-06,
+      "logits/chosen": -2.5793073177337646,
+      "logits/rejected": -2.0290324687957764,
+      "logps/chosen": -425.0152282714844,
+      "logps/rejected": -606.7701416015625,
+      "loss": 0.1412,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.23049192130565643,
+      "rewards/margins": 0.20440904796123505,
+      "rewards/rejected": -0.43490099906921387,
+      "step": 3270
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.7989404266298614e-06,
+      "logits/chosen": -2.2553319931030273,
+      "logits/rejected": -2.1567931175231934,
+      "logps/chosen": -516.1847534179688,
+      "logps/rejected": -646.5059204101562,
+      "loss": 0.1402,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3488321602344513,
+      "rewards/margins": 0.13486294448375702,
+      "rewards/rejected": -0.4836950898170471,
+      "step": 3280
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.7899827444800824e-06,
+      "logits/chosen": -2.341078519821167,
+      "logits/rejected": -2.0495963096618652,
+      "logps/chosen": -604.7028198242188,
+      "logps/rejected": -710.20361328125,
+      "loss": 0.1285,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3399490714073181,
+      "rewards/margins": 0.09981698542833328,
+      "rewards/rejected": -0.4397660195827484,
+      "step": 3290
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7810024344079087e-06,
+      "logits/chosen": -2.299022674560547,
+      "logits/rejected": -1.9393131732940674,
+      "logps/chosen": -586.1444091796875,
+      "logps/rejected": -737.9298095703125,
+      "loss": 0.123,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3641575872898102,
+      "rewards/margins": 0.15938778221607208,
+      "rewards/rejected": -0.5235453844070435,
+      "step": 3300
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7719996539392934e-06,
+      "logits/chosen": -2.3596556186676025,
+      "logits/rejected": -2.1719517707824707,
+      "logps/chosen": -577.60693359375,
+      "logps/rejected": -666.4290771484375,
+      "loss": 0.1613,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.35797300934791565,
+      "rewards/margins": 0.11917600780725479,
+      "rewards/rejected": -0.47714900970458984,
+      "step": 3310
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7629745609943454e-06,
+      "logits/chosen": -2.1949996948242188,
+      "logits/rejected": -1.904266595840454,
+      "logps/chosen": -559.0316162109375,
+      "logps/rejected": -681.52197265625,
+      "loss": 0.2083,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.38445940613746643,
+      "rewards/margins": 0.11225737631320953,
+      "rewards/rejected": -0.4967167377471924,
+      "step": 3320
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7539273138845646e-06,
+      "logits/chosen": -2.252361297607422,
+      "logits/rejected": -1.9683983325958252,
+      "logps/chosen": -642.1447143554688,
+      "logps/rejected": -769.8976440429688,
+      "loss": 0.1144,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.4322238564491272,
+      "rewards/margins": 0.12790094316005707,
+      "rewards/rejected": -0.5601248145103455,
+      "step": 3330
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.744858071310063e-06,
+      "logits/chosen": -2.132174491882324,
+      "logits/rejected": -1.724561333656311,
+      "logps/chosen": -614.332763671875,
+      "logps/rejected": -723.3372802734375,
+      "loss": 0.1948,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.44415584206581116,
+      "rewards/margins": 0.12096796184778214,
+      "rewards/rejected": -0.5651237368583679,
+      "step": 3340
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7357669923567796e-06,
+      "logits/chosen": -2.4439380168914795,
+      "logits/rejected": -1.8819854259490967,
+      "logps/chosen": -657.61474609375,
+      "logps/rejected": -778.1177368164062,
+      "loss": 0.1178,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.4067533612251282,
+      "rewards/margins": 0.16185703873634338,
+      "rewards/rejected": -0.5686103701591492,
+      "step": 3350
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.726654236493693e-06,
+      "logits/chosen": -2.0641698837280273,
+      "logits/rejected": -1.5317274332046509,
+      "logps/chosen": -585.912353515625,
+      "logps/rejected": -715.9376831054688,
+      "loss": 0.1214,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.41456523537635803,
+      "rewards/margins": 0.16445979475975037,
+      "rewards/rejected": -0.5790249109268188,
+      "step": 3360
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.71751996357002e-06,
+      "logits/chosen": -2.3743832111358643,
+      "logits/rejected": -1.7962068319320679,
+      "logps/chosen": -559.3345947265625,
+      "logps/rejected": -671.3294677734375,
+      "loss": 0.1027,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.35539510846138,
+      "rewards/margins": 0.12307320535182953,
+      "rewards/rejected": -0.4784683287143707,
+      "step": 3370
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.7083643338124148e-06,
+      "logits/chosen": -2.2380123138427734,
+      "logits/rejected": -1.6683757305145264,
+      "logps/chosen": -602.3743896484375,
+      "logps/rejected": -748.6544799804688,
+      "loss": 0.1391,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.4354441165924072,
+      "rewards/margins": 0.16299614310264587,
+      "rewards/rejected": -0.5984402298927307,
+      "step": 3380
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6991875078221566e-06,
+      "logits/chosen": -2.258378505706787,
+      "logits/rejected": -1.7123279571533203,
+      "logps/chosen": -690.5413208007812,
+      "logps/rejected": -832.8619995117188,
+      "loss": 0.094,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.4548878073692322,
+      "rewards/margins": 0.1940111219882965,
+      "rewards/rejected": -0.6488989591598511,
+      "step": 3390
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6899896465723352e-06,
+      "logits/chosen": -2.2446978092193604,
+      "logits/rejected": -1.775933861732483,
+      "logps/chosen": -554.7044677734375,
+      "logps/rejected": -615.2047119140625,
+      "loss": 0.1147,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.37552911043167114,
+      "rewards/margins": 0.12343206256628036,
+      "rewards/rejected": -0.4989612102508545,
+      "step": 3400
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6807709114050224e-06,
+      "logits/chosen": -2.1251072883605957,
+      "logits/rejected": -1.9448951482772827,
+      "logps/chosen": -752.5145263671875,
+      "logps/rejected": -832.1990356445312,
+      "loss": 0.1535,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.5412966012954712,
+      "rewards/margins": 0.05970805138349533,
+      "rewards/rejected": -0.6010046005249023,
+      "step": 3410
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6715314640284465e-06,
+      "logits/chosen": -2.2015316486358643,
+      "logits/rejected": -1.6253087520599365,
+      "logps/chosen": -735.8681030273438,
+      "logps/rejected": -901.7691650390625,
+      "loss": 0.132,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.5181323289871216,
+      "rewards/margins": 0.18187302350997925,
+      "rewards/rejected": -0.7000053524971008,
+      "step": 3420
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6622714665141555e-06,
+      "logits/chosen": -2.118595600128174,
+      "logits/rejected": -1.8773285150527954,
+      "logps/chosen": -594.6820068359375,
+      "logps/rejected": -723.7277221679688,
+      "loss": 0.1788,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.40029376745224,
+      "rewards/margins": 0.12403945624828339,
+      "rewards/rejected": -0.5243332386016846,
+      "step": 3430
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6529910812941688e-06,
+      "logits/chosen": -2.307635545730591,
+      "logits/rejected": -1.765216588973999,
+      "logps/chosen": -611.1375732421875,
+      "logps/rejected": -734.8345947265625,
+      "loss": 0.0924,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.3667663037776947,
+      "rewards/margins": 0.15591205656528473,
+      "rewards/rejected": -0.5226783156394958,
+      "step": 3440
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6436904711581358e-06,
+      "logits/chosen": -2.0896542072296143,
+      "logits/rejected": -1.7076025009155273,
+      "logps/chosen": -463.38458251953125,
+      "logps/rejected": -604.485107421875,
+      "loss": 0.1016,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.25780436396598816,
+      "rewards/margins": 0.1584676206111908,
+      "rewards/rejected": -0.41627198457717896,
+      "step": 3450
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.6343697992504745e-06,
+      "logits/chosen": -2.1812429428100586,
+      "logits/rejected": -1.7760646343231201,
+      "logps/chosen": -491.634033203125,
+      "logps/rejected": -603.2550659179688,
+      "loss": 0.1358,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.29424309730529785,
+      "rewards/margins": 0.14275255799293518,
+      "rewards/rejected": -0.43699565529823303,
+      "step": 3460
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.6250292290675103e-06,
+      "logits/chosen": -2.193786144256592,
+      "logits/rejected": -2.0568480491638184,
+      "logps/chosen": -541.5204467773438,
+      "logps/rejected": -563.2637939453125,
+      "loss": 0.1152,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.32292261719703674,
+      "rewards/margins": 0.052775122225284576,
+      "rewards/rejected": -0.37569770216941833,
+      "step": 3470
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.6156689244546135e-06,
+      "logits/chosen": -2.3388359546661377,
+      "logits/rejected": -1.9526780843734741,
+      "logps/chosen": -525.0833129882812,
+      "logps/rejected": -639.524658203125,
+      "loss": 0.1314,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.2768683433532715,
+      "rewards/margins": 0.12221084535121918,
+      "rewards/rejected": -0.39907920360565186,
+      "step": 3480
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.606289049603317e-06,
+      "logits/chosen": -2.326199531555176,
+      "logits/rejected": -1.9685226678848267,
+      "logps/chosen": -495.380859375,
+      "logps/rejected": -594.8028564453125,
+      "loss": 0.2052,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.33412912487983704,
+      "rewards/margins": 0.06082766130566597,
+      "rewards/rejected": -0.3949567377567291,
+      "step": 3490
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.596889769048442e-06,
+      "logits/chosen": -2.288808822631836,
+      "logits/rejected": -2.1472086906433105,
+      "logps/chosen": -479.8212890625,
+      "logps/rejected": -604.1199951171875,
+      "loss": 0.1399,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.28404390811920166,
+      "rewards/margins": 0.12190987914800644,
+      "rewards/rejected": -0.4059537351131439,
+      "step": 3500
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.587471247665211e-06,
+      "logits/chosen": -2.1581714153289795,
+      "logits/rejected": -1.6125532388687134,
+      "logps/chosen": -553.27197265625,
+      "logps/rejected": -684.0994262695312,
+      "loss": 0.1347,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.3265748918056488,
+      "rewards/margins": 0.14639294147491455,
+      "rewards/rejected": -0.47296780347824097,
+      "step": 3510
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.578033650666354e-06,
+      "logits/chosen": -2.2027840614318848,
+      "logits/rejected": -1.961639404296875,
+      "logps/chosen": -555.2904052734375,
+      "logps/rejected": -654.9388427734375,
+      "loss": 0.1271,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.3394280672073364,
+      "rewards/margins": 0.11397820711135864,
+      "rewards/rejected": -0.45340627431869507,
+      "step": 3520
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.56857714359921e-06,
+      "logits/chosen": -2.23160982131958,
+      "logits/rejected": -1.6669390201568604,
+      "logps/chosen": -624.6026611328125,
+      "logps/rejected": -675.472412109375,
+      "loss": 0.1616,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.36295104026794434,
+      "rewards/margins": 0.11474791914224625,
+      "rewards/rejected": -0.4776989817619324,
+      "step": 3530
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.5591018923428273e-06,
+      "logits/chosen": -2.105347156524658,
+      "logits/rejected": -1.813043236732483,
+      "logps/chosen": -457.2730407714844,
+      "logps/rejected": -523.214599609375,
+      "loss": 0.143,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.27354902029037476,
+      "rewards/margins": 0.08861085772514343,
+      "rewards/rejected": -0.3621598780155182,
+      "step": 3540
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.5496080631050494e-06,
+      "logits/chosen": -2.30836820602417,
+      "logits/rejected": -1.9932880401611328,
+      "logps/chosen": -485.6265563964844,
+      "logps/rejected": -571.0599365234375,
+      "loss": 0.1723,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.28763633966445923,
+      "rewards/margins": 0.0949304848909378,
+      "rewards/rejected": -0.3825668692588806,
+      "step": 3550
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.5400958224196e-06,
+      "logits/chosen": -2.0381979942321777,
+      "logits/rejected": -1.9254405498504639,
+      "logps/chosen": -470.24346923828125,
+      "logps/rejected": -575.70849609375,
+      "loss": 0.1038,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.28994056582450867,
+      "rewards/margins": 0.09728164970874786,
+      "rewards/rejected": -0.3872222304344177,
+      "step": 3560
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.5305653371431635e-06,
+      "logits/chosen": -2.0645923614501953,
+      "logits/rejected": -1.7110099792480469,
+      "logps/chosen": -669.3499145507812,
+      "logps/rejected": -744.7472534179688,
+      "loss": 0.1418,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.4642150402069092,
+      "rewards/margins": 0.09605596959590912,
+      "rewards/rejected": -0.5602710247039795,
+      "step": 3570
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.52101677445246e-06,
+      "logits/chosen": -2.0632002353668213,
+      "logits/rejected": -1.7261145114898682,
+      "logps/chosen": -670.3865356445312,
+      "logps/rejected": -761.6419677734375,
+      "loss": 0.1149,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.43943580985069275,
+      "rewards/margins": 0.11705873161554337,
+      "rewards/rejected": -0.5564945936203003,
+      "step": 3580
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.5114503018413055e-06,
+      "logits/chosen": -2.2619094848632812,
+      "logits/rejected": -1.8394790887832642,
+      "logps/chosen": -538.2764892578125,
+      "logps/rejected": -605.486083984375,
+      "loss": 0.127,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.333726704120636,
+      "rewards/margins": 0.09332706779241562,
+      "rewards/rejected": -0.4270537793636322,
+      "step": 3590
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.5018660871176815e-06,
+      "logits/chosen": -2.316187620162964,
+      "logits/rejected": -1.7859646081924438,
+      "logps/chosen": -575.8388671875,
+      "logps/rejected": -600.463623046875,
+      "loss": 0.1161,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3067213296890259,
+      "rewards/margins": 0.0944039523601532,
+      "rewards/rejected": -0.4011252820491791,
+      "step": 3600
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.4922642984007888e-06,
+      "logits/chosen": -2.122157096862793,
+      "logits/rejected": -1.5312693119049072,
+      "logps/chosen": -574.7163696289062,
+      "logps/rejected": -682.40380859375,
+      "loss": 0.1841,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.33856090903282166,
+      "rewards/margins": 0.17644380033016205,
+      "rewards/rejected": -0.5150047540664673,
+      "step": 3610
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.4826451041180963e-06,
+      "logits/chosen": -2.1528148651123047,
+      "logits/rejected": -1.916869878768921,
+      "logps/chosen": -490.4339294433594,
+      "logps/rejected": -609.12744140625,
+      "loss": 0.1352,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3152255117893219,
+      "rewards/margins": 0.11505546420812607,
+      "rewards/rejected": -0.43028098344802856,
+      "step": 3620
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4730086730023904e-06,
+      "logits/chosen": -2.206726551055908,
+      "logits/rejected": -1.9309971332550049,
+      "logps/chosen": -471.94573974609375,
+      "logps/rejected": -559.0374755859375,
+      "loss": 0.1737,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.23736457526683807,
+      "rewards/margins": 0.11887004226446152,
+      "rewards/rejected": -0.356234610080719,
+      "step": 3630
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4633551740888122e-06,
+      "logits/chosen": -2.4420673847198486,
+      "logits/rejected": -1.59353768825531,
+      "logps/chosen": -585.2825927734375,
+      "logps/rejected": -699.0955200195312,
+      "loss": 0.0716,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.3031196594238281,
+      "rewards/margins": 0.19079551100730896,
+      "rewards/rejected": -0.4939151704311371,
+      "step": 3640
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4536847767118926e-06,
+      "logits/chosen": -2.2045276165008545,
+      "logits/rejected": -1.8025071620941162,
+      "logps/chosen": -556.0777587890625,
+      "logps/rejected": -635.7542114257812,
+      "loss": 0.1673,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.36056894063949585,
+      "rewards/margins": 0.11475469172000885,
+      "rewards/rejected": -0.4753236174583435,
+      "step": 3650
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.443997650502586e-06,
+      "logits/chosen": -2.0511951446533203,
+      "logits/rejected": -1.7067844867706299,
+      "logps/chosen": -643.9922485351562,
+      "logps/rejected": -691.350830078125,
+      "loss": 0.1503,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.4573690891265869,
+      "rewards/margins": 0.0998232364654541,
+      "rewards/rejected": -0.5571922659873962,
+      "step": 3660
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.434293965385287e-06,
+      "logits/chosen": -2.154787063598633,
+      "logits/rejected": -1.9054477214813232,
+      "logps/chosen": -541.8016967773438,
+      "logps/rejected": -594.554443359375,
+      "loss": 0.1459,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.31952032446861267,
+      "rewards/margins": 0.09629140049219131,
+      "rewards/rejected": -0.4158117175102234,
+      "step": 3670
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4245738915748584e-06,
+      "logits/chosen": -2.434523105621338,
+      "logits/rejected": -2.161329984664917,
+      "logps/chosen": -530.0040893554688,
+      "logps/rejected": -651.8059692382812,
+      "loss": 0.1307,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.30816537141799927,
+      "rewards/margins": 0.11906369775533676,
+      "rewards/rejected": -0.4272290766239166,
+      "step": 3680
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4148375995736395e-06,
+      "logits/chosen": -2.1189913749694824,
+      "logits/rejected": -1.6555522680282593,
+      "logps/chosen": -712.1950073242188,
+      "logps/rejected": -819.061767578125,
+      "loss": 0.1211,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.4779934883117676,
+      "rewards/margins": 0.15066465735435486,
+      "rewards/rejected": -0.6286581158638,
+      "step": 3690
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4050852601684563e-06,
+      "logits/chosen": -1.9637362957000732,
+      "logits/rejected": -1.4346816539764404,
+      "logps/chosen": -663.6091918945312,
+      "logps/rejected": -772.9646606445312,
+      "loss": 0.1794,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.4800654947757721,
+      "rewards/margins": 0.13248559832572937,
+      "rewards/rejected": -0.6125510931015015,
+      "step": 3700
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3953170444276283e-06,
+      "logits/chosen": -2.255424737930298,
+      "logits/rejected": -1.8064028024673462,
+      "logps/chosen": -652.1144409179688,
+      "logps/rejected": -751.6597900390625,
+      "loss": 0.1247,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.41208648681640625,
+      "rewards/margins": 0.13556751608848572,
+      "rewards/rejected": -0.5476540327072144,
+      "step": 3710
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.385533123697966e-06,
+      "logits/chosen": -1.9411197900772095,
+      "logits/rejected": -1.8104366064071655,
+      "logps/chosen": -576.5701904296875,
+      "logps/rejected": -744.0640258789062,
+      "loss": 0.1033,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.3834145665168762,
+      "rewards/margins": 0.15638017654418945,
+      "rewards/rejected": -0.5397947430610657,
+      "step": 3720
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.375733669601763e-06,
+      "logits/chosen": -2.228116273880005,
+      "logits/rejected": -1.8068163394927979,
+      "logps/chosen": -675.9473876953125,
+      "logps/rejected": -718.5662841796875,
+      "loss": 0.1199,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.43388399481773376,
+      "rewards/margins": 0.10030052810907364,
+      "rewards/rejected": -0.5341845750808716,
+      "step": 3730
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3659188540337884e-06,
+      "logits/chosen": -2.2735633850097656,
+      "logits/rejected": -1.9548370838165283,
+      "logps/chosen": -449.8229064941406,
+      "logps/rejected": -577.3477172851562,
+      "loss": 0.1156,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.25816214084625244,
+      "rewards/margins": 0.12060244381427765,
+      "rewards/rejected": -0.3787645697593689,
+      "step": 3740
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3560888491582736e-06,
+      "logits/chosen": -2.126878261566162,
+      "logits/rejected": -1.881369948387146,
+      "logps/chosen": -495.64825439453125,
+      "logps/rejected": -639.0670166015625,
+      "loss": 0.157,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.324956476688385,
+      "rewards/margins": 0.1283748298883438,
+      "rewards/rejected": -0.4533312916755676,
+      "step": 3750
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3462438274058856e-06,
+      "logits/chosen": -2.029367446899414,
+      "logits/rejected": -1.7525581121444702,
+      "logps/chosen": -564.4833984375,
+      "logps/rejected": -715.7012329101562,
+      "loss": 0.1174,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.34159308671951294,
+      "rewards/margins": 0.1452983021736145,
+      "rewards/rejected": -0.48689141869544983,
+      "step": 3760
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3363839614707094e-06,
+      "logits/chosen": -2.1096203327178955,
+      "logits/rejected": -1.9339758157730103,
+      "logps/chosen": -610.3280029296875,
+      "logps/rejected": -739.428955078125,
+      "loss": 0.1466,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3321211636066437,
+      "rewards/margins": 0.12011714279651642,
+      "rewards/rejected": -0.4522382616996765,
+      "step": 3770
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.326509424307214e-06,
+      "logits/chosen": -2.17301607131958,
+      "logits/rejected": -1.8869720697402954,
+      "logps/chosen": -576.1785888671875,
+      "logps/rejected": -704.5159912109375,
+      "loss": 0.1464,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.36916476488113403,
+      "rewards/margins": 0.15194693207740784,
+      "rewards/rejected": -0.521111786365509,
+      "step": 3780
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3166203891272204e-06,
+      "logits/chosen": -2.3157355785369873,
+      "logits/rejected": -1.9209871292114258,
+      "logps/chosen": -652.081787109375,
+      "logps/rejected": -775.4389038085938,
+      "loss": 0.13,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.36450713872909546,
+      "rewards/margins": 0.17267830669879913,
+      "rewards/rejected": -0.5371854901313782,
+      "step": 3790
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.306717029396863e-06,
+      "logits/chosen": -2.145477056503296,
+      "logits/rejected": -1.81096613407135,
+      "logps/chosen": -677.7953491210938,
+      "logps/rejected": -735.6583862304688,
+      "loss": 0.1616,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.42701902985572815,
+      "rewards/margins": 0.10124798119068146,
+      "rewards/rejected": -0.528266966342926,
+      "step": 3800
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.2967995188335487e-06,
+      "logits/chosen": -2.3144187927246094,
+      "logits/rejected": -2.0936036109924316,
+      "logps/chosen": -481.04296875,
+      "logps/rejected": -592.5134887695312,
+      "loss": 0.1517,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.32679641246795654,
+      "rewards/margins": 0.12168729305267334,
+      "rewards/rejected": -0.4484837055206299,
+      "step": 3810
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.2868680314029056e-06,
+      "logits/chosen": -2.321157693862915,
+      "logits/rejected": -1.9773813486099243,
+      "logps/chosen": -618.4539794921875,
+      "logps/rejected": -719.310302734375,
+      "loss": 0.1004,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.36911845207214355,
+      "rewards/margins": 0.1299140900373459,
+      "rewards/rejected": -0.49903255701065063,
+      "step": 3820
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.2769227413157346e-06,
+      "logits/chosen": -2.163858413696289,
+      "logits/rejected": -1.881376028060913,
+      "logps/chosen": -553.7333374023438,
+      "logps/rejected": -634.0310668945312,
+      "loss": 0.1819,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.31492236256599426,
+      "rewards/margins": 0.14754173159599304,
+      "rewards/rejected": -0.4624640941619873,
+      "step": 3830
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.266963823024951e-06,
+      "logits/chosen": -1.985674262046814,
+      "logits/rejected": -1.6714332103729248,
+      "logps/chosen": -554.2951049804688,
+      "logps/rejected": -664.5439453125,
+      "loss": 0.1266,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.36099833250045776,
+      "rewards/margins": 0.1292954832315445,
+      "rewards/rejected": -0.49029380083084106,
+      "step": 3840
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.2569914512225294e-06,
+      "logits/chosen": -2.6219682693481445,
+      "logits/rejected": -1.9104496240615845,
+      "logps/chosen": -544.3038940429688,
+      "logps/rejected": -625.7306518554688,
+      "loss": 0.2211,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.2861507534980774,
+      "rewards/margins": 0.15220731496810913,
+      "rewards/rejected": -0.4383581280708313,
+      "step": 3850
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.2470058008364335e-06,
+      "logits/chosen": -2.145233392715454,
+      "logits/rejected": -1.6807218790054321,
+      "logps/chosen": -651.1583862304688,
+      "logps/rejected": -762.2835693359375,
+      "loss": 0.0936,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.39982348680496216,
+      "rewards/margins": 0.14731648564338684,
+      "rewards/rejected": -0.5471399426460266,
+      "step": 3860
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.2370070470275493e-06,
+      "logits/chosen": -2.2440435886383057,
+      "logits/rejected": -1.8730751276016235,
+      "logps/chosen": -593.9110107421875,
+      "logps/rejected": -748.48681640625,
+      "loss": 0.1565,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.40593594312667847,
+      "rewards/margins": 0.14334239065647125,
+      "rewards/rejected": -0.5492783784866333,
+      "step": 3870
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.226995365186616e-06,
+      "logits/chosen": -2.154481887817383,
+      "logits/rejected": -1.7816855907440186,
+      "logps/chosen": -531.9981689453125,
+      "logps/rejected": -623.5601806640625,
+      "loss": 0.1249,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.3522031903266907,
+      "rewards/margins": 0.13410066068172455,
+      "rewards/rejected": -0.48630380630493164,
+      "step": 3880
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.216970930931144e-06,
+      "logits/chosen": -2.343778371810913,
+      "logits/rejected": -1.945330023765564,
+      "logps/chosen": -482.8518981933594,
+      "logps/rejected": -581.4893798828125,
+      "loss": 0.1361,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.29796892404556274,
+      "rewards/margins": 0.09733384847640991,
+      "rewards/rejected": -0.39530277252197266,
+      "step": 3890
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.2069339201023398e-06,
+      "logits/chosen": -2.281834602355957,
+      "logits/rejected": -2.2298338413238525,
+      "logps/chosen": -614.375,
+      "logps/rejected": -702.3538818359375,
+      "loss": 0.1208,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.3901093900203705,
+      "rewards/margins": 0.0884203165769577,
+      "rewards/rejected": -0.478529691696167,
+      "step": 3900
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.196884508762016e-06,
+      "logits/chosen": -2.1160051822662354,
+      "logits/rejected": -1.8963677883148193,
+      "logps/chosen": -612.60009765625,
+      "logps/rejected": -675.6456298828125,
+      "loss": 0.2207,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.40564996004104614,
+      "rewards/margins": 0.11190341413021088,
+      "rewards/rejected": -0.5175533890724182,
+      "step": 3910
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.186822873189508e-06,
+      "logits/chosen": -2.0276057720184326,
+      "logits/rejected": -1.6573140621185303,
+      "logps/chosen": -607.2412109375,
+      "logps/rejected": -751.7598266601562,
+      "loss": 0.1048,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.41555699706077576,
+      "rewards/margins": 0.16056308150291443,
+      "rewards/rejected": -0.576120138168335,
+      "step": 3920
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.1767491898785795e-06,
+      "logits/chosen": -2.361463785171509,
+      "logits/rejected": -1.906935453414917,
+      "logps/chosen": -552.9371337890625,
+      "logps/rejected": -592.3452758789062,
+      "loss": 0.1763,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.337443083524704,
+      "rewards/margins": 0.11611269414424896,
+      "rewards/rejected": -0.45355576276779175,
+      "step": 3930
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.166663635534325e-06,
+      "logits/chosen": -2.203479051589966,
+      "logits/rejected": -2.042908191680908,
+      "logps/chosen": -536.9729614257812,
+      "logps/rejected": -639.6466064453125,
+      "loss": 0.0704,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.3332824110984802,
+      "rewards/margins": 0.08808855712413788,
+      "rewards/rejected": -0.4213709235191345,
+      "step": 3940
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.1565663870700735e-06,
+      "logits/chosen": -2.0548367500305176,
+      "logits/rejected": -1.8372876644134521,
+      "logps/chosen": -609.2579345703125,
+      "logps/rejected": -734.8900146484375,
+      "loss": 0.1336,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.41414204239845276,
+      "rewards/margins": 0.11156319081783295,
+      "rewards/rejected": -0.5257052183151245,
+      "step": 3950
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.1464576216042832e-06,
+      "logits/chosen": -2.280349016189575,
+      "logits/rejected": -1.8611831665039062,
+      "logps/chosen": -621.3673706054688,
+      "logps/rejected": -714.0742797851562,
+      "loss": 0.1067,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.3787679970264435,
+      "rewards/margins": 0.1551198959350586,
+      "rewards/rejected": -0.5338879227638245,
+      "step": 3960
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.1363375164574343e-06,
+      "logits/chosen": -2.3183341026306152,
+      "logits/rejected": -2.0496323108673096,
+      "logps/chosen": -500.1532287597656,
+      "logps/rejected": -573.7332763671875,
+      "loss": 0.1816,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3042662441730499,
+      "rewards/margins": 0.08853505551815033,
+      "rewards/rejected": -0.3928012549877167,
+      "step": 3970
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.126206249148921e-06,
+      "logits/chosen": -2.132477283477783,
+      "logits/rejected": -1.8276363611221313,
+      "logps/chosen": -572.4835815429688,
+      "logps/rejected": -736.7272338867188,
+      "loss": 0.1048,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3354787826538086,
+      "rewards/margins": 0.1652851551771164,
+      "rewards/rejected": -0.5007639527320862,
+      "step": 3980
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.1160639973939337e-06,
+      "logits/chosen": -2.477264404296875,
+      "logits/rejected": -2.1663661003112793,
+      "logps/chosen": -467.11767578125,
+      "logps/rejected": -523.8724365234375,
+      "loss": 0.1934,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.21022024750709534,
+      "rewards/margins": 0.08616986125707626,
+      "rewards/rejected": -0.2963901162147522,
+      "step": 3990
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.105910939100345e-06,
+      "logits/chosen": -2.548734426498413,
+      "logits/rejected": -2.0472655296325684,
+      "logps/chosen": -482.7063903808594,
+      "logps/rejected": -667.7801513671875,
+      "loss": 0.1321,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.23916666209697723,
+      "rewards/margins": 0.20540814101696014,
+      "rewards/rejected": -0.44457483291625977,
+      "step": 4000
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.095747252365588e-06,
+      "logits/chosen": -2.2769620418548584,
+      "logits/rejected": -1.908728003501892,
+      "logps/chosen": -463.02801513671875,
+      "logps/rejected": -556.0867919921875,
+      "loss": 0.1297,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.23721089959144592,
+      "rewards/margins": 0.1076841801404953,
+      "rewards/rejected": -0.34489503502845764,
+      "step": 4010
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.0855731154735326e-06,
+      "logits/chosen": -2.043519973754883,
+      "logits/rejected": -1.7572590112686157,
+      "logps/chosen": -509.5525817871094,
+      "logps/rejected": -650.0911865234375,
+      "loss": 0.1972,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.3402670621871948,
+      "rewards/margins": 0.15102212131023407,
+      "rewards/rejected": -0.4912891983985901,
+      "step": 4020
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.0753887068913545e-06,
+      "logits/chosen": -2.2707228660583496,
+      "logits/rejected": -1.912559151649475,
+      "logps/chosen": -500.855224609375,
+      "logps/rejected": -574.0370483398438,
+      "loss": 0.1803,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.2886977791786194,
+      "rewards/margins": 0.0991811752319336,
+      "rewards/rejected": -0.387878954410553,
+      "step": 4030
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.0651942052664117e-06,
+      "logits/chosen": -2.0734620094299316,
+      "logits/rejected": -1.6979328393936157,
+      "logps/chosen": -594.78271484375,
+      "logps/rejected": -680.763671875,
+      "loss": 0.1438,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.35500848293304443,
+      "rewards/margins": 0.1231137067079544,
+      "rewards/rejected": -0.47812214493751526,
+      "step": 4040
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.0549897894231058e-06,
+      "logits/chosen": -2.239816665649414,
+      "logits/rejected": -1.9973691701889038,
+      "logps/chosen": -625.9370727539062,
+      "logps/rejected": -675.5764770507812,
+      "loss": 0.0963,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3629154860973358,
+      "rewards/margins": 0.0827527791261673,
+      "rewards/rejected": -0.4456682801246643,
+      "step": 4050
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.0447756383597438e-06,
+      "logits/chosen": -2.213779926300049,
+      "logits/rejected": -1.6821537017822266,
+      "logps/chosen": -548.8224487304688,
+      "logps/rejected": -622.7188720703125,
+      "loss": 0.1718,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.3775545656681061,
+      "rewards/margins": 0.12418852746486664,
+      "rewards/rejected": -0.5017431378364563,
+      "step": 4060
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.034551931245404e-06,
+      "logits/chosen": -2.2327404022216797,
+      "logits/rejected": -1.6847496032714844,
+      "logps/chosen": -707.8839721679688,
+      "logps/rejected": -742.4009399414062,
+      "loss": 0.1234,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.4029286503791809,
+      "rewards/margins": 0.1285533607006073,
+      "rewards/rejected": -0.5314820408821106,
+      "step": 4070
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.0243188474167884e-06,
+      "logits/chosen": -2.2276296615600586,
+      "logits/rejected": -1.778869867324829,
+      "logps/chosen": -495.7210998535156,
+      "logps/rejected": -618.3314819335938,
+      "loss": 0.1643,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.28643786907196045,
+      "rewards/margins": 0.15960052609443665,
+      "rewards/rejected": -0.4460384249687195,
+      "step": 4080
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.014076566375078e-06,
+      "logits/chosen": -2.287414789199829,
+      "logits/rejected": -2.0237205028533936,
+      "logps/chosen": -618.1912841796875,
+      "logps/rejected": -643.1911010742188,
+      "loss": 0.1821,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.36938709020614624,
+      "rewards/margins": 0.07493145018815994,
+      "rewards/rejected": -0.44431859254837036,
+      "step": 4090
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.003825267782785e-06,
+      "logits/chosen": -2.4438891410827637,
+      "logits/rejected": -2.005458354949951,
+      "logps/chosen": -509.8524475097656,
+      "logps/rejected": -653.6589965820312,
+      "loss": 0.0887,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.32322680950164795,
+      "rewards/margins": 0.17616334557533264,
+      "rewards/rejected": -0.4993901252746582,
+      "step": 4100
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 2.993565131460602e-06,
+      "logits/chosen": -2.1488864421844482,
+      "logits/rejected": -1.8533220291137695,
+      "logps/chosen": -524.0560913085938,
+      "logps/rejected": -621.2794189453125,
+      "loss": 0.1332,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3051479756832123,
+      "rewards/margins": 0.11268274486064911,
+      "rewards/rejected": -0.41783076524734497,
+      "step": 4110
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 2.9832963373842434e-06,
+      "logits/chosen": -2.1603903770446777,
+      "logits/rejected": -1.9370009899139404,
+      "logps/chosen": -515.6595458984375,
+      "logps/rejected": -611.8275146484375,
+      "loss": 0.1092,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.33213692903518677,
+      "rewards/margins": 0.10028378665447235,
+      "rewards/rejected": -0.4324207305908203,
+      "step": 4120
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.973019065681294e-06,
+      "logits/chosen": -2.281172275543213,
+      "logits/rejected": -1.80695378780365,
+      "logps/chosen": -529.9024658203125,
+      "logps/rejected": -643.1387939453125,
+      "loss": 0.1411,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.3373478055000305,
+      "rewards/margins": 0.14949987828731537,
+      "rewards/rejected": -0.4868476390838623,
+      "step": 4130
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.9627334966280474e-06,
+      "logits/chosen": -2.3365895748138428,
+      "logits/rejected": -1.919284462928772,
+      "logps/chosen": -565.9381103515625,
+      "logps/rejected": -616.6011352539062,
+      "loss": 0.1445,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.33689266443252563,
+      "rewards/margins": 0.1010858416557312,
+      "rewards/rejected": -0.43797844648361206,
+      "step": 4140
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.952439810646341e-06,
+      "logits/chosen": -2.2541048526763916,
+      "logits/rejected": -1.8350908756256104,
+      "logps/chosen": -572.382568359375,
+      "logps/rejected": -689.2306518554688,
+      "loss": 0.1333,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.3678056597709656,
+      "rewards/margins": 0.15155570209026337,
+      "rewards/rejected": -0.5193613171577454,
+      "step": 4150
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.942138188300394e-06,
+      "logits/chosen": -2.0759081840515137,
+      "logits/rejected": -1.6488910913467407,
+      "logps/chosen": -582.0780639648438,
+      "logps/rejected": -727.6575927734375,
+      "loss": 0.171,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3770234286785126,
+      "rewards/margins": 0.16119059920310974,
+      "rewards/rejected": -0.5382140874862671,
+      "step": 4160
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.931828810293642e-06,
+      "logits/chosen": -2.308687925338745,
+      "logits/rejected": -1.7430492639541626,
+      "logps/chosen": -524.843505859375,
+      "logps/rejected": -658.0339965820312,
+      "loss": 0.1537,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3190454840660095,
+      "rewards/margins": 0.17034713923931122,
+      "rewards/rejected": -0.48939257860183716,
+      "step": 4170
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.92151185746556e-06,
+      "logits/chosen": -2.180727958679199,
+      "logits/rejected": -1.8375177383422852,
+      "logps/chosen": -588.4954833984375,
+      "logps/rejected": -669.7674560546875,
+      "loss": 0.1627,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3628826141357422,
+      "rewards/margins": 0.08934937417507172,
+      "rewards/rejected": -0.4522319734096527,
+      "step": 4180
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.911187510788498e-06,
+      "logits/chosen": -2.1842234134674072,
+      "logits/rejected": -1.8807151317596436,
+      "logps/chosen": -540.1070556640625,
+      "logps/rejected": -620.3232421875,
+      "loss": 0.1044,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.2997668385505676,
+      "rewards/margins": 0.12294472754001617,
+      "rewards/rejected": -0.4227116107940674,
+      "step": 4190
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.9008559513645033e-06,
+      "logits/chosen": -2.21677827835083,
+      "logits/rejected": -1.954911470413208,
+      "logps/chosen": -551.2860107421875,
+      "logps/rejected": -641.84423828125,
+      "loss": 0.1355,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.33010560274124146,
+      "rewards/margins": 0.12660209834575653,
+      "rewards/rejected": -0.4567077159881592,
+      "step": 4200
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.890517360422144e-06,
+      "logits/chosen": -2.207214832305908,
+      "logits/rejected": -1.9762712717056274,
+      "logps/chosen": -511.46832275390625,
+      "logps/rejected": -596.9978637695312,
+      "loss": 0.1365,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.2954466938972473,
+      "rewards/margins": 0.10667997598648071,
+      "rewards/rejected": -0.40212664008140564,
+      "step": 4210
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.880171919313327e-06,
+      "logits/chosen": -2.370978593826294,
+      "logits/rejected": -1.755467176437378,
+      "logps/chosen": -467.592041015625,
+      "logps/rejected": -531.8112182617188,
+      "loss": 0.1113,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.2423483431339264,
+      "rewards/margins": 0.15210387110710144,
+      "rewards/rejected": -0.39445218443870544,
+      "step": 4220
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.869819809510125e-06,
+      "logits/chosen": -2.241830825805664,
+      "logits/rejected": -1.936621069908142,
+      "logps/chosen": -484.8067321777344,
+      "logps/rejected": -562.88037109375,
+      "loss": 0.1523,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.3006126284599304,
+      "rewards/margins": 0.08912810683250427,
+      "rewards/rejected": -0.3897407650947571,
+      "step": 4230
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8594612126015825e-06,
+      "logits/chosen": -2.372626304626465,
+      "logits/rejected": -2.041419744491577,
+      "logps/chosen": -469.9847717285156,
+      "logps/rejected": -650.1815185546875,
+      "loss": 0.1209,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.24346506595611572,
+      "rewards/margins": 0.16278813779354095,
+      "rewards/rejected": -0.4062531888484955,
+      "step": 4240
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.84909631029054e-06,
+      "logits/chosen": -2.0417003631591797,
+      "logits/rejected": -1.7644426822662354,
+      "logps/chosen": -600.1260986328125,
+      "logps/rejected": -758.5798950195312,
+      "loss": 0.1627,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.3970033824443817,
+      "rewards/margins": 0.15082235634326935,
+      "rewards/rejected": -0.5478257536888123,
+      "step": 4250
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.838725284390441e-06,
+      "logits/chosen": -2.0364885330200195,
+      "logits/rejected": -1.8718957901000977,
+      "logps/chosen": -585.9464111328125,
+      "logps/rejected": -737.6552734375,
+      "loss": 0.129,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.371036171913147,
+      "rewards/margins": 0.1429629623889923,
+      "rewards/rejected": -0.5139991641044617,
+      "step": 4260
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.828348316822144e-06,
+      "logits/chosen": -2.1091666221618652,
+      "logits/rejected": -1.8892055749893188,
+      "logps/chosen": -473.142333984375,
+      "logps/rejected": -652.0009765625,
+      "loss": 0.1205,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.30975714325904846,
+      "rewards/margins": 0.13321761786937714,
+      "rewards/rejected": -0.4429748058319092,
+      "step": 4270
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.817965589610733e-06,
+      "logits/chosen": -2.0144519805908203,
+      "logits/rejected": -1.759075403213501,
+      "logps/chosen": -567.5418090820312,
+      "logps/rejected": -702.8500366210938,
+      "loss": 0.1155,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.4076942503452301,
+      "rewards/margins": 0.12074587494134903,
+      "rewards/rejected": -0.5284401178359985,
+      "step": 4280
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.807577284882324e-06,
+      "logits/chosen": -2.1255810260772705,
+      "logits/rejected": -1.5518189668655396,
+      "logps/chosen": -496.31646728515625,
+      "logps/rejected": -672.8104248046875,
+      "loss": 0.1342,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.329243540763855,
+      "rewards/margins": 0.1847027987241745,
+      "rewards/rejected": -0.5139464139938354,
+      "step": 4290
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.797183584860867e-06,
+      "logits/chosen": -2.1264777183532715,
+      "logits/rejected": -1.7686008214950562,
+      "logps/chosen": -423.479736328125,
+      "logps/rejected": -507.6529235839844,
+      "loss": 0.1977,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.2590286433696747,
+      "rewards/margins": 0.09827350080013275,
+      "rewards/rejected": -0.357302188873291,
+      "step": 4300
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7867846718649538e-06,
+      "logits/chosen": -1.96920907497406,
+      "logits/rejected": -1.6666243076324463,
+      "logps/chosen": -475.94219970703125,
+      "logps/rejected": -708.38427734375,
+      "loss": 0.1031,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.27250292897224426,
+      "rewards/margins": 0.20877785980701447,
+      "rewards/rejected": -0.48128074407577515,
+      "step": 4310
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7763807283046195e-06,
+      "logits/chosen": -2.269808292388916,
+      "logits/rejected": -2.094205379486084,
+      "logps/chosen": -465.37811279296875,
+      "logps/rejected": -589.6844482421875,
+      "loss": 0.1385,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.2890546917915344,
+      "rewards/margins": 0.1288965791463852,
+      "rewards/rejected": -0.4179512560367584,
+      "step": 4320
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.76597193667814e-06,
+      "logits/chosen": -2.3413045406341553,
+      "logits/rejected": -1.8694992065429688,
+      "logps/chosen": -561.702880859375,
+      "logps/rejected": -669.148193359375,
+      "loss": 0.1222,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.3053972125053406,
+      "rewards/margins": 0.131010502576828,
+      "rewards/rejected": -0.4364077150821686,
+      "step": 4330
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7555584795688328e-06,
+      "logits/chosen": -2.1611485481262207,
+      "logits/rejected": -1.8126842975616455,
+      "logps/chosen": -542.7488403320312,
+      "logps/rejected": -652.1010131835938,
+      "loss": 0.1597,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.3296564519405365,
+      "rewards/margins": 0.12688103318214417,
+      "rewards/rejected": -0.4565374255180359,
+      "step": 4340
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7451405396418544e-06,
+      "logits/chosen": -2.1406474113464355,
+      "logits/rejected": -1.6467218399047852,
+      "logps/chosen": -545.9462280273438,
+      "logps/rejected": -604.2755126953125,
+      "loss": 0.1516,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3393089473247528,
+      "rewards/margins": 0.10570643842220306,
+      "rewards/rejected": -0.44501543045043945,
+      "step": 4350
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.734718299640994e-06,
+      "logits/chosen": -2.402259111404419,
+      "logits/rejected": -2.119187831878662,
+      "logps/chosen": -498.740478515625,
+      "logps/rejected": -608.8494873046875,
+      "loss": 0.1406,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.2847122550010681,
+      "rewards/margins": 0.12144307792186737,
+      "rewards/rejected": -0.4061553478240967,
+      "step": 4360
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.724291942385472e-06,
+      "logits/chosen": -2.4201648235321045,
+      "logits/rejected": -1.8801660537719727,
+      "logps/chosen": -534.5179443359375,
+      "logps/rejected": -644.624267578125,
+      "loss": 0.0876,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.26577314734458923,
+      "rewards/margins": 0.16301746666431427,
+      "rewards/rejected": -0.4287906289100647,
+      "step": 4370
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.713861650766729e-06,
+      "logits/chosen": -2.2512550354003906,
+      "logits/rejected": -1.7787246704101562,
+      "logps/chosen": -511.293701171875,
+      "logps/rejected": -629.5089111328125,
+      "loss": 0.1633,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.31032973527908325,
+      "rewards/margins": 0.14793090522289276,
+      "rewards/rejected": -0.4582606256008148,
+      "step": 4380
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.703427607745219e-06,
+      "logits/chosen": -2.4362640380859375,
+      "logits/rejected": -1.9733850955963135,
+      "logps/chosen": -525.0638427734375,
+      "logps/rejected": -591.4035034179688,
+      "loss": 0.1675,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.2703065276145935,
+      "rewards/margins": 0.10110236704349518,
+      "rewards/rejected": -0.3714088797569275,
+      "step": 4390
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6929899963472005e-06,
+      "logits/chosen": -2.2381815910339355,
+      "logits/rejected": -1.7432883977890015,
+      "logps/chosen": -523.2630615234375,
+      "logps/rejected": -647.0036010742188,
+      "loss": 0.1238,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.323739618062973,
+      "rewards/margins": 0.1632293164730072,
+      "rewards/rejected": -0.486968994140625,
+      "step": 4400
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6825489996615278e-06,
+      "logits/chosen": -2.015223979949951,
+      "logits/rejected": -1.7316118478775024,
+      "logps/chosen": -580.5408325195312,
+      "logps/rejected": -703.6740112304688,
+      "loss": 0.1186,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3971772789955139,
+      "rewards/margins": 0.14447876811027527,
+      "rewards/rejected": -0.5416560173034668,
+      "step": 4410
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6721048008364343e-06,
+      "logits/chosen": -2.1667702198028564,
+      "logits/rejected": -1.683070182800293,
+      "logps/chosen": -666.6958618164062,
+      "logps/rejected": -793.5103759765625,
+      "loss": 0.1462,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.4387953281402588,
+      "rewards/margins": 0.16940779983997345,
+      "rewards/rejected": -0.608203113079071,
+      "step": 4420
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6616575830763247e-06,
+      "logits/chosen": -2.1974172592163086,
+      "logits/rejected": -1.7128534317016602,
+      "logps/chosen": -683.750732421875,
+      "logps/rejected": -815.8909301757812,
+      "loss": 0.1627,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.47468703985214233,
+      "rewards/margins": 0.14315280318260193,
+      "rewards/rejected": -0.6178398728370667,
+      "step": 4430
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.651207529638561e-06,
+      "logits/chosen": -1.9084030389785767,
+      "logits/rejected": -1.472893238067627,
+      "logps/chosen": -650.3751831054688,
+      "logps/rejected": -732.6022338867188,
+      "loss": 0.1224,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.41856008768081665,
+      "rewards/margins": 0.1350983828306198,
+      "rewards/rejected": -0.5536584854125977,
+      "step": 4440
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.640754823830242e-06,
+      "logits/chosen": -2.3581013679504395,
+      "logits/rejected": -2.0822043418884277,
+      "logps/chosen": -673.743896484375,
+      "logps/rejected": -669.1588745117188,
+      "loss": 0.1139,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3935781717300415,
+      "rewards/margins": 0.08010774850845337,
+      "rewards/rejected": -0.47368597984313965,
+      "step": 4450
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.6302996490049983e-06,
+      "logits/chosen": -2.2223429679870605,
+      "logits/rejected": -1.7170603275299072,
+      "logps/chosen": -607.3829345703125,
+      "logps/rejected": -716.4435424804688,
+      "loss": 0.125,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.38815176486968994,
+      "rewards/margins": 0.11930838972330093,
+      "rewards/rejected": -0.5074602365493774,
+      "step": 4460
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.619842188559765e-06,
+      "logits/chosen": -2.0616865158081055,
+      "logits/rejected": -1.6523540019989014,
+      "logps/chosen": -514.1375732421875,
+      "logps/rejected": -699.1463623046875,
+      "loss": 0.1472,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.3626302480697632,
+      "rewards/margins": 0.19071173667907715,
+      "rewards/rejected": -0.5533419847488403,
+      "step": 4470
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.609382625931575e-06,
+      "logits/chosen": -2.0505645275115967,
+      "logits/rejected": -1.7823690176010132,
+      "logps/chosen": -632.944091796875,
+      "logps/rejected": -797.3037109375,
+      "loss": 0.1552,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.43152347207069397,
+      "rewards/margins": 0.15618659555912018,
+      "rewards/rejected": -0.5877100825309753,
+      "step": 4480
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.59892114459433e-06,
+      "logits/chosen": -2.027765989303589,
+      "logits/rejected": -2.119673252105713,
+      "logps/chosen": -600.08544921875,
+      "logps/rejected": -747.7369384765625,
+      "loss": 0.1525,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.4025164246559143,
+      "rewards/margins": 0.11830423772335052,
+      "rewards/rejected": -0.5208206176757812,
+      "step": 4490
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.588457928055592e-06,
+      "logits/chosen": -1.825016975402832,
+      "logits/rejected": -1.3659065961837769,
+      "logps/chosen": -580.4990844726562,
+      "logps/rejected": -752.3088989257812,
+      "loss": 0.0955,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.36483579874038696,
+      "rewards/margins": 0.21505002677440643,
+      "rewards/rejected": -0.5798857808113098,
+      "step": 4500
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5779931598533624e-06,
+      "logits/chosen": -2.070948839187622,
+      "logits/rejected": -1.6930170059204102,
+      "logps/chosen": -607.9359741210938,
+      "logps/rejected": -687.7504272460938,
+      "loss": 0.1777,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.38371676206588745,
+      "rewards/margins": 0.11133384704589844,
+      "rewards/rejected": -0.4950506091117859,
+      "step": 4510
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.567527023552857e-06,
+      "logits/chosen": -2.084954261779785,
+      "logits/rejected": -1.813650131225586,
+      "logps/chosen": -643.3284301757812,
+      "logps/rejected": -728.577880859375,
+      "loss": 0.0717,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.3763432502746582,
+      "rewards/margins": 0.14553704857826233,
+      "rewards/rejected": -0.5218802690505981,
+      "step": 4520
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5570597027432907e-06,
+      "logits/chosen": -2.151637554168701,
+      "logits/rejected": -1.610276460647583,
+      "logps/chosen": -551.1524658203125,
+      "logps/rejected": -651.0610961914062,
+      "loss": 0.1219,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.34246599674224854,
+      "rewards/margins": 0.1529092937707901,
+      "rewards/rejected": -0.49537524580955505,
+      "step": 4530
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5465913810346575e-06,
+      "logits/chosen": -1.9505962133407593,
+      "logits/rejected": -1.7450811862945557,
+      "logps/chosen": -618.9695434570312,
+      "logps/rejected": -745.1346435546875,
+      "loss": 0.1537,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.3996599018573761,
+      "rewards/margins": 0.11709467321634293,
+      "rewards/rejected": -0.5167545676231384,
+      "step": 4540
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.536122242054507e-06,
+      "logits/chosen": -2.146873950958252,
+      "logits/rejected": -1.4876574277877808,
+      "logps/chosen": -571.4114990234375,
+      "logps/rejected": -693.3803100585938,
+      "loss": 0.1217,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.36150923371315,
+      "rewards/margins": 0.16997110843658447,
+      "rewards/rejected": -0.5314803719520569,
+      "step": 4550
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.525652469444727e-06,
+      "logits/chosen": -2.233916759490967,
+      "logits/rejected": -1.761114478111267,
+      "logps/chosen": -495.17401123046875,
+      "logps/rejected": -585.968994140625,
+      "loss": 0.1458,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.3195408880710602,
+      "rewards/margins": 0.1199750155210495,
+      "rewards/rejected": -0.4395158886909485,
+      "step": 4560
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.5151822468583165e-06,
+      "logits/chosen": -1.9932966232299805,
+      "logits/rejected": -1.5087451934814453,
+      "logps/chosen": -573.6025390625,
+      "logps/rejected": -696.4073486328125,
+      "loss": 0.0722,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.37811872363090515,
+      "rewards/margins": 0.17516979575157166,
+      "rewards/rejected": -0.5532885193824768,
+      "step": 4570
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.5047117579561703e-06,
+      "logits/chosen": -1.9902255535125732,
+      "logits/rejected": -1.7906286716461182,
+      "logps/chosen": -798.6557006835938,
+      "logps/rejected": -900.3358154296875,
+      "loss": 0.1466,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.5353633165359497,
+      "rewards/margins": 0.11944514513015747,
+      "rewards/rejected": -0.654808521270752,
+      "step": 4580
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.494241186403854e-06,
+      "logits/chosen": -2.1336703300476074,
+      "logits/rejected": -1.9880857467651367,
+      "logps/chosen": -534.771484375,
+      "logps/rejected": -620.3009033203125,
+      "loss": 0.1919,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.36856359243392944,
+      "rewards/margins": 0.10053074359893799,
+      "rewards/rejected": -0.46909427642822266,
+      "step": 4590
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.4837707158683833e-06,
+      "logits/chosen": -1.9466358423233032,
+      "logits/rejected": -1.766659140586853,
+      "logps/chosen": -683.0281982421875,
+      "logps/rejected": -803.1266479492188,
+      "loss": 0.1315,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.48551732301712036,
+      "rewards/margins": 0.11497100442647934,
+      "rewards/rejected": -0.6004883050918579,
+      "step": 4600
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.473300530015e-06,
+      "logits/chosen": -2.4355952739715576,
+      "logits/rejected": -2.095571517944336,
+      "logps/chosen": -561.1920166015625,
+      "logps/rejected": -672.131103515625,
+      "loss": 0.156,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.316837340593338,
+      "rewards/margins": 0.12641747295856476,
+      "rewards/rejected": -0.44325485825538635,
+      "step": 4610
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.4628308125039557e-06,
+      "logits/chosen": -2.103797435760498,
+      "logits/rejected": -1.6855539083480835,
+      "logps/chosen": -620.6937866210938,
+      "logps/rejected": -723.973876953125,
+      "loss": 0.151,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3569558262825012,
+      "rewards/margins": 0.14904451370239258,
+      "rewards/rejected": -0.5060003399848938,
+      "step": 4620
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.452361746987284e-06,
+      "logits/chosen": -2.039536952972412,
+      "logits/rejected": -1.8618084192276,
+      "logps/chosen": -631.0521850585938,
+      "logps/rejected": -751.0166625976562,
+      "loss": 0.0947,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.41549959778785706,
+      "rewards/margins": 0.11577264964580536,
+      "rewards/rejected": -0.531272292137146,
+      "step": 4630
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.4418935171055818e-06,
+      "logits/chosen": -2.1718621253967285,
+      "logits/rejected": -1.848221778869629,
+      "logps/chosen": -496.31488037109375,
+      "logps/rejected": -618.4346923828125,
+      "loss": 0.1148,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.33470824360847473,
+      "rewards/margins": 0.12179449945688248,
+      "rewards/rejected": -0.456502765417099,
+      "step": 4640
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.43142630648479e-06,
+      "logits/chosen": -2.1049323081970215,
+      "logits/rejected": -1.7634950876235962,
+      "logps/chosen": -691.8548583984375,
+      "logps/rejected": -845.0455932617188,
+      "loss": 0.0872,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.4345971643924713,
+      "rewards/margins": 0.12043090909719467,
+      "rewards/rejected": -0.5550281405448914,
+      "step": 4650
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.4209602987329685e-06,
+      "logits/chosen": -1.9521276950836182,
+      "logits/rejected": -1.4278624057769775,
+      "logps/chosen": -531.8436279296875,
+      "logps/rejected": -626.9532470703125,
+      "loss": 0.0844,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.34457993507385254,
+      "rewards/margins": 0.1545027792453766,
+      "rewards/rejected": -0.4990827143192291,
+      "step": 4660
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.410495677437076e-06,
+      "logits/chosen": -2.1297354698181152,
+      "logits/rejected": -2.012040376663208,
+      "logps/chosen": -541.9227905273438,
+      "logps/rejected": -636.0309448242188,
+      "loss": 0.1847,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.35125699639320374,
+      "rewards/margins": 0.1137257069349289,
+      "rewards/rejected": -0.46498265862464905,
+      "step": 4670
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.400032626159756e-06,
+      "logits/chosen": -2.1385817527770996,
+      "logits/rejected": -1.8811590671539307,
+      "logps/chosen": -592.0183715820312,
+      "logps/rejected": -700.8385620117188,
+      "loss": 0.1792,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.4057946801185608,
+      "rewards/margins": 0.12076137959957123,
+      "rewards/rejected": -0.5265560150146484,
+      "step": 4680
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.3895713284361065e-06,
+      "logits/chosen": -2.304227828979492,
+      "logits/rejected": -1.7567590475082397,
+      "logps/chosen": -567.26806640625,
+      "logps/rejected": -729.3734130859375,
+      "loss": 0.1006,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.3536512553691864,
+      "rewards/margins": 0.2043255865573883,
+      "rewards/rejected": -0.5579768419265747,
+      "step": 4690
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.3791119677704676e-06,
+      "logits/chosen": -2.326404571533203,
+      "logits/rejected": -1.695572853088379,
+      "logps/chosen": -656.3429565429688,
+      "logps/rejected": -747.4685668945312,
+      "loss": 0.1423,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.40097013115882874,
+      "rewards/margins": 0.14889459311962128,
+      "rewards/rejected": -0.5498647093772888,
+      "step": 4700
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.3686547276332046e-06,
+      "logits/chosen": -2.231799364089966,
+      "logits/rejected": -1.7230803966522217,
+      "logps/chosen": -671.7639770507812,
+      "logps/rejected": -765.615234375,
+      "loss": 0.1474,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.44814324378967285,
+      "rewards/margins": 0.1409330815076828,
+      "rewards/rejected": -0.5890763998031616,
+      "step": 4710
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.3581997914574807e-06,
+      "logits/chosen": -2.079420566558838,
+      "logits/rejected": -1.6395925283432007,
+      "logps/chosen": -621.941162109375,
+      "logps/rejected": -724.62109375,
+      "loss": 0.1359,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.4237375259399414,
+      "rewards/margins": 0.1329127997159958,
+      "rewards/rejected": -0.5566502809524536,
+      "step": 4720
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.3477473426360463e-06,
+      "logits/chosen": -2.3243088722229004,
+      "logits/rejected": -1.7983930110931396,
+      "logps/chosen": -613.1595458984375,
+      "logps/rejected": -746.106201171875,
+      "loss": 0.12,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.39571815729141235,
+      "rewards/margins": 0.18775278329849243,
+      "rewards/rejected": -0.5834709405899048,
+      "step": 4730
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.337297564518024e-06,
+      "logits/chosen": -2.3540751934051514,
+      "logits/rejected": -2.03450083732605,
+      "logps/chosen": -641.3284912109375,
+      "logps/rejected": -711.5076904296875,
+      "loss": 0.1221,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.37791794538497925,
+      "rewards/margins": 0.1164165586233139,
+      "rewards/rejected": -0.49433451890945435,
+      "step": 4740
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.326850640405684e-06,
+      "logits/chosen": -2.066770553588867,
+      "logits/rejected": -1.443925142288208,
+      "logps/chosen": -612.4591674804688,
+      "logps/rejected": -798.1064453125,
+      "loss": 0.1003,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.3162711262702942,
+      "rewards/margins": 0.24829897284507751,
+      "rewards/rejected": -0.5645700693130493,
+      "step": 4750
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.3164067535512353e-06,
+      "logits/chosen": -2.090834140777588,
+      "logits/rejected": -1.6742346286773682,
+      "logps/chosen": -610.352294921875,
+      "logps/rejected": -661.9411010742188,
+      "loss": 0.1373,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.37288302183151245,
+      "rewards/margins": 0.1150321215391159,
+      "rewards/rejected": -0.48791512846946716,
+      "step": 4760
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.3059660871536123e-06,
+      "logits/chosen": -1.9306840896606445,
+      "logits/rejected": -1.5933589935302734,
+      "logps/chosen": -571.6657104492188,
+      "logps/rejected": -695.8966674804688,
+      "loss": 0.1244,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3844566345214844,
+      "rewards/margins": 0.1267177015542984,
+      "rewards/rejected": -0.5111743211746216,
+      "step": 4770
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.2955288243552543e-06,
+      "logits/chosen": -2.3082027435302734,
+      "logits/rejected": -1.7937629222869873,
+      "logps/chosen": -682.3204956054688,
+      "logps/rejected": -660.8674926757812,
+      "loss": 0.1326,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3952789306640625,
+      "rewards/margins": 0.08656209707260132,
+      "rewards/rejected": -0.4818410277366638,
+      "step": 4780
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.285095148238899e-06,
+      "logits/chosen": -2.163402795791626,
+      "logits/rejected": -1.9141143560409546,
+      "logps/chosen": -610.841796875,
+      "logps/rejected": -689.5853881835938,
+      "loss": 0.1701,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.37611109018325806,
+      "rewards/margins": 0.11169426143169403,
+      "rewards/rejected": -0.4878053069114685,
+      "step": 4790
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2746652418243714e-06,
+      "logits/chosen": -2.1892688274383545,
+      "logits/rejected": -1.9378070831298828,
+      "logps/chosen": -646.8173217773438,
+      "logps/rejected": -737.2486572265625,
+      "loss": 0.1058,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.37569350004196167,
+      "rewards/margins": 0.11736834049224854,
+      "rewards/rejected": -0.4930618703365326,
+      "step": 4800
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2642392880653677e-06,
+      "logits/chosen": -2.1913487911224365,
+      "logits/rejected": -2.1414527893066406,
+      "logps/chosen": -562.5335083007812,
+      "logps/rejected": -600.5672607421875,
+      "loss": 0.1577,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.3508028984069824,
+      "rewards/margins": 0.07039102166891098,
+      "rewards/rejected": -0.4211938977241516,
+      "step": 4810
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.25381746984625e-06,
+      "logits/chosen": -2.19293475151062,
+      "logits/rejected": -1.8960790634155273,
+      "logps/chosen": -522.1268310546875,
+      "logps/rejected": -658.8265991210938,
+      "loss": 0.1255,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.30920180678367615,
+      "rewards/margins": 0.13133490085601807,
+      "rewards/rejected": -0.4405367374420166,
+      "step": 4820
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2433999699788404e-06,
+      "logits/chosen": -2.2714991569519043,
+      "logits/rejected": -1.8668155670166016,
+      "logps/chosen": -553.7095947265625,
+      "logps/rejected": -590.1159057617188,
+      "loss": 0.1353,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.3420394957065582,
+      "rewards/margins": 0.08509062230587006,
+      "rewards/rejected": -0.4271301329135895,
+      "step": 4830
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2329869711992093e-06,
+      "logits/chosen": -2.1053225994110107,
+      "logits/rejected": -1.9494432210922241,
+      "logps/chosen": -485.41375732421875,
+      "logps/rejected": -606.818603515625,
+      "loss": 0.1255,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3072953224182129,
+      "rewards/margins": 0.09817321598529816,
+      "rewards/rejected": -0.40546855330467224,
+      "step": 4840
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2225786561644724e-06,
+      "logits/chosen": -1.996241807937622,
+      "logits/rejected": -1.842153549194336,
+      "logps/chosen": -539.3800048828125,
+      "logps/rejected": -673.2063598632812,
+      "loss": 0.0987,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.33509454131126404,
+      "rewards/margins": 0.14485691487789154,
+      "rewards/rejected": -0.4799514412879944,
+      "step": 4850
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.212175207449589e-06,
+      "logits/chosen": -2.1658384799957275,
+      "logits/rejected": -1.5615532398223877,
+      "logps/chosen": -537.4581298828125,
+      "logps/rejected": -696.8929443359375,
+      "loss": 0.109,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.3726101517677307,
+      "rewards/margins": 0.1857399195432663,
+      "rewards/rejected": -0.5583500266075134,
+      "step": 4860
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2017768075441544e-06,
+      "logits/chosen": -2.113062858581543,
+      "logits/rejected": -2.0136196613311768,
+      "logps/chosen": -590.265869140625,
+      "logps/rejected": -701.2208862304688,
+      "loss": 0.0937,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3913181722164154,
+      "rewards/margins": 0.10938689857721329,
+      "rewards/rejected": -0.5007050633430481,
+      "step": 4870
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.191383638849201e-06,
+      "logits/chosen": -1.839450478553772,
+      "logits/rejected": -1.7185344696044922,
+      "logps/chosen": -558.6953125,
+      "logps/rejected": -721.52587890625,
+      "loss": 0.1337,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.38823121786117554,
+      "rewards/margins": 0.15455672144889832,
+      "rewards/rejected": -0.5427879691123962,
+      "step": 4880
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.180995883674003e-06,
+      "logits/chosen": -2.320197820663452,
+      "logits/rejected": -2.0700695514678955,
+      "logps/chosen": -602.8727416992188,
+      "logps/rejected": -683.3446655273438,
+      "loss": 0.1255,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.35641875863075256,
+      "rewards/margins": 0.13076770305633545,
+      "rewards/rejected": -0.4871864914894104,
+      "step": 4890
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1706137242328708e-06,
+      "logits/chosen": -2.1988813877105713,
+      "logits/rejected": -2.024855613708496,
+      "logps/chosen": -512.3671875,
+      "logps/rejected": -634.8638916015625,
+      "loss": 0.1269,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.3405269980430603,
+      "rewards/margins": 0.11310543864965439,
+      "rewards/rejected": -0.4536324441432953,
+      "step": 4900
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1602373426419593e-06,
+      "logits/chosen": -2.2114062309265137,
+      "logits/rejected": -1.8391650915145874,
+      "logps/chosen": -530.4862060546875,
+      "logps/rejected": -691.3091430664062,
+      "loss": 0.1449,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.33998486399650574,
+      "rewards/margins": 0.18002228438854218,
+      "rewards/rejected": -0.5200071334838867,
+      "step": 4910
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.149866920916075e-06,
+      "logits/chosen": -2.1867594718933105,
+      "logits/rejected": -1.9130885601043701,
+      "logps/chosen": -535.1444091796875,
+      "logps/rejected": -605.7540893554688,
+      "loss": 0.1116,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.29843541979789734,
+      "rewards/margins": 0.10633637011051178,
+      "rewards/rejected": -0.4047718048095703,
+      "step": 4920
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1395026409654776e-06,
+      "logits/chosen": -2.3580214977264404,
+      "logits/rejected": -2.013789653778076,
+      "logps/chosen": -531.3629150390625,
+      "logps/rejected": -580.8612060546875,
+      "loss": 0.1167,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.29877519607543945,
+      "rewards/margins": 0.09707503765821457,
+      "rewards/rejected": -0.395850270986557,
+      "step": 4930
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.129144684592694e-06,
+      "logits/chosen": -2.209461212158203,
+      "logits/rejected": -1.7003713846206665,
+      "logps/chosen": -503.0797424316406,
+      "logps/rejected": -584.7760620117188,
+      "loss": 0.1355,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3278839886188507,
+      "rewards/margins": 0.10966457426548004,
+      "rewards/rejected": -0.43754857778549194,
+      "step": 4940
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1187932334893282e-06,
+      "logits/chosen": -2.333221673965454,
+      "logits/rejected": -2.0828678607940674,
+      "logps/chosen": -506.39459228515625,
+      "logps/rejected": -594.1668701171875,
+      "loss": 0.1454,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3198184072971344,
+      "rewards/margins": 0.09863405674695969,
+      "rewards/rejected": -0.4184524416923523,
+      "step": 4950
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.1084484692328726e-06,
+      "logits/chosen": -2.0685410499572754,
+      "logits/rejected": -1.8335059881210327,
+      "logps/chosen": -727.5977783203125,
+      "logps/rejected": -853.7720947265625,
+      "loss": 0.0814,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.4701920449733734,
+      "rewards/margins": 0.1254054754972458,
+      "rewards/rejected": -0.595597505569458,
+      "step": 4960
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0981105732835227e-06,
+      "logits/chosen": -2.245957851409912,
+      "logits/rejected": -1.660552740097046,
+      "logps/chosen": -599.4436645507812,
+      "logps/rejected": -720.6624755859375,
+      "loss": 0.1808,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.3824935555458069,
+      "rewards/margins": 0.19680753350257874,
+      "rewards/rejected": -0.5793010592460632,
+      "step": 4970
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.087779726980999e-06,
+      "logits/chosen": -2.3509912490844727,
+      "logits/rejected": -1.8343099355697632,
+      "logps/chosen": -643.5232543945312,
+      "logps/rejected": -751.803466796875,
+      "loss": 0.0719,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.4002252221107483,
+      "rewards/margins": 0.14421038329601288,
+      "rewards/rejected": -0.5444355607032776,
+      "step": 4980
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.077456111541359e-06,
+      "logits/chosen": -2.2299487590789795,
+      "logits/rejected": -1.7281221151351929,
+      "logps/chosen": -602.8156127929688,
+      "logps/rejected": -692.3547973632812,
+      "loss": 0.1705,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.36152979731559753,
+      "rewards/margins": 0.15708096325397491,
+      "rewards/rejected": -0.5186107754707336,
+      "step": 4990
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.067139908053821e-06,
+      "logits/chosen": -2.3910698890686035,
+      "logits/rejected": -2.04085111618042,
+      "logps/chosen": -599.81591796875,
+      "logps/rejected": -694.18603515625,
+      "loss": 0.137,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.36251357197761536,
+      "rewards/margins": 0.12564414739608765,
+      "rewards/rejected": -0.4881577491760254,
+      "step": 5000
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.056831297477592e-06,
+      "logits/chosen": -2.126121997833252,
+      "logits/rejected": -2.0639469623565674,
+      "logps/chosen": -743.83154296875,
+      "logps/rejected": -769.8225708007812,
+      "loss": 0.1388,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.48225560784339905,
+      "rewards/margins": 0.06981122493743896,
+      "rewards/rejected": -0.5520668625831604,
+      "step": 5010
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.046530460638687e-06,
+      "logits/chosen": -2.3220863342285156,
+      "logits/rejected": -1.921440839767456,
+      "logps/chosen": -643.3884887695312,
+      "logps/rejected": -750.0999755859375,
+      "loss": 0.1211,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.37827813625335693,
+      "rewards/margins": 0.16364845633506775,
+      "rewards/rejected": -0.5419266223907471,
+      "step": 5020
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.036237578226761e-06,
+      "logits/chosen": -1.9888349771499634,
+      "logits/rejected": -1.596352458000183,
+      "logps/chosen": -563.1626586914062,
+      "logps/rejected": -674.4523315429688,
+      "loss": 0.1562,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.3902806043624878,
+      "rewards/margins": 0.133496955037117,
+      "rewards/rejected": -0.5237776041030884,
+      "step": 5030
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0259528307919385e-06,
+      "logits/chosen": -2.2416653633117676,
+      "logits/rejected": -1.5866806507110596,
+      "logps/chosen": -587.1439208984375,
+      "logps/rejected": -659.7548828125,
+      "loss": 0.1681,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.3323143422603607,
+      "rewards/margins": 0.15864050388336182,
+      "rewards/rejected": -0.4909548759460449,
+      "step": 5040
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.015676398741644e-06,
+      "logits/chosen": -2.150412082672119,
+      "logits/rejected": -1.5077117681503296,
+      "logps/chosen": -706.4153442382812,
+      "logps/rejected": -789.60986328125,
+      "loss": 0.1364,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.431532084941864,
+      "rewards/margins": 0.1571730077266693,
+      "rewards/rejected": -0.5887050628662109,
+      "step": 5050
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.005408462337443e-06,
+      "logits/chosen": -2.2040181159973145,
+      "logits/rejected": -1.6936447620391846,
+      "logps/chosen": -634.9398803710938,
+      "logps/rejected": -798.7047119140625,
+      "loss": 0.1548,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.4336550235748291,
+      "rewards/margins": 0.19133055210113525,
+      "rewards/rejected": -0.6249855756759644,
+      "step": 5060
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.9951492016918745e-06,
+      "logits/chosen": -2.1265008449554443,
+      "logits/rejected": -1.671492576599121,
+      "logps/chosen": -580.5635986328125,
+      "logps/rejected": -706.6000366210938,
+      "loss": 0.17,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.43969279527664185,
+      "rewards/margins": 0.13454791903495789,
+      "rewards/rejected": -0.5742406845092773,
+      "step": 5070
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.984898796765294e-06,
+      "logits/chosen": -2.083684206008911,
+      "logits/rejected": -1.5871495008468628,
+      "logps/chosen": -541.4649658203125,
+      "logps/rejected": -668.8524169921875,
+      "loss": 0.1822,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.392850399017334,
+      "rewards/margins": 0.145137220621109,
+      "rewards/rejected": -0.5379875898361206,
+      "step": 5080
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.974657427362717e-06,
+      "logits/chosen": -1.9917738437652588,
+      "logits/rejected": -1.7782930135726929,
+      "logps/chosen": -694.8370971679688,
+      "logps/rejected": -757.8206787109375,
+      "loss": 0.0995,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.4584885537624359,
+      "rewards/margins": 0.09417570382356644,
+      "rewards/rejected": -0.5526642203330994,
+      "step": 5090
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.9644252731306653e-06,
+      "logits/chosen": -2.1260459423065186,
+      "logits/rejected": -1.6753833293914795,
+      "logps/chosen": -767.4806518554688,
+      "logps/rejected": -822.35693359375,
+      "loss": 0.1257,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.4604128897190094,
+      "rewards/margins": 0.12554161250591278,
+      "rewards/rejected": -0.5859545469284058,
+      "step": 5100
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.954202513554013e-06,
+      "logits/chosen": -2.2659480571746826,
+      "logits/rejected": -2.011653423309326,
+      "logps/chosen": -561.1275024414062,
+      "logps/rejected": -716.3524169921875,
+      "loss": 0.1593,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.3647858202457428,
+      "rewards/margins": 0.14984898269176483,
+      "rewards/rejected": -0.5146347880363464,
+      "step": 5110
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.943989327952841e-06,
+      "logits/chosen": -2.316166400909424,
+      "logits/rejected": -1.857942819595337,
+      "logps/chosen": -722.9827270507812,
+      "logps/rejected": -805.546875,
+      "loss": 0.086,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.4280776381492615,
+      "rewards/margins": 0.12500518560409546,
+      "rewards/rejected": -0.5530828237533569,
+      "step": 5120
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.9337858954792917e-06,
+      "logits/chosen": -2.0795562267303467,
+      "logits/rejected": -1.8410499095916748,
+      "logps/chosen": -642.4514770507812,
+      "logps/rejected": -797.9846801757812,
+      "loss": 0.1243,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.44671449065208435,
+      "rewards/margins": 0.16672448813915253,
+      "rewards/rejected": -0.6134389638900757,
+      "step": 5130
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.9235923951144246e-06,
+      "logits/chosen": -2.2457327842712402,
+      "logits/rejected": -1.8359851837158203,
+      "logps/chosen": -594.8767700195312,
+      "logps/rejected": -669.226318359375,
+      "loss": 0.1186,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.35655707120895386,
+      "rewards/margins": 0.10433490574359894,
+      "rewards/rejected": -0.4608920216560364,
+      "step": 5140
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.9134090056650764e-06,
+      "logits/chosen": -2.35942006111145,
+      "logits/rejected": -2.0687317848205566,
+      "logps/chosen": -570.7220458984375,
+      "logps/rejected": -665.747314453125,
+      "loss": 0.197,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3417629897594452,
+      "rewards/margins": 0.12745366990566254,
+      "rewards/rejected": -0.4692166745662689,
+      "step": 5150
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.9032359057607272e-06,
+      "logits/chosen": -2.4121201038360596,
+      "logits/rejected": -1.7959457635879517,
+      "logps/chosen": -587.4695434570312,
+      "logps/rejected": -735.8468017578125,
+      "loss": 0.0919,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.2974987328052521,
+      "rewards/margins": 0.19046206772327423,
+      "rewards/rejected": -0.4879607558250427,
+      "step": 5160
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.8930732738503652e-06,
+      "logits/chosen": -2.3014063835144043,
+      "logits/rejected": -1.9863837957382202,
+      "logps/chosen": -542.3372802734375,
+      "logps/rejected": -612.9231567382812,
+      "loss": 0.1028,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.33853524923324585,
+      "rewards/margins": 0.13003009557724,
+      "rewards/rejected": -0.46856531500816345,
+      "step": 5170
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.8829212881993553e-06,
+      "logits/chosen": -2.4339759349823,
+      "logits/rejected": -2.0722947120666504,
+      "logps/chosen": -551.5011596679688,
+      "logps/rejected": -639.1527099609375,
+      "loss": 0.0693,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.32050183415412903,
+      "rewards/margins": 0.12614518404006958,
+      "rewards/rejected": -0.4466470181941986,
+      "step": 5180
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.872780126886316e-06,
+      "logits/chosen": -2.372493267059326,
+      "logits/rejected": -1.82937490940094,
+      "logps/chosen": -562.5084838867188,
+      "logps/rejected": -664.5687255859375,
+      "loss": 0.1271,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.32636040449142456,
+      "rewards/margins": 0.1591111272573471,
+      "rewards/rejected": -0.48547154664993286,
+      "step": 5190
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.8626499677999915e-06,
+      "logits/chosen": -2.105699300765991,
+      "logits/rejected": -2.0460121631622314,
+      "logps/chosen": -603.1366577148438,
+      "logps/rejected": -734.0123901367188,
+      "loss": 0.1318,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.39799219369888306,
+      "rewards/margins": 0.1266644448041916,
+      "rewards/rejected": -0.5246566534042358,
+      "step": 5200
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8525309886361332e-06,
+      "logits/chosen": -2.2633297443389893,
+      "logits/rejected": -1.6591545343399048,
+      "logps/chosen": -543.9161376953125,
+      "logps/rejected": -729.98828125,
+      "loss": 0.1028,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.38227465748786926,
+      "rewards/margins": 0.2150934636592865,
+      "rewards/rejected": -0.5973681211471558,
+      "step": 5210
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8424233668943844e-06,
+      "logits/chosen": -2.0686941146850586,
+      "logits/rejected": -1.9637184143066406,
+      "logps/chosen": -558.0516967773438,
+      "logps/rejected": -708.8333740234375,
+      "loss": 0.12,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3659864068031311,
+      "rewards/margins": 0.15353603661060333,
+      "rewards/rejected": -0.5195223689079285,
+      "step": 5220
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8323272798751629e-06,
+      "logits/chosen": -2.0943186283111572,
+      "logits/rejected": -1.8889824151992798,
+      "logps/chosen": -644.1373291015625,
+      "logps/rejected": -746.6627197265625,
+      "loss": 0.1395,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.43741488456726074,
+      "rewards/margins": 0.12883993983268738,
+      "rewards/rejected": -0.566254734992981,
+      "step": 5230
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.822242904676552e-06,
+      "logits/chosen": -2.26570463180542,
+      "logits/rejected": -1.908050537109375,
+      "logps/chosen": -577.956298828125,
+      "logps/rejected": -682.3907470703125,
+      "loss": 0.0757,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.41112637519836426,
+      "rewards/margins": 0.10414554178714752,
+      "rewards/rejected": -0.5152719020843506,
+      "step": 5240
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8121704181911989e-06,
+      "logits/chosen": -2.3262784481048584,
+      "logits/rejected": -2.0528576374053955,
+      "logps/chosen": -669.1859741210938,
+      "logps/rejected": -698.4793701171875,
+      "loss": 0.1236,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.4042871594429016,
+      "rewards/margins": 0.07768969237804413,
+      "rewards/rejected": -0.48197680711746216,
+      "step": 5250
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8021099971032046e-06,
+      "logits/chosen": -2.0667221546173096,
+      "logits/rejected": -1.5488338470458984,
+      "logps/chosen": -582.1514892578125,
+      "logps/rejected": -671.4440307617188,
+      "loss": 0.1118,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.38886600732803345,
+      "rewards/margins": 0.13451814651489258,
+      "rewards/rejected": -0.5233842134475708,
+      "step": 5260
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.7920618178850269e-06,
+      "logits/chosen": -2.2863402366638184,
+      "logits/rejected": -2.0318825244903564,
+      "logps/chosen": -618.773193359375,
+      "logps/rejected": -732.3018188476562,
+      "loss": 0.106,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3683752417564392,
+      "rewards/margins": 0.1530582755804062,
+      "rewards/rejected": -0.5214334726333618,
+      "step": 5270
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.7820260567943904e-06,
+      "logits/chosen": -2.196944236755371,
+      "logits/rejected": -2.017430543899536,
+      "logps/chosen": -541.4947509765625,
+      "logps/rejected": -666.664306640625,
+      "loss": 0.1848,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.41327619552612305,
+      "rewards/margins": 0.12232637405395508,
+      "rewards/rejected": -0.5356025695800781,
+      "step": 5280
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.7720028898711852e-06,
+      "logits/chosen": -2.135990858078003,
+      "logits/rejected": -1.638816237449646,
+      "logps/chosen": -561.9149169921875,
+      "logps/rejected": -675.7244873046875,
+      "loss": 0.1392,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3478553295135498,
+      "rewards/margins": 0.15960641205310822,
+      "rewards/rejected": -0.507461667060852,
+      "step": 5290
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7619924929343857e-06,
+      "logits/chosen": -2.175929307937622,
+      "logits/rejected": -1.8564764261245728,
+      "logps/chosen": -592.1212158203125,
+      "logps/rejected": -740.3436279296875,
+      "loss": 0.1235,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.3666415810585022,
+      "rewards/margins": 0.1334037035703659,
+      "rewards/rejected": -0.5000452995300293,
+      "step": 5300
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7519950415789661e-06,
+      "logits/chosen": -2.0560708045959473,
+      "logits/rejected": -1.8349571228027344,
+      "logps/chosen": -560.9859619140625,
+      "logps/rejected": -754.5137329101562,
+      "loss": 0.1613,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.3551250398159027,
+      "rewards/margins": 0.16058704257011414,
+      "rewards/rejected": -0.5157120823860168,
+      "step": 5310
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7420107111728167e-06,
+      "logits/chosen": -2.2044639587402344,
+      "logits/rejected": -1.9750274419784546,
+      "logps/chosen": -462.22998046875,
+      "logps/rejected": -597.7857666015625,
+      "loss": 0.0738,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.2953898310661316,
+      "rewards/margins": 0.1337275207042694,
+      "rewards/rejected": -0.429117351770401,
+      "step": 5320
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7320396768536695e-06,
+      "logits/chosen": -2.295987367630005,
+      "logits/rejected": -1.7668756246566772,
+      "logps/chosen": -637.4358520507812,
+      "logps/rejected": -741.8875122070312,
+      "loss": 0.0991,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.41747063398361206,
+      "rewards/margins": 0.14269547164440155,
+      "rewards/rejected": -0.5601661205291748,
+      "step": 5330
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7220821135260301e-06,
+      "logits/chosen": -2.1273655891418457,
+      "logits/rejected": -1.5581985712051392,
+      "logps/chosen": -613.4246826171875,
+      "logps/rejected": -717.3649291992188,
+      "loss": 0.076,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.423337459564209,
+      "rewards/margins": 0.1381046324968338,
+      "rewards/rejected": -0.5614420771598816,
+      "step": 5340
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7121381958581018e-06,
+      "logits/chosen": -2.28882098197937,
+      "logits/rejected": -1.8115923404693604,
+      "logps/chosen": -726.3345947265625,
+      "logps/rejected": -777.6907348632812,
+      "loss": 0.1117,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.48346003890037537,
+      "rewards/margins": 0.13243672251701355,
+      "rewards/rejected": -0.6158968210220337,
+      "step": 5350
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7022080982787259e-06,
+      "logits/chosen": -2.186537265777588,
+      "logits/rejected": -1.6854503154754639,
+      "logps/chosen": -664.9749145507812,
+      "logps/rejected": -777.406005859375,
+      "loss": 0.0971,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.446157842874527,
+      "rewards/margins": 0.14880336821079254,
+      "rewards/rejected": -0.5949612259864807,
+      "step": 5360
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.692291994974326e-06,
+      "logits/chosen": -2.2510595321655273,
+      "logits/rejected": -1.7074207067489624,
+      "logps/chosen": -621.3232421875,
+      "logps/rejected": -734.2008666992188,
+      "loss": 0.13,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3647621273994446,
+      "rewards/margins": 0.1666307896375656,
+      "rewards/rejected": -0.5313929319381714,
+      "step": 5370
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.682390059885845e-06,
+      "logits/chosen": -2.2437822818756104,
+      "logits/rejected": -1.6662489175796509,
+      "logps/chosen": -657.34521484375,
+      "logps/rejected": -770.3602294921875,
+      "loss": 0.0886,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.4201204180717468,
+      "rewards/margins": 0.20016567409038544,
+      "rewards/rejected": -0.6202861070632935,
+      "step": 5380
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6725024667056965e-06,
+      "logits/chosen": -2.125859498977661,
+      "logits/rejected": -1.6562635898590088,
+      "logps/chosen": -594.5446166992188,
+      "logps/rejected": -625.6378784179688,
+      "loss": 0.1683,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3732602000236511,
+      "rewards/margins": 0.1052890419960022,
+      "rewards/rejected": -0.4785493016242981,
+      "step": 5390
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6626293888747238e-06,
+      "logits/chosen": -2.2732555866241455,
+      "logits/rejected": -1.6359913349151611,
+      "logps/chosen": -562.03369140625,
+      "logps/rejected": -714.646484375,
+      "loss": 0.1111,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.34102416038513184,
+      "rewards/margins": 0.1775747686624527,
+      "rewards/rejected": -0.5185989141464233,
+      "step": 5400
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.652770999579148e-06,
+      "logits/chosen": -2.218611001968384,
+      "logits/rejected": -1.8803329467773438,
+      "logps/chosen": -565.531982421875,
+      "logps/rejected": -695.0848388671875,
+      "loss": 0.1205,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3656262159347534,
+      "rewards/margins": 0.13394705951213837,
+      "rewards/rejected": -0.499573290348053,
+      "step": 5410
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6429274717475358e-06,
+      "logits/chosen": -2.2132153511047363,
+      "logits/rejected": -1.8499853610992432,
+      "logps/chosen": -545.2037353515625,
+      "logps/rejected": -643.0628051757812,
+      "loss": 0.0961,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.30922383069992065,
+      "rewards/margins": 0.16129609942436218,
+      "rewards/rejected": -0.47051987051963806,
+      "step": 5420
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6330989780477673e-06,
+      "logits/chosen": -2.1837105751037598,
+      "logits/rejected": -1.7655189037322998,
+      "logps/chosen": -592.931396484375,
+      "logps/rejected": -705.00634765625,
+      "loss": 0.1583,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.39096584916114807,
+      "rewards/margins": 0.12922033667564392,
+      "rewards/rejected": -0.520186185836792,
+      "step": 5430
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6232856908840033e-06,
+      "logits/chosen": -2.538215160369873,
+      "logits/rejected": -1.9228191375732422,
+      "logps/chosen": -507.7865295410156,
+      "logps/rejected": -601.995361328125,
+      "loss": 0.0911,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.2888845205307007,
+      "rewards/margins": 0.15696683526039124,
+      "rewards/rejected": -0.44585132598876953,
+      "step": 5440
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.613487782393661e-06,
+      "logits/chosen": -2.2866086959838867,
+      "logits/rejected": -1.841134786605835,
+      "logps/chosen": -517.4513549804688,
+      "logps/rejected": -667.3543090820312,
+      "loss": 0.112,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.3022264540195465,
+      "rewards/margins": 0.15514351427555084,
+      "rewards/rejected": -0.45736995339393616,
+      "step": 5450
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.6037054244444007e-06,
+      "logits/chosen": -2.2672483921051025,
+      "logits/rejected": -1.903223991394043,
+      "logps/chosen": -561.9781494140625,
+      "logps/rejected": -636.1325073242188,
+      "loss": 0.152,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3652775287628174,
+      "rewards/margins": 0.08305509388446808,
+      "rewards/rejected": -0.44833260774612427,
+      "step": 5460
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.593938788631103e-06,
+      "logits/chosen": -2.03192400932312,
+      "logits/rejected": -1.6650311946868896,
+      "logps/chosen": -493.18194580078125,
+      "logps/rejected": -695.05029296875,
+      "loss": 0.0899,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.3039739429950714,
+      "rewards/margins": 0.1745443046092987,
+      "rewards/rejected": -0.47851818799972534,
+      "step": 5470
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5841880462728626e-06,
+      "logits/chosen": -2.1529488563537598,
+      "logits/rejected": -1.8345768451690674,
+      "logps/chosen": -522.617919921875,
+      "logps/rejected": -683.61474609375,
+      "loss": 0.1333,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.28847092390060425,
+      "rewards/margins": 0.17859551310539246,
+      "rewards/rejected": -0.46706637740135193,
+      "step": 5480
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5744533684099861e-06,
+      "logits/chosen": -2.355879306793213,
+      "logits/rejected": -1.89114511013031,
+      "logps/chosen": -551.9974365234375,
+      "logps/rejected": -636.5489501953125,
+      "loss": 0.1691,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3352873921394348,
+      "rewards/margins": 0.10984688997268677,
+      "rewards/rejected": -0.44513431191444397,
+      "step": 5490
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5647349258009857e-06,
+      "logits/chosen": -2.0358262062072754,
+      "logits/rejected": -1.8236891031265259,
+      "logps/chosen": -617.5446166992188,
+      "logps/rejected": -746.7028198242188,
+      "loss": 0.0848,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3904941976070404,
+      "rewards/margins": 0.12986770272254944,
+      "rewards/rejected": -0.5203619003295898,
+      "step": 5500
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.555032888919586e-06,
+      "logits/chosen": -2.0381500720977783,
+      "logits/rejected": -1.6796925067901611,
+      "logps/chosen": -584.2960815429688,
+      "logps/rejected": -690.2525024414062,
+      "loss": 0.2384,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.37364262342453003,
+      "rewards/margins": 0.13294579088687897,
+      "rewards/rejected": -0.5065884590148926,
+      "step": 5510
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5453474279517383e-06,
+      "logits/chosen": -2.056273937225342,
+      "logits/rejected": -1.8743489980697632,
+      "logps/chosen": -565.3619384765625,
+      "logps/rejected": -669.6155395507812,
+      "loss": 0.119,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.3746207058429718,
+      "rewards/margins": 0.12104348838329315,
+      "rewards/rejected": -0.49566420912742615,
+      "step": 5520
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5356787127926285e-06,
+      "logits/chosen": -2.261429786682129,
+      "logits/rejected": -1.6400963068008423,
+      "logps/chosen": -658.01220703125,
+      "logps/rejected": -781.8679809570312,
+      "loss": 0.098,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3911864757537842,
+      "rewards/margins": 0.1944567859172821,
+      "rewards/rejected": -0.5856431722640991,
+      "step": 5530
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.526026913043699e-06,
+      "logits/chosen": -2.0749402046203613,
+      "logits/rejected": -2.0158350467681885,
+      "logps/chosen": -532.8299560546875,
+      "logps/rejected": -650.2535400390625,
+      "loss": 0.0782,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.3548316955566406,
+      "rewards/margins": 0.12620936334133148,
+      "rewards/rejected": -0.4810411036014557,
+      "step": 5540
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.5163921980096791e-06,
+      "logits/chosen": -2.0629875659942627,
+      "logits/rejected": -1.9767868518829346,
+      "logps/chosen": -620.8287353515625,
+      "logps/rejected": -718.6694946289062,
+      "loss": 0.123,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.4190002977848053,
+      "rewards/margins": 0.08350223302841187,
+      "rewards/rejected": -0.5025025010108948,
+      "step": 5550
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.5067747366956065e-06,
+      "logits/chosen": -2.3835396766662598,
+      "logits/rejected": -2.049961566925049,
+      "logps/chosen": -501.70330810546875,
+      "logps/rejected": -598.4969482421875,
+      "loss": 0.1501,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.29090631008148193,
+      "rewards/margins": 0.15174177289009094,
+      "rewards/rejected": -0.4426480829715729,
+      "step": 5560
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4971746978038671e-06,
+      "logits/chosen": -2.158270835876465,
+      "logits/rejected": -2.1244850158691406,
+      "logps/chosen": -543.0271606445312,
+      "logps/rejected": -678.8372192382812,
+      "loss": 0.1159,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.33038273453712463,
+      "rewards/margins": 0.12986037135124207,
+      "rewards/rejected": -0.4602430760860443,
+      "step": 5570
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4875922497312384e-06,
+      "logits/chosen": -2.137779474258423,
+      "logits/rejected": -1.6454992294311523,
+      "logps/chosen": -526.1381225585938,
+      "logps/rejected": -670.2523193359375,
+      "loss": 0.0791,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3154241442680359,
+      "rewards/margins": 0.16663704812526703,
+      "rewards/rejected": -0.4820612370967865,
+      "step": 5580
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4780275605659308e-06,
+      "logits/chosen": -2.154015302658081,
+      "logits/rejected": -1.6270949840545654,
+      "logps/chosen": -501.64788818359375,
+      "logps/rejected": -664.1883544921875,
+      "loss": 0.1011,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.3323962688446045,
+      "rewards/margins": 0.1896507889032364,
+      "rewards/rejected": -0.5220469832420349,
+      "step": 5590
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.46848079808464e-06,
+      "logits/chosen": -2.088256359100342,
+      "logits/rejected": -1.7928571701049805,
+      "logps/chosen": -593.34033203125,
+      "logps/rejected": -670.0963745117188,
+      "loss": 0.1633,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.35423874855041504,
+      "rewards/margins": 0.12422002851963043,
+      "rewards/rejected": -0.47845879197120667,
+      "step": 5600
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4589521297496085e-06,
+      "logits/chosen": -2.1464853286743164,
+      "logits/rejected": -1.932496428489685,
+      "logps/chosen": -597.0916748046875,
+      "logps/rejected": -707.6761474609375,
+      "loss": 0.1521,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.356023371219635,
+      "rewards/margins": 0.10124766826629639,
+      "rewards/rejected": -0.457271009683609,
+      "step": 5610
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4494417227056811e-06,
+      "logits/chosen": -2.3297324180603027,
+      "logits/rejected": -1.866176962852478,
+      "logps/chosen": -506.70953369140625,
+      "logps/rejected": -682.1671142578125,
+      "loss": 0.0791,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.3113633394241333,
+      "rewards/margins": 0.1794387698173523,
+      "rewards/rejected": -0.4908021092414856,
+      "step": 5620
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.4399497437773786e-06,
+      "logits/chosen": -2.2159767150878906,
+      "logits/rejected": -1.7067703008651733,
+      "logps/chosen": -620.3472900390625,
+      "logps/rejected": -782.3670654296875,
+      "loss": 0.1021,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.4018901288509369,
+      "rewards/margins": 0.1893962025642395,
+      "rewards/rejected": -0.591286301612854,
+      "step": 5630
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.4304763594659694e-06,
+      "logits/chosen": -2.3427064418792725,
+      "logits/rejected": -1.7328617572784424,
+      "logps/chosen": -598.8403930664062,
+      "logps/rejected": -682.558837890625,
+      "loss": 0.1715,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.34609344601631165,
+      "rewards/margins": 0.14814725518226624,
+      "rewards/rejected": -0.4942407011985779,
+      "step": 5640
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.4210217359465483e-06,
+      "logits/chosen": -2.3395252227783203,
+      "logits/rejected": -2.0148463249206543,
+      "logps/chosen": -534.9087524414062,
+      "logps/rejected": -620.61767578125,
+      "loss": 0.156,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3284892737865448,
+      "rewards/margins": 0.08220402896404266,
+      "rewards/rejected": -0.41069331765174866,
+      "step": 5650
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.4115860390651204e-06,
+      "logits/chosen": -2.2537014484405518,
+      "logits/rejected": -1.432426929473877,
+      "logps/chosen": -608.4447021484375,
+      "logps/rejected": -742.2864990234375,
+      "loss": 0.1272,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.3906776010990143,
+      "rewards/margins": 0.21027517318725586,
+      "rewards/rejected": -0.6009527444839478,
+      "step": 5660
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.4021694343356992e-06,
+      "logits/chosen": -2.3975398540496826,
+      "logits/rejected": -1.9052507877349854,
+      "logps/chosen": -529.54052734375,
+      "logps/rejected": -666.8858642578125,
+      "loss": 0.0997,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.33328336477279663,
+      "rewards/margins": 0.1553386151790619,
+      "rewards/rejected": -0.4886220097541809,
+      "step": 5670
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.3927720869373912e-06,
+      "logits/chosen": -2.0599472522735596,
+      "logits/rejected": -1.7738994359970093,
+      "logps/chosen": -579.2435302734375,
+      "logps/rejected": -697.3091430664062,
+      "loss": 0.0939,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.34287673234939575,
+      "rewards/margins": 0.1265747845172882,
+      "rewards/rejected": -0.46945157647132874,
+      "step": 5680
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.383394161711509e-06,
+      "logits/chosen": -2.054938793182373,
+      "logits/rejected": -1.7687804698944092,
+      "logps/chosen": -546.76171875,
+      "logps/rejected": -690.7644653320312,
+      "loss": 0.1094,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.35319188237190247,
+      "rewards/margins": 0.1568966805934906,
+      "rewards/rejected": -0.5100885629653931,
+      "step": 5690
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.3740358231586752e-06,
+      "logits/chosen": -2.237222194671631,
+      "logits/rejected": -1.8938148021697998,
+      "logps/chosen": -566.2021484375,
+      "logps/rejected": -674.2051391601562,
+      "loss": 0.1559,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.38061046600341797,
+      "rewards/margins": 0.149147629737854,
+      "rewards/rejected": -0.5297580361366272,
+      "step": 5700
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3646972354359379e-06,
+      "logits/chosen": -2.3532071113586426,
+      "logits/rejected": -1.9048734903335571,
+      "logps/chosen": -548.6280517578125,
+      "logps/rejected": -681.6298828125,
+      "loss": 0.1277,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.35211119055747986,
+      "rewards/margins": 0.1571749746799469,
+      "rewards/rejected": -0.509286105632782,
+      "step": 5710
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3553785623538873e-06,
+      "logits/chosen": -2.115138530731201,
+      "logits/rejected": -1.9435638189315796,
+      "logps/chosen": -500.4805603027344,
+      "logps/rejected": -654.8005981445312,
+      "loss": 0.1267,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3329200744628906,
+      "rewards/margins": 0.14238497614860535,
+      "rewards/rejected": -0.47530508041381836,
+      "step": 5720
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.346079967373792e-06,
+      "logits/chosen": -2.0034701824188232,
+      "logits/rejected": -1.9602371454238892,
+      "logps/chosen": -489.84820556640625,
+      "logps/rejected": -591.6983642578125,
+      "loss": 0.0961,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.3333253264427185,
+      "rewards/margins": 0.10935930907726288,
+      "rewards/rejected": -0.4426846504211426,
+      "step": 5730
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3368016136047194e-06,
+      "logits/chosen": -2.143648862838745,
+      "logits/rejected": -1.8776613473892212,
+      "logps/chosen": -617.1884155273438,
+      "logps/rejected": -668.1665649414062,
+      "loss": 0.1793,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.36267703771591187,
+      "rewards/margins": 0.11167631298303604,
+      "rewards/rejected": -0.4743533134460449,
+      "step": 5740
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3275436638006838e-06,
+      "logits/chosen": -2.244128704071045,
+      "logits/rejected": -1.8278095722198486,
+      "logps/chosen": -564.4967041015625,
+      "logps/rejected": -703.3328857421875,
+      "loss": 0.1332,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3400401175022125,
+      "rewards/margins": 0.1413845270872116,
+      "rewards/rejected": -0.48142462968826294,
+      "step": 5750
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3183062803577872e-06,
+      "logits/chosen": -2.241028308868408,
+      "logits/rejected": -1.855151891708374,
+      "logps/chosen": -514.6339111328125,
+      "logps/rejected": -676.3679809570312,
+      "loss": 0.1371,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.3447694182395935,
+      "rewards/margins": 0.196563258767128,
+      "rewards/rejected": -0.5413327217102051,
+      "step": 5760
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3090896253113736e-06,
+      "logits/chosen": -2.2188217639923096,
+      "logits/rejected": -2.020664691925049,
+      "logps/chosen": -509.14990234375,
+      "logps/rejected": -585.9461669921875,
+      "loss": 0.1599,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.3009883761405945,
+      "rewards/margins": 0.09804406017065048,
+      "rewards/rejected": -0.3990324139595032,
+      "step": 5770
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.2998938603331796e-06,
+      "logits/chosen": -2.152019739151001,
+      "logits/rejected": -1.8972949981689453,
+      "logps/chosen": -610.6968994140625,
+      "logps/rejected": -716.9977416992188,
+      "loss": 0.0925,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.41787225008010864,
+      "rewards/margins": 0.09503994137048721,
+      "rewards/rejected": -0.5129121541976929,
+      "step": 5780
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.2907191467285118e-06,
+      "logits/chosen": -2.161350727081299,
+      "logits/rejected": -1.9723711013793945,
+      "logps/chosen": -499.3922424316406,
+      "logps/rejected": -745.1807861328125,
+      "loss": 0.146,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.28003165125846863,
+      "rewards/margins": 0.15189263224601746,
+      "rewards/rejected": -0.4319242537021637,
+      "step": 5790
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2815656454334013e-06,
+      "logits/chosen": -2.1615190505981445,
+      "logits/rejected": -2.063441038131714,
+      "logps/chosen": -572.2648315429688,
+      "logps/rejected": -684.9200439453125,
+      "loss": 0.1573,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.36520278453826904,
+      "rewards/margins": 0.11442582309246063,
+      "rewards/rejected": -0.47962862253189087,
+      "step": 5800
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.272433517011793e-06,
+      "logits/chosen": -2.228452444076538,
+      "logits/rejected": -1.968274712562561,
+      "logps/chosen": -632.6124267578125,
+      "logps/rejected": -719.0970458984375,
+      "loss": 0.0852,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3688015341758728,
+      "rewards/margins": 0.11153985559940338,
+      "rewards/rejected": -0.480341374874115,
+      "step": 5810
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2633229216527235e-06,
+      "logits/chosen": -2.1732847690582275,
+      "logits/rejected": -1.6696808338165283,
+      "logps/chosen": -505.33184814453125,
+      "logps/rejected": -671.8098754882812,
+      "loss": 0.1274,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.3174054026603699,
+      "rewards/margins": 0.19355422258377075,
+      "rewards/rejected": -0.5109595656394958,
+      "step": 5820
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.254234019167514e-06,
+      "logits/chosen": -2.143200397491455,
+      "logits/rejected": -1.7109419107437134,
+      "logps/chosen": -591.1917114257812,
+      "logps/rejected": -732.4910278320312,
+      "loss": 0.0837,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.35197946429252625,
+      "rewards/margins": 0.18254925310611725,
+      "rewards/rejected": -0.5345287322998047,
+      "step": 5830
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.24516696898696e-06,
+      "logits/chosen": -2.1101744174957275,
+      "logits/rejected": -1.7346477508544922,
+      "logps/chosen": -623.3919677734375,
+      "logps/rejected": -784.92626953125,
+      "loss": 0.1069,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.4072991907596588,
+      "rewards/margins": 0.16031184792518616,
+      "rewards/rejected": -0.567611038684845,
+      "step": 5840
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2361219301585487e-06,
+      "logits/chosen": -2.3174877166748047,
+      "logits/rejected": -1.904543161392212,
+      "logps/chosen": -559.4212646484375,
+      "logps/rejected": -680.2595825195312,
+      "loss": 0.1099,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3236214816570282,
+      "rewards/margins": 0.1729937493801117,
+      "rewards/rejected": -0.4966152310371399,
+      "step": 5850
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2270990613436522e-06,
+      "logits/chosen": -2.217374086380005,
+      "logits/rejected": -1.8524898290634155,
+      "logps/chosen": -509.1732482910156,
+      "logps/rejected": -638.6177978515625,
+      "loss": 0.1501,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.32199567556381226,
+      "rewards/margins": 0.11517663300037384,
+      "rewards/rejected": -0.4371722638607025,
+      "step": 5860
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2180985208147571e-06,
+      "logits/chosen": -2.1892879009246826,
+      "logits/rejected": -1.988385796546936,
+      "logps/chosen": -490.54339599609375,
+      "logps/rejected": -646.9719848632812,
+      "loss": 0.0834,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.316831111907959,
+      "rewards/margins": 0.14619895815849304,
+      "rewards/rejected": -0.46303004026412964,
+      "step": 5870
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.2091204664526831e-06,
+      "logits/chosen": -2.226302146911621,
+      "logits/rejected": -1.807069182395935,
+      "logps/chosen": -618.5587768554688,
+      "logps/rejected": -682.0938110351562,
+      "loss": 0.2029,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3742113709449768,
+      "rewards/margins": 0.13137498497962952,
+      "rewards/rejected": -0.5055863261222839,
+      "step": 5880
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.2001650557438143e-06,
+      "logits/chosen": -2.2408766746520996,
+      "logits/rejected": -1.956112265586853,
+      "logps/chosen": -630.4677124023438,
+      "logps/rejected": -772.59716796875,
+      "loss": 0.1129,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.380460649728775,
+      "rewards/margins": 0.1700032353401184,
+      "rewards/rejected": -0.5504637956619263,
+      "step": 5890
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1912324457773336e-06,
+      "logits/chosen": -2.274834156036377,
+      "logits/rejected": -1.88030207157135,
+      "logps/chosen": -554.555419921875,
+      "logps/rejected": -725.6541748046875,
+      "loss": 0.1094,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.35316720604896545,
+      "rewards/margins": 0.1630186140537262,
+      "rewards/rejected": -0.5161858201026917,
+      "step": 5900
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.182322793242476e-06,
+      "logits/chosen": -2.262993097305298,
+      "logits/rejected": -2.0412886142730713,
+      "logps/chosen": -586.8534545898438,
+      "logps/rejected": -728.2939453125,
+      "loss": 0.1399,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.404453843832016,
+      "rewards/margins": 0.12682022154331207,
+      "rewards/rejected": -0.5312740802764893,
+      "step": 5910
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1734362544257686e-06,
+      "logits/chosen": -2.095017910003662,
+      "logits/rejected": -1.5635570287704468,
+      "logps/chosen": -601.4953002929688,
+      "logps/rejected": -652.2698364257812,
+      "loss": 0.1153,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.3327557444572449,
+      "rewards/margins": 0.13259822130203247,
+      "rewards/rejected": -0.4653540253639221,
+      "step": 5920
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1645729852082977e-06,
+      "logits/chosen": -2.3760483264923096,
+      "logits/rejected": -1.8564010858535767,
+      "logps/chosen": -627.1971435546875,
+      "logps/rejected": -751.6751708984375,
+      "loss": 0.1702,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.42715543508529663,
+      "rewards/margins": 0.15015287697315216,
+      "rewards/rejected": -0.5773082971572876,
+      "step": 5930
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1557331410629708e-06,
+      "logits/chosen": -2.4716744422912598,
+      "logits/rejected": -1.741478681564331,
+      "logps/chosen": -554.5077514648438,
+      "logps/rejected": -648.3279418945312,
+      "loss": 0.1061,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.32236966490745544,
+      "rewards/margins": 0.16489025950431824,
+      "rewards/rejected": -0.4872599244117737,
+      "step": 5940
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1469168770517913e-06,
+      "logits/chosen": -2.4337592124938965,
+      "logits/rejected": -1.8899034261703491,
+      "logps/chosen": -574.9594116210938,
+      "logps/rejected": -646.1371459960938,
+      "loss": 0.1352,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.3693484663963318,
+      "rewards/margins": 0.12260559946298599,
+      "rewards/rejected": -0.49195408821105957,
+      "step": 5950
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.1381243478231336e-06,
+      "logits/chosen": -2.344062328338623,
+      "logits/rejected": -1.9013738632202148,
+      "logps/chosen": -614.05322265625,
+      "logps/rejected": -648.7421264648438,
+      "loss": 0.1526,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.35756710171699524,
+      "rewards/margins": 0.11178120225667953,
+      "rewards/rejected": -0.469348281621933,
+      "step": 5960
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.1293557076090403e-06,
+      "logits/chosen": -2.3604280948638916,
+      "logits/rejected": -1.8050537109375,
+      "logps/chosen": -537.1051635742188,
+      "logps/rejected": -670.773681640625,
+      "loss": 0.1185,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.3040335178375244,
+      "rewards/margins": 0.1627335101366043,
+      "rewards/rejected": -0.4667670726776123,
+      "step": 5970
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.1206111102225043e-06,
+      "logits/chosen": -2.238236427307129,
+      "logits/rejected": -1.9439430236816406,
+      "logps/chosen": -665.1473388671875,
+      "logps/rejected": -785.346923828125,
+      "loss": 0.12,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3865990936756134,
+      "rewards/margins": 0.1437794715166092,
+      "rewards/rejected": -0.5303784608840942,
+      "step": 5980
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.1118907090547805e-06,
+      "logits/chosen": -2.2979445457458496,
+      "logits/rejected": -1.8327503204345703,
+      "logps/chosen": -620.1920166015625,
+      "logps/rejected": -761.3084716796875,
+      "loss": 0.0972,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.37477365136146545,
+      "rewards/margins": 0.19091708958148956,
+      "rewards/rejected": -0.565690815448761,
+      "step": 5990
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.1031946570726912e-06,
+      "logits/chosen": -2.160611629486084,
+      "logits/rejected": -2.038691520690918,
+      "logps/chosen": -696.7026977539062,
+      "logps/rejected": -803.025146484375,
+      "loss": 0.1038,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.4737313389778137,
+      "rewards/margins": 0.10619950294494629,
+      "rewards/rejected": -0.5799307823181152,
+      "step": 6000
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.094523106815944e-06,
+      "logits/chosen": -2.098212957382202,
+      "logits/rejected": -1.7056735754013062,
+      "logps/chosen": -597.8346557617188,
+      "logps/rejected": -761.1981811523438,
+      "loss": 0.1325,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.3751162588596344,
+      "rewards/margins": 0.1747501790523529,
+      "rewards/rejected": -0.5498664379119873,
+      "step": 6010
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.0858762103944511e-06,
+      "logits/chosen": -2.0707337856292725,
+      "logits/rejected": -1.757552146911621,
+      "logps/chosen": -688.3229370117188,
+      "logps/rejected": -756.0613403320312,
+      "loss": 0.1043,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.4600822329521179,
+      "rewards/margins": 0.09591424465179443,
+      "rewards/rejected": -0.5559965372085571,
+      "step": 6020
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.0772541194856732e-06,
+      "logits/chosen": -2.3866918087005615,
+      "logits/rejected": -1.76297926902771,
+      "logps/chosen": -642.9715576171875,
+      "logps/rejected": -747.2301635742188,
+      "loss": 0.0667,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.36399585008621216,
+      "rewards/margins": 0.16233620047569275,
+      "rewards/rejected": -0.5263320207595825,
+      "step": 6030
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.068656985331943e-06,
+      "logits/chosen": -2.2231695652008057,
+      "logits/rejected": -1.8431459665298462,
+      "logps/chosen": -605.690673828125,
+      "logps/rejected": -787.3533935546875,
+      "loss": 0.1075,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.4007289409637451,
+      "rewards/margins": 0.18494176864624023,
+      "rewards/rejected": -0.5856707692146301,
+      "step": 6040
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.060084958737825e-06,
+      "logits/chosen": -2.2790846824645996,
+      "logits/rejected": -1.6443449258804321,
+      "logps/chosen": -519.8569946289062,
+      "logps/rejected": -669.1280517578125,
+      "loss": 0.1168,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.3220061659812927,
+      "rewards/margins": 0.17372606694698334,
+      "rewards/rejected": -0.49573221802711487,
+      "step": 6050
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0515381900674643e-06,
+      "logits/chosen": -2.2725367546081543,
+      "logits/rejected": -1.8912683725357056,
+      "logps/chosen": -563.7933959960938,
+      "logps/rejected": -727.9124755859375,
+      "loss": 0.1137,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.3516715168952942,
+      "rewards/margins": 0.15827377140522003,
+      "rewards/rejected": -0.509945273399353,
+      "step": 6060
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.04301682924195e-06,
+      "logits/chosen": -2.1667473316192627,
+      "logits/rejected": -1.629547119140625,
+      "logps/chosen": -520.6829833984375,
+      "logps/rejected": -679.8859252929688,
+      "loss": 0.136,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3583363890647888,
+      "rewards/margins": 0.193718820810318,
+      "rewards/rejected": -0.5520552396774292,
+      "step": 6070
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.034521025736686e-06,
+      "logits/chosen": -2.208749532699585,
+      "logits/rejected": -1.7252041101455688,
+      "logps/chosen": -545.7933959960938,
+      "logps/rejected": -690.5831298828125,
+      "loss": 0.1692,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.36778688430786133,
+      "rewards/margins": 0.1479516178369522,
+      "rewards/rejected": -0.5157384872436523,
+      "step": 6080
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0260509285787694e-06,
+      "logits/chosen": -2.341977596282959,
+      "logits/rejected": -1.6944694519042969,
+      "logps/chosen": -573.2259521484375,
+      "logps/rejected": -683.978759765625,
+      "loss": 0.1008,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.36088132858276367,
+      "rewards/margins": 0.16303828358650208,
+      "rewards/rejected": -0.5239195823669434,
+      "step": 6090
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0176066863443726e-06,
+      "logits/chosen": -2.115180253982544,
+      "logits/rejected": -1.761863112449646,
+      "logps/chosen": -615.5621337890625,
+      "logps/rejected": -688.4131469726562,
+      "loss": 0.131,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.40311139822006226,
+      "rewards/margins": 0.12240666151046753,
+      "rewards/rejected": -0.5255180597305298,
+      "step": 6100
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0091884471561424e-06,
+      "logits/chosen": -2.123635768890381,
+      "logits/rejected": -1.830743432044983,
+      "logps/chosen": -614.26513671875,
+      "logps/rejected": -695.25146484375,
+      "loss": 0.1628,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.4025048613548279,
+      "rewards/margins": 0.1111200675368309,
+      "rewards/rejected": -0.5136249661445618,
+      "step": 6110
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0007963586806e-06,
+      "logits/chosen": -2.092599630355835,
+      "logits/rejected": -1.6244100332260132,
+      "logps/chosen": -661.3244018554688,
+      "logps/rejected": -798.8341064453125,
+      "loss": 0.1152,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.4307892322540283,
+      "rewards/margins": 0.1681210696697235,
+      "rewards/rejected": -0.5989102125167847,
+      "step": 6120
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.924305681255484e-07,
+      "logits/chosen": -2.117426633834839,
+      "logits/rejected": -1.6414989233016968,
+      "logps/chosen": -604.1846313476562,
+      "logps/rejected": -759.3634033203125,
+      "loss": 0.1323,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.39068490266799927,
+      "rewards/margins": 0.19260461628437042,
+      "rewards/rejected": -0.5832895040512085,
+      "step": 6130
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.840912222374932e-07,
+      "logits/chosen": -2.2299506664276123,
+      "logits/rejected": -1.6416250467300415,
+      "logps/chosen": -626.3302001953125,
+      "logps/rejected": -757.1051025390625,
+      "loss": 0.1299,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.40157923102378845,
+      "rewards/margins": 0.1709556132555008,
+      "rewards/rejected": -0.5725348591804504,
+      "step": 6140
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.757784672990668e-07,
+      "logits/chosen": -1.979491949081421,
+      "logits/rejected": -1.4871678352355957,
+      "logps/chosen": -662.2730712890625,
+      "logps/rejected": -771.0643310546875,
+      "loss": 0.1377,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.4613346457481384,
+      "rewards/margins": 0.15380756556987762,
+      "rewards/rejected": -0.6151422262191772,
+      "step": 6150
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.674924491264632e-07,
+      "logits/chosen": -2.0712523460388184,
+      "logits/rejected": -1.842908263206482,
+      "logps/chosen": -566.8055419921875,
+      "logps/rejected": -667.7489013671875,
+      "loss": 0.114,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.39151254296302795,
+      "rewards/margins": 0.11860235035419464,
+      "rewards/rejected": -0.5101149082183838,
+      "step": 6160
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.59233313066878e-07,
+      "logits/chosen": -2.295414924621582,
+      "logits/rejected": -1.7716766595840454,
+      "logps/chosen": -594.5213623046875,
+      "logps/rejected": -732.1650390625,
+      "loss": 0.0641,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.3785635232925415,
+      "rewards/margins": 0.16181840002536774,
+      "rewards/rejected": -0.540381908416748,
+      "step": 6170
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.510012039959632e-07,
+      "logits/chosen": -2.161921977996826,
+      "logits/rejected": -1.7852888107299805,
+      "logps/chosen": -654.4505004882812,
+      "logps/rejected": -756.0235595703125,
+      "loss": 0.1187,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.41412264108657837,
+      "rewards/margins": 0.14051425457000732,
+      "rewards/rejected": -0.5546368956565857,
+      "step": 6180
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.427962663152821e-07,
+      "logits/chosen": -2.1516811847686768,
+      "logits/rejected": -1.8562183380126953,
+      "logps/chosen": -667.8128051757812,
+      "logps/rejected": -737.7991943359375,
+      "loss": 0.1186,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.4085676074028015,
+      "rewards/margins": 0.13456174731254578,
+      "rewards/rejected": -0.5431293249130249,
+      "step": 6190
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.346186439497778e-07,
+      "logits/chosen": -2.1450953483581543,
+      "logits/rejected": -1.7751076221466064,
+      "logps/chosen": -602.6964111328125,
+      "logps/rejected": -693.8768310546875,
+      "loss": 0.1886,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.4082205295562744,
+      "rewards/margins": 0.11635033786296844,
+      "rewards/rejected": -0.5245708227157593,
+      "step": 6200
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 9.264684803452484e-07,
+      "logits/chosen": -2.1668758392333984,
+      "logits/rejected": -1.8443607091903687,
+      "logps/chosen": -610.8843994140625,
+      "logps/rejected": -699.7890625,
+      "loss": 0.126,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.3591611683368683,
+      "rewards/margins": 0.10757863521575928,
+      "rewards/rejected": -0.46673980355262756,
+      "step": 6210
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 9.183459184658317e-07,
+      "logits/chosen": -2.0819430351257324,
+      "logits/rejected": -1.7518011331558228,
+      "logps/chosen": -649.2342529296875,
+      "logps/rejected": -719.9200439453125,
+      "loss": 0.1074,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.43511080741882324,
+      "rewards/margins": 0.08323594182729721,
+      "rewards/rejected": -0.5183467268943787,
+      "step": 6220
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 9.102511007914924e-07,
+      "logits/chosen": -2.086353063583374,
+      "logits/rejected": -1.5733944177627563,
+      "logps/chosen": -555.7533569335938,
+      "logps/rejected": -699.146728515625,
+      "loss": 0.1383,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.38916483521461487,
+      "rewards/margins": 0.1736629158258438,
+      "rewards/rejected": -0.5628277063369751,
+      "step": 6230
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 9.021841693155343e-07,
+      "logits/chosen": -2.23599910736084,
+      "logits/rejected": -1.840063452720642,
+      "logps/chosen": -588.5497436523438,
+      "logps/rejected": -712.6214599609375,
+      "loss": 0.1291,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.36313143372535706,
+      "rewards/margins": 0.15469108521938324,
+      "rewards/rejected": -0.5178225040435791,
+      "step": 6240
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.94145265542094e-07,
+      "logits/chosen": -2.335198402404785,
+      "logits/rejected": -2.099552869796753,
+      "logps/chosen": -637.7589721679688,
+      "logps/rejected": -750.11474609375,
+      "loss": 0.0867,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.36321356892585754,
+      "rewards/margins": 0.15715032815933228,
+      "rewards/rejected": -0.5203639268875122,
+      "step": 6250
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.861345304836727e-07,
+      "logits/chosen": -2.139549493789673,
+      "logits/rejected": -2.0422310829162598,
+      "logps/chosen": -681.3778686523438,
+      "logps/rejected": -796.6890258789062,
+      "loss": 0.1087,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.4570437967777252,
+      "rewards/margins": 0.09793035686016083,
+      "rewards/rejected": -0.5549741983413696,
+      "step": 6260
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.781521046586541e-07,
+      "logits/chosen": -2.156123638153076,
+      "logits/rejected": -1.6848316192626953,
+      "logps/chosen": -598.8487548828125,
+      "logps/rejected": -724.5496215820312,
+      "loss": 0.1385,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.396481454372406,
+      "rewards/margins": 0.15511992573738098,
+      "rewards/rejected": -0.5516014099121094,
+      "step": 6270
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.701981280888444e-07,
+      "logits/chosen": -2.0162224769592285,
+      "logits/rejected": -1.8342578411102295,
+      "logps/chosen": -579.2422485351562,
+      "logps/rejected": -722.4891967773438,
+      "loss": 0.1614,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.37111929059028625,
+      "rewards/margins": 0.14426055550575256,
+      "rewards/rejected": -0.5153799057006836,
+      "step": 6280
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.622727402970097e-07,
+      "logits/chosen": -1.9541709423065186,
+      "logits/rejected": -1.8992236852645874,
+      "logps/chosen": -662.0847778320312,
+      "logps/rejected": -788.9730224609375,
+      "loss": 0.0856,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.4542261064052582,
+      "rewards/margins": 0.08948315680027008,
+      "rewards/rejected": -0.5437092185020447,
+      "step": 6290
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.543760803044393e-07,
+      "logits/chosen": -2.0333831310272217,
+      "logits/rejected": -1.6173734664916992,
+      "logps/chosen": -646.6117553710938,
+      "logps/rejected": -818.1234130859375,
+      "loss": 0.116,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.4436543583869934,
+      "rewards/margins": 0.1918451488018036,
+      "rewards/rejected": -0.6354994773864746,
+      "step": 6300
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.465082866284951e-07,
+      "logits/chosen": -2.2854881286621094,
+      "logits/rejected": -1.7396091222763062,
+      "logps/chosen": -634.8878784179688,
+      "logps/rejected": -762.9354858398438,
+      "loss": 0.1194,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.4202674925327301,
+      "rewards/margins": 0.1719280183315277,
+      "rewards/rejected": -0.5921956300735474,
+      "step": 6310
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.386694972801904e-07,
+      "logits/chosen": -2.049105167388916,
+      "logits/rejected": -1.6528642177581787,
+      "logps/chosen": -653.4500732421875,
+      "logps/rejected": -783.7057495117188,
+      "loss": 0.1251,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.4330308437347412,
+      "rewards/margins": 0.16803285479545593,
+      "rewards/rejected": -0.6010637283325195,
+      "step": 6320
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.308598497617648e-07,
+      "logits/chosen": -2.0765178203582764,
+      "logits/rejected": -1.7483161687850952,
+      "logps/chosen": -504.6463928222656,
+      "logps/rejected": -633.7625732421875,
+      "loss": 0.0701,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.37099042534828186,
+      "rewards/margins": 0.1265013962984085,
+      "rewards/rejected": -0.49749183654785156,
+      "step": 6330
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.230794810642753e-07,
+      "logits/chosen": -2.173595905303955,
+      "logits/rejected": -1.7059295177459717,
+      "logps/chosen": -648.63671875,
+      "logps/rejected": -713.3856811523438,
+      "loss": 0.1158,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.4014323651790619,
+      "rewards/margins": 0.11388696730136871,
+      "rewards/rejected": -0.5153193473815918,
+      "step": 6340
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.153285276651876e-07,
+      "logits/chosen": -2.324289321899414,
+      "logits/rejected": -2.057042360305786,
+      "logps/chosen": -535.645751953125,
+      "logps/rejected": -677.26513671875,
+      "loss": 0.0965,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.34391599893569946,
+      "rewards/margins": 0.12264589220285416,
+      "rewards/rejected": -0.46656185388565063,
+      "step": 6350
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.076071255259918e-07,
+      "logits/chosen": -2.132390260696411,
+      "logits/rejected": -1.6475098133087158,
+      "logps/chosen": -628.40185546875,
+      "logps/rejected": -739.6697998046875,
+      "loss": 0.1085,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.4293893277645111,
+      "rewards/margins": 0.145646870136261,
+      "rewards/rejected": -0.5750362277030945,
+      "step": 6360
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 7.999154100898063e-07,
+      "logits/chosen": -2.0084128379821777,
+      "logits/rejected": -1.9067214727401733,
+      "logps/chosen": -617.2889404296875,
+      "logps/rejected": -804.7889404296875,
+      "loss": 0.0917,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.45773690938949585,
+      "rewards/margins": 0.14870281517505646,
+      "rewards/rejected": -0.6064397692680359,
+      "step": 6370
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.922535162790095e-07,
+      "logits/chosen": -2.283172607421875,
+      "logits/rejected": -2.0394046306610107,
+      "logps/chosen": -548.1370239257812,
+      "logps/rejected": -695.6280517578125,
+      "loss": 0.1141,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.35664018988609314,
+      "rewards/margins": 0.1481877863407135,
+      "rewards/rejected": -0.5048280358314514,
+      "step": 6380
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.846215784928721e-07,
+      "logits/chosen": -2.226024866104126,
+      "logits/rejected": -1.9169858694076538,
+      "logps/chosen": -544.6981811523438,
+      "logps/rejected": -734.1502075195312,
+      "loss": 0.1011,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.35820525884628296,
+      "rewards/margins": 0.19244256615638733,
+      "rewards/rejected": -0.5506478548049927,
+      "step": 6390
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.770197306051968e-07,
+      "logits/chosen": -2.3547351360321045,
+      "logits/rejected": -1.6171627044677734,
+      "logps/chosen": -587.7491455078125,
+      "logps/rejected": -724.0132446289062,
+      "loss": 0.108,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -0.3620437979698181,
+      "rewards/margins": 0.1923428624868393,
+      "rewards/rejected": -0.5543866157531738,
+      "step": 6400
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.694481059619705e-07,
+      "logits/chosen": -2.1217620372772217,
+      "logits/rejected": -1.8178207874298096,
+      "logps/chosen": -597.6928100585938,
+      "logps/rejected": -752.03662109375,
+      "loss": 0.1255,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3996157646179199,
+      "rewards/margins": 0.16985362768173218,
+      "rewards/rejected": -0.5694694519042969,
+      "step": 6410
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.619068373790306e-07,
+      "logits/chosen": -2.293581962585449,
+      "logits/rejected": -1.7962539196014404,
+      "logps/chosen": -588.2032470703125,
+      "logps/rejected": -713.5232543945312,
+      "loss": 0.1345,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.35084909200668335,
+      "rewards/margins": 0.15288697183132172,
+      "rewards/rejected": -0.503736138343811,
+      "step": 6420
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.543960571397257e-07,
+      "logits/chosen": -2.2539820671081543,
+      "logits/rejected": -1.9140384197235107,
+      "logps/chosen": -570.0762329101562,
+      "logps/rejected": -692.5343017578125,
+      "loss": 0.0704,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3793059289455414,
+      "rewards/margins": 0.13058127462863922,
+      "rewards/rejected": -0.5098872184753418,
+      "step": 6430
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.469158969926038e-07,
+      "logits/chosen": -2.306323766708374,
+      "logits/rejected": -1.8341777324676514,
+      "logps/chosen": -598.2462158203125,
+      "logps/rejected": -711.7991943359375,
+      "loss": 0.0801,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.38286226987838745,
+      "rewards/margins": 0.12677903473377228,
+      "rewards/rejected": -0.5096412897109985,
+      "step": 6440
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.39466488149097e-07,
+      "logits/chosen": -2.269989013671875,
+      "logits/rejected": -1.656507134437561,
+      "logps/chosen": -568.6844482421875,
+      "logps/rejected": -695.0865478515625,
+      "loss": 0.1177,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3659561574459076,
+      "rewards/margins": 0.16830545663833618,
+      "rewards/rejected": -0.5342615842819214,
+      "step": 6450
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 7.320479612812218e-07,
+      "logits/chosen": -2.276284694671631,
+      "logits/rejected": -1.7564642429351807,
+      "logps/chosen": -480.1602478027344,
+      "logps/rejected": -599.44189453125,
+      "loss": 0.066,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3121473789215088,
+      "rewards/margins": 0.14222674071788788,
+      "rewards/rejected": -0.4543740153312683,
+      "step": 6460
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 7.246604465192825e-07,
+      "logits/chosen": -2.119725465774536,
+      "logits/rejected": -1.5150271654129028,
+      "logps/chosen": -629.1981811523438,
+      "logps/rejected": -742.7669067382812,
+      "loss": 0.133,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.4192887246608734,
+      "rewards/margins": 0.18765732645988464,
+      "rewards/rejected": -0.6069461107254028,
+      "step": 6470
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 7.173040734495973e-07,
+      "logits/chosen": -2.1277499198913574,
+      "logits/rejected": -1.7308118343353271,
+      "logps/chosen": -659.2029418945312,
+      "logps/rejected": -792.1743774414062,
+      "loss": 0.1134,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.418001264333725,
+      "rewards/margins": 0.11754250526428223,
+      "rewards/rejected": -0.5355437994003296,
+      "step": 6480
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 7.099789711122149e-07,
+      "logits/chosen": -2.2341220378875732,
+      "logits/rejected": -1.8366056680679321,
+      "logps/chosen": -651.04736328125,
+      "logps/rejected": -767.5491943359375,
+      "loss": 0.1119,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.4018731117248535,
+      "rewards/margins": 0.1543024629354477,
+      "rewards/rejected": -0.5561755895614624,
+      "step": 6490
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 7.02685267998659e-07,
+      "logits/chosen": -2.043992280960083,
+      "logits/rejected": -1.795344352722168,
+      "logps/chosen": -531.8258056640625,
+      "logps/rejected": -637.4591064453125,
+      "loss": 0.1778,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3571312427520752,
+      "rewards/margins": 0.10287632048130035,
+      "rewards/rejected": -0.46000751852989197,
+      "step": 6500
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 6.954230920496702e-07,
+      "logits/chosen": -2.1191444396972656,
+      "logits/rejected": -1.762515664100647,
+      "logps/chosen": -628.6137084960938,
+      "logps/rejected": -803.2864379882812,
+      "loss": 0.0959,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.4733254313468933,
+      "rewards/margins": 0.17033693194389343,
+      "rewards/rejected": -0.6436623930931091,
+      "step": 6510
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 6.881925706529641e-07,
+      "logits/chosen": -2.386753559112549,
+      "logits/rejected": -1.7375595569610596,
+      "logps/chosen": -617.1004638671875,
+      "logps/rejected": -758.0531005859375,
+      "loss": 0.0838,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.411329984664917,
+      "rewards/margins": 0.18558369576931,
+      "rewards/rejected": -0.5969136953353882,
+      "step": 6520
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 6.809938306409925e-07,
+      "logits/chosen": -2.028801679611206,
+      "logits/rejected": -1.7875490188598633,
+      "logps/chosen": -618.8595581054688,
+      "logps/rejected": -736.8878173828125,
+      "loss": 0.1057,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.41217726469039917,
+      "rewards/margins": 0.1439303755760193,
+      "rewards/rejected": -0.5561076402664185,
+      "step": 6530
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 6.738269982887266e-07,
+      "logits/chosen": -2.291637420654297,
+      "logits/rejected": -1.830788016319275,
+      "logps/chosen": -644.20751953125,
+      "logps/rejected": -745.6409912109375,
+      "loss": 0.1011,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3701763451099396,
+      "rewards/margins": 0.1719406396150589,
+      "rewards/rejected": -0.5421169996261597,
+      "step": 6540
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.66692199311432e-07,
+      "logits/chosen": -2.292654275894165,
+      "logits/rejected": -1.9484418630599976,
+      "logps/chosen": -598.6500244140625,
+      "logps/rejected": -674.4227294921875,
+      "loss": 0.1206,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3510053753852844,
+      "rewards/margins": 0.13906903564929962,
+      "rewards/rejected": -0.4900744557380676,
+      "step": 6550
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.595895588624717e-07,
+      "logits/chosen": -2.3947482109069824,
+      "logits/rejected": -1.9516847133636475,
+      "logps/chosen": -519.2133178710938,
+      "logps/rejected": -666.8939208984375,
+      "loss": 0.1252,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.29892998933792114,
+      "rewards/margins": 0.18055732548236847,
+      "rewards/rejected": -0.4794873297214508,
+      "step": 6560
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.525192015311069e-07,
+      "logits/chosen": -2.258301019668579,
+      "logits/rejected": -1.8692222833633423,
+      "logps/chosen": -612.4740600585938,
+      "logps/rejected": -744.18408203125,
+      "loss": 0.0709,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.38903966546058655,
+      "rewards/margins": 0.15283407270908356,
+      "rewards/rejected": -0.5418737530708313,
+      "step": 6570
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.454812513403127e-07,
+      "logits/chosen": -2.395693302154541,
+      "logits/rejected": -1.9158267974853516,
+      "logps/chosen": -522.1923217773438,
+      "logps/rejected": -630.0458374023438,
+      "loss": 0.0909,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.33386439085006714,
+      "rewards/margins": 0.14329631626605988,
+      "rewards/rejected": -0.4771607518196106,
+      "step": 6580
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.384758317445991e-07,
+      "logits/chosen": -2.218947649002075,
+      "logits/rejected": -1.5449450016021729,
+      "logps/chosen": -560.5776977539062,
+      "logps/rejected": -712.8621826171875,
+      "loss": 0.1186,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.3123001158237457,
+      "rewards/margins": 0.2384105622768402,
+      "rewards/rejected": -0.5507106781005859,
+      "step": 6590
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.31503065627854e-07,
+      "logits/chosen": -2.067082166671753,
+      "logits/rejected": -1.753090262413025,
+      "logps/chosen": -615.6964111328125,
+      "logps/rejected": -765.8560791015625,
+      "loss": 0.1353,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3984375298023224,
+      "rewards/margins": 0.14735157787799835,
+      "rewards/rejected": -0.5457891225814819,
+      "step": 6600
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.245630753011767e-07,
+      "logits/chosen": -2.351511001586914,
+      "logits/rejected": -1.8430366516113281,
+      "logps/chosen": -592.149658203125,
+      "logps/rejected": -701.5977783203125,
+      "loss": 0.1205,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.33855703473091125,
+      "rewards/margins": 0.16403348743915558,
+      "rewards/rejected": -0.502590537071228,
+      "step": 6610
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.176559825007408e-07,
+      "logits/chosen": -2.3234474658966064,
+      "logits/rejected": -1.964428186416626,
+      "logps/chosen": -586.528564453125,
+      "logps/rejected": -699.1837158203125,
+      "loss": 0.0996,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.3417760729789734,
+      "rewards/margins": 0.14765329658985138,
+      "rewards/rejected": -0.48942941427230835,
+      "step": 6620
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 6.107819083856559e-07,
+      "logits/chosen": -2.317223310470581,
+      "logits/rejected": -1.836854338645935,
+      "logps/chosen": -610.1551513671875,
+      "logps/rejected": -670.6759643554688,
+      "loss": 0.1941,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3576057553291321,
+      "rewards/margins": 0.11452789604663849,
+      "rewards/rejected": -0.4721336364746094,
+      "step": 6630
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 6.039409735358418e-07,
+      "logits/chosen": -2.1314139366149902,
+      "logits/rejected": -1.8690674304962158,
+      "logps/chosen": -608.4373779296875,
+      "logps/rejected": -756.8580322265625,
+      "loss": 0.0985,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.38978224992752075,
+      "rewards/margins": 0.1923934668302536,
+      "rewards/rejected": -0.5821757316589355,
+      "step": 6640
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.971332979499112e-07,
+      "logits/chosen": -2.1543240547180176,
+      "logits/rejected": -1.804102897644043,
+      "logps/chosen": -525.6702880859375,
+      "logps/rejected": -630.8278198242188,
+      "loss": 0.1054,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.34058088064193726,
+      "rewards/margins": 0.14997749030590057,
+      "rewards/rejected": -0.490558385848999,
+      "step": 6650
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.903590010430732e-07,
+      "logits/chosen": -2.202484130859375,
+      "logits/rejected": -1.6558809280395508,
+      "logps/chosen": -538.0249633789062,
+      "logps/rejected": -650.339111328125,
+      "loss": 0.1524,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.33461135625839233,
+      "rewards/margins": 0.15871895849704742,
+      "rewards/rejected": -0.49333029985427856,
+      "step": 6660
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.836182016450273e-07,
+      "logits/chosen": -2.1734023094177246,
+      "logits/rejected": -1.8794763088226318,
+      "logps/chosen": -602.8240966796875,
+      "logps/rejected": -679.4175415039062,
+      "loss": 0.2011,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.37417346239089966,
+      "rewards/margins": 0.13355031609535217,
+      "rewards/rejected": -0.5077236890792847,
+      "step": 6670
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.769110179978874e-07,
+      "logits/chosen": -2.253901958465576,
+      "logits/rejected": -2.1010782718658447,
+      "logps/chosen": -505.3155822753906,
+      "logps/rejected": -616.96435546875,
+      "loss": 0.1624,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.33282509446144104,
+      "rewards/margins": 0.09407733380794525,
+      "rewards/rejected": -0.4269024431705475,
+      "step": 6680
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.702375677541037e-07,
+      "logits/chosen": -2.105555772781372,
+      "logits/rejected": -1.9600732326507568,
+      "logps/chosen": -520.289794921875,
+      "logps/rejected": -658.5758056640625,
+      "loss": 0.0981,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.31658869981765747,
+      "rewards/margins": 0.1489449441432953,
+      "rewards/rejected": -0.46553364396095276,
+      "step": 6690
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.635979679744006e-07,
+      "logits/chosen": -2.0489306449890137,
+      "logits/rejected": -1.714437484741211,
+      "logps/chosen": -512.5379028320312,
+      "logps/rejected": -627.3717041015625,
+      "loss": 0.0804,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.32006925344467163,
+      "rewards/margins": 0.13043050467967987,
+      "rewards/rejected": -0.4504997730255127,
+      "step": 6700
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.569923351257223e-07,
+      "logits/chosen": -2.195006847381592,
+      "logits/rejected": -1.8707275390625,
+      "logps/chosen": -525.9947509765625,
+      "logps/rejected": -692.6181030273438,
+      "loss": 0.1416,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.32367023825645447,
+      "rewards/margins": 0.14642134308815002,
+      "rewards/rejected": -0.4700915217399597,
+      "step": 6710
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.504207850791912e-07,
+      "logits/chosen": -2.194368839263916,
+      "logits/rejected": -1.6521762609481812,
+      "logps/chosen": -589.457763671875,
+      "logps/rejected": -698.20166015625,
+      "loss": 0.1303,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.3594959080219269,
+      "rewards/margins": 0.17847326397895813,
+      "rewards/rejected": -0.5379691123962402,
+      "step": 6720
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.438834331080725e-07,
+      "logits/chosen": -2.1895432472229004,
+      "logits/rejected": -2.0507566928863525,
+      "logps/chosen": -561.9774780273438,
+      "logps/rejected": -703.1712036132812,
+      "loss": 0.1242,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.3589351773262024,
+      "rewards/margins": 0.14330419898033142,
+      "rewards/rejected": -0.5022393465042114,
+      "step": 6730
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.373803938857558e-07,
+      "logits/chosen": -2.1139910221099854,
+      "logits/rejected": -1.7783534526824951,
+      "logps/chosen": -687.7551879882812,
+      "logps/rejected": -740.35205078125,
+      "loss": 0.192,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.480221688747406,
+      "rewards/margins": 0.10100038349628448,
+      "rewards/rejected": -0.5812220573425293,
+      "step": 6740
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.309117814837409e-07,
+      "logits/chosen": -2.236236333847046,
+      "logits/rejected": -1.7292554378509521,
+      "logps/chosen": -500.69610595703125,
+      "logps/rejected": -605.5383911132812,
+      "loss": 0.1164,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.288241446018219,
+      "rewards/margins": 0.15512248873710632,
+      "rewards/rejected": -0.4433639645576477,
+      "step": 6750
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.244777093696385e-07,
+      "logits/chosen": -2.340095281600952,
+      "logits/rejected": -1.8143020868301392,
+      "logps/chosen": -517.8221435546875,
+      "logps/rejected": -707.0513305664062,
+      "loss": 0.0709,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.33382245898246765,
+      "rewards/margins": 0.20808658003807068,
+      "rewards/rejected": -0.5419090986251831,
+      "step": 6760
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.180782904051787e-07,
+      "logits/chosen": -2.0912747383117676,
+      "logits/rejected": -2.001460552215576,
+      "logps/chosen": -600.0858154296875,
+      "logps/rejected": -720.568115234375,
+      "loss": 0.1061,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.39635854959487915,
+      "rewards/margins": 0.1263102889060974,
+      "rewards/rejected": -0.5226688981056213,
+      "step": 6770
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.117136368442322e-07,
+      "logits/chosen": -2.2086360454559326,
+      "logits/rejected": -1.776958703994751,
+      "logps/chosen": -560.9840698242188,
+      "logps/rejected": -677.6492309570312,
+      "loss": 0.1051,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.39273813366889954,
+      "rewards/margins": 0.13489307463169098,
+      "rewards/rejected": -0.5276311635971069,
+      "step": 6780
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.053838603308403e-07,
+      "logits/chosen": -2.453263282775879,
+      "logits/rejected": -2.0356826782226562,
+      "logps/chosen": -680.1113891601562,
+      "logps/rejected": -780.4449462890625,
+      "loss": 0.1473,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.42405804991722107,
+      "rewards/margins": 0.10753035545349121,
+      "rewards/rejected": -0.5315884351730347,
+      "step": 6790
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.99089071897256e-07,
+      "logits/chosen": -2.162928819656372,
+      "logits/rejected": -1.749582052230835,
+      "logps/chosen": -575.4002685546875,
+      "logps/rejected": -765.1943359375,
+      "loss": 0.1173,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.36823928356170654,
+      "rewards/margins": 0.21135230362415314,
+      "rewards/rejected": -0.5795915722846985,
+      "step": 6800
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.92829381961999e-07,
+      "logits/chosen": -1.996819257736206,
+      "logits/rejected": -1.7093244791030884,
+      "logps/chosen": -614.5445556640625,
+      "logps/rejected": -739.2743530273438,
+      "loss": 0.1045,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.41910457611083984,
+      "rewards/margins": 0.14283986389636993,
+      "rewards/rejected": -0.5619444847106934,
+      "step": 6810
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.866049003279163e-07,
+      "logits/chosen": -2.271427869796753,
+      "logits/rejected": -1.8434674739837646,
+      "logps/chosen": -529.5025024414062,
+      "logps/rejected": -622.4044189453125,
+      "loss": 0.1481,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.324138343334198,
+      "rewards/margins": 0.1336182802915573,
+      "rewards/rejected": -0.4577566981315613,
+      "step": 6820
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.80415736180257e-07,
+      "logits/chosen": -1.8748546838760376,
+      "logits/rejected": -1.7685054540634155,
+      "logps/chosen": -563.4625244140625,
+      "logps/rejected": -707.8074951171875,
+      "loss": 0.0958,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.4074874818325043,
+      "rewards/margins": 0.1413240283727646,
+      "rewards/rejected": -0.5488115549087524,
+      "step": 6830
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.7426199808475735e-07,
+      "logits/chosen": -2.145573616027832,
+      "logits/rejected": -1.8281844854354858,
+      "logps/chosen": -577.68115234375,
+      "logps/rejected": -691.8658447265625,
+      "loss": 0.1533,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.3581429123878479,
+      "rewards/margins": 0.14514704048633575,
+      "rewards/rejected": -0.5032899379730225,
+      "step": 6840
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.6814379398573613e-07,
+      "logits/chosen": -2.174830198287964,
+      "logits/rejected": -1.886239767074585,
+      "logps/chosen": -600.62255859375,
+      "logps/rejected": -750.0690307617188,
+      "loss": 0.1335,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.3584972023963928,
+      "rewards/margins": 0.13578909635543823,
+      "rewards/rejected": -0.49428629875183105,
+      "step": 6850
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.6206123120419944e-07,
+      "logits/chosen": -1.9931304454803467,
+      "logits/rejected": -1.6433311700820923,
+      "logps/chosen": -594.0074462890625,
+      "logps/rejected": -761.8284912109375,
+      "loss": 0.0921,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3809780478477478,
+      "rewards/margins": 0.1647835522890091,
+      "rewards/rejected": -0.5457615852355957,
+      "step": 6860
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.5601441643596145e-07,
+      "logits/chosen": -2.231766939163208,
+      "logits/rejected": -1.7959997653961182,
+      "logps/chosen": -557.9501953125,
+      "logps/rejected": -681.713623046875,
+      "loss": 0.1331,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.34355828166007996,
+      "rewards/margins": 0.1582074761390686,
+      "rewards/rejected": -0.5017658472061157,
+      "step": 6870
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.500034557497709e-07,
+      "logits/chosen": -2.0769877433776855,
+      "logits/rejected": -1.5688457489013672,
+      "logps/chosen": -662.0718383789062,
+      "logps/rejected": -761.7461547851562,
+      "loss": 0.1204,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.42594462633132935,
+      "rewards/margins": 0.1603473275899887,
+      "rewards/rejected": -0.5862919688224792,
+      "step": 6880
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.4402845458545037e-07,
+      "logits/chosen": -2.074845790863037,
+      "logits/rejected": -1.7770464420318604,
+      "logps/chosen": -602.3145751953125,
+      "logps/rejected": -735.3319702148438,
+      "loss": 0.0857,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.3946993947029114,
+      "rewards/margins": 0.14674441516399384,
+      "rewards/rejected": -0.5414438247680664,
+      "step": 6890
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.380895177520475e-07,
+      "logits/chosen": -2.3463380336761475,
+      "logits/rejected": -1.5683977603912354,
+      "logps/chosen": -662.3276977539062,
+      "logps/rejected": -755.8602905273438,
+      "loss": 0.1022,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.39082303643226624,
+      "rewards/margins": 0.14671273529529572,
+      "rewards/rejected": -0.5375357866287231,
+      "step": 6900
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.3218674942599655e-07,
+      "logits/chosen": -2.1968612670898438,
+      "logits/rejected": -1.8132545948028564,
+      "logps/chosen": -528.6552734375,
+      "logps/rejected": -672.707275390625,
+      "loss": 0.1504,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.2994438409805298,
+      "rewards/margins": 0.17884543538093567,
+      "rewards/rejected": -0.4782892167568207,
+      "step": 6910
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.263202531492877e-07,
+      "logits/chosen": -2.261199712753296,
+      "logits/rejected": -1.9339559078216553,
+      "logps/chosen": -593.5693969726562,
+      "logps/rejected": -671.1707763671875,
+      "loss": 0.1158,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3875558078289032,
+      "rewards/margins": 0.11357506364583969,
+      "rewards/rejected": -0.5011309385299683,
+      "step": 6920
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.204901318276586e-07,
+      "logits/chosen": -2.260776996612549,
+      "logits/rejected": -1.8444395065307617,
+      "logps/chosen": -618.3177490234375,
+      "logps/rejected": -750.3486938476562,
+      "loss": 0.0995,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.3657168447971344,
+      "rewards/margins": 0.16786041855812073,
+      "rewards/rejected": -0.5335772037506104,
+      "step": 6930
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.146964877287804e-07,
+      "logits/chosen": -2.299511194229126,
+      "logits/rejected": -1.794007658958435,
+      "logps/chosen": -664.4022827148438,
+      "logps/rejected": -739.2222290039062,
+      "loss": 0.1379,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.3554743826389313,
+      "rewards/margins": 0.15275630354881287,
+      "rewards/rejected": -0.5082306861877441,
+      "step": 6940
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.089394224804691e-07,
+      "logits/chosen": -2.2870771884918213,
+      "logits/rejected": -1.8556206226348877,
+      "logps/chosen": -555.3814697265625,
+      "logps/rejected": -658.9181518554688,
+      "loss": 0.1183,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3631711006164551,
+      "rewards/margins": 0.14692138135433197,
+      "rewards/rejected": -0.5100924968719482,
+      "step": 6950
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 4.032190370689018e-07,
+      "logits/chosen": -2.250535011291504,
+      "logits/rejected": -1.6975818872451782,
+      "logps/chosen": -632.96044921875,
+      "logps/rejected": -673.8742065429688,
+      "loss": 0.1391,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.41129255294799805,
+      "rewards/margins": 0.10245345532894135,
+      "rewards/rejected": -0.5137460231781006,
+      "step": 6960
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.9753543183684573e-07,
+      "logits/chosen": -2.0790281295776367,
+      "logits/rejected": -1.9222500324249268,
+      "logps/chosen": -650.9879760742188,
+      "logps/rejected": -789.86328125,
+      "loss": 0.1382,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.4133787751197815,
+      "rewards/margins": 0.11907651275396347,
+      "rewards/rejected": -0.5324552655220032,
+      "step": 6970
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.9188870648189437e-07,
+      "logits/chosen": -2.265200614929199,
+      "logits/rejected": -1.9175264835357666,
+      "logps/chosen": -602.2708740234375,
+      "logps/rejected": -712.5992431640625,
+      "loss": 0.1673,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3798312544822693,
+      "rewards/margins": 0.11531635373830795,
+      "rewards/rejected": -0.4951475262641907,
+      "step": 6980
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.862789600547268e-07,
+      "logits/chosen": -2.2080492973327637,
+      "logits/rejected": -1.6112253665924072,
+      "logps/chosen": -549.698974609375,
+      "logps/rejected": -650.6605224609375,
+      "loss": 0.1497,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.3674562871456146,
+      "rewards/margins": 0.1465298980474472,
+      "rewards/rejected": -0.5139862298965454,
+      "step": 6990
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.8070629095736e-07,
+      "logits/chosen": -2.25105881690979,
+      "logits/rejected": -2.093738079071045,
+      "logps/chosen": -638.0523681640625,
+      "logps/rejected": -769.5989379882812,
+      "loss": 0.1155,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.40703916549682617,
+      "rewards/margins": 0.13328908383846283,
+      "rewards/rejected": -0.5403282046318054,
+      "step": 7000
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.7517079694143145e-07,
+      "logits/chosen": -2.152463912963867,
+      "logits/rejected": -1.815333604812622,
+      "logps/chosen": -458.2982482910156,
+      "logps/rejected": -615.7095947265625,
+      "loss": 0.1301,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.27499282360076904,
+      "rewards/margins": 0.15706054866313934,
+      "rewards/rejected": -0.4320533871650696,
+      "step": 7010
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.696725751064778e-07,
+      "logits/chosen": -2.143343687057495,
+      "logits/rejected": -1.882947564125061,
+      "logps/chosen": -562.9153442382812,
+      "logps/rejected": -683.5399169921875,
+      "loss": 0.1579,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.35540255904197693,
+      "rewards/margins": 0.15081916749477386,
+      "rewards/rejected": -0.5062217712402344,
+      "step": 7020
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.6421172189823884e-07,
+      "logits/chosen": -2.3430562019348145,
+      "logits/rejected": -2.05299711227417,
+      "logps/chosen": -618.5864868164062,
+      "logps/rejected": -675.1802978515625,
+      "loss": 0.0989,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3553038239479065,
+      "rewards/margins": 0.11355333030223846,
+      "rewards/rejected": -0.46885713934898376,
+      "step": 7030
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.587883331069575e-07,
+      "logits/chosen": -2.115288496017456,
+      "logits/rejected": -1.8080313205718994,
+      "logps/chosen": -695.3529052734375,
+      "logps/rejected": -757.877197265625,
+      "loss": 0.0898,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.44722700119018555,
+      "rewards/margins": 0.10343305766582489,
+      "rewards/rejected": -0.5506600141525269,
+      "step": 7040
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.5340250386570547e-07,
+      "logits/chosen": -2.242435932159424,
+      "logits/rejected": -1.8907146453857422,
+      "logps/chosen": -675.7139892578125,
+      "logps/rejected": -786.8787231445312,
+      "loss": 0.1022,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.4404304027557373,
+      "rewards/margins": 0.1441318392753601,
+      "rewards/rejected": -0.5845622420310974,
+      "step": 7050
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.480543286487126e-07,
+      "logits/chosen": -2.2037107944488525,
+      "logits/rejected": -1.9478371143341064,
+      "logps/chosen": -598.0003051757812,
+      "logps/rejected": -767.5926513671875,
+      "loss": 0.106,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.39436954259872437,
+      "rewards/margins": 0.17273525893688202,
+      "rewards/rejected": -0.5671048760414124,
+      "step": 7060
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.4274390126971035e-07,
+      "logits/chosen": -2.1428067684173584,
+      "logits/rejected": -1.9086189270019531,
+      "logps/chosen": -512.329833984375,
+      "logps/rejected": -604.9511108398438,
+      "loss": 0.1389,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.3192521333694458,
+      "rewards/margins": 0.12495450675487518,
+      "rewards/rejected": -0.4442066252231598,
+      "step": 7070
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.374713148802827e-07,
+      "logits/chosen": -2.2220511436462402,
+      "logits/rejected": -1.7696044445037842,
+      "logps/chosen": -567.55419921875,
+      "logps/rejected": -731.9110107421875,
+      "loss": 0.1238,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.35002821683883667,
+      "rewards/margins": 0.194062739610672,
+      "rewards/rejected": -0.544090986251831,
+      "step": 7080
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.3223666196823963e-07,
+      "logits/chosen": -2.352914333343506,
+      "logits/rejected": -1.751535415649414,
+      "logps/chosen": -711.7252807617188,
+      "logps/rejected": -742.9156494140625,
+      "loss": 0.141,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.4321292042732239,
+      "rewards/margins": 0.11960384994745255,
+      "rewards/rejected": -0.5517330169677734,
+      "step": 7090
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.27040034355986e-07,
+      "logits/chosen": -2.08139967918396,
+      "logits/rejected": -1.9922263622283936,
+      "logps/chosen": -599.2244262695312,
+      "logps/rejected": -712.5352783203125,
+      "loss": 0.1155,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.4031258225440979,
+      "rewards/margins": 0.12135348469018936,
+      "rewards/rejected": -0.5244792699813843,
+      "step": 7100
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.218815231989167e-07,
+      "logits/chosen": -2.2119665145874023,
+      "logits/rejected": -2.0059220790863037,
+      "logps/chosen": -575.0491333007812,
+      "logps/rejected": -669.5467529296875,
+      "loss": 0.1063,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.3527541756629944,
+      "rewards/margins": 0.12519457936286926,
+      "rewards/rejected": -0.47794875502586365,
+      "step": 7110
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.1676121898381597e-07,
+      "logits/chosen": -1.9564775228500366,
+      "logits/rejected": -1.7620960474014282,
+      "logps/chosen": -570.0572509765625,
+      "logps/rejected": -706.173095703125,
+      "loss": 0.102,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.330929696559906,
+      "rewards/margins": 0.1388026773929596,
+      "rewards/rejected": -0.4697323441505432,
+      "step": 7120
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 3.1167921152727096e-07,
+      "logits/chosen": -2.2482354640960693,
+      "logits/rejected": -1.8644301891326904,
+      "logps/chosen": -561.4937744140625,
+      "logps/rejected": -625.3980712890625,
+      "loss": 0.1744,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.33693429827690125,
+      "rewards/margins": 0.10655763000249863,
+      "rewards/rejected": -0.44349199533462524,
+      "step": 7130
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 3.066355899740925e-07,
+      "logits/chosen": -2.171375274658203,
+      "logits/rejected": -1.8623323440551758,
+      "logps/chosen": -586.8325805664062,
+      "logps/rejected": -729.2609252929688,
+      "loss": 0.1004,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.37973159551620483,
+      "rewards/margins": 0.13614942133426666,
+      "rewards/rejected": -0.5158810615539551,
+      "step": 7140
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 3.0163044279575865e-07,
+      "logits/chosen": -2.3011696338653564,
+      "logits/rejected": -1.8217014074325562,
+      "logps/chosen": -588.6195678710938,
+      "logps/rejected": -641.781005859375,
+      "loss": 0.1333,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.3605493903160095,
+      "rewards/margins": 0.14432016015052795,
+      "rewards/rejected": -0.5048695802688599,
+      "step": 7150
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.966638577888548e-07,
+      "logits/chosen": -2.186237096786499,
+      "logits/rejected": -1.8374769687652588,
+      "logps/chosen": -610.1043090820312,
+      "logps/rejected": -737.0577392578125,
+      "loss": 0.1072,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.39031893014907837,
+      "rewards/margins": 0.12962813675403595,
+      "rewards/rejected": -0.5199470520019531,
+      "step": 7160
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.917359220735386e-07,
+      "logits/chosen": -2.009209156036377,
+      "logits/rejected": -1.8715111017227173,
+      "logps/chosen": -542.7819213867188,
+      "logps/rejected": -579.1976928710938,
+      "loss": 0.1796,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.38193702697753906,
+      "rewards/margins": 0.05297418311238289,
+      "rewards/rejected": -0.43491119146347046,
+      "step": 7170
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.8684672209201067e-07,
+      "logits/chosen": -2.1968748569488525,
+      "logits/rejected": -1.7906357049942017,
+      "logps/chosen": -564.443603515625,
+      "logps/rejected": -637.7664794921875,
+      "loss": 0.1444,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.3547224998474121,
+      "rewards/margins": 0.11605121940374374,
+      "rewards/rejected": -0.47077369689941406,
+      "step": 7180
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.819963436069986e-07,
+      "logits/chosen": -2.311122417449951,
+      "logits/rejected": -1.9739761352539062,
+      "logps/chosen": -627.9908447265625,
+      "logps/rejected": -655.0501098632812,
+      "loss": 0.1271,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.37448355555534363,
+      "rewards/margins": 0.08943120390176773,
+      "rewards/rejected": -0.46391478180885315,
+      "step": 7190
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.771848717002498e-07,
+      "logits/chosen": -2.0168354511260986,
+      "logits/rejected": -1.9805551767349243,
+      "logps/chosen": -592.3665771484375,
+      "logps/rejected": -758.9692993164062,
+      "loss": 0.106,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.4095228314399719,
+      "rewards/margins": 0.14836850762367249,
+      "rewards/rejected": -0.5578913688659668,
+      "step": 7200
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.724123907710444e-07,
+      "logits/chosen": -1.9882259368896484,
+      "logits/rejected": -1.7495872974395752,
+      "logps/chosen": -504.8055114746094,
+      "logps/rejected": -649.2545166015625,
+      "loss": 0.1328,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.3554796278476715,
+      "rewards/margins": 0.14399318397045135,
+      "rewards/rejected": -0.49947279691696167,
+      "step": 7210
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.6767898453470886e-07,
+      "logits/chosen": -2.289686679840088,
+      "logits/rejected": -1.8368587493896484,
+      "logps/chosen": -497.95306396484375,
+      "logps/rejected": -614.7998046875,
+      "loss": 0.1081,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.2991212010383606,
+      "rewards/margins": 0.1488068848848343,
+      "rewards/rejected": -0.4479281008243561,
+      "step": 7220
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.629847360211518e-07,
+      "logits/chosen": -2.1904821395874023,
+      "logits/rejected": -1.8454691171646118,
+      "logps/chosen": -565.637939453125,
+      "logps/rejected": -688.1617431640625,
+      "loss": 0.1109,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3660406768321991,
+      "rewards/margins": 0.13481226563453674,
+      "rewards/rejected": -0.5008529424667358,
+      "step": 7230
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.5832972757340565e-07,
+      "logits/chosen": -2.253572940826416,
+      "logits/rejected": -2.002485513687134,
+      "logps/chosen": -530.50830078125,
+      "logps/rejected": -633.1273193359375,
+      "loss": 0.1269,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3323044180870056,
+      "rewards/margins": 0.09866641461849213,
+      "rewards/rejected": -0.43097084760665894,
+      "step": 7240
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.53714040846183e-07,
+      "logits/chosen": -2.240408182144165,
+      "logits/rejected": -1.8204805850982666,
+      "logps/chosen": -554.327392578125,
+      "logps/rejected": -644.2651977539062,
+      "loss": 0.1344,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.30777883529663086,
+      "rewards/margins": 0.16053727269172668,
+      "rewards/rejected": -0.46831607818603516,
+      "step": 7250
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.491377568044434e-07,
+      "logits/chosen": -2.355710983276367,
+      "logits/rejected": -1.9031877517700195,
+      "logps/chosen": -641.8040161132812,
+      "logps/rejected": -705.9580078125,
+      "loss": 0.0716,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.38147348165512085,
+      "rewards/margins": 0.12000813335180283,
+      "rewards/rejected": -0.5014816522598267,
+      "step": 7260
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.4460095572197476e-07,
+      "logits/chosen": -2.2468624114990234,
+      "logits/rejected": -1.8987802267074585,
+      "logps/chosen": -498.08905029296875,
+      "logps/rejected": -649.2863159179688,
+      "loss": 0.0956,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.3040371835231781,
+      "rewards/margins": 0.15534478425979614,
+      "rewards/rejected": -0.45938199758529663,
+      "step": 7270
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.401037171799819e-07,
+      "logits/chosen": -2.2177844047546387,
+      "logits/rejected": -1.7069318294525146,
+      "logps/chosen": -531.2175903320312,
+      "logps/rejected": -609.7860107421875,
+      "loss": 0.1532,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.30343884229660034,
+      "rewards/margins": 0.13269317150115967,
+      "rewards/rejected": -0.4361320436000824,
+      "step": 7280
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.3564612006569482e-07,
+      "logits/chosen": -2.2926876544952393,
+      "logits/rejected": -2.057739496231079,
+      "logps/chosen": -574.4547119140625,
+      "logps/rejected": -687.0198974609375,
+      "loss": 0.1497,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3552076816558838,
+      "rewards/margins": 0.12386985123157501,
+      "rewards/rejected": -0.4790775179862976,
+      "step": 7290
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.3122824257098275e-07,
+      "logits/chosen": -2.053837299346924,
+      "logits/rejected": -1.6040732860565186,
+      "logps/chosen": -500.86981201171875,
+      "logps/rejected": -650.2252807617188,
+      "loss": 0.0686,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.31567278504371643,
+      "rewards/margins": 0.17183324694633484,
+      "rewards/rejected": -0.4875060021877289,
+      "step": 7300
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.2685016219098187e-07,
+      "logits/chosen": -2.3289382457733154,
+      "logits/rejected": -1.7380855083465576,
+      "logps/chosen": -543.6334228515625,
+      "logps/rejected": -655.8740234375,
+      "loss": 0.1167,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.3171786665916443,
+      "rewards/margins": 0.18444454669952393,
+      "rewards/rejected": -0.5016232132911682,
+      "step": 7310
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.2251195572273758e-07,
+      "logits/chosen": -2.4833521842956543,
+      "logits/rejected": -1.8792396783828735,
+      "logps/chosen": -620.5154418945312,
+      "logps/rejected": -695.6890869140625,
+      "loss": 0.1075,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.31977012753486633,
+      "rewards/margins": 0.1624891310930252,
+      "rewards/rejected": -0.48225921392440796,
+      "step": 7320
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.18213699263857e-07,
+      "logits/chosen": -2.131669282913208,
+      "logits/rejected": -1.8313696384429932,
+      "logps/chosen": -598.1660766601562,
+      "logps/rejected": -781.2696533203125,
+      "loss": 0.0956,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.3545466661453247,
+      "rewards/margins": 0.17925594747066498,
+      "rewards/rejected": -0.5338026285171509,
+      "step": 7330
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.1395546821117192e-07,
+      "logits/chosen": -2.1405208110809326,
+      "logits/rejected": -1.8110458850860596,
+      "logps/chosen": -573.0201416015625,
+      "logps/rejected": -665.4700317382812,
+      "loss": 0.1561,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3399235010147095,
+      "rewards/margins": 0.13527777791023254,
+      "rewards/rejected": -0.475201278924942,
+      "step": 7340
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.097373372594197e-07,
+      "logits/chosen": -2.2694177627563477,
+      "logits/rejected": -1.8251430988311768,
+      "logps/chosen": -616.990234375,
+      "logps/rejected": -708.8989868164062,
+      "loss": 0.1449,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.3801937699317932,
+      "rewards/margins": 0.12442214787006378,
+      "rewards/rejected": -0.5046159625053406,
+      "step": 7350
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.0555938039993145e-07,
+      "logits/chosen": -2.464862823486328,
+      "logits/rejected": -2.0087287425994873,
+      "logps/chosen": -570.2470092773438,
+      "logps/rejected": -645.2218017578125,
+      "loss": 0.1017,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.2910137474536896,
+      "rewards/margins": 0.134342223405838,
+      "rewards/rejected": -0.4253559708595276,
+      "step": 7360
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.0142167091933368e-07,
+      "logits/chosen": -2.0791285037994385,
+      "logits/rejected": -1.9840514659881592,
+      "logps/chosen": -555.2263793945312,
+      "logps/rejected": -672.3201293945312,
+      "loss": 0.1481,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -0.36741846799850464,
+      "rewards/margins": 0.09931338578462601,
+      "rewards/rejected": -0.46673184633255005,
+      "step": 7370
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.973242813982626e-07,
+      "logits/chosen": -1.9962129592895508,
+      "logits/rejected": -1.6627185344696045,
+      "logps/chosen": -550.5191650390625,
+      "logps/rejected": -674.954345703125,
+      "loss": 0.0944,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.3760848343372345,
+      "rewards/margins": 0.1467878520488739,
+      "rewards/rejected": -0.5228726863861084,
+      "step": 7380
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.932672837100924e-07,
+      "logits/chosen": -2.3511760234832764,
+      "logits/rejected": -1.6492153406143188,
+      "logps/chosen": -577.3499145507812,
+      "logps/rejected": -705.6130981445312,
+      "loss": 0.1378,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.3647750914096832,
+      "rewards/margins": 0.16916489601135254,
+      "rewards/rejected": -0.5339399576187134,
+      "step": 7390
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.8925074901967406e-07,
+      "logits/chosen": -2.2768845558166504,
+      "logits/rejected": -1.6779342889785767,
+      "logps/chosen": -573.1290283203125,
+      "logps/rejected": -682.6331176757812,
+      "loss": 0.0733,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.33616867661476135,
+      "rewards/margins": 0.1551557034254074,
+      "rewards/rejected": -0.49132436513900757,
+      "step": 7400
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.8527474778208458e-07,
+      "logits/chosen": -2.0803885459899902,
+      "logits/rejected": -1.8996890783309937,
+      "logps/chosen": -471.09710693359375,
+      "logps/rejected": -569.2593994140625,
+      "loss": 0.1448,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.33374708890914917,
+      "rewards/margins": 0.09361626952886581,
+      "rewards/rejected": -0.4273633360862732,
+      "step": 7410
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.813393497413951e-07,
+      "logits/chosen": -2.2025227546691895,
+      "logits/rejected": -1.828955888748169,
+      "logps/chosen": -568.7623291015625,
+      "logps/rejected": -660.5936279296875,
+      "loss": 0.0734,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3270300626754761,
+      "rewards/margins": 0.13636808097362518,
+      "rewards/rejected": -0.46339812874794006,
+      "step": 7420
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.7744462392944472e-07,
+      "logits/chosen": -2.430055618286133,
+      "logits/rejected": -1.8941357135772705,
+      "logps/chosen": -607.2894287109375,
+      "logps/rejected": -682.6880493164062,
+      "loss": 0.1261,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.3658744692802429,
+      "rewards/margins": 0.11388440430164337,
+      "rewards/rejected": -0.4797588288784027,
+      "step": 7430
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.7359063866463048e-07,
+      "logits/chosen": -2.3431077003479004,
+      "logits/rejected": -1.8584009408950806,
+      "logps/chosen": -519.372802734375,
+      "logps/rejected": -628.9778442382812,
+      "loss": 0.1601,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.320263147354126,
+      "rewards/margins": 0.16633442044258118,
+      "rewards/rejected": -0.48659759759902954,
+      "step": 7440
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.6977746155070946e-07,
+      "logits/chosen": -2.0926170349121094,
+      "logits/rejected": -2.043325424194336,
+      "logps/chosen": -590.1150512695312,
+      "logps/rejected": -726.1919555664062,
+      "loss": 0.1436,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.3946055769920349,
+      "rewards/margins": 0.11078107357025146,
+      "rewards/rejected": -0.5053867101669312,
+      "step": 7450
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.6600515947561207e-07,
+      "logits/chosen": -2.329251766204834,
+      "logits/rejected": -1.7020574808120728,
+      "logps/chosen": -587.83251953125,
+      "logps/rejected": -654.4178466796875,
+      "loss": 0.1838,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.34926360845565796,
+      "rewards/margins": 0.1450013369321823,
+      "rewards/rejected": -0.4942649304866791,
+      "step": 7460
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.6227379861026738e-07,
+      "logits/chosen": -2.307389974594116,
+      "logits/rejected": -1.8387680053710938,
+      "logps/chosen": -565.7305297851562,
+      "logps/rejected": -694.8583984375,
+      "loss": 0.1438,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.35628753900527954,
+      "rewards/margins": 0.14819283783435822,
+      "rewards/rejected": -0.5044804215431213,
+      "step": 7470
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.5858344440744745e-07,
+      "logits/chosen": -2.3156940937042236,
+      "logits/rejected": -1.8351157903671265,
+      "logps/chosen": -619.236328125,
+      "logps/rejected": -722.6455688476562,
+      "loss": 0.1374,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.3889729380607605,
+      "rewards/margins": 0.11890371888875961,
+      "rewards/rejected": -0.5078766942024231,
+      "step": 7480
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.5493416160061254e-07,
+      "logits/chosen": -2.4197380542755127,
+      "logits/rejected": -1.9365612268447876,
+      "logps/chosen": -618.9156494140625,
+      "logps/rejected": -704.7053833007812,
+      "loss": 0.1814,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.3765208125114441,
+      "rewards/margins": 0.13450196385383606,
+      "rewards/rejected": -0.511022686958313,
+      "step": 7490
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.5132601420278086e-07,
+      "logits/chosen": -2.1727294921875,
+      "logits/rejected": -1.8300291299819946,
+      "logps/chosen": -607.3361206054688,
+      "logps/rejected": -657.9014892578125,
+      "loss": 0.0882,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.38796499371528625,
+      "rewards/margins": 0.10588376224040985,
+      "rewards/rejected": -0.4938487112522125,
+      "step": 7500
+    },
+    {
+      "epoch": 0.9,
+      "step": 7501,
+      "total_flos": 0.0,
+      "train_loss": 2.6126050373789884e-05,
+      "train_runtime": 6.3979,
+      "train_samples_per_second": 4585.774,
+      "train_steps_per_second": 1146.475
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 7335,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 100,
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}