diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,10756 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 1.0,
+  "eval_steps": 7000,
+  "global_step": 7642,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.535947712418301e-09,
+      "logits/chosen": -2.8119401931762695,
+      "logits/rejected": -2.736701011657715,
+      "logps/chosen": -234.14996337890625,
+      "logps/rejected": -256.3080749511719,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 6.535947712418302e-08,
+      "logits/chosen": -2.7925682067871094,
+      "logits/rejected": -2.772902727127075,
+      "logps/chosen": -291.9549255371094,
+      "logps/rejected": -262.1678161621094,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.4722222089767456,
+      "rewards/chosen": 0.0001956633641384542,
+      "rewards/margins": 0.0004478075134102255,
+      "rewards/rejected": -0.0002521441492717713,
+      "step": 10
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.3071895424836603e-07,
+      "logits/chosen": -2.824028253555298,
+      "logits/rejected": -2.7898128032684326,
+      "logps/chosen": -281.1107482910156,
+      "logps/rejected": -285.3966064453125,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -0.0003439037536736578,
+      "rewards/margins": 0.00010349955118726939,
+      "rewards/rejected": -0.00044740340672433376,
+      "step": 20
+    },
+    {
+      "epoch": 0.0,
+      "learning_rate": 1.9607843137254904e-07,
+      "logits/chosen": -2.800112247467041,
+      "logits/rejected": -2.7216060161590576,
+      "logps/chosen": -250.15048217773438,
+      "logps/rejected": -229.4697723388672,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.0006494318367913365,
+      "rewards/margins": 0.0007226697052828968,
+      "rewards/rejected": -7.323784666368738e-05,
+      "step": 30
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 2.6143790849673207e-07,
+      "logits/chosen": -2.877162218093872,
+      "logits/rejected": -2.7966651916503906,
+      "logps/chosen": -281.16851806640625,
+      "logps/rejected": -261.5294189453125,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 0.0001101084053516388,
+      "rewards/margins": 0.00034566796966828406,
+      "rewards/rejected": -0.00023555960797239095,
+      "step": 40
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 3.267973856209151e-07,
+      "logits/chosen": -2.8416740894317627,
+      "logits/rejected": -2.7994861602783203,
+      "logps/chosen": -240.7175750732422,
+      "logps/rejected": -241.4182891845703,
+      "loss": 0.693,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 3.8643287552986294e-05,
+      "rewards/margins": 0.00020237448916304857,
+      "rewards/rejected": -0.0001637311070226133,
+      "step": 50
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 3.921568627450981e-07,
+      "logits/chosen": -2.790154457092285,
+      "logits/rejected": -2.7840394973754883,
+      "logps/chosen": -297.1316223144531,
+      "logps/rejected": -259.43011474609375,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.0009555624565109611,
+      "rewards/margins": 0.0019151580054312944,
+      "rewards/rejected": -0.0009595954907126725,
+      "step": 60
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 4.5751633986928105e-07,
+      "logits/chosen": -2.826920509338379,
+      "logits/rejected": -2.8098552227020264,
+      "logps/chosen": -261.5799560546875,
+      "logps/rejected": -218.6430206298828,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -4.932899173581973e-05,
+      "rewards/margins": 0.0004831492842640728,
+      "rewards/rejected": -0.0005324783269315958,
+      "step": 70
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.228758169934641e-07,
+      "logits/chosen": -2.7809548377990723,
+      "logits/rejected": -2.7622580528259277,
+      "logps/chosen": -307.6874084472656,
+      "logps/rejected": -304.4044189453125,
+      "loss": 0.6934,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -0.00024535844568163157,
+      "rewards/margins": -0.0004294017271604389,
+      "rewards/rejected": 0.0001840432669268921,
+      "step": 80
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 5.882352941176471e-07,
+      "logits/chosen": -2.8685429096221924,
+      "logits/rejected": -2.829183340072632,
+      "logps/chosen": -270.9972229003906,
+      "logps/rejected": -240.2500457763672,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.0006658152560703456,
+      "rewards/margins": 0.0009269308065995574,
+      "rewards/rejected": -0.00026111555052921176,
+      "step": 90
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 6.535947712418302e-07,
+      "logits/chosen": -2.7856380939483643,
+      "logits/rejected": -2.790788173675537,
+      "logps/chosen": -243.0207977294922,
+      "logps/rejected": -250.99288940429688,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.00031755497911944985,
+      "rewards/margins": 0.00046528532402589917,
+      "rewards/rejected": -0.0007828402449376881,
+      "step": 100
+    },
+    {
+      "epoch": 0.01,
+      "learning_rate": 7.189542483660131e-07,
+      "logits/chosen": -2.8396081924438477,
+      "logits/rejected": -2.7743194103240967,
+      "logps/chosen": -283.7100830078125,
+      "logps/rejected": -268.7243957519531,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.00016692141070961952,
+      "rewards/margins": 0.00037348843761719763,
+      "rewards/rejected": -0.0005404098192229867,
+      "step": 110
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 7.843137254901962e-07,
+      "logits/chosen": -2.829432964324951,
+      "logits/rejected": -2.7791907787323,
+      "logps/chosen": -305.06939697265625,
+      "logps/rejected": -245.28713989257812,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -0.0004921638173982501,
+      "rewards/margins": -0.0002835090272128582,
+      "rewards/rejected": -0.00020865481928922236,
+      "step": 120
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 8.496732026143792e-07,
+      "logits/chosen": -2.83703351020813,
+      "logits/rejected": -2.8530797958374023,
+      "logps/chosen": -267.7164611816406,
+      "logps/rejected": -239.90087890625,
+      "loss": 0.693,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.00030112435342743993,
+      "rewards/margins": 0.0002027704322244972,
+      "rewards/rejected": 9.83539066510275e-05,
+      "step": 130
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.150326797385621e-07,
+      "logits/chosen": -2.8001341819763184,
+      "logits/rejected": -2.784090280532837,
+      "logps/chosen": -278.6336364746094,
+      "logps/rejected": -247.541748046875,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.0003716406936291605,
+      "rewards/margins": -0.00020624040917027742,
+      "rewards/rejected": -0.00016540024080313742,
+      "step": 140
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 9.80392156862745e-07,
+      "logits/chosen": -2.8891284465789795,
+      "logits/rejected": -2.8073394298553467,
+      "logps/chosen": -333.697998046875,
+      "logps/rejected": -301.34906005859375,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -5.339364361134358e-05,
+      "rewards/margins": 0.0006288294680416584,
+      "rewards/rejected": -0.0006822230061516166,
+      "step": 150
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.0457516339869283e-06,
+      "logits/chosen": -2.8700673580169678,
+      "logits/rejected": -2.7817797660827637,
+      "logps/chosen": -260.9744873046875,
+      "logps/rejected": -239.41921997070312,
+      "loss": 0.693,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": 0.00016844880883581936,
+      "rewards/margins": 0.00031576715991832316,
+      "rewards/rejected": -0.00014731836563441902,
+      "step": 160
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.111111111111111e-06,
+      "logits/chosen": -2.775143623352051,
+      "logits/rejected": -2.7452282905578613,
+      "logps/chosen": -244.25729370117188,
+      "logps/rejected": -227.45126342773438,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": 7.676922541577369e-05,
+      "rewards/margins": 0.00042827180004678667,
+      "rewards/rejected": -0.0003515026473905891,
+      "step": 170
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.1764705882352942e-06,
+      "logits/chosen": -2.874192953109741,
+      "logits/rejected": -2.7526564598083496,
+      "logps/chosen": -349.8937072753906,
+      "logps/rejected": -274.16436767578125,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 2.268646494485438e-05,
+      "rewards/margins": 0.0009162778733298182,
+      "rewards/rejected": -0.0008935912628658116,
+      "step": 180
+    },
+    {
+      "epoch": 0.02,
+      "learning_rate": 1.2418300653594772e-06,
+      "logits/chosen": -2.7780871391296387,
+      "logits/rejected": -2.7367794513702393,
+      "logps/chosen": -279.318115234375,
+      "logps/rejected": -227.4193878173828,
+      "loss": 0.693,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 1.0177784133702517e-05,
+      "rewards/margins": 0.0003014899848494679,
+      "rewards/rejected": -0.00029131225892342627,
+      "step": 190
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.3071895424836604e-06,
+      "logits/chosen": -2.853095769882202,
+      "logits/rejected": -2.747596025466919,
+      "logps/chosen": -260.20391845703125,
+      "logps/rejected": -265.9258728027344,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.0002617259160615504,
+      "rewards/margins": 0.0018547021318227053,
+      "rewards/rejected": -0.0021164279896765947,
+      "step": 200
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.3725490196078434e-06,
+      "logits/chosen": -2.917522430419922,
+      "logits/rejected": -2.8804614543914795,
+      "logps/chosen": -284.3828430175781,
+      "logps/rejected": -261.9861145019531,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.0004467476101126522,
+      "rewards/margins": 0.0007827846566215158,
+      "rewards/rejected": -0.0003360370174050331,
+      "step": 210
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.4379084967320261e-06,
+      "logits/chosen": -2.7833666801452637,
+      "logits/rejected": -2.7220213413238525,
+      "logps/chosen": -285.97283935546875,
+      "logps/rejected": -261.6575012207031,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.00043272305629216135,
+      "rewards/margins": 0.002436288632452488,
+      "rewards/rejected": -0.0020035658963024616,
+      "step": 220
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.5032679738562091e-06,
+      "logits/chosen": -2.7486743927001953,
+      "logits/rejected": -2.7189061641693115,
+      "logps/chosen": -233.9739227294922,
+      "logps/rejected": -277.0560607910156,
+      "loss": 0.693,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.00024150774697773159,
+      "rewards/margins": 0.00027872290229424834,
+      "rewards/rejected": -0.0005202307365834713,
+      "step": 230
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.5686274509803923e-06,
+      "logits/chosen": -2.798421859741211,
+      "logits/rejected": -2.7745471000671387,
+      "logps/chosen": -250.8818817138672,
+      "logps/rejected": -268.77203369140625,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.00021761751850135624,
+      "rewards/margins": 0.0008384333923459053,
+      "rewards/rejected": -0.001056050998158753,
+      "step": 240
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.6339869281045753e-06,
+      "logits/chosen": -2.8311660289764404,
+      "logits/rejected": -2.7577476501464844,
+      "logps/chosen": -321.122314453125,
+      "logps/rejected": -275.36285400390625,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0012902079615741968,
+      "rewards/margins": 0.002782456111162901,
+      "rewards/rejected": -0.0014922486152499914,
+      "step": 250
+    },
+    {
+      "epoch": 0.03,
+      "learning_rate": 1.6993464052287585e-06,
+      "logits/chosen": -2.854680061340332,
+      "logits/rejected": -2.807109832763672,
+      "logps/chosen": -296.96356201171875,
+      "logps/rejected": -262.04254150390625,
+      "loss": 0.6923,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.000237447748077102,
+      "rewards/margins": 0.0017886876594275236,
+      "rewards/rejected": -0.0015512399841099977,
+      "step": 260
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.7647058823529414e-06,
+      "logits/chosen": -2.7997894287109375,
+      "logits/rejected": -2.751993417739868,
+      "logps/chosen": -251.9638671875,
+      "logps/rejected": -231.748046875,
+      "loss": 0.6925,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.0009734320337884128,
+      "rewards/margins": 0.0013420346658676863,
+      "rewards/rejected": -0.00231546675786376,
+      "step": 270
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.8300653594771242e-06,
+      "logits/chosen": -2.877136707305908,
+      "logits/rejected": -2.793832302093506,
+      "logps/chosen": -320.9353942871094,
+      "logps/rejected": -288.93310546875,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.0004052144940942526,
+      "rewards/margins": 0.0022023480851203203,
+      "rewards/rejected": -0.0017971337074413896,
+      "step": 280
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.8954248366013072e-06,
+      "logits/chosen": -2.820586681365967,
+      "logits/rejected": -2.79305362701416,
+      "logps/chosen": -292.45770263671875,
+      "logps/rejected": -271.52386474609375,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.0009529812377877533,
+      "rewards/margins": 0.002716564806178212,
+      "rewards/rejected": -0.001763583510182798,
+      "step": 290
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 1.96078431372549e-06,
+      "logits/chosen": -2.9215292930603027,
+      "logits/rejected": -2.8291478157043457,
+      "logps/chosen": -302.40496826171875,
+      "logps/rejected": -252.5030517578125,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.0021990996319800615,
+      "rewards/margins": 0.0027074983809143305,
+      "rewards/rejected": -0.0005083988653495908,
+      "step": 300
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.0261437908496734e-06,
+      "logits/chosen": -2.8122951984405518,
+      "logits/rejected": -2.793062686920166,
+      "logps/chosen": -298.7405700683594,
+      "logps/rejected": -289.6739807128906,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": 0.0009888180065900087,
+      "rewards/margins": 0.0015734133776277304,
+      "rewards/rejected": -0.0005845952546223998,
+      "step": 310
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.0915032679738565e-06,
+      "logits/chosen": -2.852717161178589,
+      "logits/rejected": -2.7725658416748047,
+      "logps/chosen": -278.22052001953125,
+      "logps/rejected": -279.6014099121094,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": 0.0023006487172096968,
+      "rewards/margins": 0.001985882641747594,
+      "rewards/rejected": 0.0003147665411233902,
+      "step": 320
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.1568627450980393e-06,
+      "logits/chosen": -2.8605873584747314,
+      "logits/rejected": -2.792485237121582,
+      "logps/chosen": -301.82171630859375,
+      "logps/rejected": -254.5903778076172,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.0033324025571346283,
+      "rewards/margins": 0.002957900520414114,
+      "rewards/rejected": 0.00037450186209753156,
+      "step": 330
+    },
+    {
+      "epoch": 0.04,
+      "learning_rate": 2.222222222222222e-06,
+      "logits/chosen": -2.8137636184692383,
+      "logits/rejected": -2.708789110183716,
+      "logps/chosen": -275.42730712890625,
+      "logps/rejected": -231.0994110107422,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.004821686539798975,
+      "rewards/margins": 0.00611809641122818,
+      "rewards/rejected": -0.0012964101042598486,
+      "step": 340
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.2875816993464053e-06,
+      "logits/chosen": -2.8824973106384277,
+      "logits/rejected": -2.7771410942077637,
+      "logps/chosen": -327.0098571777344,
+      "logps/rejected": -270.22283935546875,
+      "loss": 0.6912,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.004658971913158894,
+      "rewards/margins": 0.003912682645022869,
+      "rewards/rejected": 0.0007462892099283636,
+      "step": 350
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.3529411764705885e-06,
+      "logits/chosen": -2.8761162757873535,
+      "logits/rejected": -2.8172099590301514,
+      "logps/chosen": -263.3186950683594,
+      "logps/rejected": -260.9554138183594,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.004147452302277088,
+      "rewards/margins": 0.004154963418841362,
+      "rewards/rejected": -7.510674095101422e-06,
+      "step": 360
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.4183006535947716e-06,
+      "logits/chosen": -2.8148701190948486,
+      "logits/rejected": -2.8049988746643066,
+      "logps/chosen": -267.3759765625,
+      "logps/rejected": -245.693603515625,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.004090488888323307,
+      "rewards/margins": 0.005000332370400429,
+      "rewards/rejected": -0.0009098442387767136,
+      "step": 370
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.4836601307189544e-06,
+      "logits/chosen": -2.840355157852173,
+      "logits/rejected": -2.809810161590576,
+      "logps/chosen": -290.0486755371094,
+      "logps/rejected": -255.69473266601562,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.0061897882260382175,
+      "rewards/margins": 0.005519126541912556,
+      "rewards/rejected": 0.0006706616259180009,
+      "step": 380
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.549019607843137e-06,
+      "logits/chosen": -2.7894577980041504,
+      "logits/rejected": -2.7253575325012207,
+      "logps/chosen": -308.2140808105469,
+      "logps/rejected": -250.3750762939453,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.007893599569797516,
+      "rewards/margins": 0.009147431701421738,
+      "rewards/rejected": -0.0012538309674710035,
+      "step": 390
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.6143790849673208e-06,
+      "logits/chosen": -2.8778653144836426,
+      "logits/rejected": -2.8764147758483887,
+      "logps/chosen": -291.77850341796875,
+      "logps/rejected": -301.2120361328125,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.008383329026401043,
+      "rewards/margins": 0.004777857102453709,
+      "rewards/rejected": 0.003605470759794116,
+      "step": 400
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.6797385620915036e-06,
+      "logits/chosen": -2.783327579498291,
+      "logits/rejected": -2.7389895915985107,
+      "logps/chosen": -264.718505859375,
+      "logps/rejected": -219.590576171875,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": 0.007756390608847141,
+      "rewards/margins": 0.005835867486894131,
+      "rewards/rejected": 0.001920522772707045,
+      "step": 410
+    },
+    {
+      "epoch": 0.05,
+      "learning_rate": 2.7450980392156867e-06,
+      "logits/chosen": -2.8096060752868652,
+      "logits/rejected": -2.757349729537964,
+      "logps/chosen": -284.1047058105469,
+      "logps/rejected": -283.6143798828125,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.008391691371798515,
+      "rewards/margins": 0.007876824587583542,
+      "rewards/rejected": 0.0005148652126081288,
+      "step": 420
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 2.8104575163398695e-06,
+      "logits/chosen": -2.848623752593994,
+      "logits/rejected": -2.7812159061431885,
+      "logps/chosen": -286.5287170410156,
+      "logps/rejected": -270.6346435546875,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.00907193310558796,
+      "rewards/margins": 0.007010035216808319,
+      "rewards/rejected": 0.0020618971902877092,
+      "step": 430
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 2.8758169934640523e-06,
+      "logits/chosen": -2.902583599090576,
+      "logits/rejected": -2.88633394241333,
+      "logps/chosen": -276.67352294921875,
+      "logps/rejected": -250.07455444335938,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.009775395505130291,
+      "rewards/margins": 0.007532182149589062,
+      "rewards/rejected": 0.0022432133555412292,
+      "step": 440
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 2.9411764705882355e-06,
+      "logits/chosen": -2.880934476852417,
+      "logits/rejected": -2.878293514251709,
+      "logps/chosen": -292.5964660644531,
+      "logps/rejected": -307.35797119140625,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.009976732544600964,
+      "rewards/margins": 0.006814384367316961,
+      "rewards/rejected": 0.0031623479444533587,
+      "step": 450
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.0065359477124182e-06,
+      "logits/chosen": -2.7925028800964355,
+      "logits/rejected": -2.7293059825897217,
+      "logps/chosen": -260.94281005859375,
+      "logps/rejected": -249.77169799804688,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.013284455053508282,
+      "rewards/margins": 0.008283937349915504,
+      "rewards/rejected": 0.005000515840947628,
+      "step": 460
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.071895424836602e-06,
+      "logits/chosen": -2.824601173400879,
+      "logits/rejected": -2.8208603858947754,
+      "logps/chosen": -281.3301696777344,
+      "logps/rejected": -260.6267395019531,
+      "loss": 0.6872,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.01699897274374962,
+      "rewards/margins": 0.012018144130706787,
+      "rewards/rejected": 0.004980829078704119,
+      "step": 470
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.1372549019607846e-06,
+      "logits/chosen": -2.7856502532958984,
+      "logits/rejected": -2.7704176902770996,
+      "logps/chosen": -290.65362548828125,
+      "logps/rejected": -242.20700073242188,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.013819393701851368,
+      "rewards/margins": 0.010202933102846146,
+      "rewards/rejected": 0.0036164610646665096,
+      "step": 480
+    },
+    {
+      "epoch": 0.06,
+      "learning_rate": 3.2026143790849674e-06,
+      "logits/chosen": -2.8649086952209473,
+      "logits/rejected": -2.7738921642303467,
+      "logps/chosen": -277.52166748046875,
+      "logps/rejected": -226.4635467529297,
+      "loss": 0.6851,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.014747830107808113,
+      "rewards/margins": 0.0163627527654171,
+      "rewards/rejected": -0.0016149235889315605,
+      "step": 490
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.2679738562091506e-06,
+      "logits/chosen": -2.830211639404297,
+      "logits/rejected": -2.759883403778076,
+      "logps/chosen": -256.46429443359375,
+      "logps/rejected": -236.2734832763672,
+      "loss": 0.6882,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.012899341993033886,
+      "rewards/margins": 0.01006043516099453,
+      "rewards/rejected": 0.0028389065992087126,
+      "step": 500
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.3333333333333333e-06,
+      "logits/chosen": -2.767530918121338,
+      "logits/rejected": -2.7557694911956787,
+      "logps/chosen": -267.4305725097656,
+      "logps/rejected": -260.91925048828125,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.01752861775457859,
+      "rewards/margins": 0.010302940383553505,
+      "rewards/rejected": 0.0072256773710250854,
+      "step": 510
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.398692810457517e-06,
+      "logits/chosen": -2.791713237762451,
+      "logits/rejected": -2.686591386795044,
+      "logps/chosen": -256.9478454589844,
+      "logps/rejected": -262.20465087890625,
+      "loss": 0.6864,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": 0.016486337408423424,
+      "rewards/margins": 0.013938124291598797,
+      "rewards/rejected": 0.0025482119526714087,
+      "step": 520
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.4640522875816997e-06,
+      "logits/chosen": -2.763822555541992,
+      "logits/rejected": -2.7151129245758057,
+      "logps/chosen": -265.5009460449219,
+      "logps/rejected": -241.47598266601562,
+      "loss": 0.6847,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.02259974554181099,
+      "rewards/margins": 0.01733039692044258,
+      "rewards/rejected": 0.005269350949674845,
+      "step": 530
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.529411764705883e-06,
+      "logits/chosen": -2.8222291469573975,
+      "logits/rejected": -2.734029531478882,
+      "logps/chosen": -293.8299560546875,
+      "logps/rejected": -264.50653076171875,
+      "loss": 0.684,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.02375880442559719,
+      "rewards/margins": 0.018576953560113907,
+      "rewards/rejected": 0.005181849934160709,
+      "step": 540
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.5947712418300657e-06,
+      "logits/chosen": -2.7996761798858643,
+      "logits/rejected": -2.7677814960479736,
+      "logps/chosen": -257.5671081542969,
+      "logps/rejected": -246.5146026611328,
+      "loss": 0.6829,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.029493218287825584,
+      "rewards/margins": 0.021040869876742363,
+      "rewards/rejected": 0.008452351205050945,
+      "step": 550
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.6601307189542484e-06,
+      "logits/chosen": -2.841050624847412,
+      "logits/rejected": -2.7818596363067627,
+      "logps/chosen": -288.5670166015625,
+      "logps/rejected": -299.4884033203125,
+      "loss": 0.6824,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.03232832998037338,
+      "rewards/margins": 0.022060425952076912,
+      "rewards/rejected": 0.01026790589094162,
+      "step": 560
+    },
+    {
+      "epoch": 0.07,
+      "learning_rate": 3.7254901960784316e-06,
+      "logits/chosen": -2.8778281211853027,
+      "logits/rejected": -2.8253567218780518,
+      "logps/chosen": -267.6673583984375,
+      "logps/rejected": -256.3402099609375,
+      "loss": 0.681,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.03789962828159332,
+      "rewards/margins": 0.024803100153803825,
+      "rewards/rejected": 0.013096528127789497,
+      "step": 570
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.7908496732026144e-06,
+      "logits/chosen": -2.850435256958008,
+      "logits/rejected": -2.7622811794281006,
+      "logps/chosen": -311.47882080078125,
+      "logps/rejected": -284.8739318847656,
+      "loss": 0.6796,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.051514703780412674,
+      "rewards/margins": 0.027963165193796158,
+      "rewards/rejected": 0.023551538586616516,
+      "step": 580
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.856209150326798e-06,
+      "logits/chosen": -2.862598419189453,
+      "logits/rejected": -2.7575337886810303,
+      "logps/chosen": -292.02423095703125,
+      "logps/rejected": -256.5790100097656,
+      "loss": 0.6808,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.050418369472026825,
+      "rewards/margins": 0.025475427508354187,
+      "rewards/rejected": 0.02494293823838234,
+      "step": 590
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.92156862745098e-06,
+      "logits/chosen": -2.7841854095458984,
+      "logits/rejected": -2.665539264678955,
+      "logps/chosen": -259.4479675292969,
+      "logps/rejected": -247.23379516601562,
+      "loss": 0.6839,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.04178684577345848,
+      "rewards/margins": 0.019426746293902397,
+      "rewards/rejected": 0.022360099479556084,
+      "step": 600
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 3.986928104575164e-06,
+      "logits/chosen": -2.7963860034942627,
+      "logits/rejected": -2.7074332237243652,
+      "logps/chosen": -236.59683227539062,
+      "logps/rejected": -205.6645965576172,
+      "loss": 0.6799,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.04800710082054138,
+      "rewards/margins": 0.02720050886273384,
+      "rewards/rejected": 0.020806599408388138,
+      "step": 610
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.052287581699347e-06,
+      "logits/chosen": -2.7985177040100098,
+      "logits/rejected": -2.7399001121520996,
+      "logps/chosen": -294.8608093261719,
+      "logps/rejected": -267.06072998046875,
+      "loss": 0.6801,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.050414424389600754,
+      "rewards/margins": 0.027711128816008568,
+      "rewards/rejected": 0.022703295573592186,
+      "step": 620
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.11764705882353e-06,
+      "logits/chosen": -2.768751621246338,
+      "logits/rejected": -2.753446102142334,
+      "logps/chosen": -276.5562438964844,
+      "logps/rejected": -257.9153137207031,
+      "loss": 0.6816,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.04279092699289322,
+      "rewards/margins": 0.02434687875211239,
+      "rewards/rejected": 0.01844405010342598,
+      "step": 630
+    },
+    {
+      "epoch": 0.08,
+      "learning_rate": 4.183006535947713e-06,
+      "logits/chosen": -2.8574509620666504,
+      "logits/rejected": -2.777613401412964,
+      "logps/chosen": -267.9512939453125,
+      "logps/rejected": -258.92413330078125,
+      "loss": 0.6801,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.036323048174381256,
+      "rewards/margins": 0.0279191043227911,
+      "rewards/rejected": 0.008403941988945007,
+      "step": 640
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.2483660130718954e-06,
+      "logits/chosen": -2.799482822418213,
+      "logits/rejected": -2.7799582481384277,
+      "logps/chosen": -265.56072998046875,
+      "logps/rejected": -254.2129364013672,
+      "loss": 0.6844,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.033449865877628326,
+      "rewards/margins": 0.018980346620082855,
+      "rewards/rejected": 0.014469517394900322,
+      "step": 650
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.313725490196079e-06,
+      "logits/chosen": -2.8195960521698,
+      "logits/rejected": -2.759016275405884,
+      "logps/chosen": -312.8327331542969,
+      "logps/rejected": -258.8818054199219,
+      "loss": 0.677,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.040408067405223846,
+      "rewards/margins": 0.0337202362716198,
+      "rewards/rejected": 0.006687832064926624,
+      "step": 660
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.379084967320262e-06,
+      "logits/chosen": -2.911236524581909,
+      "logits/rejected": -2.8115108013153076,
+      "logps/chosen": -292.05633544921875,
+      "logps/rejected": -312.80792236328125,
+      "loss": 0.6763,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.050738751888275146,
+      "rewards/margins": 0.03538566082715988,
+      "rewards/rejected": 0.015353086404502392,
+      "step": 670
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.444444444444444e-06,
+      "logits/chosen": -2.9012043476104736,
+      "logits/rejected": -2.8279881477355957,
+      "logps/chosen": -282.02581787109375,
+      "logps/rejected": -262.3529052734375,
+      "loss": 0.6785,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.051031775772571564,
+      "rewards/margins": 0.030756209045648575,
+      "rewards/rejected": 0.020275568589568138,
+      "step": 680
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.509803921568628e-06,
+      "logits/chosen": -2.8346965312957764,
+      "logits/rejected": -2.7855236530303955,
+      "logps/chosen": -306.7787170410156,
+      "logps/rejected": -266.53851318359375,
+      "loss": 0.6786,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.03847826272249222,
+      "rewards/margins": 0.030614981427788734,
+      "rewards/rejected": 0.007863285019993782,
+      "step": 690
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.5751633986928105e-06,
+      "logits/chosen": -2.829515218734741,
+      "logits/rejected": -2.821662187576294,
+      "logps/chosen": -281.5744323730469,
+      "logps/rejected": -271.4909362792969,
+      "loss": 0.6765,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.03147972375154495,
+      "rewards/margins": 0.035172365605831146,
+      "rewards/rejected": -0.0036926374305039644,
+      "step": 700
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.640522875816994e-06,
+      "logits/chosen": -2.8266708850860596,
+      "logits/rejected": -2.770461320877075,
+      "logps/chosen": -288.9119567871094,
+      "logps/rejected": -275.3086242675781,
+      "loss": 0.6737,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.033450257033109665,
+      "rewards/margins": 0.04209939390420914,
+      "rewards/rejected": -0.008649135008454323,
+      "step": 710
+    },
+    {
+      "epoch": 0.09,
+      "learning_rate": 4.705882352941177e-06,
+      "logits/chosen": -2.8076250553131104,
+      "logits/rejected": -2.769458055496216,
+      "logps/chosen": -328.4112548828125,
+      "logps/rejected": -306.272216796875,
+      "loss": 0.6729,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.03407909348607063,
+      "rewards/margins": 0.04409077763557434,
+      "rewards/rejected": -0.010011687874794006,
+      "step": 720
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.77124183006536e-06,
+      "logits/chosen": -2.789391040802002,
+      "logits/rejected": -2.7553000450134277,
+      "logps/chosen": -290.4982604980469,
+      "logps/rejected": -281.2314758300781,
+      "loss": 0.6736,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.029938995838165283,
+      "rewards/margins": 0.041954852640628815,
+      "rewards/rejected": -0.01201585866510868,
+      "step": 730
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.836601307189543e-06,
+      "logits/chosen": -2.781406879425049,
+      "logits/rejected": -2.696737766265869,
+      "logps/chosen": -310.0440673828125,
+      "logps/rejected": -264.82293701171875,
+      "loss": 0.6689,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.03745822235941887,
+      "rewards/margins": 0.052424948662519455,
+      "rewards/rejected": -0.014966726303100586,
+      "step": 740
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.901960784313726e-06,
+      "logits/chosen": -2.769747495651245,
+      "logits/rejected": -2.716500759124756,
+      "logps/chosen": -248.9564208984375,
+      "logps/rejected": -241.31808471679688,
+      "loss": 0.6657,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.025083869695663452,
+      "rewards/margins": 0.05821135640144348,
+      "rewards/rejected": -0.03312749043107033,
+      "step": 750
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.967320261437909e-06,
+      "logits/chosen": -2.8008737564086914,
+      "logits/rejected": -2.6803557872772217,
+      "logps/chosen": -280.2804870605469,
+      "logps/rejected": -245.4314727783203,
+      "loss": 0.6625,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.02910938300192356,
+      "rewards/margins": 0.06502566486597061,
+      "rewards/rejected": -0.0359162762761116,
+      "step": 760
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.99999347843947e-06,
+      "logits/chosen": -2.770726442337036,
+      "logits/rejected": -2.7643446922302246,
+      "logps/chosen": -297.40362548828125,
+      "logps/rejected": -277.90728759765625,
+      "loss": 0.6692,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.012996147386729717,
+      "rewards/margins": 0.05361568182706833,
+      "rewards/rejected": -0.04061952978372574,
+      "step": 770
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.999941306159375e-06,
+      "logits/chosen": -2.8241565227508545,
+      "logits/rejected": -2.717512369155884,
+      "logps/chosen": -306.24188232421875,
+      "logps/rejected": -281.04534912109375,
+      "loss": 0.6533,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.012457502074539661,
+      "rewards/margins": 0.0876227617263794,
+      "rewards/rejected": -0.07516525685787201,
+      "step": 780
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.999836962687967e-06,
+      "logits/chosen": -2.7425379753112793,
+      "logits/rejected": -2.7373032569885254,
+      "logps/chosen": -291.7939758300781,
+      "logps/rejected": -297.1993408203125,
+      "loss": 0.6622,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.007721015717834234,
+      "rewards/margins": 0.06823545694351196,
+      "rewards/rejected": -0.06051445007324219,
+      "step": 790
+    },
+    {
+      "epoch": 0.1,
+      "learning_rate": 4.999680450202786e-06,
+      "logits/chosen": -2.638617515563965,
+      "logits/rejected": -2.647702693939209,
+      "logps/chosen": -271.705810546875,
+      "logps/rejected": -293.2637634277344,
+      "loss": 0.663,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.022435462102293968,
+      "rewards/margins": 0.06659122556447983,
+      "rewards/rejected": -0.04415576159954071,
+      "step": 800
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.999471771970087e-06,
+      "logits/chosen": -2.7543206214904785,
+      "logits/rejected": -2.703037738800049,
+      "logps/chosen": -296.8338623046875,
+      "logps/rejected": -269.7286071777344,
+      "loss": 0.6646,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.03234348073601723,
+      "rewards/margins": 0.06366707384586334,
+      "rewards/rejected": -0.03132358938455582,
+      "step": 810
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.999210932344767e-06,
+      "logits/chosen": -2.856318235397339,
+      "logits/rejected": -2.7844817638397217,
+      "logps/chosen": -316.36920166015625,
+      "logps/rejected": -279.80035400390625,
+      "loss": 0.6617,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.03255653753876686,
+      "rewards/margins": 0.07101921737194061,
+      "rewards/rejected": -0.03846268355846405,
+      "step": 820
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.998897936770281e-06,
+      "logits/chosen": -2.7458736896514893,
+      "logits/rejected": -2.7834811210632324,
+      "logps/chosen": -245.2959442138672,
+      "logps/rejected": -280.3979797363281,
+      "loss": 0.6666,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.015771931037306786,
+      "rewards/margins": 0.061041127890348434,
+      "rewards/rejected": -0.07681305706501007,
+      "step": 830
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.998532791778521e-06,
+      "logits/chosen": -2.6956088542938232,
+      "logits/rejected": -2.616732120513916,
+      "logps/chosen": -307.7881164550781,
+      "logps/rejected": -250.6544189453125,
+      "loss": 0.6473,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.03812180086970329,
+      "rewards/margins": 0.1033933013677597,
+      "rewards/rejected": -0.1415151059627533,
+      "step": 840
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.9981155049896885e-06,
+      "logits/chosen": -2.7655982971191406,
+      "logits/rejected": -2.72393798828125,
+      "logps/chosen": -296.951904296875,
+      "logps/rejected": -322.07928466796875,
+      "loss": 0.6622,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.06325708329677582,
+      "rewards/margins": 0.0730038732290268,
+      "rewards/rejected": -0.1362609565258026,
+      "step": 850
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.997646085112126e-06,
+      "logits/chosen": -2.776740550994873,
+      "logits/rejected": -2.7122843265533447,
+      "logps/chosen": -261.51904296875,
+      "logps/rejected": -266.55230712890625,
+      "loss": 0.6575,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.0611543282866478,
+      "rewards/margins": 0.08332563936710358,
+      "rewards/rejected": -0.14447996020317078,
+      "step": 860
+    },
+    {
+      "epoch": 0.11,
+      "learning_rate": 4.997124541942141e-06,
+      "logits/chosen": -2.768399953842163,
+      "logits/rejected": -2.7058193683624268,
+      "logps/chosen": -291.50299072265625,
+      "logps/rejected": -289.19683837890625,
+      "loss": 0.6575,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.02319035865366459,
+      "rewards/margins": 0.08150584995746613,
+      "rewards/rejected": -0.058315493166446686,
+      "step": 870
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.996550886363801e-06,
+      "logits/chosen": -2.747941255569458,
+      "logits/rejected": -2.7044715881347656,
+      "logps/chosen": -263.110595703125,
+      "logps/rejected": -282.12908935546875,
+      "loss": 0.6802,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.025197049602866173,
+      "rewards/margins": 0.03696311265230179,
+      "rewards/rejected": -0.011766057461500168,
+      "step": 880
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.995925130348706e-06,
+      "logits/chosen": -2.7597579956054688,
+      "logits/rejected": -2.7350759506225586,
+      "logps/chosen": -263.8006591796875,
+      "logps/rejected": -274.7923889160156,
+      "loss": 0.6746,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": 0.050489284098148346,
+      "rewards/margins": 0.047527458518743515,
+      "rewards/rejected": 0.002961826976388693,
+      "step": 890
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.995247286955734e-06,
+      "logits/chosen": -2.7716643810272217,
+      "logits/rejected": -2.686768054962158,
+      "logps/chosen": -258.016357421875,
+      "logps/rejected": -232.6066436767578,
+      "loss": 0.651,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.004680367652326822,
+      "rewards/margins": 0.09733270108699799,
+      "rewards/rejected": -0.10201307386159897,
+      "step": 900
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.994517370330779e-06,
+      "logits/chosen": -2.697218894958496,
+      "logits/rejected": -2.7090718746185303,
+      "logps/chosen": -286.7010192871094,
+      "logps/rejected": -322.4683837890625,
+      "loss": 0.6511,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.02778555080294609,
+      "rewards/margins": 0.09914027899503708,
+      "rewards/rejected": -0.12692582607269287,
+      "step": 910
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.993735395706446e-06,
+      "logits/chosen": -2.757145404815674,
+      "logits/rejected": -2.6732077598571777,
+      "logps/chosen": -246.65292358398438,
+      "logps/rejected": -255.90878295898438,
+      "loss": 0.6447,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.0350009985268116,
+      "rewards/margins": 0.11443473398685455,
+      "rewards/rejected": -0.14943572878837585,
+      "step": 920
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.992901379401737e-06,
+      "logits/chosen": -2.834156036376953,
+      "logits/rejected": -2.807741165161133,
+      "logps/chosen": -320.0839538574219,
+      "logps/rejected": -301.0787353515625,
+      "loss": 0.6461,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.007714649196714163,
+      "rewards/margins": 0.11063899844884872,
+      "rewards/rejected": -0.10292434692382812,
+      "step": 930
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.992015338821711e-06,
+      "logits/chosen": -2.7422382831573486,
+      "logits/rejected": -2.656106948852539,
+      "logps/chosen": -293.58984375,
+      "logps/rejected": -289.71282958984375,
+      "loss": 0.6459,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.01883086748421192,
+      "rewards/margins": 0.11053939908742905,
+      "rewards/rejected": -0.12937027215957642,
+      "step": 940
+    },
+    {
+      "epoch": 0.12,
+      "learning_rate": 4.991077292457117e-06,
+      "logits/chosen": -2.769420862197876,
+      "logits/rejected": -2.6854679584503174,
+      "logps/chosen": -288.72186279296875,
+      "logps/rejected": -270.62603759765625,
+      "loss": 0.6348,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.017589321359992027,
+      "rewards/margins": 0.13609319925308228,
+      "rewards/rejected": -0.15368251502513885,
+      "step": 950
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.990087259884016e-06,
+      "logits/chosen": -2.769928216934204,
+      "logits/rejected": -2.7173972129821777,
+      "logps/chosen": -261.3431701660156,
+      "logps/rejected": -248.7234649658203,
+      "loss": 0.6266,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.011517312377691269,
+      "rewards/margins": 0.15848945081233978,
+      "rewards/rejected": -0.17000675201416016,
+      "step": 960
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.989045261763362e-06,
+      "logits/chosen": -2.780383586883545,
+      "logits/rejected": -2.758472204208374,
+      "logps/chosen": -287.06842041015625,
+      "logps/rejected": -283.26629638671875,
+      "loss": 0.6245,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.04591415077447891,
+      "rewards/margins": 0.1571168601512909,
+      "rewards/rejected": -0.11120273917913437,
+      "step": 970
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.98795131984058e-06,
+      "logits/chosen": -2.67490553855896,
+      "logits/rejected": -2.652907371520996,
+      "logps/chosen": -270.46405029296875,
+      "logps/rejected": -253.50790405273438,
+      "loss": 0.6393,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.038785211741924286,
+      "rewards/margins": 0.12635438144207,
+      "rewards/rejected": -0.08756916224956512,
+      "step": 980
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.986805456945107e-06,
+      "logits/chosen": -2.7391343116760254,
+      "logits/rejected": -2.678925037384033,
+      "logps/chosen": -229.54867553710938,
+      "logps/rejected": -227.01904296875,
+      "loss": 0.6239,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.08784683048725128,
+      "rewards/margins": 0.16485615074634552,
+      "rewards/rejected": -0.07700931280851364,
+      "step": 990
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.985607696989919e-06,
+      "logits/chosen": -2.7790751457214355,
+      "logits/rejected": -2.7710416316986084,
+      "logps/chosen": -281.12677001953125,
+      "logps/rejected": -282.32000732421875,
+      "loss": 0.6446,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.013560568913817406,
+      "rewards/margins": 0.12122897803783417,
+      "rewards/rejected": -0.10766839981079102,
+      "step": 1000
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.984358064971026e-06,
+      "logits/chosen": -2.7263317108154297,
+      "logits/rejected": -2.6554925441741943,
+      "logps/chosen": -285.2276916503906,
+      "logps/rejected": -264.2181396484375,
+      "loss": 0.6624,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.08757422119379044,
+      "rewards/margins": 0.09389257431030273,
+      "rewards/rejected": -0.18146678805351257,
+      "step": 1010
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.983056586966958e-06,
+      "logits/chosen": -2.7499728202819824,
+      "logits/rejected": -2.659660577774048,
+      "logps/chosen": -328.3389587402344,
+      "logps/rejected": -275.9952392578125,
+      "loss": 0.5988,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.01166592352092266,
+      "rewards/margins": 0.2222457230091095,
+      "rewards/rejected": -0.23391160368919373,
+      "step": 1020
+    },
+    {
+      "epoch": 0.13,
+      "learning_rate": 4.981703290138215e-06,
+      "logits/chosen": -2.8197715282440186,
+      "logits/rejected": -2.7556724548339844,
+      "logps/chosen": -320.1171875,
+      "logps/rejected": -312.1669006347656,
+      "loss": 0.6182,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.03866108879446983,
+      "rewards/margins": 0.19545957446098328,
+      "rewards/rejected": -0.23412065207958221,
+      "step": 1030
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.980298202726706e-06,
+      "logits/chosen": -2.8647043704986572,
+      "logits/rejected": -2.7961294651031494,
+      "logps/chosen": -325.565185546875,
+      "logps/rejected": -302.745361328125,
+      "loss": 0.5974,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.0439925454556942,
+      "rewards/margins": 0.2330087125301361,
+      "rewards/rejected": -0.2770012617111206,
+      "step": 1040
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.978841354055148e-06,
+      "logits/chosen": -2.7200796604156494,
+      "logits/rejected": -2.6861369609832764,
+      "logps/chosen": -282.4591979980469,
+      "logps/rejected": -276.94720458984375,
+      "loss": 0.6547,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.07256370037794113,
+      "rewards/margins": 0.10875082015991211,
+      "rewards/rejected": -0.18131451308727264,
+      "step": 1050
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.977332774526471e-06,
+      "logits/chosen": -2.735813856124878,
+      "logits/rejected": -2.590101718902588,
+      "logps/chosen": -285.9839782714844,
+      "logps/rejected": -247.0618438720703,
+      "loss": 0.6228,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.00786397885531187,
+      "rewards/margins": 0.1807597130537033,
+      "rewards/rejected": -0.18862371146678925,
+      "step": 1060
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.97577249562317e-06,
+      "logits/chosen": -2.6723811626434326,
+      "logits/rejected": -2.637002468109131,
+      "logps/chosen": -265.9768981933594,
+      "logps/rejected": -268.60333251953125,
+      "loss": 0.6157,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.09876110404729843,
+      "rewards/margins": 0.184899240732193,
+      "rewards/rejected": -0.08613810688257217,
+      "step": 1070
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.974160549906652e-06,
+      "logits/chosen": -2.671915054321289,
+      "logits/rejected": -2.6104953289031982,
+      "logps/chosen": -258.3193664550781,
+      "logps/rejected": -253.4278106689453,
+      "loss": 0.6351,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": 0.03933478146791458,
+      "rewards/margins": 0.14855535328388214,
+      "rewards/rejected": -0.10922057926654816,
+      "step": 1080
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.972496971016559e-06,
+      "logits/chosen": -2.6837778091430664,
+      "logits/rejected": -2.6769649982452393,
+      "logps/chosen": -267.5766296386719,
+      "logps/rejected": -249.96322631835938,
+      "loss": 0.6117,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.0017799038905650377,
+      "rewards/margins": 0.2136697769165039,
+      "rewards/rejected": -0.21544966101646423,
+      "step": 1090
+    },
+    {
+      "epoch": 0.14,
+      "learning_rate": 4.9707817936700635e-06,
+      "logits/chosen": -2.710200071334839,
+      "logits/rejected": -2.6542322635650635,
+      "logps/chosen": -295.22698974609375,
+      "logps/rejected": -294.02398681640625,
+      "loss": 0.6224,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.13250088691711426,
+      "rewards/margins": 0.18700218200683594,
+      "rewards/rejected": -0.3195030987262726,
+      "step": 1100
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.969015053661142e-06,
+      "logits/chosen": -2.7028110027313232,
+      "logits/rejected": -2.6644277572631836,
+      "logps/chosen": -307.25677490234375,
+      "logps/rejected": -293.72113037109375,
+      "loss": 0.6474,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.1993504762649536,
+      "rewards/margins": 0.12248294055461884,
+      "rewards/rejected": -0.32183343172073364,
+      "step": 1110
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.967196787859835e-06,
+      "logits/chosen": -2.7485759258270264,
+      "logits/rejected": -2.658440113067627,
+      "logps/chosen": -312.5169982910156,
+      "logps/rejected": -319.265625,
+      "loss": 0.6383,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.1994025707244873,
+      "rewards/margins": 0.14066831767559052,
+      "rewards/rejected": -0.340070903301239,
+      "step": 1120
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.965327034211469e-06,
+      "logits/chosen": -2.7448534965515137,
+      "logits/rejected": -2.675642728805542,
+      "logps/chosen": -326.31427001953125,
+      "logps/rejected": -300.79913330078125,
+      "loss": 0.5981,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.1741999387741089,
+      "rewards/margins": 0.23407408595085144,
+      "rewards/rejected": -0.4082740247249603,
+      "step": 1130
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.96340583173587e-06,
+      "logits/chosen": -2.72843861579895,
+      "logits/rejected": -2.5941829681396484,
+      "logps/chosen": -264.9902648925781,
+      "logps/rejected": -255.72195434570312,
+      "loss": 0.61,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.14372649788856506,
+      "rewards/margins": 0.2092621773481369,
+      "rewards/rejected": -0.35298866033554077,
+      "step": 1140
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.96143322052655e-06,
+      "logits/chosen": -2.6982827186584473,
+      "logits/rejected": -2.744253635406494,
+      "logps/chosen": -301.6868591308594,
+      "logps/rejected": -296.90576171875,
+      "loss": 0.6398,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.04231922701001167,
+      "rewards/margins": 0.14572744071483612,
+      "rewards/rejected": -0.1880466639995575,
+      "step": 1150
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.959409241749864e-06,
+      "logits/chosen": -2.7407147884368896,
+      "logits/rejected": -2.635188579559326,
+      "logps/chosen": -284.01605224609375,
+      "logps/rejected": -298.7756652832031,
+      "loss": 0.6447,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.006490018218755722,
+      "rewards/margins": 0.13500967621803284,
+      "rewards/rejected": -0.14149968326091766,
+      "step": 1160
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.957333937644159e-06,
+      "logits/chosen": -2.771390914916992,
+      "logits/rejected": -2.711841344833374,
+      "logps/chosen": -301.462890625,
+      "logps/rejected": -291.5288391113281,
+      "loss": 0.6167,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.023664310574531555,
+      "rewards/margins": 0.19239868223667145,
+      "rewards/rejected": -0.216062992811203,
+      "step": 1170
+    },
+    {
+      "epoch": 0.15,
+      "learning_rate": 4.955207351518885e-06,
+      "logits/chosen": -2.7537341117858887,
+      "logits/rejected": -2.748892068862915,
+      "logps/chosen": -276.0171813964844,
+      "logps/rejected": -251.59872436523438,
+      "loss": 0.6549,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.040898025035858154,
+      "rewards/margins": 0.1072196513414383,
+      "rewards/rejected": -0.14811769127845764,
+      "step": 1180
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.953029527753699e-06,
+      "logits/chosen": -2.7324886322021484,
+      "logits/rejected": -2.730910301208496,
+      "logps/chosen": -249.3578338623047,
+      "logps/rejected": -329.9027099609375,
+      "loss": 0.6606,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.04136783257126808,
+      "rewards/margins": 0.10689421743154526,
+      "rewards/rejected": -0.14826205372810364,
+      "step": 1190
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.95080051179753e-06,
+      "logits/chosen": -2.6676039695739746,
+      "logits/rejected": -2.6622016429901123,
+      "logps/chosen": -255.8192901611328,
+      "logps/rejected": -264.82177734375,
+      "loss": 0.6406,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.03838808462023735,
+      "rewards/margins": 0.13400152325630188,
+      "rewards/rejected": -0.09561344981193542,
+      "step": 1200
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.948520350167637e-06,
+      "logits/chosen": -2.685241222381592,
+      "logits/rejected": -2.642319679260254,
+      "logps/chosen": -277.08966064453125,
+      "logps/rejected": -274.6904602050781,
+      "loss": 0.6442,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.02941305562853813,
+      "rewards/margins": 0.14138095080852509,
+      "rewards/rejected": -0.11196788400411606,
+      "step": 1210
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.946189090448639e-06,
+      "logits/chosen": -2.6670005321502686,
+      "logits/rejected": -2.6389825344085693,
+      "logps/chosen": -285.34619140625,
+      "logps/rejected": -245.8319854736328,
+      "loss": 0.679,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.007370577659457922,
+      "rewards/margins": 0.05299089103937149,
+      "rewards/rejected": -0.04562031105160713,
+      "step": 1220
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.943806781291515e-06,
+      "logits/chosen": -2.723661422729492,
+      "logits/rejected": -2.669473886489868,
+      "logps/chosen": -269.0576171875,
+      "logps/rejected": -249.31478881835938,
+      "loss": 0.6069,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.06089981645345688,
+      "rewards/margins": 0.21457311511039734,
+      "rewards/rejected": -0.15367329120635986,
+      "step": 1230
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.941373472412595e-06,
+      "logits/chosen": -2.739572048187256,
+      "logits/rejected": -2.7109625339508057,
+      "logps/chosen": -330.05181884765625,
+      "logps/rejected": -323.09814453125,
+      "loss": 0.6355,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.03341181203722954,
+      "rewards/margins": 0.16557389497756958,
+      "rewards/rejected": -0.19898571074008942,
+      "step": 1240
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.938889214592521e-06,
+      "logits/chosen": -2.7377562522888184,
+      "logits/rejected": -2.699403762817383,
+      "logps/chosen": -336.968994140625,
+      "logps/rejected": -311.3940124511719,
+      "loss": 0.6593,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.027558648958802223,
+      "rewards/margins": 0.1004289835691452,
+      "rewards/rejected": -0.12798762321472168,
+      "step": 1250
+    },
+    {
+      "epoch": 0.16,
+      "learning_rate": 4.936354059675186e-06,
+      "logits/chosen": -2.7607648372650146,
+      "logits/rejected": -2.694934129714966,
+      "logps/chosen": -293.25653076171875,
+      "logps/rejected": -270.6409912109375,
+      "loss": 0.6225,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.06491407006978989,
+      "rewards/margins": 0.18064187467098236,
+      "rewards/rejected": -0.24555595219135284,
+      "step": 1260
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.933768060566654e-06,
+      "logits/chosen": -2.751033306121826,
+      "logits/rejected": -2.6835689544677734,
+      "logps/chosen": -284.2615661621094,
+      "logps/rejected": -282.6009826660156,
+      "loss": 0.6041,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.043512895703315735,
+      "rewards/margins": 0.22329530119895935,
+      "rewards/rejected": -0.2668081820011139,
+      "step": 1270
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.931131271234052e-06,
+      "logits/chosen": -2.709684371948242,
+      "logits/rejected": -2.6038787364959717,
+      "logps/chosen": -286.12042236328125,
+      "logps/rejected": -258.8558044433594,
+      "loss": 0.6336,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.15650294721126556,
+      "rewards/margins": 0.1588658094406128,
+      "rewards/rejected": -0.31536874175071716,
+      "step": 1280
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.928443746704448e-06,
+      "logits/chosen": -2.702178478240967,
+      "logits/rejected": -2.6311123371124268,
+      "logps/chosen": -330.3631286621094,
+      "logps/rejected": -302.4740295410156,
+      "loss": 0.6058,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.12174341827630997,
+      "rewards/margins": 0.2224629670381546,
+      "rewards/rejected": -0.3442063629627228,
+      "step": 1290
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.925705543063703e-06,
+      "logits/chosen": -2.6648240089416504,
+      "logits/rejected": -2.6492886543273926,
+      "logps/chosen": -296.5184631347656,
+      "logps/rejected": -348.2719421386719,
+      "loss": 0.6081,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.07323465496301651,
+      "rewards/margins": 0.2226163148880005,
+      "rewards/rejected": -0.2958509624004364,
+      "step": 1300
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.922916717455297e-06,
+      "logits/chosen": -2.7255916595458984,
+      "logits/rejected": -2.632220506668091,
+      "logps/chosen": -319.171142578125,
+      "logps/rejected": -273.6234436035156,
+      "loss": 0.6111,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.0864974707365036,
+      "rewards/margins": 0.21783606708049774,
+      "rewards/rejected": -0.30433353781700134,
+      "step": 1310
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.920077328079136e-06,
+      "logits/chosen": -2.7409908771514893,
+      "logits/rejected": -2.66633677482605,
+      "logps/chosen": -323.6951904296875,
+      "logps/rejected": -287.40032958984375,
+      "loss": 0.6502,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.23855753242969513,
+      "rewards/margins": 0.13359208405017853,
+      "rewards/rejected": -0.3721495568752289,
+      "step": 1320
+    },
+    {
+      "epoch": 0.17,
+      "learning_rate": 4.9171874341903445e-06,
+      "logits/chosen": -2.714674711227417,
+      "logits/rejected": -2.6588025093078613,
+      "logps/chosen": -311.8841247558594,
+      "logps/rejected": -330.3231201171875,
+      "loss": 0.6096,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.303422749042511,
+      "rewards/margins": 0.19621005654335022,
+      "rewards/rejected": -0.4996328353881836,
+      "step": 1330
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.914247096098019e-06,
+      "logits/chosen": -2.6881370544433594,
+      "logits/rejected": -2.678586959838867,
+      "logps/chosen": -307.6955871582031,
+      "logps/rejected": -317.8343811035156,
+      "loss": 0.6745,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.3028998374938965,
+      "rewards/margins": 0.07659934461116791,
+      "rewards/rejected": -0.3794991672039032,
+      "step": 1340
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.911256375163977e-06,
+      "logits/chosen": -2.6150739192962646,
+      "logits/rejected": -2.658870220184326,
+      "logps/chosen": -248.73880004882812,
+      "logps/rejected": -290.62152099609375,
+      "loss": 0.6131,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.14969298243522644,
+      "rewards/margins": 0.21568353474140167,
+      "rewards/rejected": -0.3653765320777893,
+      "step": 1350
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.908215333801474e-06,
+      "logits/chosen": -2.6679298877716064,
+      "logits/rejected": -2.6431031227111816,
+      "logps/chosen": -265.43896484375,
+      "logps/rejected": -249.8400421142578,
+      "loss": 0.6204,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.04471645504236221,
+      "rewards/margins": 0.19784018397331238,
+      "rewards/rejected": -0.2425566464662552,
+      "step": 1360
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.9051240354739004e-06,
+      "logits/chosen": -2.6677639484405518,
+      "logits/rejected": -2.600980281829834,
+      "logps/chosen": -323.40484619140625,
+      "logps/rejected": -307.5841369628906,
+      "loss": 0.6647,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.12204649299383163,
+      "rewards/margins": 0.08674965798854828,
+      "rewards/rejected": -0.2087961733341217,
+      "step": 1370
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.901982544693457e-06,
+      "logits/chosen": -2.7245945930480957,
+      "logits/rejected": -2.6861279010772705,
+      "logps/chosen": -310.47869873046875,
+      "logps/rejected": -307.6448059082031,
+      "loss": 0.5954,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.03722001612186432,
+      "rewards/margins": 0.2571137547492981,
+      "rewards/rejected": -0.294333815574646,
+      "step": 1380
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.898790927019809e-06,
+      "logits/chosen": -2.588104486465454,
+      "logits/rejected": -2.5740177631378174,
+      "logps/chosen": -293.8135986328125,
+      "logps/rejected": -302.7144775390625,
+      "loss": 0.6412,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.14841726422309875,
+      "rewards/margins": 0.157002255320549,
+      "rewards/rejected": -0.30541953444480896,
+      "step": 1390
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.895549249058718e-06,
+      "logits/chosen": -2.6189510822296143,
+      "logits/rejected": -2.510685682296753,
+      "logps/chosen": -277.7233581542969,
+      "logps/rejected": -276.20172119140625,
+      "loss": 0.6085,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.02877277508378029,
+      "rewards/margins": 0.22848379611968994,
+      "rewards/rejected": -0.25725656747817993,
+      "step": 1400
+    },
+    {
+      "epoch": 0.18,
+      "learning_rate": 4.892257578460656e-06,
+      "logits/chosen": -2.700005054473877,
+      "logits/rejected": -2.651778221130371,
+      "logps/chosen": -278.5904541015625,
+      "logps/rejected": -286.35882568359375,
+      "loss": 0.6394,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.028505906462669373,
+      "rewards/margins": 0.16637474298477173,
+      "rewards/rejected": -0.1948806345462799,
+      "step": 1410
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.888915983919383e-06,
+      "logits/chosen": -2.7137064933776855,
+      "logits/rejected": -2.706315755844116,
+      "logps/chosen": -324.4673767089844,
+      "logps/rejected": -329.97222900390625,
+      "loss": 0.6013,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.03984386846423149,
+      "rewards/margins": 0.22730712592601776,
+      "rewards/rejected": -0.18746325373649597,
+      "step": 1420
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.885524535170525e-06,
+      "logits/chosen": -2.637549877166748,
+      "logits/rejected": -2.6319408416748047,
+      "logps/chosen": -298.5765075683594,
+      "logps/rejected": -275.1312561035156,
+      "loss": 0.6268,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.02735677734017372,
+      "rewards/margins": 0.16681280732154846,
+      "rewards/rejected": -0.1941695660352707,
+      "step": 1430
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.882083302990113e-06,
+      "logits/chosen": -2.7360949516296387,
+      "logits/rejected": -2.733842134475708,
+      "logps/chosen": -330.0787048339844,
+      "logps/rejected": -308.88775634765625,
+      "loss": 0.6229,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.01667322963476181,
+      "rewards/margins": 0.18382708728313446,
+      "rewards/rejected": -0.20050033926963806,
+      "step": 1440
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.878592359193104e-06,
+      "logits/chosen": -2.6717278957366943,
+      "logits/rejected": -2.705718994140625,
+      "logps/chosen": -275.1805114746094,
+      "logps/rejected": -264.8807678222656,
+      "loss": 0.6135,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.006876927800476551,
+      "rewards/margins": 0.20822569727897644,
+      "rewards/rejected": -0.21510262787342072,
+      "step": 1450
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.875051776631888e-06,
+      "logits/chosen": -2.683805465698242,
+      "logits/rejected": -2.6308257579803467,
+      "logps/chosen": -365.669921875,
+      "logps/rejected": -333.26983642578125,
+      "loss": 0.6226,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.10370182991027832,
+      "rewards/margins": 0.19327282905578613,
+      "rewards/rejected": -0.29697468876838684,
+      "step": 1460
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.871461629194764e-06,
+      "logits/chosen": -2.6987807750701904,
+      "logits/rejected": -2.5851948261260986,
+      "logps/chosen": -288.1730041503906,
+      "logps/rejected": -265.02886962890625,
+      "loss": 0.6462,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.25078457593917847,
+      "rewards/margins": 0.13921622931957245,
+      "rewards/rejected": -0.3900007903575897,
+      "step": 1470
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.8678219918043984e-06,
+      "logits/chosen": -2.6955254077911377,
+      "logits/rejected": -2.6151630878448486,
+      "logps/chosen": -294.0158996582031,
+      "logps/rejected": -263.16607666015625,
+      "loss": 0.6311,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.15440413355827332,
+      "rewards/margins": 0.1786438524723053,
+      "rewards/rejected": -0.333048015832901,
+      "step": 1480
+    },
+    {
+      "epoch": 0.19,
+      "learning_rate": 4.864132940416262e-06,
+      "logits/chosen": -2.7128043174743652,
+      "logits/rejected": -2.6179797649383545,
+      "logps/chosen": -321.33819580078125,
+      "logps/rejected": -288.50836181640625,
+      "loss": 0.5874,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.056278448551893234,
+      "rewards/margins": 0.2687918245792389,
+      "rewards/rejected": -0.3250702917575836,
+      "step": 1490
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.860394552017044e-06,
+      "logits/chosen": -2.6810142993927,
+      "logits/rejected": -2.6712329387664795,
+      "logps/chosen": -300.34637451171875,
+      "logps/rejected": -333.18939208984375,
+      "loss": 0.5938,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.11966991424560547,
+      "rewards/margins": 0.259331077337265,
+      "rewards/rejected": -0.37900102138519287,
+      "step": 1500
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.856606904623047e-06,
+      "logits/chosen": -2.651618242263794,
+      "logits/rejected": -2.633094072341919,
+      "logps/chosen": -250.1557159423828,
+      "logps/rejected": -281.1156311035156,
+      "loss": 0.6346,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.19381049275398254,
+      "rewards/margins": 0.16430801153182983,
+      "rewards/rejected": -0.35811847448349,
+      "step": 1510
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.852770077278557e-06,
+      "logits/chosen": -2.647432327270508,
+      "logits/rejected": -2.59525203704834,
+      "logps/chosen": -283.68304443359375,
+      "logps/rejected": -277.7024841308594,
+      "loss": 0.5746,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.043497271835803986,
+      "rewards/margins": 0.3027854263782501,
+      "rewards/rejected": -0.3462826907634735,
+      "step": 1520
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.848884150054196e-06,
+      "logits/chosen": -2.6727054119110107,
+      "logits/rejected": -2.6422343254089355,
+      "logps/chosen": -280.96405029296875,
+      "logps/rejected": -258.3843688964844,
+      "loss": 0.659,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.13501325249671936,
+      "rewards/margins": 0.12726901471614838,
+      "rewards/rejected": -0.26228228211402893,
+      "step": 1530
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.8449492040452495e-06,
+      "logits/chosen": -2.6572389602661133,
+      "logits/rejected": -2.613257646560669,
+      "logps/chosen": -305.8749084472656,
+      "logps/rejected": -282.15838623046875,
+      "loss": 0.5836,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.11882825195789337,
+      "rewards/margins": 0.28394126892089844,
+      "rewards/rejected": -0.4027694761753082,
+      "step": 1540
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.840965321369973e-06,
+      "logits/chosen": -2.6655185222625732,
+      "logits/rejected": -2.553612470626831,
+      "logps/chosen": -291.9546203613281,
+      "logps/rejected": -279.5517883300781,
+      "loss": 0.6149,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.08108758926391602,
+      "rewards/margins": 0.21086159348487854,
+      "rewards/rejected": -0.29194921255111694,
+      "step": 1550
+    },
+    {
+      "epoch": 0.2,
+      "learning_rate": 4.8369325851678795e-06,
+      "logits/chosen": -2.6508326530456543,
+      "logits/rejected": -2.5720787048339844,
+      "logps/chosen": -302.269287109375,
+      "logps/rejected": -300.6006774902344,
+      "loss": 0.5975,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.11006896197795868,
+      "rewards/margins": 0.2537834346294403,
+      "rewards/rejected": -0.3638524115085602,
+      "step": 1560
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.832851079598007e-06,
+      "logits/chosen": -2.6661791801452637,
+      "logits/rejected": -2.6303372383117676,
+      "logps/chosen": -269.0194396972656,
+      "logps/rejected": -305.2875061035156,
+      "loss": 0.6233,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.1653946489095688,
+      "rewards/margins": 0.20300225913524628,
+      "rewards/rejected": -0.36839693784713745,
+      "step": 1570
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.828720889837158e-06,
+      "logits/chosen": -2.609922170639038,
+      "logits/rejected": -2.557817220687866,
+      "logps/chosen": -293.16900634765625,
+      "logps/rejected": -262.88818359375,
+      "loss": 0.6351,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.09230937063694,
+      "rewards/margins": 0.17000973224639893,
+      "rewards/rejected": -0.26231908798217773,
+      "step": 1580
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.824542102078125e-06,
+      "logits/chosen": -2.5694689750671387,
+      "logits/rejected": -2.575117588043213,
+      "logps/chosen": -280.21990966796875,
+      "logps/rejected": -269.1886291503906,
+      "loss": 0.6342,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.09446844458580017,
+      "rewards/margins": 0.16695217788219452,
+      "rewards/rejected": -0.2614205777645111,
+      "step": 1590
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.820314803527888e-06,
+      "logits/chosen": -2.5533907413482666,
+      "logits/rejected": -2.4679887294769287,
+      "logps/chosen": -276.68109130859375,
+      "logps/rejected": -282.217529296875,
+      "loss": 0.6114,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.20810618996620178,
+      "rewards/margins": 0.2135452777147293,
+      "rewards/rejected": -0.4216514229774475,
+      "step": 1600
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.816039082405799e-06,
+      "logits/chosen": -2.56892728805542,
+      "logits/rejected": -2.552664279937744,
+      "logps/chosen": -284.3816223144531,
+      "logps/rejected": -301.64312744140625,
+      "loss": 0.6136,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.08640266954898834,
+      "rewards/margins": 0.21293416619300842,
+      "rewards/rejected": -0.29933685064315796,
+      "step": 1610
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.81171502794174e-06,
+      "logits/chosen": -2.6324715614318848,
+      "logits/rejected": -2.62526535987854,
+      "logps/chosen": -319.0238952636719,
+      "logps/rejected": -304.2762756347656,
+      "loss": 0.6255,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.020495552569627762,
+      "rewards/margins": 0.18524014949798584,
+      "rewards/rejected": -0.16474460065364838,
+      "step": 1620
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.8073427303742584e-06,
+      "logits/chosen": -2.692636728286743,
+      "logits/rejected": -2.626201868057251,
+      "logps/chosen": -238.89102172851562,
+      "logps/rejected": -248.6376190185547,
+      "loss": 0.6008,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.05814732238650322,
+      "rewards/margins": 0.229304701089859,
+      "rewards/rejected": -0.17115740478038788,
+      "step": 1630
+    },
+    {
+      "epoch": 0.21,
+      "learning_rate": 4.802922280948685e-06,
+      "logits/chosen": -2.598919630050659,
+      "logits/rejected": -2.584041118621826,
+      "logps/chosen": -249.4228057861328,
+      "logps/rejected": -305.5156555175781,
+      "loss": 0.6302,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.07586696743965149,
+      "rewards/margins": 0.18719211220741272,
+      "rewards/rejected": -0.2630590796470642,
+      "step": 1640
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.798453771915231e-06,
+      "logits/chosen": -2.7062296867370605,
+      "logits/rejected": -2.612534761428833,
+      "logps/chosen": -295.24114990234375,
+      "logps/rejected": -292.77862548828125,
+      "loss": 0.6023,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.09026648104190826,
+      "rewards/margins": 0.24725332856178284,
+      "rewards/rejected": -0.3375197649002075,
+      "step": 1650
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.793937296527062e-06,
+      "logits/chosen": -2.726835250854492,
+      "logits/rejected": -2.634275436401367,
+      "logps/chosen": -342.76947021484375,
+      "logps/rejected": -351.55487060546875,
+      "loss": 0.6336,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.13259488344192505,
+      "rewards/margins": 0.19262318313121796,
+      "rewards/rejected": -0.3252180516719818,
+      "step": 1660
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.78937294903835e-06,
+      "logits/chosen": -2.6562981605529785,
+      "logits/rejected": -2.5573320388793945,
+      "logps/chosen": -272.51300048828125,
+      "logps/rejected": -273.8321838378906,
+      "loss": 0.5999,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.03216207027435303,
+      "rewards/margins": 0.2476842850446701,
+      "rewards/rejected": -0.2798463702201843,
+      "step": 1670
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.78476082470231e-06,
+      "logits/chosen": -2.6850619316101074,
+      "logits/rejected": -2.5989696979522705,
+      "logps/chosen": -273.6483154296875,
+      "logps/rejected": -272.3609313964844,
+      "loss": 0.6028,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.09215790778398514,
+      "rewards/margins": 0.25344574451446533,
+      "rewards/rejected": -0.34560370445251465,
+      "step": 1680
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.780101019769212e-06,
+      "logits/chosen": -2.637836217880249,
+      "logits/rejected": -2.5876827239990234,
+      "logps/chosen": -322.16937255859375,
+      "logps/rejected": -303.0202941894531,
+      "loss": 0.6205,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.14790192246437073,
+      "rewards/margins": 0.220493346452713,
+      "rewards/rejected": -0.36839523911476135,
+      "step": 1690
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.775393631484368e-06,
+      "logits/chosen": -2.594028949737549,
+      "logits/rejected": -2.533957004547119,
+      "logps/chosen": -318.33087158203125,
+      "logps/rejected": -338.8320617675781,
+      "loss": 0.5687,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.16223548352718353,
+      "rewards/margins": 0.3275444507598877,
+      "rewards/rejected": -0.48977988958358765,
+      "step": 1700
+    },
+    {
+      "epoch": 0.22,
+      "learning_rate": 4.770638758086105e-06,
+      "logits/chosen": -2.630344867706299,
+      "logits/rejected": -2.57621693611145,
+      "logps/chosen": -315.2181091308594,
+      "logps/rejected": -303.3475646972656,
+      "loss": 0.6312,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3247630298137665,
+      "rewards/margins": 0.18050554394721985,
+      "rewards/rejected": -0.5052685737609863,
+      "step": 1710
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.7658364988037184e-06,
+      "logits/chosen": -2.6764087677001953,
+      "logits/rejected": -2.6398842334747314,
+      "logps/chosen": -338.4540100097656,
+      "logps/rejected": -328.9268493652344,
+      "loss": 0.6441,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.3214486241340637,
+      "rewards/margins": 0.1526016741991043,
+      "rewards/rejected": -0.47405022382736206,
+      "step": 1720
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.760986953855395e-06,
+      "logits/chosen": -2.6666629314422607,
+      "logits/rejected": -2.6785898208618164,
+      "logps/chosen": -288.61676025390625,
+      "logps/rejected": -316.87115478515625,
+      "loss": 0.6188,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.1931554675102234,
+      "rewards/margins": 0.21359868347644806,
+      "rewards/rejected": -0.40675416588783264,
+      "step": 1730
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.756090224446127e-06,
+      "logits/chosen": -2.6678307056427,
+      "logits/rejected": -2.5994620323181152,
+      "logps/chosen": -321.52008056640625,
+      "logps/rejected": -317.11248779296875,
+      "loss": 0.6017,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.011467398144304752,
+      "rewards/margins": 0.23510918021202087,
+      "rewards/rejected": -0.24657657742500305,
+      "step": 1740
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.7511464127655945e-06,
+      "logits/chosen": -2.704003095626831,
+      "logits/rejected": -2.5501084327697754,
+      "logps/chosen": -289.18798828125,
+      "logps/rejected": -285.47894287109375,
+      "loss": 0.5844,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.09155084937810898,
+      "rewards/margins": 0.2973455786705017,
+      "rewards/rejected": -0.3888964056968689,
+      "step": 1750
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.74615562198604e-06,
+      "logits/chosen": -2.62406063079834,
+      "logits/rejected": -2.512871742248535,
+      "logps/chosen": -274.460693359375,
+      "logps/rejected": -273.2636413574219,
+      "loss": 0.5697,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.05653535574674606,
+      "rewards/margins": 0.3312791883945465,
+      "rewards/rejected": -0.38781455159187317,
+      "step": 1760
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.741117956260107e-06,
+      "logits/chosen": -2.583909749984741,
+      "logits/rejected": -2.52388596534729,
+      "logps/chosen": -281.722900390625,
+      "logps/rejected": -303.64447021484375,
+      "loss": 0.6017,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.03865355998277664,
+      "rewards/margins": 0.26362842321395874,
+      "rewards/rejected": -0.30228191614151,
+      "step": 1770
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.736033520718672e-06,
+      "logits/chosen": -2.6637024879455566,
+      "logits/rejected": -2.548191547393799,
+      "logps/chosen": -312.50286865234375,
+      "logps/rejected": -310.6639709472656,
+      "loss": 0.5958,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.020857496187090874,
+      "rewards/margins": 0.26661014556884766,
+      "rewards/rejected": -0.2874676585197449,
+      "step": 1780
+    },
+    {
+      "epoch": 0.23,
+      "learning_rate": 4.730902421468652e-06,
+      "logits/chosen": -2.6668648719787598,
+      "logits/rejected": -2.5857529640197754,
+      "logps/chosen": -269.2945861816406,
+      "logps/rejected": -300.3023376464844,
+      "loss": 0.5973,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.07311777770519257,
+      "rewards/margins": 0.2713143229484558,
+      "rewards/rejected": -0.34443214535713196,
+      "step": 1790
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.7257247655907854e-06,
+      "logits/chosen": -2.625272274017334,
+      "logits/rejected": -2.609955310821533,
+      "logps/chosen": -245.3594512939453,
+      "logps/rejected": -281.7267761230469,
+      "loss": 0.6064,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.04326234385371208,
+      "rewards/margins": 0.24693512916564941,
+      "rewards/rejected": -0.2901974618434906,
+      "step": 1800
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.720500661137397e-06,
+      "logits/chosen": -2.6524150371551514,
+      "logits/rejected": -2.6066842079162598,
+      "logps/chosen": -279.03106689453125,
+      "logps/rejected": -302.11151123046875,
+      "loss": 0.6356,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.14594458043575287,
+      "rewards/margins": 0.19176334142684937,
+      "rewards/rejected": -0.33770790696144104,
+      "step": 1810
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.71523021713015e-06,
+      "logits/chosen": -2.7315337657928467,
+      "logits/rejected": -2.7280869483947754,
+      "logps/chosen": -397.20501708984375,
+      "logps/rejected": -421.51171875,
+      "loss": 0.653,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.24775946140289307,
+      "rewards/margins": 0.15556971728801727,
+      "rewards/rejected": -0.4033292233943939,
+      "step": 1820
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.709913543557761e-06,
+      "logits/chosen": -2.6619479656219482,
+      "logits/rejected": -2.6229560375213623,
+      "logps/chosen": -338.3002014160156,
+      "logps/rejected": -339.80291748046875,
+      "loss": 0.6151,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.17326846718788147,
+      "rewards/margins": 0.2287912666797638,
+      "rewards/rejected": -0.40205973386764526,
+      "step": 1830
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.704550751373715e-06,
+      "logits/chosen": -2.606283664703369,
+      "logits/rejected": -2.5262932777404785,
+      "logps/chosen": -370.41290283203125,
+      "logps/rejected": -341.31890869140625,
+      "loss": 0.5954,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.05165138095617294,
+      "rewards/margins": 0.2767849862575531,
+      "rewards/rejected": -0.3284364342689514,
+      "step": 1840
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.699141952493941e-06,
+      "logits/chosen": -2.6392226219177246,
+      "logits/rejected": -2.60284686088562,
+      "logps/chosen": -278.09844970703125,
+      "logps/rejected": -297.26910400390625,
+      "loss": 0.627,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.043605975806713104,
+      "rewards/margins": 0.18779481947422028,
+      "rewards/rejected": -0.23140080273151398,
+      "step": 1850
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.6936872597944814e-06,
+      "logits/chosen": -2.7057807445526123,
+      "logits/rejected": -2.5877327919006348,
+      "logps/chosen": -319.1133117675781,
+      "logps/rejected": -272.9107971191406,
+      "loss": 0.5656,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.06138920783996582,
+      "rewards/margins": 0.3460971415042877,
+      "rewards/rejected": -0.4074863791465759,
+      "step": 1860
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 4.688186787109136e-06,
+      "logits/chosen": -2.6357007026672363,
+      "logits/rejected": -2.601635456085205,
+      "logps/chosen": -320.27459716796875,
+      "logps/rejected": -336.41729736328125,
+      "loss": 0.5987,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.11088709533214569,
+      "rewards/margins": 0.2718547284603119,
+      "rewards/rejected": -0.3827418088912964,
+      "step": 1870
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.682640649227085e-06,
+      "logits/chosen": -2.6405344009399414,
+      "logits/rejected": -2.596005916595459,
+      "logps/chosen": -287.4984436035156,
+      "logps/rejected": -277.5819091796875,
+      "loss": 0.5966,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.06364084780216217,
+      "rewards/margins": 0.260766863822937,
+      "rewards/rejected": -0.3244077265262604,
+      "step": 1880
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.677048961890492e-06,
+      "logits/chosen": -2.555321216583252,
+      "logits/rejected": -2.5089797973632812,
+      "logps/chosen": -297.0433044433594,
+      "logps/rejected": -288.5172424316406,
+      "loss": 0.6122,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.10188271850347519,
+      "rewards/margins": 0.22268228232860565,
+      "rewards/rejected": -0.32456502318382263,
+      "step": 1890
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.671411841792096e-06,
+      "logits/chosen": -2.586582660675049,
+      "logits/rejected": -2.5934300422668457,
+      "logps/chosen": -326.0126953125,
+      "logps/rejected": -349.7406311035156,
+      "loss": 0.6123,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.13140635192394257,
+      "rewards/margins": 0.23367634415626526,
+      "rewards/rejected": -0.36508268117904663,
+      "step": 1900
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.665729406572764e-06,
+      "logits/chosen": -2.611443281173706,
+      "logits/rejected": -2.581946849822998,
+      "logps/chosen": -260.5864562988281,
+      "logps/rejected": -278.5222473144531,
+      "loss": 0.6254,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.1393495798110962,
+      "rewards/margins": 0.2319023609161377,
+      "rewards/rejected": -0.3712519705295563,
+      "step": 1910
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.660001774819048e-06,
+      "logits/chosen": -2.6996841430664062,
+      "logits/rejected": -2.6368303298950195,
+      "logps/chosen": -331.8706970214844,
+      "logps/rejected": -291.9696350097656,
+      "loss": 0.5923,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.14025358855724335,
+      "rewards/margins": 0.2745034098625183,
+      "rewards/rejected": -0.41475701332092285,
+      "step": 1920
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.654229066060702e-06,
+      "logits/chosen": -2.633701801300049,
+      "logits/rejected": -2.559678554534912,
+      "logps/chosen": -300.74444580078125,
+      "logps/rejected": -277.4322814941406,
+      "loss": 0.6305,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.23295915126800537,
+      "rewards/margins": 0.18096324801445007,
+      "rewards/rejected": -0.41392236948013306,
+      "step": 1930
+    },
+    {
+      "epoch": 0.25,
+      "learning_rate": 4.648411400768193e-06,
+      "logits/chosen": -2.695106267929077,
+      "logits/rejected": -2.577064037322998,
+      "logps/chosen": -321.78302001953125,
+      "logps/rejected": -268.8973083496094,
+      "loss": 0.5919,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.2493828982114792,
+      "rewards/margins": 0.26992055773735046,
+      "rewards/rejected": -0.5193034410476685,
+      "step": 1940
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.642548900350182e-06,
+      "logits/chosen": -2.6117231845855713,
+      "logits/rejected": -2.592233419418335,
+      "logps/chosen": -268.0585021972656,
+      "logps/rejected": -281.26214599609375,
+      "loss": 0.6129,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.21964256465435028,
+      "rewards/margins": 0.22322368621826172,
+      "rewards/rejected": -0.4428662657737732,
+      "step": 1950
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.636641687150994e-06,
+      "logits/chosen": -2.6418070793151855,
+      "logits/rejected": -2.6547489166259766,
+      "logps/chosen": -319.56707763671875,
+      "logps/rejected": -310.83111572265625,
+      "loss": 0.5911,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.052779704332351685,
+      "rewards/margins": 0.2844510078430176,
+      "rewards/rejected": -0.33723071217536926,
+      "step": 1960
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.6306898844480615e-06,
+      "logits/chosen": -2.591348171234131,
+      "logits/rejected": -2.551347255706787,
+      "logps/chosen": -326.96929931640625,
+      "logps/rejected": -301.774169921875,
+      "loss": 0.6208,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.013970981352031231,
+      "rewards/margins": 0.19149453938007355,
+      "rewards/rejected": -0.20546552538871765,
+      "step": 1970
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.624693616449358e-06,
+      "logits/chosen": -2.6167454719543457,
+      "logits/rejected": -2.561824321746826,
+      "logps/chosen": -237.9170684814453,
+      "logps/rejected": -248.0195770263672,
+      "loss": 0.6245,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.016004789620637894,
+      "rewards/margins": 0.20452181994915009,
+      "rewards/rejected": -0.22052660584449768,
+      "step": 1980
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.6186530082908e-06,
+      "logits/chosen": -2.6295902729034424,
+      "logits/rejected": -2.5565896034240723,
+      "logps/chosen": -324.8574523925781,
+      "logps/rejected": -332.98321533203125,
+      "loss": 0.6018,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.06827336549758911,
+      "rewards/margins": 0.2616387903690338,
+      "rewards/rejected": -0.32991212606430054,
+      "step": 1990
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.612568186033633e-06,
+      "logits/chosen": -2.6730666160583496,
+      "logits/rejected": -2.6284358501434326,
+      "logps/chosen": -275.6248474121094,
+      "logps/rejected": -302.05133056640625,
+      "loss": 0.6355,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.19102105498313904,
+      "rewards/margins": 0.19359147548675537,
+      "rewards/rejected": -0.384612500667572,
+      "step": 2000
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.6064392766618125e-06,
+      "logits/chosen": -2.5352156162261963,
+      "logits/rejected": -2.493509292602539,
+      "logps/chosen": -313.5549011230469,
+      "logps/rejected": -313.4109191894531,
+      "loss": 0.6075,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.13206224143505096,
+      "rewards/margins": 0.24549846351146698,
+      "rewards/rejected": -0.37756067514419556,
+      "step": 2010
+    },
+    {
+      "epoch": 0.26,
+      "learning_rate": 4.60026640807934e-06,
+      "logits/chosen": -2.6311490535736084,
+      "logits/rejected": -2.584712028503418,
+      "logps/chosen": -300.6962890625,
+      "logps/rejected": -284.7915954589844,
+      "loss": 0.5934,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.05578576400876045,
+      "rewards/margins": 0.26223909854888916,
+      "rewards/rejected": -0.3180248439311981,
+      "step": 2020
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.594049709107604e-06,
+      "logits/chosen": -2.6556649208068848,
+      "logits/rejected": -2.5575225353240967,
+      "logps/chosen": -321.55718994140625,
+      "logps/rejected": -326.5420837402344,
+      "loss": 0.6111,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.04723922163248062,
+      "rewards/margins": 0.24892227351665497,
+      "rewards/rejected": -0.2961614727973938,
+      "step": 2030
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.587789309482687e-06,
+      "logits/chosen": -2.555610179901123,
+      "logits/rejected": -2.5157666206359863,
+      "logps/chosen": -251.49795532226562,
+      "logps/rejected": -312.0182189941406,
+      "loss": 0.5922,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 0.00901651382446289,
+      "rewards/margins": 0.2695215344429016,
+      "rewards/rejected": -0.2605050206184387,
+      "step": 2040
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.581485339852659e-06,
+      "logits/chosen": -2.5391087532043457,
+      "logits/rejected": -2.49125075340271,
+      "logps/chosen": -318.1247253417969,
+      "logps/rejected": -346.09893798828125,
+      "loss": 0.622,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.04015789553523064,
+      "rewards/margins": 0.20342564582824707,
+      "rewards/rejected": -0.2435835301876068,
+      "step": 2050
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.5751379317748514e-06,
+      "logits/chosen": -2.621710777282715,
+      "logits/rejected": -2.577556610107422,
+      "logps/chosen": -278.97332763671875,
+      "logps/rejected": -281.84423828125,
+      "loss": 0.665,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.07916220277547836,
+      "rewards/margins": 0.11006591469049454,
+      "rewards/rejected": -0.1892281323671341,
+      "step": 2060
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.56874721771311e-06,
+      "logits/chosen": -2.6533901691436768,
+      "logits/rejected": -2.591190814971924,
+      "logps/chosen": -298.0502014160156,
+      "logps/rejected": -282.8466491699219,
+      "loss": 0.6364,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.07570312172174454,
+      "rewards/margins": 0.18112945556640625,
+      "rewards/rejected": -0.2568325996398926,
+      "step": 2070
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.562313331035032e-06,
+      "logits/chosen": -2.5666067600250244,
+      "logits/rejected": -2.517413854598999,
+      "logps/chosen": -270.82623291015625,
+      "logps/rejected": -313.10089111328125,
+      "loss": 0.5991,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.13663598895072937,
+      "rewards/margins": 0.26162397861480713,
+      "rewards/rejected": -0.3982599377632141,
+      "step": 2080
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.555836406009183e-06,
+      "logits/chosen": -2.6103110313415527,
+      "logits/rejected": -2.5872230529785156,
+      "logps/chosen": -277.1881408691406,
+      "logps/rejected": -276.60321044921875,
+      "loss": 0.6379,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.3089580237865448,
+      "rewards/margins": 0.18561621010303497,
+      "rewards/rejected": -0.4945742189884186,
+      "step": 2090
+    },
+    {
+      "epoch": 0.27,
+      "learning_rate": 4.5493165778022945e-06,
+      "logits/chosen": -2.598449230194092,
+      "logits/rejected": -2.4996886253356934,
+      "logps/chosen": -285.7212219238281,
+      "logps/rejected": -276.3428649902344,
+      "loss": 0.5979,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.2876005470752716,
+      "rewards/margins": 0.25197282433509827,
+      "rewards/rejected": -0.5395733714103699,
+      "step": 2100
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.542753982476443e-06,
+      "logits/chosen": -2.6144790649414062,
+      "logits/rejected": -2.5341458320617676,
+      "logps/chosen": -333.5113830566406,
+      "logps/rejected": -315.1654357910156,
+      "loss": 0.6069,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.19754566252231598,
+      "rewards/margins": 0.2584041953086853,
+      "rewards/rejected": -0.4559497833251953,
+      "step": 2110
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.53614875698621e-06,
+      "logits/chosen": -2.6930184364318848,
+      "logits/rejected": -2.6099038124084473,
+      "logps/chosen": -339.15850830078125,
+      "logps/rejected": -353.9617919921875,
+      "loss": 0.5874,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.18686847388744354,
+      "rewards/margins": 0.29116055369377136,
+      "rewards/rejected": -0.47802895307540894,
+      "step": 2120
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.529501039175824e-06,
+      "logits/chosen": -2.6591010093688965,
+      "logits/rejected": -2.622368812561035,
+      "logps/chosen": -260.2812194824219,
+      "logps/rejected": -276.27301025390625,
+      "loss": 0.6283,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.22271648049354553,
+      "rewards/margins": 0.2172001302242279,
+      "rewards/rejected": -0.43991661071777344,
+      "step": 2130
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.522810967776287e-06,
+      "logits/chosen": -2.5809121131896973,
+      "logits/rejected": -2.4882516860961914,
+      "logps/chosen": -309.24237060546875,
+      "logps/rejected": -266.2737731933594,
+      "loss": 0.5756,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.15023306012153625,
+      "rewards/margins": 0.31738346815109253,
+      "rewards/rejected": -0.4676164984703064,
+      "step": 2140
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.516078682402473e-06,
+      "logits/chosen": -2.643094539642334,
+      "logits/rejected": -2.5485596656799316,
+      "logps/chosen": -314.19610595703125,
+      "logps/rejected": -315.14959716796875,
+      "loss": 0.5837,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.13438525795936584,
+      "rewards/margins": 0.3204689025878906,
+      "rewards/rejected": -0.45485416054725647,
+      "step": 2150
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.509304323550221e-06,
+      "logits/chosen": -2.6377241611480713,
+      "logits/rejected": -2.58573055267334,
+      "logps/chosen": -336.39727783203125,
+      "logps/rejected": -337.25836181640625,
+      "loss": 0.598,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.07529301941394806,
+      "rewards/margins": 0.2836524248123169,
+      "rewards/rejected": -0.35894545912742615,
+      "step": 2160
+    },
+    {
+      "epoch": 0.28,
+      "learning_rate": 4.502488032593398e-06,
+      "logits/chosen": -2.674180746078491,
+      "logits/rejected": -2.6048054695129395,
+      "logps/chosen": -309.07586669921875,
+      "logps/rejected": -303.78118896484375,
+      "loss": 0.6104,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.17862166464328766,
+      "rewards/margins": 0.2347385436296463,
+      "rewards/rejected": -0.41336020827293396,
+      "step": 2170
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.495629951780951e-06,
+      "logits/chosen": -2.639866590499878,
+      "logits/rejected": -2.5290236473083496,
+      "logps/chosen": -366.51873779296875,
+      "logps/rejected": -307.1587829589844,
+      "loss": 0.5744,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.09607136249542236,
+      "rewards/margins": 0.31942200660705566,
+      "rewards/rejected": -0.4154933989048004,
+      "step": 2180
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.488730224233941e-06,
+      "logits/chosen": -2.5855698585510254,
+      "logits/rejected": -2.532564640045166,
+      "logps/chosen": -241.4503631591797,
+      "logps/rejected": -297.1728515625,
+      "loss": 0.5433,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.060798972845077515,
+      "rewards/margins": 0.3830801844596863,
+      "rewards/rejected": -0.4438791275024414,
+      "step": 2190
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.481788993942547e-06,
+      "logits/chosen": -2.6381936073303223,
+      "logits/rejected": -2.560931444168091,
+      "logps/chosen": -291.14691162109375,
+      "logps/rejected": -283.0338134765625,
+      "loss": 0.5887,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.036971576511859894,
+      "rewards/margins": 0.2973865270614624,
+      "rewards/rejected": -0.3343580961227417,
+      "step": 2200
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.474806405763076e-06,
+      "logits/chosen": -2.61448073387146,
+      "logits/rejected": -2.582944869995117,
+      "logps/chosen": -296.05999755859375,
+      "logps/rejected": -295.82708740234375,
+      "loss": 0.6029,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.09153171628713608,
+      "rewards/margins": 0.2685583233833313,
+      "rewards/rejected": -0.36009007692337036,
+      "step": 2210
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.4677826054149235e-06,
+      "logits/chosen": -2.60630464553833,
+      "logits/rejected": -2.5313713550567627,
+      "logps/chosen": -331.80938720703125,
+      "logps/rejected": -329.1781921386719,
+      "loss": 0.6199,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.0674784705042839,
+      "rewards/margins": 0.21762514114379883,
+      "rewards/rejected": -0.28510361909866333,
+      "step": 2220
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.460717739477543e-06,
+      "logits/chosen": -2.6077935695648193,
+      "logits/rejected": -2.596501588821411,
+      "logps/chosen": -279.3420104980469,
+      "logps/rejected": -335.58514404296875,
+      "loss": 0.6248,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.09416131675243378,
+      "rewards/margins": 0.20481404662132263,
+      "rewards/rejected": -0.2989753782749176,
+      "step": 2230
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.4536119553873866e-06,
+      "logits/chosen": -2.5776143074035645,
+      "logits/rejected": -2.5966944694519043,
+      "logps/chosen": -279.759033203125,
+      "logps/rejected": -303.5380554199219,
+      "loss": 0.5708,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.07991547882556915,
+      "rewards/margins": 0.31736770272254944,
+      "rewards/rejected": -0.3972831666469574,
+      "step": 2240
+    },
+    {
+      "epoch": 0.29,
+      "learning_rate": 4.446465401434824e-06,
+      "logits/chosen": -2.574833631515503,
+      "logits/rejected": -2.5545363426208496,
+      "logps/chosen": -261.7332458496094,
+      "logps/rejected": -322.7508544921875,
+      "loss": 0.6126,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.12314830720424652,
+      "rewards/margins": 0.24508614838123322,
+      "rewards/rejected": -0.3682344853878021,
+      "step": 2250
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.43927822676105e-06,
+      "logits/chosen": -2.6496713161468506,
+      "logits/rejected": -2.55104660987854,
+      "logps/chosen": -315.08941650390625,
+      "logps/rejected": -301.93963623046875,
+      "loss": 0.606,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.10801728814840317,
+      "rewards/margins": 0.2596587538719177,
+      "rewards/rejected": -0.3676760792732239,
+      "step": 2260
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.432050581354972e-06,
+      "logits/chosen": -2.6128652095794678,
+      "logits/rejected": -2.5364413261413574,
+      "logps/chosen": -288.2715759277344,
+      "logps/rejected": -312.6794128417969,
+      "loss": 0.6312,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.11209984123706818,
+      "rewards/margins": 0.20748715102672577,
+      "rewards/rejected": -0.31958699226379395,
+      "step": 2270
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.424782616050078e-06,
+      "logits/chosen": -2.5691895484924316,
+      "logits/rejected": -2.5039467811584473,
+      "logps/chosen": -304.8255615234375,
+      "logps/rejected": -299.4909973144531,
+      "loss": 0.6188,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.12285183370113373,
+      "rewards/margins": 0.24723884463310242,
+      "rewards/rejected": -0.37009069323539734,
+      "step": 2280
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.4174744825212954e-06,
+      "logits/chosen": -2.654545307159424,
+      "logits/rejected": -2.6522722244262695,
+      "logps/chosen": -268.68853759765625,
+      "logps/rejected": -301.3064270019531,
+      "loss": 0.6721,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -0.12379419803619385,
+      "rewards/margins": 0.1324099600315094,
+      "rewards/rejected": -0.25620415806770325,
+      "step": 2290
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.410126333281815e-06,
+      "logits/chosen": -2.612492322921753,
+      "logits/rejected": -2.530735969543457,
+      "logps/chosen": -347.1034240722656,
+      "logps/rejected": -383.4774475097656,
+      "loss": 0.6024,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.1915418654680252,
+      "rewards/margins": 0.253628134727478,
+      "rewards/rejected": -0.44517001509666443,
+      "step": 2300
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.402738321679918e-06,
+      "logits/chosen": -2.591958999633789,
+      "logits/rejected": -2.5409371852874756,
+      "logps/chosen": -315.42889404296875,
+      "logps/rejected": -279.4637756347656,
+      "loss": 0.5729,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.1491278111934662,
+      "rewards/margins": 0.3320808708667755,
+      "rewards/rejected": -0.4812086224555969,
+      "step": 2310
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.395310601895772e-06,
+      "logits/chosen": -2.6729795932769775,
+      "logits/rejected": -2.599367618560791,
+      "logps/chosen": -299.4936218261719,
+      "logps/rejected": -317.6011657714844,
+      "loss": 0.5713,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.07269485294818878,
+      "rewards/margins": 0.34723931550979614,
+      "rewards/rejected": -0.4199342131614685,
+      "step": 2320
+    },
+    {
+      "epoch": 0.3,
+      "learning_rate": 4.38784332893821e-06,
+      "logits/chosen": -2.6256155967712402,
+      "logits/rejected": -2.6133885383605957,
+      "logps/chosen": -215.117431640625,
+      "logps/rejected": -251.8661346435547,
+      "loss": 0.5667,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.02816862240433693,
+      "rewards/margins": 0.34232696890830994,
+      "rewards/rejected": -0.37049558758735657,
+      "step": 2330
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.380336658641503e-06,
+      "logits/chosen": -2.68818998336792,
+      "logits/rejected": -2.5473694801330566,
+      "logps/chosen": -303.960205078125,
+      "logps/rejected": -294.9482116699219,
+      "loss": 0.5786,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.12416895478963852,
+      "rewards/margins": 0.325573205947876,
+      "rewards/rejected": -0.4497421383857727,
+      "step": 2340
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.372790747662101e-06,
+      "logits/chosen": -2.584380626678467,
+      "logits/rejected": -2.5486741065979004,
+      "logps/chosen": -300.21343994140625,
+      "logps/rejected": -303.4608459472656,
+      "loss": 0.597,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.19595685601234436,
+      "rewards/margins": 0.27592629194259644,
+      "rewards/rejected": -0.4718831479549408,
+      "step": 2350
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.365205753475367e-06,
+      "logits/chosen": -2.6563780307769775,
+      "logits/rejected": -2.5540294647216797,
+      "logps/chosen": -364.38006591796875,
+      "logps/rejected": -324.7938537597656,
+      "loss": 0.5552,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.12454191595315933,
+      "rewards/margins": 0.3732561469078064,
+      "rewards/rejected": -0.49779802560806274,
+      "step": 2360
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.35758183437229e-06,
+      "logits/chosen": -2.67205810546875,
+      "logits/rejected": -2.640204429626465,
+      "logps/chosen": -352.1476745605469,
+      "logps/rejected": -363.07635498046875,
+      "loss": 0.6129,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.0900491401553154,
+      "rewards/margins": 0.23913398385047913,
+      "rewards/rejected": -0.32918310165405273,
+      "step": 2370
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.3499191494561835e-06,
+      "logits/chosen": -2.492128849029541,
+      "logits/rejected": -2.4748432636260986,
+      "logps/chosen": -302.64288330078125,
+      "logps/rejected": -281.3841552734375,
+      "loss": 0.6168,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.16875137388706207,
+      "rewards/margins": 0.2410108596086502,
+      "rewards/rejected": -0.4097622334957123,
+      "step": 2380
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.3422178586393615e-06,
+      "logits/chosen": -2.695197343826294,
+      "logits/rejected": -2.610597610473633,
+      "logps/chosen": -344.9779052734375,
+      "logps/rejected": -317.9693298339844,
+      "loss": 0.5728,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.14025068283081055,
+      "rewards/margins": 0.3393901586532593,
+      "rewards/rejected": -0.47964081168174744,
+      "step": 2390
+    },
+    {
+      "epoch": 0.31,
+      "learning_rate": 4.334478122639804e-06,
+      "logits/chosen": -2.7056970596313477,
+      "logits/rejected": -2.5337796211242676,
+      "logps/chosen": -290.0587463378906,
+      "logps/rejected": -255.80770874023438,
+      "loss": 0.5309,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.21059250831604004,
+      "rewards/margins": 0.4257400631904602,
+      "rewards/rejected": -0.6363325119018555,
+      "step": 2400
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.3267001029778015e-06,
+      "logits/chosen": -2.661207914352417,
+      "logits/rejected": -2.580709934234619,
+      "logps/chosen": -324.47607421875,
+      "logps/rejected": -276.6221008300781,
+      "loss": 0.6177,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.16842873394489288,
+      "rewards/margins": 0.22736510634422302,
+      "rewards/rejected": -0.3957938551902771,
+      "step": 2410
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.318883961972585e-06,
+      "logits/chosen": -2.573106288909912,
+      "logits/rejected": -2.560438394546509,
+      "logps/chosen": -282.1749267578125,
+      "logps/rejected": -294.05517578125,
+      "loss": 0.5966,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.08466902375221252,
+      "rewards/margins": 0.2872466742992401,
+      "rewards/rejected": -0.371915727853775,
+      "step": 2420
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.311029862738942e-06,
+      "logits/chosen": -2.666644811630249,
+      "logits/rejected": -2.5996875762939453,
+      "logps/chosen": -313.3420715332031,
+      "logps/rejected": -310.0482482910156,
+      "loss": 0.5964,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.2359074056148529,
+      "rewards/margins": 0.2667393088340759,
+      "rewards/rejected": -0.502646803855896,
+      "step": 2430
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.303137969183804e-06,
+      "logits/chosen": -2.5801563262939453,
+      "logits/rejected": -2.5598385334014893,
+      "logps/chosen": -306.23089599609375,
+      "logps/rejected": -307.0191345214844,
+      "loss": 0.5652,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.27654871344566345,
+      "rewards/margins": 0.3418423533439636,
+      "rewards/rejected": -0.6183910965919495,
+      "step": 2440
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.295208446002832e-06,
+      "logits/chosen": -2.5977237224578857,
+      "logits/rejected": -2.5126519203186035,
+      "logps/chosen": -267.31317138671875,
+      "logps/rejected": -269.39459228515625,
+      "loss": 0.6163,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.27999770641326904,
+      "rewards/margins": 0.251874178647995,
+      "rewards/rejected": -0.5318719148635864,
+      "step": 2450
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.287241458676981e-06,
+      "logits/chosen": -2.5850372314453125,
+      "logits/rejected": -2.5397372245788574,
+      "logps/chosen": -341.08837890625,
+      "logps/rejected": -335.3498229980469,
+      "loss": 0.5491,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.07927463203668594,
+      "rewards/margins": 0.3927169740200043,
+      "rewards/rejected": -0.4719915986061096,
+      "step": 2460
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.279237173469043e-06,
+      "logits/chosen": -2.62870192527771,
+      "logits/rejected": -2.5097358226776123,
+      "logps/chosen": -279.8306579589844,
+      "logps/rejected": -270.1935119628906,
+      "loss": 0.5839,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.003627000842243433,
+      "rewards/margins": 0.3063355088233948,
+      "rewards/rejected": -0.3027085065841675,
+      "step": 2470
+    },
+    {
+      "epoch": 0.32,
+      "learning_rate": 4.271195757420177e-06,
+      "logits/chosen": -2.515432357788086,
+      "logits/rejected": -2.4704678058624268,
+      "logps/chosen": -289.6210021972656,
+      "logps/rejected": -326.5202941894531,
+      "loss": 0.6182,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.06860475242137909,
+      "rewards/margins": 0.24095562100410461,
+      "rewards/rejected": -0.3095603585243225,
+      "step": 2480
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.263117378346425e-06,
+      "logits/chosen": -2.5774495601654053,
+      "logits/rejected": -2.5021750926971436,
+      "logps/chosen": -311.0116271972656,
+      "logps/rejected": -292.1097717285156,
+      "loss": 0.584,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.014823523350059986,
+      "rewards/margins": 0.3134276270866394,
+      "rewards/rejected": -0.2986041307449341,
+      "step": 2490
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.255002204835208e-06,
+      "logits/chosen": -2.5983493328094482,
+      "logits/rejected": -2.5537943840026855,
+      "logps/chosen": -308.4456481933594,
+      "logps/rejected": -303.02313232421875,
+      "loss": 0.598,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.04211214929819107,
+      "rewards/margins": 0.2478078305721283,
+      "rewards/rejected": -0.20569567382335663,
+      "step": 2500
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.246850406241812e-06,
+      "logits/chosen": -2.590212821960449,
+      "logits/rejected": -2.523480176925659,
+      "logps/chosen": -226.5638427734375,
+      "logps/rejected": -281.7768249511719,
+      "loss": 0.5845,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.0037656850181519985,
+      "rewards/margins": 0.3136346936225891,
+      "rewards/rejected": -0.31740036606788635,
+      "step": 2510
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.2386621526858465e-06,
+      "logits/chosen": -2.6058316230773926,
+      "logits/rejected": -2.569547653198242,
+      "logps/chosen": -246.8446807861328,
+      "logps/rejected": -230.9818878173828,
+      "loss": 0.6196,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.07652827352285385,
+      "rewards/margins": 0.22620157897472382,
+      "rewards/rejected": -0.30272990465164185,
+      "step": 2520
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.2304376150477015e-06,
+      "logits/chosen": -2.6098198890686035,
+      "logits/rejected": -2.605799436569214,
+      "logps/chosen": -299.2288818359375,
+      "logps/rejected": -321.2333984375,
+      "loss": 0.5655,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.0636453628540039,
+      "rewards/margins": 0.34476250410079956,
+      "rewards/rejected": -0.40840786695480347,
+      "step": 2530
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.222176964964977e-06,
+      "logits/chosen": -2.513763904571533,
+      "logits/rejected": -2.499044418334961,
+      "logps/chosen": -303.6919250488281,
+      "logps/rejected": -313.796630859375,
+      "loss": 0.6241,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.17382441461086273,
+      "rewards/margins": 0.2192266434431076,
+      "rewards/rejected": -0.3930510878562927,
+      "step": 2540
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.213880374828903e-06,
+      "logits/chosen": -2.6237854957580566,
+      "logits/rejected": -2.5704426765441895,
+      "logps/chosen": -268.98028564453125,
+      "logps/rejected": -309.5152893066406,
+      "loss": 0.5798,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.2150762379169464,
+      "rewards/margins": 0.32503241300582886,
+      "rewards/rejected": -0.5401086211204529,
+      "step": 2550
+    },
+    {
+      "epoch": 0.33,
+      "learning_rate": 4.2055480177807406e-06,
+      "logits/chosen": -2.471327543258667,
+      "logits/rejected": -2.480410099029541,
+      "logps/chosen": -254.95065307617188,
+      "logps/rejected": -263.35546875,
+      "loss": 0.6478,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.22877395153045654,
+      "rewards/margins": 0.16371914744377136,
+      "rewards/rejected": -0.39249303936958313,
+      "step": 2560
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.1971800677081696e-06,
+      "logits/chosen": -2.5488550662994385,
+      "logits/rejected": -2.5174922943115234,
+      "logps/chosen": -302.5681457519531,
+      "logps/rejected": -358.01531982421875,
+      "loss": 0.6376,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.2010851353406906,
+      "rewards/margins": 0.20292583107948303,
+      "rewards/rejected": -0.4040110111236572,
+      "step": 2570
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.188776699241661e-06,
+      "logits/chosen": -2.66471529006958,
+      "logits/rejected": -2.5682389736175537,
+      "logps/chosen": -312.4477233886719,
+      "logps/rejected": -308.2301330566406,
+      "loss": 0.6231,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.15150368213653564,
+      "rewards/margins": 0.22309520840644836,
+      "rewards/rejected": -0.374598890542984,
+      "step": 2580
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.180338087750827e-06,
+      "logits/chosen": -2.6020894050598145,
+      "logits/rejected": -2.5996291637420654,
+      "logps/chosen": -284.0779724121094,
+      "logps/rejected": -323.916015625,
+      "loss": 0.6058,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.13083168864250183,
+      "rewards/margins": 0.26478689908981323,
+      "rewards/rejected": -0.39561858773231506,
+      "step": 2590
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.1718644093407704e-06,
+      "logits/chosen": -2.5531649589538574,
+      "logits/rejected": -2.49448823928833,
+      "logps/chosen": -299.306884765625,
+      "logps/rejected": -307.28021240234375,
+      "loss": 0.5747,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.13294611871242523,
+      "rewards/margins": 0.3297271132469177,
+      "rewards/rejected": -0.46267324686050415,
+      "step": 2600
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.163355840848401e-06,
+      "logits/chosen": -2.6424007415771484,
+      "logits/rejected": -2.5573110580444336,
+      "logps/chosen": -327.47198486328125,
+      "logps/rejected": -339.07952880859375,
+      "loss": 0.6121,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.2437247484922409,
+      "rewards/margins": 0.2633318603038788,
+      "rewards/rejected": -0.5070565938949585,
+      "step": 2610
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.154812559838748e-06,
+      "logits/chosen": -2.613345146179199,
+      "logits/rejected": -2.5385689735412598,
+      "logps/chosen": -282.72186279296875,
+      "logps/rejected": -289.22430419921875,
+      "loss": 0.5582,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.17926782369613647,
+      "rewards/margins": 0.37267833948135376,
+      "rewards/rejected": -0.551946222782135,
+      "step": 2620
+    },
+    {
+      "epoch": 0.34,
+      "learning_rate": 4.146234744601259e-06,
+      "logits/chosen": -2.573025941848755,
+      "logits/rejected": -2.489048480987549,
+      "logps/chosen": -279.2339172363281,
+      "logps/rejected": -317.5060119628906,
+      "loss": 0.5931,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.2666030824184418,
+      "rewards/margins": 0.30796945095062256,
+      "rewards/rejected": -0.5745726227760315,
+      "step": 2630
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.137622574146071e-06,
+      "logits/chosen": -2.6189424991607666,
+      "logits/rejected": -2.544630765914917,
+      "logps/chosen": -281.3956604003906,
+      "logps/rejected": -271.474609375,
+      "loss": 0.5875,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.1797598898410797,
+      "rewards/margins": 0.2901497185230255,
+      "rewards/rejected": -0.46990957856178284,
+      "step": 2640
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.12897622820028e-06,
+      "logits/chosen": -2.6096343994140625,
+      "logits/rejected": -2.529465913772583,
+      "logps/chosen": -328.5892028808594,
+      "logps/rejected": -329.6146545410156,
+      "loss": 0.5909,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.17160935699939728,
+      "rewards/margins": 0.30293500423431396,
+      "rewards/rejected": -0.47454437613487244,
+      "step": 2650
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.120295887204191e-06,
+      "logits/chosen": -2.558011293411255,
+      "logits/rejected": -2.462571382522583,
+      "logps/chosen": -287.7781066894531,
+      "logps/rejected": -327.0203857421875,
+      "loss": 0.6213,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.26796939969062805,
+      "rewards/margins": 0.26254338026046753,
+      "rewards/rejected": -0.530512809753418,
+      "step": 2660
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.111581732307548e-06,
+      "logits/chosen": -2.5581307411193848,
+      "logits/rejected": -2.516145944595337,
+      "logps/chosen": -261.4139709472656,
+      "logps/rejected": -289.26312255859375,
+      "loss": 0.6114,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.12782561779022217,
+      "rewards/margins": 0.2583548426628113,
+      "rewards/rejected": -0.38618040084838867,
+      "step": 2670
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.1028339453657595e-06,
+      "logits/chosen": -2.595445156097412,
+      "logits/rejected": -2.4503002166748047,
+      "logps/chosen": -331.8076171875,
+      "logps/rejected": -298.85797119140625,
+      "loss": 0.6173,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.01653856225311756,
+      "rewards/margins": 0.2292560338973999,
+      "rewards/rejected": -0.2457946091890335,
+      "step": 2680
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.094052708936096e-06,
+      "logits/chosen": -2.6281116008758545,
+      "logits/rejected": -2.5902490615844727,
+      "logps/chosen": -301.99542236328125,
+      "logps/rejected": -347.1701354980469,
+      "loss": 0.5754,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.0312839113175869,
+      "rewards/margins": 0.3134475350379944,
+      "rewards/rejected": -0.2821635901927948,
+      "step": 2690
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.0852382062738874e-06,
+      "logits/chosen": -2.6223397254943848,
+      "logits/rejected": -2.54856538772583,
+      "logps/chosen": -308.45208740234375,
+      "logps/rejected": -327.242919921875,
+      "loss": 0.6068,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.11781201511621475,
+      "rewards/margins": 0.25472837686538696,
+      "rewards/rejected": -0.3725403845310211,
+      "step": 2700
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 4.076390621328693e-06,
+      "logits/chosen": -2.657325029373169,
+      "logits/rejected": -2.5487172603607178,
+      "logps/chosen": -336.58111572265625,
+      "logps/rejected": -323.90789794921875,
+      "loss": 0.572,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.11925222724676132,
+      "rewards/margins": 0.3534049093723297,
+      "rewards/rejected": -0.47265711426734924,
+      "step": 2710
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.067510138740467e-06,
+      "logits/chosen": -2.544666290283203,
+      "logits/rejected": -2.519986391067505,
+      "logps/chosen": -321.7418518066406,
+      "logps/rejected": -332.93170166015625,
+      "loss": 0.5988,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.07618260383605957,
+      "rewards/margins": 0.28174036741256714,
+      "rewards/rejected": -0.3579230308532715,
+      "step": 2720
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.058596943835703e-06,
+      "logits/chosen": -2.6016604900360107,
+      "logits/rejected": -2.6012988090515137,
+      "logps/chosen": -310.2478332519531,
+      "logps/rejected": -288.92999267578125,
+      "loss": 0.6302,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.16724756360054016,
+      "rewards/margins": 0.22624871134757996,
+      "rewards/rejected": -0.3934962749481201,
+      "step": 2730
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.049651222623568e-06,
+      "logits/chosen": -2.6430485248565674,
+      "logits/rejected": -2.595076084136963,
+      "logps/chosen": -294.1294860839844,
+      "logps/rejected": -281.5249938964844,
+      "loss": 0.592,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.12644225358963013,
+      "rewards/margins": 0.29058462381362915,
+      "rewards/rejected": -0.41702690720558167,
+      "step": 2740
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.040673161792014e-06,
+      "logits/chosen": -2.6344447135925293,
+      "logits/rejected": -2.561687469482422,
+      "logps/chosen": -333.14874267578125,
+      "logps/rejected": -345.97308349609375,
+      "loss": 0.6019,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.16730010509490967,
+      "rewards/margins": 0.2885555326938629,
+      "rewards/rejected": -0.45585566759109497,
+      "step": 2750
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.031662948703896e-06,
+      "logits/chosen": -2.5788815021514893,
+      "logits/rejected": -2.5059776306152344,
+      "logps/chosen": -304.4678955078125,
+      "logps/rejected": -323.04669189453125,
+      "loss": 0.5621,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.1394096463918686,
+      "rewards/margins": 0.39707663655281067,
+      "rewards/rejected": -0.5364862084388733,
+      "step": 2760
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.022620771393047e-06,
+      "logits/chosen": -2.610898971557617,
+      "logits/rejected": -2.534040927886963,
+      "logps/chosen": -317.138916015625,
+      "logps/rejected": -307.34979248046875,
+      "loss": 0.5988,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.22270476818084717,
+      "rewards/margins": 0.2881709039211273,
+      "rewards/rejected": -0.5108756422996521,
+      "step": 2770
+    },
+    {
+      "epoch": 0.36,
+      "learning_rate": 4.013546818560362e-06,
+      "logits/chosen": -2.562182903289795,
+      "logits/rejected": -2.4286561012268066,
+      "logps/chosen": -285.60455322265625,
+      "logps/rejected": -283.19146728515625,
+      "loss": 0.6279,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.2095610797405243,
+      "rewards/margins": 0.22186429798603058,
+      "rewards/rejected": -0.43142542243003845,
+      "step": 2780
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 4.00444127956986e-06,
+      "logits/chosen": -2.626108407974243,
+      "logits/rejected": -2.513409376144409,
+      "logps/chosen": -270.6407165527344,
+      "logps/rejected": -281.5836486816406,
+      "loss": 0.5709,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.10766404867172241,
+      "rewards/margins": 0.34720560908317566,
+      "rewards/rejected": -0.45486965775489807,
+      "step": 2790
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.9953043444447255e-06,
+      "logits/chosen": -2.6467347145080566,
+      "logits/rejected": -2.6300857067108154,
+      "logps/chosen": -321.7974853515625,
+      "logps/rejected": -316.6775207519531,
+      "loss": 0.6048,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.0664382353425026,
+      "rewards/margins": 0.2652502655982971,
+      "rewards/rejected": -0.3316884934902191,
+      "step": 2800
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.986136203863355e-06,
+      "logits/chosen": -2.578612804412842,
+      "logits/rejected": -2.5796585083007812,
+      "logps/chosen": -326.597412109375,
+      "logps/rejected": -333.0013122558594,
+      "loss": 0.5689,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.10297360271215439,
+      "rewards/margins": 0.3589724898338318,
+      "rewards/rejected": -0.46194615960121155,
+      "step": 2810
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.976937049155365e-06,
+      "logits/chosen": -2.583843231201172,
+      "logits/rejected": -2.518981456756592,
+      "logps/chosen": -299.9917907714844,
+      "logps/rejected": -333.1919250488281,
+      "loss": 0.5889,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.2250460386276245,
+      "rewards/margins": 0.28562989830970764,
+      "rewards/rejected": -0.5106759071350098,
+      "step": 2820
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.967707072297608e-06,
+      "logits/chosen": -2.4864184856414795,
+      "logits/rejected": -2.4462053775787354,
+      "logps/chosen": -321.8039855957031,
+      "logps/rejected": -348.07257080078125,
+      "loss": 0.5901,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.21852532029151917,
+      "rewards/margins": 0.3042159378528595,
+      "rewards/rejected": -0.5227411985397339,
+      "step": 2830
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.958446465910159e-06,
+      "logits/chosen": -2.6602659225463867,
+      "logits/rejected": -2.6174426078796387,
+      "logps/chosen": -293.2267150878906,
+      "logps/rejected": -327.6724548339844,
+      "loss": 0.5649,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.162770077586174,
+      "rewards/margins": 0.35363587737083435,
+      "rewards/rejected": -0.5164059400558472,
+      "step": 2840
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.9491554232523066e-06,
+      "logits/chosen": -2.5172951221466064,
+      "logits/rejected": -2.5395772457122803,
+      "logps/chosen": -237.58596801757812,
+      "logps/rejected": -280.90936279296875,
+      "loss": 0.6409,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.09324173629283905,
+      "rewards/margins": 0.17134135961532593,
+      "rewards/rejected": -0.26458311080932617,
+      "step": 2850
+    },
+    {
+      "epoch": 0.37,
+      "learning_rate": 3.939834138218505e-06,
+      "logits/chosen": -2.5699565410614014,
+      "logits/rejected": -2.539430618286133,
+      "logps/chosen": -296.15435791015625,
+      "logps/rejected": -304.0159912109375,
+      "loss": 0.5863,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.07466890662908554,
+      "rewards/margins": 0.3267366290092468,
+      "rewards/rejected": -0.4014055132865906,
+      "step": 2860
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.930482805334339e-06,
+      "logits/chosen": -2.576266288757324,
+      "logits/rejected": -2.5219125747680664,
+      "logps/chosen": -270.07916259765625,
+      "logps/rejected": -286.56329345703125,
+      "loss": 0.5929,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.11337437480688095,
+      "rewards/margins": 0.3078486919403076,
+      "rewards/rejected": -0.42122307419776917,
+      "step": 2870
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.921101619752464e-06,
+      "logits/chosen": -2.56925630569458,
+      "logits/rejected": -2.4922568798065186,
+      "logps/chosen": -298.41705322265625,
+      "logps/rejected": -284.24822998046875,
+      "loss": 0.5846,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.12777367234230042,
+      "rewards/margins": 0.3138543665409088,
+      "rewards/rejected": -0.44162803888320923,
+      "step": 2880
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.911690777248525e-06,
+      "logits/chosen": -2.5973918437957764,
+      "logits/rejected": -2.5152206420898438,
+      "logps/chosen": -317.0423278808594,
+      "logps/rejected": -344.5504455566406,
+      "loss": 0.5635,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.1820289045572281,
+      "rewards/margins": 0.34764498472213745,
+      "rewards/rejected": -0.5296738743782043,
+      "step": 2890
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.902250474217079e-06,
+      "logits/chosen": -2.5988688468933105,
+      "logits/rejected": -2.5440027713775635,
+      "logps/chosen": -282.93194580078125,
+      "logps/rejected": -286.84552001953125,
+      "loss": 0.5789,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.1756581962108612,
+      "rewards/margins": 0.3192526698112488,
+      "rewards/rejected": -0.4949108064174652,
+      "step": 2900
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.892780907667495e-06,
+      "logits/chosen": -2.577561140060425,
+      "logits/rejected": -2.531710147857666,
+      "logps/chosen": -286.76776123046875,
+      "logps/rejected": -304.3691711425781,
+      "loss": 0.5707,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.08928398787975311,
+      "rewards/margins": 0.3487761914730072,
+      "rewards/rejected": -0.4380602240562439,
+      "step": 2910
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.883282275219837e-06,
+      "logits/chosen": -2.6716904640197754,
+      "logits/rejected": -2.611128330230713,
+      "logps/chosen": -314.375732421875,
+      "logps/rejected": -306.6675109863281,
+      "loss": 0.5819,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.2186862677335739,
+      "rewards/margins": 0.3232896625995636,
+      "rewards/rejected": -0.5419759750366211,
+      "step": 2920
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.873754775100751e-06,
+      "logits/chosen": -2.653162717819214,
+      "logits/rejected": -2.6087594032287598,
+      "logps/chosen": -304.66351318359375,
+      "logps/rejected": -310.02197265625,
+      "loss": 0.5732,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.10551968961954117,
+      "rewards/margins": 0.3658735454082489,
+      "rewards/rejected": -0.47139328718185425,
+      "step": 2930
+    },
+    {
+      "epoch": 0.38,
+      "learning_rate": 3.8641986061393145e-06,
+      "logits/chosen": -2.6352903842926025,
+      "logits/rejected": -2.610074758529663,
+      "logps/chosen": -279.8460693359375,
+      "logps/rejected": -332.79949951171875,
+      "loss": 0.5703,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.06143326312303543,
+      "rewards/margins": 0.3413320481777191,
+      "rewards/rejected": -0.40276533365249634,
+      "step": 2940
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.854613967762898e-06,
+      "logits/chosen": -2.6187663078308105,
+      "logits/rejected": -2.5134811401367188,
+      "logps/chosen": -344.96331787109375,
+      "logps/rejected": -329.61187744140625,
+      "loss": 0.5857,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.14645352959632874,
+      "rewards/margins": 0.32061487436294556,
+      "rewards/rejected": -0.4670684337615967,
+      "step": 2950
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.845001059992999e-06,
+      "logits/chosen": -2.6192657947540283,
+      "logits/rejected": -2.552077531814575,
+      "logps/chosen": -311.8778381347656,
+      "logps/rejected": -297.2644348144531,
+      "loss": 0.5896,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.15005072951316833,
+      "rewards/margins": 0.30447477102279663,
+      "rewards/rejected": -0.45452556014060974,
+      "step": 2960
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.835360083441067e-06,
+      "logits/chosen": -2.526991367340088,
+      "logits/rejected": -2.5425732135772705,
+      "logps/chosen": -274.29150390625,
+      "logps/rejected": -287.47314453125,
+      "loss": 0.5667,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.07174061238765717,
+      "rewards/margins": 0.3619951605796814,
+      "rewards/rejected": -0.43373578786849976,
+      "step": 2970
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.825691239304318e-06,
+      "logits/chosen": -2.6114232540130615,
+      "logits/rejected": -2.539461851119995,
+      "logps/chosen": -290.63873291015625,
+      "logps/rejected": -314.540771484375,
+      "loss": 0.6013,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.1670713722705841,
+      "rewards/margins": 0.2761486768722534,
+      "rewards/rejected": -0.4432200491428375,
+      "step": 2980
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.8159947293615385e-06,
+      "logits/chosen": -2.6737751960754395,
+      "logits/rejected": -2.6144914627075195,
+      "logps/chosen": -328.87603759765625,
+      "logps/rejected": -316.49005126953125,
+      "loss": 0.5786,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.11168032884597778,
+      "rewards/margins": 0.3224450647830963,
+      "rewards/rejected": -0.4341253638267517,
+      "step": 2990
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.806270755968866e-06,
+      "logits/chosen": -2.6640400886535645,
+      "logits/rejected": -2.600228786468506,
+      "logps/chosen": -360.91729736328125,
+      "logps/rejected": -320.532470703125,
+      "loss": 0.5779,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.20109805464744568,
+      "rewards/margins": 0.3598141074180603,
+      "rewards/rejected": -0.5609121322631836,
+      "step": 3000
+    },
+    {
+      "epoch": 0.39,
+      "learning_rate": 3.7965195220555784e-06,
+      "logits/chosen": -2.526873826980591,
+      "logits/rejected": -2.491114854812622,
+      "logps/chosen": -295.8037109375,
+      "logps/rejected": -310.78448486328125,
+      "loss": 0.6371,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.4205259382724762,
+      "rewards/margins": 0.1770562380552292,
+      "rewards/rejected": -0.5975821018218994,
+      "step": 3010
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.786741231119847e-06,
+      "logits/chosen": -2.6449503898620605,
+      "logits/rejected": -2.5297398567199707,
+      "logps/chosen": -324.0474548339844,
+      "logps/rejected": -314.4382629394531,
+      "loss": 0.6363,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.3445380926132202,
+      "rewards/margins": 0.20752224326133728,
+      "rewards/rejected": -0.5520604252815247,
+      "step": 3020
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7769360872244992e-06,
+      "logits/chosen": -2.5686895847320557,
+      "logits/rejected": -2.5801048278808594,
+      "logps/chosen": -302.50579833984375,
+      "logps/rejected": -304.86566162109375,
+      "loss": 0.5934,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.3660580515861511,
+      "rewards/margins": 0.2665599286556244,
+      "rewards/rejected": -0.6326179504394531,
+      "step": 3030
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.767104294992754e-06,
+      "logits/chosen": -2.6589438915252686,
+      "logits/rejected": -2.601179599761963,
+      "logps/chosen": -342.2464904785156,
+      "logps/rejected": -275.0693664550781,
+      "loss": 0.6121,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.23833589255809784,
+      "rewards/margins": 0.259996622800827,
+      "rewards/rejected": -0.49833250045776367,
+      "step": 3040
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7572460596039524e-06,
+      "logits/chosen": -2.634748935699463,
+      "logits/rejected": -2.63116455078125,
+      "logps/chosen": -306.77130126953125,
+      "logps/rejected": -325.4991149902344,
+      "loss": 0.6115,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.29471856355667114,
+      "rewards/margins": 0.26444265246391296,
+      "rewards/rejected": -0.5591612458229065,
+      "step": 3050
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.74736158678928e-06,
+      "logits/chosen": -2.5428338050842285,
+      "logits/rejected": -2.512752056121826,
+      "logps/chosen": -399.3180236816406,
+      "logps/rejected": -338.7885437011719,
+      "loss": 0.5816,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.2622656226158142,
+      "rewards/margins": 0.3359803557395935,
+      "rewards/rejected": -0.5982460379600525,
+      "step": 3060
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7374510828274673e-06,
+      "logits/chosen": -2.6895453929901123,
+      "logits/rejected": -2.611549139022827,
+      "logps/chosen": -366.79034423828125,
+      "logps/rejected": -322.94384765625,
+      "loss": 0.6117,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.1811170130968094,
+      "rewards/margins": 0.27233922481536865,
+      "rewards/rejected": -0.453456312417984,
+      "step": 3070
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.72751475454049e-06,
+      "logits/chosen": -2.5715219974517822,
+      "logits/rejected": -2.553978681564331,
+      "logps/chosen": -312.04364013671875,
+      "logps/rejected": -315.54351806640625,
+      "loss": 0.6306,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.1746423840522766,
+      "rewards/margins": 0.21813610196113586,
+      "rewards/rejected": -0.3927784860134125,
+      "step": 3080
+    },
+    {
+      "epoch": 0.4,
+      "learning_rate": 3.7175528092892503e-06,
+      "logits/chosen": -2.546525716781616,
+      "logits/rejected": -2.478912830352783,
+      "logps/chosen": -301.1471252441406,
+      "logps/rejected": -306.3082275390625,
+      "loss": 0.6172,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.2471695840358734,
+      "rewards/margins": 0.24716773629188538,
+      "rewards/rejected": -0.49433737993240356,
+      "step": 3090
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.7075654549692498e-06,
+      "logits/chosen": -2.675877332687378,
+      "logits/rejected": -2.615410566329956,
+      "logps/chosen": -378.7645568847656,
+      "logps/rejected": -316.1293029785156,
+      "loss": 0.5447,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.12326345592737198,
+      "rewards/margins": 0.39845913648605347,
+      "rewards/rejected": -0.5217226147651672,
+      "step": 3100
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.697552900006249e-06,
+      "logits/chosen": -2.5884854793548584,
+      "logits/rejected": -2.5903048515319824,
+      "logps/chosen": -297.75372314453125,
+      "logps/rejected": -299.5635070800781,
+      "loss": 0.5918,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.23306098580360413,
+      "rewards/margins": 0.29291313886642456,
+      "rewards/rejected": -0.5259741544723511,
+      "step": 3110
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6875153533519244e-06,
+      "logits/chosen": -2.4664971828460693,
+      "logits/rejected": -2.4570648670196533,
+      "logps/chosen": -331.24591064453125,
+      "logps/rejected": -321.0543518066406,
+      "loss": 0.6014,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.1272236555814743,
+      "rewards/margins": 0.2839981019496918,
+      "rewards/rejected": -0.41122180223464966,
+      "step": 3120
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6774530244794992e-06,
+      "logits/chosen": -2.602510690689087,
+      "logits/rejected": -2.5052011013031006,
+      "logps/chosen": -318.2493896484375,
+      "logps/rejected": -285.765869140625,
+      "loss": 0.5955,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.20577561855316162,
+      "rewards/margins": 0.31061360239982605,
+      "rewards/rejected": -0.5163891911506653,
+      "step": 3130
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.667366123379378e-06,
+      "logits/chosen": -2.576422691345215,
+      "logits/rejected": -2.52091646194458,
+      "logps/chosen": -258.256591796875,
+      "logps/rejected": -256.0735778808594,
+      "loss": 0.6409,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.3037491738796234,
+      "rewards/margins": 0.17431828379631042,
+      "rewards/rejected": -0.478067547082901,
+      "step": 3140
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6572548605547607e-06,
+      "logits/chosen": -2.690229892730713,
+      "logits/rejected": -2.631052017211914,
+      "logps/chosen": -275.73553466796875,
+      "logps/rejected": -318.1167297363281,
+      "loss": 0.5933,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.16246561706066132,
+      "rewards/margins": 0.3064282536506653,
+      "rewards/rejected": -0.468893826007843,
+      "step": 3150
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.6471194470172538e-06,
+      "logits/chosen": -2.6639180183410645,
+      "logits/rejected": -2.6506240367889404,
+      "logps/chosen": -296.90521240234375,
+      "logps/rejected": -329.15533447265625,
+      "loss": 0.5911,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.13294672966003418,
+      "rewards/margins": 0.324388712644577,
+      "rewards/rejected": -0.4573354125022888,
+      "step": 3160
+    },
+    {
+      "epoch": 0.41,
+      "learning_rate": 3.636960094282461e-06,
+      "logits/chosen": -2.5740599632263184,
+      "logits/rejected": -2.5333621501922607,
+      "logps/chosen": -315.98883056640625,
+      "logps/rejected": -302.2984313964844,
+      "loss": 0.5665,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.20909647643566132,
+      "rewards/margins": 0.35148894786834717,
+      "rewards/rejected": -0.5605853796005249,
+      "step": 3170
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.6267770143655743e-06,
+      "logits/chosen": -2.733361005783081,
+      "logits/rejected": -2.6483659744262695,
+      "logps/chosen": -319.62493896484375,
+      "logps/rejected": -314.0323181152344,
+      "loss": 0.6027,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.2789066731929779,
+      "rewards/margins": 0.2787697911262512,
+      "rewards/rejected": -0.5576764345169067,
+      "step": 3180
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.6165704197769484e-06,
+      "logits/chosen": -2.651341438293457,
+      "logits/rejected": -2.6233811378479004,
+      "logps/chosen": -295.9290466308594,
+      "logps/rejected": -307.7383117675781,
+      "loss": 0.6165,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.19166718423366547,
+      "rewards/margins": 0.23387548327445984,
+      "rewards/rejected": -0.4255426824092865,
+      "step": 3190
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.606340523517663e-06,
+      "logits/chosen": -2.61683988571167,
+      "logits/rejected": -2.555131435394287,
+      "logps/chosen": -295.7688903808594,
+      "logps/rejected": -307.00250244140625,
+      "loss": 0.5885,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.21164658665657043,
+      "rewards/margins": 0.3170962333679199,
+      "rewards/rejected": -0.5287428498268127,
+      "step": 3200
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.5960875390750793e-06,
+      "logits/chosen": -2.6817736625671387,
+      "logits/rejected": -2.524237871170044,
+      "logps/chosen": -367.2226867675781,
+      "logps/rejected": -336.5373229980469,
+      "loss": 0.5423,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.16680178046226501,
+      "rewards/margins": 0.4267476201057434,
+      "rewards/rejected": -0.5935494899749756,
+      "step": 3210
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.585811680418386e-06,
+      "logits/chosen": -2.6061294078826904,
+      "logits/rejected": -2.555504083633423,
+      "logps/chosen": -315.3666076660156,
+      "logps/rejected": -319.6710510253906,
+      "loss": 0.5826,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.18171259760856628,
+      "rewards/margins": 0.32478052377700806,
+      "rewards/rejected": -0.5064931511878967,
+      "step": 3220
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.5755131619941347e-06,
+      "logits/chosen": -2.5690414905548096,
+      "logits/rejected": -2.565368175506592,
+      "logps/chosen": -281.3846435546875,
+      "logps/rejected": -278.86761474609375,
+      "loss": 0.5943,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.11518166214227676,
+      "rewards/margins": 0.3046377897262573,
+      "rewards/rejected": -0.4198194444179535,
+      "step": 3230
+    },
+    {
+      "epoch": 0.42,
+      "learning_rate": 3.565192198721759e-06,
+      "logits/chosen": -2.573634147644043,
+      "logits/rejected": -2.5383400917053223,
+      "logps/chosen": -267.2909851074219,
+      "logps/rejected": -327.7779235839844,
+      "loss": 0.5823,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.14731571078300476,
+      "rewards/margins": 0.3336889445781708,
+      "rewards/rejected": -0.48100465536117554,
+      "step": 3240
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.5548490059890965e-06,
+      "logits/chosen": -2.587608814239502,
+      "logits/rejected": -2.5181055068969727,
+      "logps/chosen": -325.87530517578125,
+      "logps/rejected": -336.8580627441406,
+      "loss": 0.5853,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.07696576416492462,
+      "rewards/margins": 0.328901469707489,
+      "rewards/rejected": -0.40586724877357483,
+      "step": 3250
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.5444837996478903e-06,
+      "logits/chosen": -2.6658148765563965,
+      "logits/rejected": -2.5645992755889893,
+      "logps/chosen": -329.1102294921875,
+      "logps/rejected": -369.98553466796875,
+      "loss": 0.6139,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.10167242586612701,
+      "rewards/margins": 0.2717786729335785,
+      "rewards/rejected": -0.3734511137008667,
+      "step": 3260
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.534096796009282e-06,
+      "logits/chosen": -2.5413317680358887,
+      "logits/rejected": -2.5287163257598877,
+      "logps/chosen": -282.94744873046875,
+      "logps/rejected": -298.67779541015625,
+      "loss": 0.6326,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.10002797842025757,
+      "rewards/margins": 0.21359138190746307,
+      "rewards/rejected": -0.31361934542655945,
+      "step": 3270
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.5236882118393046e-06,
+      "logits/chosen": -2.6598095893859863,
+      "logits/rejected": -2.616147756576538,
+      "logps/chosen": -322.4290466308594,
+      "logps/rejected": -309.7900390625,
+      "loss": 0.6074,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.05319022387266159,
+      "rewards/margins": 0.25976449251174927,
+      "rewards/rejected": -0.31295478343963623,
+      "step": 3280
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.5132582643543513e-06,
+      "logits/chosen": -2.5804498195648193,
+      "logits/rejected": -2.5233707427978516,
+      "logps/chosen": -299.89520263671875,
+      "logps/rejected": -306.67291259765625,
+      "loss": 0.584,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.0694229006767273,
+      "rewards/margins": 0.3088359832763672,
+      "rewards/rejected": -0.37825894355773926,
+      "step": 3290
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.5028071712166456e-06,
+      "logits/chosen": -2.6169161796569824,
+      "logits/rejected": -2.5239739418029785,
+      "logps/chosen": -313.2078552246094,
+      "logps/rejected": -282.47259521484375,
+      "loss": 0.6046,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.08295028656721115,
+      "rewards/margins": 0.28517070412635803,
+      "rewards/rejected": -0.3681209683418274,
+      "step": 3300
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.4923351505297008e-06,
+      "logits/chosen": -2.702962875366211,
+      "logits/rejected": -2.6618094444274902,
+      "logps/chosen": -317.6420593261719,
+      "logps/rejected": -325.88385009765625,
+      "loss": 0.5678,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.051228463649749756,
+      "rewards/margins": 0.3347882628440857,
+      "rewards/rejected": -0.38601672649383545,
+      "step": 3310
+    },
+    {
+      "epoch": 0.43,
+      "learning_rate": 3.481842420833766e-06,
+      "logits/chosen": -2.6977882385253906,
+      "logits/rejected": -2.620340347290039,
+      "logps/chosen": -359.4865417480469,
+      "logps/rejected": -308.225341796875,
+      "loss": 0.5757,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.09152285754680634,
+      "rewards/margins": 0.3334515690803528,
+      "rewards/rejected": -0.4249744415283203,
+      "step": 3320
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4713292011012645e-06,
+      "logits/chosen": -2.570391893386841,
+      "logits/rejected": -2.5014007091522217,
+      "logps/chosen": -328.263916015625,
+      "logps/rejected": -321.28582763671875,
+      "loss": 0.6362,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.21234555542469025,
+      "rewards/margins": 0.22594225406646729,
+      "rewards/rejected": -0.43828779458999634,
+      "step": 3330
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4607957107322277e-06,
+      "logits/chosen": -2.6263375282287598,
+      "logits/rejected": -2.503077745437622,
+      "logps/chosen": -272.3758544921875,
+      "logps/rejected": -280.59808349609375,
+      "loss": 0.5937,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.24431531131267548,
+      "rewards/margins": 0.28803953528404236,
+      "rewards/rejected": -0.5323548316955566,
+      "step": 3340
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4502421695497112e-06,
+      "logits/chosen": -2.523693561553955,
+      "logits/rejected": -2.5273375511169434,
+      "logps/chosen": -297.10186767578125,
+      "logps/rejected": -303.70953369140625,
+      "loss": 0.6195,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.10780465602874756,
+      "rewards/margins": 0.2236817181110382,
+      "rewards/rejected": -0.33148640394210815,
+      "step": 3350
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4396687977952137e-06,
+      "logits/chosen": -2.583043098449707,
+      "logits/rejected": -2.4975414276123047,
+      "logps/chosen": -280.9002685546875,
+      "logps/rejected": -288.56317138671875,
+      "loss": 0.5827,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.07089909166097641,
+      "rewards/margins": 0.32813531160354614,
+      "rewards/rejected": -0.39903444051742554,
+      "step": 3360
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.429075816124075e-06,
+      "logits/chosen": -2.6454784870147705,
+      "logits/rejected": -2.616804838180542,
+      "logps/chosen": -312.5958557128906,
+      "logps/rejected": -326.32574462890625,
+      "loss": 0.5821,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.08628349006175995,
+      "rewards/margins": 0.3039689362049103,
+      "rewards/rejected": -0.3902524709701538,
+      "step": 3370
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.418463445600874e-06,
+      "logits/chosen": -2.6286797523498535,
+      "logits/rejected": -2.576748847961426,
+      "logps/chosen": -249.4486541748047,
+      "logps/rejected": -278.88665771484375,
+      "loss": 0.5938,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.12657615542411804,
+      "rewards/margins": 0.30838608741760254,
+      "rewards/rejected": -0.4349622130393982,
+      "step": 3380
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.4078319076948173e-06,
+      "logits/chosen": -2.5853271484375,
+      "logits/rejected": -2.532613515853882,
+      "logps/chosen": -270.4429626464844,
+      "logps/rejected": -287.1538391113281,
+      "loss": 0.5569,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.09764670580625534,
+      "rewards/margins": 0.38367223739624023,
+      "rewards/rejected": -0.48131895065307617,
+      "step": 3390
+    },
+    {
+      "epoch": 0.44,
+      "learning_rate": 3.3971814242751123e-06,
+      "logits/chosen": -2.6251912117004395,
+      "logits/rejected": -2.540031671524048,
+      "logps/chosen": -319.8332214355469,
+      "logps/rejected": -303.64959716796875,
+      "loss": 0.5906,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.051111720502376556,
+      "rewards/margins": 0.3035109043121338,
+      "rewards/rejected": -0.35462266206741333,
+      "step": 3400
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.386512217606339e-06,
+      "logits/chosen": -2.6400036811828613,
+      "logits/rejected": -2.606139659881592,
+      "logps/chosen": -295.169677734375,
+      "logps/rejected": -290.8325500488281,
+      "loss": 0.5647,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.13387389481067657,
+      "rewards/margins": 0.35568928718566895,
+      "rewards/rejected": -0.4895631670951843,
+      "step": 3410
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.375824510343816e-06,
+      "logits/chosen": -2.673135757446289,
+      "logits/rejected": -2.594059705734253,
+      "logps/chosen": -327.8915710449219,
+      "logps/rejected": -360.8843994140625,
+      "loss": 0.5968,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.2103693187236786,
+      "rewards/margins": 0.29845374822616577,
+      "rewards/rejected": -0.508823037147522,
+      "step": 3420
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3651185255289466e-06,
+      "logits/chosen": -2.6686813831329346,
+      "logits/rejected": -2.6119613647460938,
+      "logps/chosen": -343.3296203613281,
+      "logps/rejected": -357.33099365234375,
+      "loss": 0.5752,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.2293224334716797,
+      "rewards/margins": 0.3290192484855652,
+      "rewards/rejected": -0.5583416819572449,
+      "step": 3430
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.354394486584568e-06,
+      "logits/chosen": -2.6448490619659424,
+      "logits/rejected": -2.576442003250122,
+      "logps/chosen": -342.14288330078125,
+      "logps/rejected": -366.2242431640625,
+      "loss": 0.5648,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.1682378649711609,
+      "rewards/margins": 0.3552810549736023,
+      "rewards/rejected": -0.5235189199447632,
+      "step": 3440
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3436526173102913e-06,
+      "logits/chosen": -2.6141438484191895,
+      "logits/rejected": -2.551224946975708,
+      "logps/chosen": -284.8623352050781,
+      "logps/rejected": -258.30010986328125,
+      "loss": 0.5993,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.34725385904312134,
+      "rewards/margins": 0.2650673985481262,
+      "rewards/rejected": -0.6123212575912476,
+      "step": 3450
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3328931418778254e-06,
+      "logits/chosen": -2.604269504547119,
+      "logits/rejected": -2.5947041511535645,
+      "logps/chosen": -305.6647644042969,
+      "logps/rejected": -332.5096740722656,
+      "loss": 0.5498,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.19446662068367004,
+      "rewards/margins": 0.3958137631416321,
+      "rewards/rejected": -0.5902804136276245,
+      "step": 3460
+    },
+    {
+      "epoch": 0.45,
+      "learning_rate": 3.3221162848263028e-06,
+      "logits/chosen": -2.6202986240386963,
+      "logits/rejected": -2.5820131301879883,
+      "logps/chosen": -330.19268798828125,
+      "logps/rejected": -314.20220947265625,
+      "loss": 0.5456,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.1648586094379425,
+      "rewards/margins": 0.4006679952144623,
+      "rewards/rejected": -0.56552654504776,
+      "step": 3470
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.3113222710575914e-06,
+      "logits/chosen": -2.6502761840820312,
+      "logits/rejected": -2.6139705181121826,
+      "logps/chosen": -331.75701904296875,
+      "logps/rejected": -325.5454406738281,
+      "loss": 0.6496,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.24137596786022186,
+      "rewards/margins": 0.18592584133148193,
+      "rewards/rejected": -0.427301824092865,
+      "step": 3480
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.300511325831603e-06,
+      "logits/chosen": -2.560652256011963,
+      "logits/rejected": -2.567096710205078,
+      "logps/chosen": -299.5778503417969,
+      "logps/rejected": -328.95452880859375,
+      "loss": 0.603,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.14705495536327362,
+      "rewards/margins": 0.28899452090263367,
+      "rewards/rejected": -0.43604952096939087,
+      "step": 3490
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.289683674761592e-06,
+      "logits/chosen": -2.57741117477417,
+      "logits/rejected": -2.541520833969116,
+      "logps/chosen": -303.9013671875,
+      "logps/rejected": -296.300048828125,
+      "loss": 0.5909,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.17057619988918304,
+      "rewards/margins": 0.2913939952850342,
+      "rewards/rejected": -0.4619702398777008,
+      "step": 3500
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.2788395438094444e-06,
+      "logits/chosen": -2.6372673511505127,
+      "logits/rejected": -2.577939510345459,
+      "logps/chosen": -312.88665771484375,
+      "logps/rejected": -320.70599365234375,
+      "loss": 0.5781,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.19582369923591614,
+      "rewards/margins": 0.34374529123306274,
+      "rewards/rejected": -0.5395689606666565,
+      "step": 3510
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.2679791592809653e-06,
+      "logits/chosen": -2.6001806259155273,
+      "logits/rejected": -2.535001754760742,
+      "logps/chosen": -268.4497375488281,
+      "logps/rejected": -306.09625244140625,
+      "loss": 0.5938,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.09515003114938736,
+      "rewards/margins": 0.2680366337299347,
+      "rewards/rejected": -0.36318665742874146,
+      "step": 3520
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.257102747821157e-06,
+      "logits/chosen": -2.5398240089416504,
+      "logits/rejected": -2.4677555561065674,
+      "logps/chosen": -326.4378356933594,
+      "logps/rejected": -336.32196044921875,
+      "loss": 0.5328,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.05473318696022034,
+      "rewards/margins": 0.43821921944618225,
+      "rewards/rejected": -0.4929524064064026,
+      "step": 3530
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.246210536409484e-06,
+      "logits/chosen": -2.534557342529297,
+      "logits/rejected": -2.486480951309204,
+      "logps/chosen": -306.159912109375,
+      "logps/rejected": -296.2869873046875,
+      "loss": 0.603,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.12035051733255386,
+      "rewards/margins": 0.2683168351650238,
+      "rewards/rejected": -0.38866734504699707,
+      "step": 3540
+    },
+    {
+      "epoch": 0.46,
+      "learning_rate": 3.235302752355142e-06,
+      "logits/chosen": -2.6506826877593994,
+      "logits/rejected": -2.5596671104431152,
+      "logps/chosen": -327.5464782714844,
+      "logps/rejected": -280.18853759765625,
+      "loss": 0.5596,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.036321885883808136,
+      "rewards/margins": 0.37242797017097473,
+      "rewards/rejected": -0.40874987840652466,
+      "step": 3550
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.2243796232923097e-06,
+      "logits/chosen": -2.6477935314178467,
+      "logits/rejected": -2.5448243618011475,
+      "logps/chosen": -344.7415466308594,
+      "logps/rejected": -357.2950134277344,
+      "loss": 0.5772,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.14982673525810242,
+      "rewards/margins": 0.3258996605873108,
+      "rewards/rejected": -0.4757264256477356,
+      "step": 3560
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.2134413771754037e-06,
+      "logits/chosen": -2.649160861968994,
+      "logits/rejected": -2.6048355102539062,
+      "logps/chosen": -293.1799621582031,
+      "logps/rejected": -304.89544677734375,
+      "loss": 0.5972,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.13853810727596283,
+      "rewards/margins": 0.28082212805747986,
+      "rewards/rejected": -0.4193602204322815,
+      "step": 3570
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.2024882422743118e-06,
+      "logits/chosen": -2.606848955154419,
+      "logits/rejected": -2.5511131286621094,
+      "logps/chosen": -260.3493347167969,
+      "logps/rejected": -300.005859375,
+      "loss": 0.55,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.11977507919073105,
+      "rewards/margins": 0.3719303011894226,
+      "rewards/rejected": -0.49170541763305664,
+      "step": 3580
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.1915204471696425e-06,
+      "logits/chosen": -2.6543116569519043,
+      "logits/rejected": -2.5203542709350586,
+      "logps/chosen": -292.5749816894531,
+      "logps/rejected": -265.5362854003906,
+      "loss": 0.5896,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.08421161025762558,
+      "rewards/margins": 0.3118685483932495,
+      "rewards/rejected": -0.3960801661014557,
+      "step": 3590
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.180538220747943e-06,
+      "logits/chosen": -2.6206531524658203,
+      "logits/rejected": -2.575209140777588,
+      "logps/chosen": -324.55828857421875,
+      "logps/rejected": -317.8137512207031,
+      "loss": 0.6134,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.22396568953990936,
+      "rewards/margins": 0.2720452845096588,
+      "rewards/rejected": -0.496010959148407,
+      "step": 3600
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.1695417921969287e-06,
+      "logits/chosen": -2.676797389984131,
+      "logits/rejected": -2.6270275115966797,
+      "logps/chosen": -284.1139831542969,
+      "logps/rejected": -323.57611083984375,
+      "loss": 0.6305,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.10977538675069809,
+      "rewards/margins": 0.21086139976978302,
+      "rewards/rejected": -0.3206367790699005,
+      "step": 3610
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 3.158531391000697e-06,
+      "logits/chosen": -2.6428816318511963,
+      "logits/rejected": -2.5296685695648193,
+      "logps/chosen": -309.40478515625,
+      "logps/rejected": -303.77777099609375,
+      "loss": 0.5774,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.14539624750614166,
+      "rewards/margins": 0.3583511412143707,
+      "rewards/rejected": -0.5037473440170288,
+      "step": 3620
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.147507246934943e-06,
+      "logits/chosen": -2.629356861114502,
+      "logits/rejected": -2.608485698699951,
+      "logps/chosen": -331.5457458496094,
+      "logps/rejected": -343.12823486328125,
+      "loss": 0.5842,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.0827968493103981,
+      "rewards/margins": 0.3230699896812439,
+      "rewards/rejected": -0.405866801738739,
+      "step": 3630
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.136469590062158e-06,
+      "logits/chosen": -2.507965564727783,
+      "logits/rejected": -2.462904453277588,
+      "logps/chosen": -279.53106689453125,
+      "logps/rejected": -273.98712158203125,
+      "loss": 0.5893,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.20209722220897675,
+      "rewards/margins": 0.30006346106529236,
+      "rewards/rejected": -0.5021606683731079,
+      "step": 3640
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.1254186507268354e-06,
+      "logits/chosen": -2.632068634033203,
+      "logits/rejected": -2.554795742034912,
+      "logps/chosen": -268.78594970703125,
+      "logps/rejected": -296.078369140625,
+      "loss": 0.5996,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.12344853579998016,
+      "rewards/margins": 0.2852582335472107,
+      "rewards/rejected": -0.40870675444602966,
+      "step": 3650
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.114354659550656e-06,
+      "logits/chosen": -2.520400047302246,
+      "logits/rejected": -2.5127463340759277,
+      "logps/chosen": -312.34429931640625,
+      "logps/rejected": -302.1906433105469,
+      "loss": 0.5584,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.180680513381958,
+      "rewards/margins": 0.3782447874546051,
+      "rewards/rejected": -0.5589253306388855,
+      "step": 3660
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.1032778474276816e-06,
+      "logits/chosen": -2.724027156829834,
+      "logits/rejected": -2.5943634510040283,
+      "logps/chosen": -303.89434814453125,
+      "logps/rejected": -288.9135437011719,
+      "loss": 0.5465,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.09625785052776337,
+      "rewards/margins": 0.4044003486633301,
+      "rewards/rejected": -0.5006581544876099,
+      "step": 3670
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.092188445519532e-06,
+      "logits/chosen": -2.5960793495178223,
+      "logits/rejected": -2.5011520385742188,
+      "logps/chosen": -316.4403991699219,
+      "logps/rejected": -289.95794677734375,
+      "loss": 0.5967,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.1785668432712555,
+      "rewards/margins": 0.3221067488193512,
+      "rewards/rejected": -0.5006736516952515,
+      "step": 3680
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.081086685250565e-06,
+      "logits/chosen": -2.6398634910583496,
+      "logits/rejected": -2.5474326610565186,
+      "logps/chosen": -283.3089904785156,
+      "logps/rejected": -266.81646728515625,
+      "loss": 0.5791,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.16495847702026367,
+      "rewards/margins": 0.33490079641342163,
+      "rewards/rejected": -0.4998592734336853,
+      "step": 3690
+    },
+    {
+      "epoch": 0.48,
+      "learning_rate": 3.0699727983030434e-06,
+      "logits/chosen": -2.6663928031921387,
+      "logits/rejected": -2.577331066131592,
+      "logps/chosen": -328.13873291015625,
+      "logps/rejected": -296.5749206542969,
+      "loss": 0.6165,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.22938323020935059,
+      "rewards/margins": 0.247257798910141,
+      "rewards/rejected": -0.4766410291194916,
+      "step": 3700
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.058847016612301e-06,
+      "logits/chosen": -2.588397741317749,
+      "logits/rejected": -2.5687057971954346,
+      "logps/chosen": -342.5824279785156,
+      "logps/rejected": -360.75421142578125,
+      "loss": 0.6192,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.12575873732566833,
+      "rewards/margins": 0.2307460755109787,
+      "rewards/rejected": -0.3565048277378082,
+      "step": 3710
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.0477095723619034e-06,
+      "logits/chosen": -2.655534029006958,
+      "logits/rejected": -2.539325475692749,
+      "logps/chosen": -280.05303955078125,
+      "logps/rejected": -290.4456481933594,
+      "loss": 0.5734,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.15500256419181824,
+      "rewards/margins": 0.3527732491493225,
+      "rewards/rejected": -0.5077757835388184,
+      "step": 3720
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.0365606979788003e-06,
+      "logits/chosen": -2.6514976024627686,
+      "logits/rejected": -2.5548408031463623,
+      "logps/chosen": -299.7529602050781,
+      "logps/rejected": -290.4812927246094,
+      "loss": 0.5962,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.12529566884040833,
+      "rewards/margins": 0.28550538420677185,
+      "rewards/rejected": -0.4108010232448578,
+      "step": 3730
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.0254006261284786e-06,
+      "logits/chosen": -2.6113336086273193,
+      "logits/rejected": -2.546644687652588,
+      "logps/chosen": -297.1475830078125,
+      "logps/rejected": -305.62060546875,
+      "loss": 0.5907,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.17782071232795715,
+      "rewards/margins": 0.3022399842739105,
+      "rewards/rejected": -0.4800606667995453,
+      "step": 3740
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.0142295897101032e-06,
+      "logits/chosen": -2.4700398445129395,
+      "logits/rejected": -2.3766896724700928,
+      "logps/chosen": -317.51214599609375,
+      "logps/rejected": -298.0809326171875,
+      "loss": 0.5401,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.1111142635345459,
+      "rewards/margins": 0.42605048418045044,
+      "rewards/rejected": -0.5371647477149963,
+      "step": 3750
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 3.0030478218516578e-06,
+      "logits/chosen": -2.561290979385376,
+      "logits/rejected": -2.503338575363159,
+      "logps/chosen": -294.38360595703125,
+      "logps/rejected": -307.7070007324219,
+      "loss": 0.5507,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.03715894743800163,
+      "rewards/margins": 0.3968127369880676,
+      "rewards/rejected": -0.43397170305252075,
+      "step": 3760
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 2.9918555559050826e-06,
+      "logits/chosen": -2.5190422534942627,
+      "logits/rejected": -2.5101046562194824,
+      "logps/chosen": -288.7281188964844,
+      "logps/rejected": -282.1548156738281,
+      "loss": 0.572,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.0820334181189537,
+      "rewards/margins": 0.3213749825954437,
+      "rewards/rejected": -0.40340837836265564,
+      "step": 3770
+    },
+    {
+      "epoch": 0.49,
+      "learning_rate": 2.980653025441399e-06,
+      "logits/chosen": -2.6211345195770264,
+      "logits/rejected": -2.5797693729400635,
+      "logps/chosen": -327.58099365234375,
+      "logps/rejected": -319.9222106933594,
+      "loss": 0.534,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.020748626440763474,
+      "rewards/margins": 0.4278421401977539,
+      "rewards/rejected": -0.44859081506729126,
+      "step": 3780
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.969440464245841e-06,
+      "logits/chosen": -2.6424736976623535,
+      "logits/rejected": -2.5715973377227783,
+      "logps/chosen": -325.75726318359375,
+      "logps/rejected": -300.81195068359375,
+      "loss": 0.6088,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.10960064083337784,
+      "rewards/margins": 0.2427833378314972,
+      "rewards/rejected": -0.35238394141197205,
+      "step": 3790
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.95821810631297e-06,
+      "logits/chosen": -2.5100483894348145,
+      "logits/rejected": -2.5134928226470947,
+      "logps/chosen": -258.62420654296875,
+      "logps/rejected": -275.17767333984375,
+      "loss": 0.5753,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.07906125485897064,
+      "rewards/margins": 0.3346591591835022,
+      "rewards/rejected": -0.41372042894363403,
+      "step": 3800
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.946986185841801e-06,
+      "logits/chosen": -2.5099751949310303,
+      "logits/rejected": -2.4276437759399414,
+      "logps/chosen": -292.8961486816406,
+      "logps/rejected": -301.64141845703125,
+      "loss": 0.6023,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.09722457826137543,
+      "rewards/margins": 0.2565222382545471,
+      "rewards/rejected": -0.35374683141708374,
+      "step": 3810
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.935744937230903e-06,
+      "logits/chosen": -2.514857769012451,
+      "logits/rejected": -2.508336067199707,
+      "logps/chosen": -317.51861572265625,
+      "logps/rejected": -301.63299560546875,
+      "loss": 0.6212,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.019365906715393066,
+      "rewards/margins": 0.22282662987709045,
+      "rewards/rejected": -0.24219253659248352,
+      "step": 3820
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.924494595073517e-06,
+      "logits/chosen": -2.577742576599121,
+      "logits/rejected": -2.547731399536133,
+      "logps/chosen": -296.11993408203125,
+      "logps/rejected": -299.71868896484375,
+      "loss": 0.5485,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.022993959486484528,
+      "rewards/margins": 0.4055466651916504,
+      "rewards/rejected": -0.38255277276039124,
+      "step": 3830
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.9132353941526575e-06,
+      "logits/chosen": -2.5459213256835938,
+      "logits/rejected": -2.481527328491211,
+      "logps/chosen": -281.5252990722656,
+      "logps/rejected": -311.25341796875,
+      "loss": 0.5793,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.024998091161251068,
+      "rewards/margins": 0.31911057233810425,
+      "rewards/rejected": -0.3441086411476135,
+      "step": 3840
+    },
+    {
+      "epoch": 0.5,
+      "learning_rate": 2.901967569436209e-06,
+      "logits/chosen": -2.562847137451172,
+      "logits/rejected": -2.5347342491149902,
+      "logps/chosen": -239.20425415039062,
+      "logps/rejected": -253.73587036132812,
+      "loss": 0.5748,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.011256558820605278,
+      "rewards/margins": 0.3290144205093384,
+      "rewards/rejected": -0.31775784492492676,
+      "step": 3850
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.89069135607203e-06,
+      "logits/chosen": -2.5373330116271973,
+      "logits/rejected": -2.4725966453552246,
+      "logps/chosen": -303.675048828125,
+      "logps/rejected": -305.76776123046875,
+      "loss": 0.5991,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.057954348623752594,
+      "rewards/margins": 0.2569880485534668,
+      "rewards/rejected": -0.3149423897266388,
+      "step": 3860
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8794069893830386e-06,
+      "logits/chosen": -2.544450044631958,
+      "logits/rejected": -2.4837541580200195,
+      "logps/chosen": -279.5870666503906,
+      "logps/rejected": -301.9145812988281,
+      "loss": 0.5991,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.14103463292121887,
+      "rewards/margins": 0.2767513692378998,
+      "rewards/rejected": -0.41778603196144104,
+      "step": 3870
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8681147048623038e-06,
+      "logits/chosen": -2.549741744995117,
+      "logits/rejected": -2.548783779144287,
+      "logps/chosen": -316.31597900390625,
+      "logps/rejected": -338.0789489746094,
+      "loss": 0.5603,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.040364861488342285,
+      "rewards/margins": 0.380311518907547,
+      "rewards/rejected": -0.4206763803958893,
+      "step": 3880
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8568147381681333e-06,
+      "logits/chosen": -2.563974142074585,
+      "logits/rejected": -2.5319700241088867,
+      "logps/chosen": -277.9092712402344,
+      "logps/rejected": -298.34063720703125,
+      "loss": 0.5911,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.1139674186706543,
+      "rewards/margins": 0.31823068857192993,
+      "rewards/rejected": -0.4321981370449066,
+      "step": 3890
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8455073251191533e-06,
+      "logits/chosen": -2.610217571258545,
+      "logits/rejected": -2.5704054832458496,
+      "logps/chosen": -297.24188232421875,
+      "logps/rejected": -322.96942138671875,
+      "loss": 0.5747,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.16270823776721954,
+      "rewards/margins": 0.3477099537849426,
+      "rewards/rejected": -0.5104181170463562,
+      "step": 3900
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8341927016893887e-06,
+      "logits/chosen": -2.578953504562378,
+      "logits/rejected": -2.552518367767334,
+      "logps/chosen": -265.6591796875,
+      "logps/rejected": -300.9714660644531,
+      "loss": 0.5899,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.157217338681221,
+      "rewards/margins": 0.308678537607193,
+      "rewards/rejected": -0.4658958315849304,
+      "step": 3910
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.822871104003335e-06,
+      "logits/chosen": -2.473323345184326,
+      "logits/rejected": -2.4271864891052246,
+      "logps/chosen": -253.3193359375,
+      "logps/rejected": -316.17352294921875,
+      "loss": 0.5923,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.08135505765676498,
+      "rewards/margins": 0.30147120356559753,
+      "rewards/rejected": -0.3828262686729431,
+      "step": 3920
+    },
+    {
+      "epoch": 0.51,
+      "learning_rate": 2.8115427683310355e-06,
+      "logits/chosen": -2.6394295692443848,
+      "logits/rejected": -2.5360267162323,
+      "logps/chosen": -328.1321716308594,
+      "logps/rejected": -334.3201904296875,
+      "loss": 0.5383,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.07986856997013092,
+      "rewards/margins": 0.42982950806617737,
+      "rewards/rejected": -0.5096980333328247,
+      "step": 3930
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.8002079310831477e-06,
+      "logits/chosen": -2.5841400623321533,
+      "logits/rejected": -2.5012383460998535,
+      "logps/chosen": -286.5207824707031,
+      "logps/rejected": -306.86181640625,
+      "loss": 0.6261,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.15493087470531464,
+      "rewards/margins": 0.21819886565208435,
+      "rewards/rejected": -0.37312978506088257,
+      "step": 3940
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7888668288060095e-06,
+      "logits/chosen": -2.590946912765503,
+      "logits/rejected": -2.49660062789917,
+      "logps/chosen": -315.48583984375,
+      "logps/rejected": -330.4048767089844,
+      "loss": 0.5251,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.08069697767496109,
+      "rewards/margins": 0.4589342474937439,
+      "rewards/rejected": -0.539631187915802,
+      "step": 3950
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7775196981767044e-06,
+      "logits/chosen": -2.507908344268799,
+      "logits/rejected": -2.4412014484405518,
+      "logps/chosen": -268.2931823730469,
+      "logps/rejected": -305.7597961425781,
+      "loss": 0.586,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.11341669410467148,
+      "rewards/margins": 0.3378104269504547,
+      "rewards/rejected": -0.4512270987033844,
+      "step": 3960
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7661667759981213e-06,
+      "logits/chosen": -2.582885265350342,
+      "logits/rejected": -2.5785489082336426,
+      "logps/chosen": -260.2806091308594,
+      "logps/rejected": -287.21099853515625,
+      "loss": 0.6067,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.1321321278810501,
+      "rewards/margins": 0.25400346517562866,
+      "rewards/rejected": -0.38613563776016235,
+      "step": 3970
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7548082991940137e-06,
+      "logits/chosen": -2.573814630508423,
+      "logits/rejected": -2.523665428161621,
+      "logps/chosen": -297.26104736328125,
+      "logps/rejected": -307.47686767578125,
+      "loss": 0.6455,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.12636759877204895,
+      "rewards/margins": 0.18434467911720276,
+      "rewards/rejected": -0.3107122778892517,
+      "step": 3980
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.743444504804051e-06,
+      "logits/chosen": -2.502840280532837,
+      "logits/rejected": -2.4645447731018066,
+      "logps/chosen": -240.093505859375,
+      "logps/rejected": -255.398193359375,
+      "loss": 0.6138,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.12067928165197372,
+      "rewards/margins": 0.24375872313976288,
+      "rewards/rejected": -0.364437997341156,
+      "step": 3990
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7320756299788788e-06,
+      "logits/chosen": -2.5586676597595215,
+      "logits/rejected": -2.46779203414917,
+      "logps/chosen": -311.4109191894531,
+      "logps/rejected": -302.7515869140625,
+      "loss": 0.6177,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.1380089819431305,
+      "rewards/margins": 0.2380903661251068,
+      "rewards/rejected": -0.37609928846359253,
+      "step": 4000
+    },
+    {
+      "epoch": 0.52,
+      "learning_rate": 2.7207019119751644e-06,
+      "logits/chosen": -2.5790040493011475,
+      "logits/rejected": -2.533700942993164,
+      "logps/chosen": -271.5816955566406,
+      "logps/rejected": -263.3572998046875,
+      "loss": 0.5593,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.14962056279182434,
+      "rewards/margins": 0.37803760170936584,
+      "rewards/rejected": -0.5276581645011902,
+      "step": 4010
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.7093235881506474e-06,
+      "logits/chosen": -2.624016761779785,
+      "logits/rejected": -2.5202982425689697,
+      "logps/chosen": -304.161376953125,
+      "logps/rejected": -330.45831298828125,
+      "loss": 0.527,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.09990578889846802,
+      "rewards/margins": 0.46382632851600647,
+      "rewards/rejected": -0.5637321472167969,
+      "step": 4020
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6979408959591863e-06,
+      "logits/chosen": -2.6023032665252686,
+      "logits/rejected": -2.630441188812256,
+      "logps/chosen": -270.9969787597656,
+      "logps/rejected": -299.814697265625,
+      "loss": 0.6012,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.13590934872627258,
+      "rewards/margins": 0.30410879850387573,
+      "rewards/rejected": -0.4400181770324707,
+      "step": 4030
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6865540729458034e-06,
+      "logits/chosen": -2.5819599628448486,
+      "logits/rejected": -2.4968574047088623,
+      "logps/chosen": -314.42242431640625,
+      "logps/rejected": -335.09222412109375,
+      "loss": 0.5585,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.09674658626317978,
+      "rewards/margins": 0.39674049615859985,
+      "rewards/rejected": -0.49348706007003784,
+      "step": 4040
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.675163356741726e-06,
+      "logits/chosen": -2.5685536861419678,
+      "logits/rejected": -2.489570140838623,
+      "logps/chosen": -271.8477478027344,
+      "logps/rejected": -307.17901611328125,
+      "loss": 0.5788,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.11576119810342789,
+      "rewards/margins": 0.3515714108943939,
+      "rewards/rejected": -0.467332661151886,
+      "step": 4050
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.6637689850594285e-06,
+      "logits/chosen": -2.575380563735962,
+      "logits/rejected": -2.495800495147705,
+      "logps/chosen": -273.8753662109375,
+      "logps/rejected": -325.6121520996094,
+      "loss": 0.6371,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.17124585807323456,
+      "rewards/margins": 0.20482996106147766,
+      "rewards/rejected": -0.376075804233551,
+      "step": 4060
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.652371195687671e-06,
+      "logits/chosen": -2.6011927127838135,
+      "logits/rejected": -2.5393528938293457,
+      "logps/chosen": -283.3669738769531,
+      "logps/rejected": -318.2293395996094,
+      "loss": 0.537,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.011819666251540184,
+      "rewards/margins": 0.45211896300315857,
+      "rewards/rejected": -0.4639386534690857,
+      "step": 4070
+    },
+    {
+      "epoch": 0.53,
+      "learning_rate": 2.64097022648654e-06,
+      "logits/chosen": -2.5448155403137207,
+      "logits/rejected": -2.4590229988098145,
+      "logps/chosen": -318.558349609375,
+      "logps/rejected": -306.23321533203125,
+      "loss": 0.5369,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.05935832858085632,
+      "rewards/margins": 0.4383729100227356,
+      "rewards/rejected": -0.49773120880126953,
+      "step": 4080
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.6295663153824774e-06,
+      "logits/chosen": -2.5590224266052246,
+      "logits/rejected": -2.446524143218994,
+      "logps/chosen": -319.48114013671875,
+      "logps/rejected": -293.68585205078125,
+      "loss": 0.5787,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.1162920817732811,
+      "rewards/margins": 0.3492317497730255,
+      "rewards/rejected": -0.4655238687992096,
+      "step": 4090
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.6181597003633218e-06,
+      "logits/chosen": -2.612055540084839,
+      "logits/rejected": -2.524919033050537,
+      "logps/chosen": -287.921630859375,
+      "logps/rejected": -300.99798583984375,
+      "loss": 0.583,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.08873952925205231,
+      "rewards/margins": 0.34233278036117554,
+      "rewards/rejected": -0.43107232451438904,
+      "step": 4100
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.606750619473342e-06,
+      "logits/chosen": -2.5327954292297363,
+      "logits/rejected": -2.481710910797119,
+      "logps/chosen": -273.19110107421875,
+      "logps/rejected": -323.3677062988281,
+      "loss": 0.5405,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.13292399048805237,
+      "rewards/margins": 0.4172719120979309,
+      "rewards/rejected": -0.5501959919929504,
+      "step": 4110
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.595339310808262e-06,
+      "logits/chosen": -2.607631206512451,
+      "logits/rejected": -2.5887436866760254,
+      "logps/chosen": -283.9256286621094,
+      "logps/rejected": -318.1768493652344,
+      "loss": 0.6062,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.20378096401691437,
+      "rewards/margins": 0.2889503538608551,
+      "rewards/rejected": -0.4927312731742859,
+      "step": 4120
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5839260125103004e-06,
+      "logits/chosen": -2.586212396621704,
+      "logits/rejected": -2.465703248977661,
+      "logps/chosen": -289.11553955078125,
+      "logps/rejected": -306.0029296875,
+      "loss": 0.5717,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.21482428908348083,
+      "rewards/margins": 0.3551693260669708,
+      "rewards/rejected": -0.5699936151504517,
+      "step": 4130
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5725109627631984e-06,
+      "logits/chosen": -2.5917022228240967,
+      "logits/rejected": -2.5772533416748047,
+      "logps/chosen": -321.9040222167969,
+      "logps/rejected": -290.53887939453125,
+      "loss": 0.6073,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.21336093544960022,
+      "rewards/margins": 0.2973124086856842,
+      "rewards/rejected": -0.5106734037399292,
+      "step": 4140
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5610943997872443e-06,
+      "logits/chosen": -2.5546934604644775,
+      "logits/rejected": -2.486725091934204,
+      "logps/chosen": -297.88653564453125,
+      "logps/rejected": -322.6026611328125,
+      "loss": 0.5851,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.24911344051361084,
+      "rewards/margins": 0.3188527226448059,
+      "rewards/rejected": -0.5679661631584167,
+      "step": 4150
+    },
+    {
+      "epoch": 0.54,
+      "learning_rate": 2.5496765618343096e-06,
+      "logits/chosen": -2.5874791145324707,
+      "logits/rejected": -2.6127076148986816,
+      "logps/chosen": -227.0814666748047,
+      "logps/rejected": -304.7928771972656,
+      "loss": 0.5717,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.004513068590313196,
+      "rewards/margins": 0.34239429235458374,
+      "rewards/rejected": -0.3469073176383972,
+      "step": 4160
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.538257687182871e-06,
+      "logits/chosen": -2.556873321533203,
+      "logits/rejected": -2.58785343170166,
+      "logps/chosen": -308.37896728515625,
+      "logps/rejected": -328.1161804199219,
+      "loss": 0.5779,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.200246661901474,
+      "rewards/margins": 0.3823387622833252,
+      "rewards/rejected": -0.5825853943824768,
+      "step": 4170
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.526838014133041e-06,
+      "logits/chosen": -2.647902488708496,
+      "logits/rejected": -2.545426845550537,
+      "logps/chosen": -288.0462341308594,
+      "logps/rejected": -299.14874267578125,
+      "loss": 0.5916,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.15408281981945038,
+      "rewards/margins": 0.3019229769706726,
+      "rewards/rejected": -0.4560058116912842,
+      "step": 4180
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.515417781001594e-06,
+      "logits/chosen": -2.5432028770446777,
+      "logits/rejected": -2.479264497756958,
+      "logps/chosen": -304.49481201171875,
+      "logps/rejected": -299.4053649902344,
+      "loss": 0.5771,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.17431829869747162,
+      "rewards/margins": 0.3661697208881378,
+      "rewards/rejected": -0.540488064289093,
+      "step": 4190
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.503997226116992e-06,
+      "logits/chosen": -2.541903018951416,
+      "logits/rejected": -2.494457483291626,
+      "logps/chosen": -281.78656005859375,
+      "logps/rejected": -319.5441589355469,
+      "loss": 0.591,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.16101603209972382,
+      "rewards/margins": 0.28700757026672363,
+      "rewards/rejected": -0.44802361726760864,
+      "step": 4200
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.4925765878144115e-06,
+      "logits/chosen": -2.6121087074279785,
+      "logits/rejected": -2.5418975353240967,
+      "logps/chosen": -272.30194091796875,
+      "logps/rejected": -299.36334228515625,
+      "loss": 0.5455,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.1072150245308876,
+      "rewards/margins": 0.41332411766052246,
+      "rewards/rejected": -0.5205391049385071,
+      "step": 4210
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.4811561044307727e-06,
+      "logits/chosen": -2.519291400909424,
+      "logits/rejected": -2.5210442543029785,
+      "logps/chosen": -271.16473388671875,
+      "logps/rejected": -321.5150146484375,
+      "loss": 0.5551,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.18764032423496246,
+      "rewards/margins": 0.39520907402038574,
+      "rewards/rejected": -0.582849383354187,
+      "step": 4220
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.469736014299758e-06,
+      "logits/chosen": -2.68046236038208,
+      "logits/rejected": -2.6409106254577637,
+      "logps/chosen": -316.367431640625,
+      "logps/rejected": -337.29241943359375,
+      "loss": 0.6024,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.19918163120746613,
+      "rewards/margins": 0.27686649560928345,
+      "rewards/rejected": -0.476048082113266,
+      "step": 4230
+    },
+    {
+      "epoch": 0.55,
+      "learning_rate": 2.458316555746846e-06,
+      "logits/chosen": -2.660388231277466,
+      "logits/rejected": -2.6214935779571533,
+      "logps/chosen": -359.70733642578125,
+      "logps/rejected": -352.8019104003906,
+      "loss": 0.5446,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.16549673676490784,
+      "rewards/margins": 0.4153873324394226,
+      "rewards/rejected": -0.5808840394020081,
+      "step": 4240
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.446897967084334e-06,
+      "logits/chosen": -2.6188812255859375,
+      "logits/rejected": -2.5265862941741943,
+      "logps/chosen": -344.84832763671875,
+      "logps/rejected": -308.1126708984375,
+      "loss": 0.6202,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.30671724677085876,
+      "rewards/margins": 0.2823528051376343,
+      "rewards/rejected": -0.5890700221061707,
+      "step": 4250
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.4354804866063684e-06,
+      "logits/chosen": -2.590623378753662,
+      "logits/rejected": -2.5362191200256348,
+      "logps/chosen": -304.72601318359375,
+      "logps/rejected": -329.4560546875,
+      "loss": 0.5606,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.17591656744480133,
+      "rewards/margins": 0.3673582375049591,
+      "rewards/rejected": -0.5432747602462769,
+      "step": 4260
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.424064352583964e-06,
+      "logits/chosen": -2.633047342300415,
+      "logits/rejected": -2.548260450363159,
+      "logps/chosen": -318.0061950683594,
+      "logps/rejected": -361.3074035644531,
+      "loss": 0.5799,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.2432432919740677,
+      "rewards/margins": 0.3581022620201111,
+      "rewards/rejected": -0.6013455390930176,
+      "step": 4270
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.4126498032600403e-06,
+      "logits/chosen": -2.6011974811553955,
+      "logits/rejected": -2.589582681655884,
+      "logps/chosen": -311.56182861328125,
+      "logps/rejected": -347.2060546875,
+      "loss": 0.5895,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.17930284142494202,
+      "rewards/margins": 0.31041544675827026,
+      "rewards/rejected": -0.4897182881832123,
+      "step": 4280
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.401237076844445e-06,
+      "logits/chosen": -2.5530214309692383,
+      "logits/rejected": -2.486758232116699,
+      "logps/chosen": -286.376708984375,
+      "logps/rejected": -269.7857666015625,
+      "loss": 0.6592,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.29157310724258423,
+      "rewards/margins": 0.1503646820783615,
+      "rewards/rejected": -0.44193777441978455,
+      "step": 4290
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.38982641150898e-06,
+      "logits/chosen": -2.459611654281616,
+      "logits/rejected": -2.449705123901367,
+      "logps/chosen": -289.855224609375,
+      "logps/rejected": -291.42974853515625,
+      "loss": 0.5489,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.18662351369857788,
+      "rewards/margins": 0.4012455344200134,
+      "rewards/rejected": -0.5878690481185913,
+      "step": 4300
+    },
+    {
+      "epoch": 0.56,
+      "learning_rate": 2.3784180453824414e-06,
+      "logits/chosen": -2.541564464569092,
+      "logits/rejected": -2.4901671409606934,
+      "logps/chosen": -295.0165100097656,
+      "logps/rejected": -319.0411682128906,
+      "loss": 0.5992,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.2873517572879791,
+      "rewards/margins": 0.30892321467399597,
+      "rewards/rejected": -0.5962749123573303,
+      "step": 4310
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.367012216545638e-06,
+      "logits/chosen": -2.588183879852295,
+      "logits/rejected": -2.5278923511505127,
+      "logps/chosen": -292.7016906738281,
+      "logps/rejected": -292.919677734375,
+      "loss": 0.5949,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.32504764199256897,
+      "rewards/margins": 0.290105402469635,
+      "rewards/rejected": -0.6151530146598816,
+      "step": 4320
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.3556091630264294e-06,
+      "logits/chosen": -2.5658657550811768,
+      "logits/rejected": -2.468454599380493,
+      "logps/chosen": -301.0626525878906,
+      "logps/rejected": -285.8448791503906,
+      "loss": 0.614,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.23350822925567627,
+      "rewards/margins": 0.25222522020339966,
+      "rewards/rejected": -0.48573341965675354,
+      "step": 4330
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.344209122794757e-06,
+      "logits/chosen": -2.6183600425720215,
+      "logits/rejected": -2.5591893196105957,
+      "logps/chosen": -311.8841247558594,
+      "logps/rejected": -347.6051940917969,
+      "loss": 0.5847,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.25667446851730347,
+      "rewards/margins": 0.3387332260608673,
+      "rewards/rejected": -0.5954076647758484,
+      "step": 4340
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.3328123337576787e-06,
+      "logits/chosen": -2.555053234100342,
+      "logits/rejected": -2.57133150100708,
+      "logps/chosen": -274.073974609375,
+      "logps/rejected": -335.1023254394531,
+      "loss": 0.572,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.21889004111289978,
+      "rewards/margins": 0.3607257604598999,
+      "rewards/rejected": -0.5796157717704773,
+      "step": 4350
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.3214190337544017e-06,
+      "logits/chosen": -2.6688897609710693,
+      "logits/rejected": -2.5526506900787354,
+      "logps/chosen": -294.30438232421875,
+      "logps/rejected": -312.80584716796875,
+      "loss": 0.5649,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.22945478558540344,
+      "rewards/margins": 0.367873877286911,
+      "rewards/rejected": -0.5973286628723145,
+      "step": 4360
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.310029460551323e-06,
+      "logits/chosen": -2.541673421859741,
+      "logits/rejected": -2.4680838584899902,
+      "logps/chosen": -332.10882568359375,
+      "logps/rejected": -301.04730224609375,
+      "loss": 0.591,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.13747885823249817,
+      "rewards/margins": 0.3306196630001068,
+      "rewards/rejected": -0.46809855103492737,
+      "step": 4370
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.2986438518370645e-06,
+      "logits/chosen": -2.6398513317108154,
+      "logits/rejected": -2.522425413131714,
+      "logps/chosen": -288.0531005859375,
+      "logps/rejected": -306.16668701171875,
+      "loss": 0.5639,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.18993988633155823,
+      "rewards/margins": 0.3859085738658905,
+      "rewards/rejected": -0.5758484601974487,
+      "step": 4380
+    },
+    {
+      "epoch": 0.57,
+      "learning_rate": 2.2872624452175123e-06,
+      "logits/chosen": -2.4865167140960693,
+      "logits/rejected": -2.4587464332580566,
+      "logps/chosen": -310.3607177734375,
+      "logps/rejected": -302.1121520996094,
+      "loss": 0.5699,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.1371636986732483,
+      "rewards/margins": 0.3356013596057892,
+      "rewards/rejected": -0.4727650582790375,
+      "step": 4390
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2758854782108584e-06,
+      "logits/chosen": -2.6290535926818848,
+      "logits/rejected": -2.5545456409454346,
+      "logps/chosen": -332.05560302734375,
+      "logps/rejected": -345.66668701171875,
+      "loss": 0.5843,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.22577285766601562,
+      "rewards/margins": 0.33528390526771545,
+      "rewards/rejected": -0.5610567331314087,
+      "step": 4400
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2645131882426458e-06,
+      "logits/chosen": -2.598134994506836,
+      "logits/rejected": -2.572866439819336,
+      "logps/chosen": -289.02679443359375,
+      "logps/rejected": -310.4499816894531,
+      "loss": 0.6291,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.10650638490915298,
+      "rewards/margins": 0.2374182641506195,
+      "rewards/rejected": -0.3439246714115143,
+      "step": 4410
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2531458126408154e-06,
+      "logits/chosen": -2.555345058441162,
+      "logits/rejected": -2.4923336505889893,
+      "logps/chosen": -289.09661865234375,
+      "logps/rejected": -285.81463623046875,
+      "loss": 0.5836,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.1535475254058838,
+      "rewards/margins": 0.3218473494052887,
+      "rewards/rejected": -0.4753949046134949,
+      "step": 4420
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2417835886307452e-06,
+      "logits/chosen": -2.6209311485290527,
+      "logits/rejected": -2.5675032138824463,
+      "logps/chosen": -281.6811828613281,
+      "logps/rejected": -297.09393310546875,
+      "loss": 0.5497,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.134135901927948,
+      "rewards/margins": 0.4027267396450043,
+      "rewards/rejected": -0.5368626713752747,
+      "step": 4430
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.2304267533303075e-06,
+      "logits/chosen": -2.5350840091705322,
+      "logits/rejected": -2.498253583908081,
+      "logps/chosen": -324.82037353515625,
+      "logps/rejected": -320.0078125,
+      "loss": 0.5815,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.17317651212215424,
+      "rewards/margins": 0.3382209241390228,
+      "rewards/rejected": -0.5113974809646606,
+      "step": 4440
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.219075543744918e-06,
+      "logits/chosen": -2.6306023597717285,
+      "logits/rejected": -2.551877021789551,
+      "logps/chosen": -299.55072021484375,
+      "logps/rejected": -293.7462463378906,
+      "loss": 0.6297,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.1527298539876938,
+      "rewards/margins": 0.24451592564582825,
+      "rewards/rejected": -0.3972457945346832,
+      "step": 4450
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.207730196762589e-06,
+      "logits/chosen": -2.5206148624420166,
+      "logits/rejected": -2.438523292541504,
+      "logps/chosen": -317.34765625,
+      "logps/rejected": -313.8121337890625,
+      "loss": 0.5385,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.11913515627384186,
+      "rewards/margins": 0.4204968512058258,
+      "rewards/rejected": -0.5396319627761841,
+      "step": 4460
+    },
+    {
+      "epoch": 0.58,
+      "learning_rate": 2.1963909491489846e-06,
+      "logits/chosen": -2.6192498207092285,
+      "logits/rejected": -2.5608458518981934,
+      "logps/chosen": -305.0476379394531,
+      "logps/rejected": -359.00408935546875,
+      "loss": 0.5902,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.11978902667760849,
+      "rewards/margins": 0.3082696795463562,
+      "rewards/rejected": -0.4280587136745453,
+      "step": 4470
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.185058037542486e-06,
+      "logits/chosen": -2.632004976272583,
+      "logits/rejected": -2.535244941711426,
+      "logps/chosen": -309.0987854003906,
+      "logps/rejected": -306.10992431640625,
+      "loss": 0.5762,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.11530432850122452,
+      "rewards/margins": 0.33437785506248474,
+      "rewards/rejected": -0.44968217611312866,
+      "step": 4480
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.173731698449244e-06,
+      "logits/chosen": -2.5994200706481934,
+      "logits/rejected": -2.546800374984741,
+      "logps/chosen": -287.562255859375,
+      "logps/rejected": -315.8476867675781,
+      "loss": 0.5776,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.11606922000646591,
+      "rewards/margins": 0.3415149450302124,
+      "rewards/rejected": -0.4575842022895813,
+      "step": 4490
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1624121682382495e-06,
+      "logits/chosen": -2.6481831073760986,
+      "logits/rejected": -2.5380067825317383,
+      "logps/chosen": -283.59710693359375,
+      "logps/rejected": -288.51251220703125,
+      "loss": 0.564,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.1574515700340271,
+      "rewards/margins": 0.3552487790584564,
+      "rewards/rejected": -0.5127003788948059,
+      "step": 4500
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.1510996831363993e-06,
+      "logits/chosen": -2.5903282165527344,
+      "logits/rejected": -2.513554811477661,
+      "logps/chosen": -285.13031005859375,
+      "logps/rejected": -265.69744873046875,
+      "loss": 0.5909,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.20109689235687256,
+      "rewards/margins": 0.3178878724575043,
+      "rewards/rejected": -0.5189847946166992,
+      "step": 4510
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.139794479223565e-06,
+      "logits/chosen": -2.50488543510437,
+      "logits/rejected": -2.5171425342559814,
+      "logps/chosen": -344.6709899902344,
+      "logps/rejected": -316.1105651855469,
+      "loss": 0.5568,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.10511822998523712,
+      "rewards/margins": 0.39226776361465454,
+      "rewards/rejected": -0.49738597869873047,
+      "step": 4520
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.128496792427669e-06,
+      "logits/chosen": -2.604703187942505,
+      "logits/rejected": -2.5967934131622314,
+      "logps/chosen": -315.81292724609375,
+      "logps/rejected": -355.1261901855469,
+      "loss": 0.5738,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.10320033133029938,
+      "rewards/margins": 0.34820085763931274,
+      "rewards/rejected": -0.45140114426612854,
+      "step": 4530
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 2.117206858519758e-06,
+      "logits/chosen": -2.6536474227905273,
+      "logits/rejected": -2.6627275943756104,
+      "logps/chosen": -279.7059631347656,
+      "logps/rejected": -333.873291015625,
+      "loss": 0.6109,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.20014262199401855,
+      "rewards/margins": 0.25761887431144714,
+      "rewards/rejected": -0.4577614665031433,
+      "step": 4540
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.1059249131090844e-06,
+      "logits/chosen": -2.520747661590576,
+      "logits/rejected": -2.4857754707336426,
+      "logps/chosen": -259.98992919921875,
+      "logps/rejected": -255.5244903564453,
+      "loss": 0.6148,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.22918829321861267,
+      "rewards/margins": 0.27222687005996704,
+      "rewards/rejected": -0.5014151334762573,
+      "step": 4550
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.094651191638189e-06,
+      "logits/chosen": -2.5376927852630615,
+      "logits/rejected": -2.4998600482940674,
+      "logps/chosen": -290.6065979003906,
+      "logps/rejected": -287.73089599609375,
+      "loss": 0.5996,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.1908431053161621,
+      "rewards/margins": 0.2888910472393036,
+      "rewards/rejected": -0.4797341227531433,
+      "step": 4560
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0833859293779867e-06,
+      "logits/chosen": -2.560824155807495,
+      "logits/rejected": -2.5106751918792725,
+      "logps/chosen": -300.39678955078125,
+      "logps/rejected": -317.60211181640625,
+      "loss": 0.601,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.15792220830917358,
+      "rewards/margins": 0.28045356273651123,
+      "rewards/rejected": -0.4383757710456848,
+      "step": 4570
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0721293614228568e-06,
+      "logits/chosen": -2.5710103511810303,
+      "logits/rejected": -2.540526866912842,
+      "logps/chosen": -258.14837646484375,
+      "logps/rejected": -329.10211181640625,
+      "loss": 0.5815,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.08148565888404846,
+      "rewards/margins": 0.30815690755844116,
+      "rewards/rejected": -0.3896425664424896,
+      "step": 4580
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.060881722685742e-06,
+      "logits/chosen": -2.6478891372680664,
+      "logits/rejected": -2.64860200881958,
+      "logps/chosen": -284.43487548828125,
+      "logps/rejected": -330.7427978515625,
+      "loss": 0.5588,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.15377451479434967,
+      "rewards/margins": 0.3742519021034241,
+      "rewards/rejected": -0.5280263423919678,
+      "step": 4590
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.049643247893235e-06,
+      "logits/chosen": -2.597916841506958,
+      "logits/rejected": -2.5824079513549805,
+      "logps/chosen": -293.86956787109375,
+      "logps/rejected": -300.90618896484375,
+      "loss": 0.5744,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.13460037112236023,
+      "rewards/margins": 0.3401843011379242,
+      "rewards/rejected": -0.4747847020626068,
+      "step": 4600
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0384141715806903e-06,
+      "logits/chosen": -2.6367084980010986,
+      "logits/rejected": -2.6011016368865967,
+      "logps/chosen": -331.17669677734375,
+      "logps/rejected": -331.61444091796875,
+      "loss": 0.5986,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.16236744821071625,
+      "rewards/margins": 0.2754122316837311,
+      "rewards/rejected": -0.43777966499328613,
+      "step": 4610
+    },
+    {
+      "epoch": 0.6,
+      "learning_rate": 2.0271947280873255e-06,
+      "logits/chosen": -2.649927854537964,
+      "logits/rejected": -2.602278232574463,
+      "logps/chosen": -283.5793151855469,
+      "logps/rejected": -301.73321533203125,
+      "loss": 0.5644,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.08142141252756119,
+      "rewards/margins": 0.3690916895866394,
+      "rewards/rejected": -0.4505131244659424,
+      "step": 4620
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.0159851515513302e-06,
+      "logits/chosen": -2.538959264755249,
+      "logits/rejected": -2.527182102203369,
+      "logps/chosen": -245.12857055664062,
+      "logps/rejected": -296.0338439941406,
+      "loss": 0.6026,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.11742006242275238,
+      "rewards/margins": 0.28807172179222107,
+      "rewards/rejected": -0.40549176931381226,
+      "step": 4630
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 2.004785675904982e-06,
+      "logits/chosen": -2.615934371948242,
+      "logits/rejected": -2.6237330436706543,
+      "logps/chosen": -315.43365478515625,
+      "logps/rejected": -326.7650451660156,
+      "loss": 0.5697,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.13643045723438263,
+      "rewards/margins": 0.3817102611064911,
+      "rewards/rejected": -0.5181406736373901,
+      "step": 4640
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.9935965348697624e-06,
+      "logits/chosen": -2.6093904972076416,
+      "logits/rejected": -2.5315451622009277,
+      "logps/chosen": -305.9197692871094,
+      "logps/rejected": -276.95196533203125,
+      "loss": 0.5806,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.1007126197218895,
+      "rewards/margins": 0.3375021815299988,
+      "rewards/rejected": -0.43821483850479126,
+      "step": 4650
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.9824179619514807e-06,
+      "logits/chosen": -2.5855777263641357,
+      "logits/rejected": -2.5461955070495605,
+      "logps/chosen": -323.4766540527344,
+      "logps/rejected": -355.25909423828125,
+      "loss": 0.6286,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.26345595717430115,
+      "rewards/margins": 0.24047298729419708,
+      "rewards/rejected": -0.5039290189743042,
+      "step": 4660
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.9712501904354004e-06,
+      "logits/chosen": -2.551574230194092,
+      "logits/rejected": -2.5038509368896484,
+      "logps/chosen": -286.7224426269531,
+      "logps/rejected": -310.63714599609375,
+      "loss": 0.5841,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.19094695150852203,
+      "rewards/margins": 0.3232021927833557,
+      "rewards/rejected": -0.5141491293907166,
+      "step": 4670
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.960093453381369e-06,
+      "logits/chosen": -2.600787401199341,
+      "logits/rejected": -2.5943074226379395,
+      "logps/chosen": -301.3719177246094,
+      "logps/rejected": -328.24664306640625,
+      "loss": 0.6778,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -0.3415972590446472,
+      "rewards/margins": 0.14613351225852966,
+      "rewards/rejected": -0.48773080110549927,
+      "step": 4680
+    },
+    {
+      "epoch": 0.61,
+      "learning_rate": 1.948947983618962e-06,
+      "logits/chosen": -2.605231523513794,
+      "logits/rejected": -2.548745632171631,
+      "logps/chosen": -320.5574035644531,
+      "logps/rejected": -319.0489501953125,
+      "loss": 0.5548,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.1902529001235962,
+      "rewards/margins": 0.40467530488967896,
+      "rewards/rejected": -0.5949281454086304,
+      "step": 4690
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.937814013742611e-06,
+      "logits/chosen": -2.5465714931488037,
+      "logits/rejected": -2.530928611755371,
+      "logps/chosen": -279.71160888671875,
+      "logps/rejected": -308.81170654296875,
+      "loss": 0.6084,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.19746831059455872,
+      "rewards/margins": 0.29210931062698364,
+      "rewards/rejected": -0.48957759141921997,
+      "step": 4700
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.9266917761067617e-06,
+      "logits/chosen": -2.614208698272705,
+      "logits/rejected": -2.529270648956299,
+      "logps/chosen": -260.76953125,
+      "logps/rejected": -305.258056640625,
+      "loss": 0.5367,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.08303768187761307,
+      "rewards/margins": 0.47025662660598755,
+      "rewards/rejected": -0.55329430103302,
+      "step": 4710
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.915581502821017e-06,
+      "logits/chosen": -2.579409122467041,
+      "logits/rejected": -2.5437111854553223,
+      "logps/chosen": -299.9549255371094,
+      "logps/rejected": -321.8433837890625,
+      "loss": 0.6395,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.23660913109779358,
+      "rewards/margins": 0.19735004007816315,
+      "rewards/rejected": -0.43395915627479553,
+      "step": 4720
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.9044834257452997e-06,
+      "logits/chosen": -2.5811142921447754,
+      "logits/rejected": -2.6148557662963867,
+      "logps/chosen": -309.6854553222656,
+      "logps/rejected": -314.71368408203125,
+      "loss": 0.5717,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.11243283748626709,
+      "rewards/margins": 0.35112327337265015,
+      "rewards/rejected": -0.4635561406612396,
+      "step": 4730
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.893397776485006e-06,
+      "logits/chosen": -2.5305540561676025,
+      "logits/rejected": -2.438352346420288,
+      "logps/chosen": -252.86416625976562,
+      "logps/rejected": -281.62713623046875,
+      "loss": 0.5635,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.15612688660621643,
+      "rewards/margins": 0.38440144062042236,
+      "rewards/rejected": -0.5405283570289612,
+      "step": 4740
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.8823247863861804e-06,
+      "logits/chosen": -2.516956090927124,
+      "logits/rejected": -2.4679129123687744,
+      "logps/chosen": -279.9268798828125,
+      "logps/rejected": -279.04852294921875,
+      "loss": 0.5488,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.09448639303445816,
+      "rewards/margins": 0.42614802718162537,
+      "rewards/rejected": -0.5206344723701477,
+      "step": 4750
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.8712646865306822e-06,
+      "logits/chosen": -2.567016124725342,
+      "logits/rejected": -2.5746636390686035,
+      "logps/chosen": -297.80621337890625,
+      "logps/rejected": -335.90631103515625,
+      "loss": 0.5905,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.15743158757686615,
+      "rewards/margins": 0.3130558431148529,
+      "rewards/rejected": -0.470487505197525,
+      "step": 4760
+    },
+    {
+      "epoch": 0.62,
+      "learning_rate": 1.8602177077313631e-06,
+      "logits/chosen": -2.5681662559509277,
+      "logits/rejected": -2.5675320625305176,
+      "logps/chosen": -276.4873352050781,
+      "logps/rejected": -332.3812561035156,
+      "loss": 0.6016,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.11791181564331055,
+      "rewards/margins": 0.2915845215320587,
+      "rewards/rejected": -0.40949636697769165,
+      "step": 4770
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8491840805272546e-06,
+      "logits/chosen": -2.599635362625122,
+      "logits/rejected": -2.546414852142334,
+      "logps/chosen": -273.6893615722656,
+      "logps/rejected": -281.02276611328125,
+      "loss": 0.589,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.1778731793165207,
+      "rewards/margins": 0.313727468252182,
+      "rewards/rejected": -0.4916006624698639,
+      "step": 4780
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8381640351787516e-06,
+      "logits/chosen": -2.480893135070801,
+      "logits/rejected": -2.4117960929870605,
+      "logps/chosen": -291.1407470703125,
+      "logps/rejected": -303.8739318847656,
+      "loss": 0.5714,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.16217121481895447,
+      "rewards/margins": 0.3526865541934967,
+      "rewards/rejected": -0.5148577690124512,
+      "step": 4790
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8271578016628122e-06,
+      "logits/chosen": -2.646735429763794,
+      "logits/rejected": -2.5718436241149902,
+      "logps/chosen": -309.89984130859375,
+      "logps/rejected": -308.4041442871094,
+      "loss": 0.5927,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.16738727688789368,
+      "rewards/margins": 0.31154894828796387,
+      "rewards/rejected": -0.47893619537353516,
+      "step": 4800
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8161656096681546e-06,
+      "logits/chosen": -2.605302572250366,
+      "logits/rejected": -2.5318710803985596,
+      "logps/chosen": -283.86724853515625,
+      "logps/rejected": -288.05987548828125,
+      "loss": 0.5448,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.03990992158651352,
+      "rewards/margins": 0.45058202743530273,
+      "rewards/rejected": -0.49049192667007446,
+      "step": 4810
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.8051876885904645e-06,
+      "logits/chosen": -2.4900870323181152,
+      "logits/rejected": -2.4417641162872314,
+      "logps/chosen": -252.48678588867188,
+      "logps/rejected": -265.570556640625,
+      "loss": 0.6218,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.1534336358308792,
+      "rewards/margins": 0.27210724353790283,
+      "rewards/rejected": -0.42554086446762085,
+      "step": 4820
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.7942242675276098e-06,
+      "logits/chosen": -2.5617733001708984,
+      "logits/rejected": -2.49678373336792,
+      "logps/chosen": -301.89056396484375,
+      "logps/rejected": -343.17474365234375,
+      "loss": 0.5615,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.17700599133968353,
+      "rewards/margins": 0.4070133566856384,
+      "rewards/rejected": -0.5840193033218384,
+      "step": 4830
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.783275575274856e-06,
+      "logits/chosen": -2.423518419265747,
+      "logits/rejected": -2.3951497077941895,
+      "logps/chosen": -299.3848571777344,
+      "logps/rejected": -320.1754150390625,
+      "loss": 0.529,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.1733754575252533,
+      "rewards/margins": 0.4372386038303375,
+      "rewards/rejected": -0.610614001750946,
+      "step": 4840
+    },
+    {
+      "epoch": 0.63,
+      "learning_rate": 1.7723418403200943e-06,
+      "logits/chosen": -2.5899882316589355,
+      "logits/rejected": -2.5356671810150146,
+      "logps/chosen": -294.1274108886719,
+      "logps/rejected": -328.3188781738281,
+      "loss": 0.5814,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.1610538810491562,
+      "rewards/margins": 0.3633432388305664,
+      "rewards/rejected": -0.5243971347808838,
+      "step": 4850
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7614232908390748e-06,
+      "logits/chosen": -2.5549492835998535,
+      "logits/rejected": -2.4538655281066895,
+      "logps/chosen": -308.332275390625,
+      "logps/rejected": -311.3489990234375,
+      "loss": 0.5669,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.22306986153125763,
+      "rewards/margins": 0.39509907364845276,
+      "rewards/rejected": -0.6181688904762268,
+      "step": 4860
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7505201546906398e-06,
+      "logits/chosen": -2.6318461894989014,
+      "logits/rejected": -2.532041549682617,
+      "logps/chosen": -309.37689208984375,
+      "logps/rejected": -309.7413024902344,
+      "loss": 0.5771,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.15767396986484528,
+      "rewards/margins": 0.3564053475856781,
+      "rewards/rejected": -0.5140793323516846,
+      "step": 4870
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7396326594119717e-06,
+      "logits/chosen": -2.537423849105835,
+      "logits/rejected": -2.5376172065734863,
+      "logps/chosen": -304.79071044921875,
+      "logps/rejected": -364.28533935546875,
+      "loss": 0.5562,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.12208513915538788,
+      "rewards/margins": 0.40494170784950256,
+      "rewards/rejected": -0.5270268321037292,
+      "step": 4880
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7287610322138449e-06,
+      "logits/chosen": -2.628302574157715,
+      "logits/rejected": -2.579782009124756,
+      "logps/chosen": -279.1582336425781,
+      "logps/rejected": -289.4959716796875,
+      "loss": 0.6031,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.17124351859092712,
+      "rewards/margins": 0.2769598364830017,
+      "rewards/rejected": -0.4482033848762512,
+      "step": 4890
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7179054999758817e-06,
+      "logits/chosen": -2.585559844970703,
+      "logits/rejected": -2.5440878868103027,
+      "logps/chosen": -313.0089416503906,
+      "logps/rejected": -309.50201416015625,
+      "loss": 0.5831,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.18334296345710754,
+      "rewards/margins": 0.32313090562820435,
+      "rewards/rejected": -0.5064738988876343,
+      "step": 4900
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.7070662892418225e-06,
+      "logits/chosen": -2.6078925132751465,
+      "logits/rejected": -2.5725595951080322,
+      "logps/chosen": -279.7121887207031,
+      "logps/rejected": -317.3712463378906,
+      "loss": 0.5599,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.09897415339946747,
+      "rewards/margins": 0.37896251678466797,
+      "rewards/rejected": -0.47793668508529663,
+      "step": 4910
+    },
+    {
+      "epoch": 0.64,
+      "learning_rate": 1.6962436262147913e-06,
+      "logits/chosen": -2.6630561351776123,
+      "logits/rejected": -2.5910372734069824,
+      "logps/chosen": -333.7852478027344,
+      "logps/rejected": -364.60394287109375,
+      "loss": 0.5951,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.24640896916389465,
+      "rewards/margins": 0.31163299083709717,
+      "rewards/rejected": -0.5580419301986694,
+      "step": 4920
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6854377367525814e-06,
+      "logits/chosen": -2.5870211124420166,
+      "logits/rejected": -2.5450308322906494,
+      "logps/chosen": -281.597412109375,
+      "logps/rejected": -317.059814453125,
+      "loss": 0.559,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.12749680876731873,
+      "rewards/margins": 0.3921080231666565,
+      "rewards/rejected": -0.5196048617362976,
+      "step": 4930
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6746488463629362e-06,
+      "logits/chosen": -2.5821316242218018,
+      "logits/rejected": -2.520230531692505,
+      "logps/chosen": -325.74493408203125,
+      "logps/rejected": -315.589111328125,
+      "loss": 0.5713,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.11354532092809677,
+      "rewards/margins": 0.38313788175582886,
+      "rewards/rejected": -0.496683269739151,
+      "step": 4940
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6638771801988483e-06,
+      "logits/chosen": -2.60164475440979,
+      "logits/rejected": -2.5188491344451904,
+      "logps/chosen": -331.33843994140625,
+      "logps/rejected": -332.96417236328125,
+      "loss": 0.5966,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.1409718245267868,
+      "rewards/margins": 0.3012249767780304,
+      "rewards/rejected": -0.4421967566013336,
+      "step": 4950
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.653122963053857e-06,
+      "logits/chosen": -2.598275899887085,
+      "logits/rejected": -2.58530592918396,
+      "logps/chosen": -275.3309326171875,
+      "logps/rejected": -292.92828369140625,
+      "loss": 0.5674,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.11479552835226059,
+      "rewards/margins": 0.3697282671928406,
+      "rewards/rejected": -0.48452383279800415,
+      "step": 4960
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6423864193573606e-06,
+      "logits/chosen": -2.623122215270996,
+      "logits/rejected": -2.5229997634887695,
+      "logps/chosen": -302.03228759765625,
+      "logps/rejected": -324.7279052734375,
+      "loss": 0.6182,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.21475405991077423,
+      "rewards/margins": 0.2553631663322449,
+      "rewards/rejected": -0.4701172411441803,
+      "step": 4970
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6316677731699286e-06,
+      "logits/chosen": -2.593745708465576,
+      "logits/rejected": -2.5143866539001465,
+      "logps/chosen": -313.32000732421875,
+      "logps/rejected": -312.0892028808594,
+      "loss": 0.5557,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.16294625401496887,
+      "rewards/margins": 0.388921320438385,
+      "rewards/rejected": -0.5518675446510315,
+      "step": 4980
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6209672481786302e-06,
+      "logits/chosen": -2.5862197875976562,
+      "logits/rejected": -2.5494134426116943,
+      "logps/chosen": -300.0966796875,
+      "logps/rejected": -338.5650329589844,
+      "loss": 0.5824,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.1950572431087494,
+      "rewards/margins": 0.3306945860385895,
+      "rewards/rejected": -0.5257518887519836,
+      "step": 4990
+    },
+    {
+      "epoch": 0.65,
+      "learning_rate": 1.6102850676923616e-06,
+      "logits/chosen": -2.5233166217803955,
+      "logits/rejected": -2.5655159950256348,
+      "logps/chosen": -231.1686553955078,
+      "logps/rejected": -282.37286376953125,
+      "loss": 0.5607,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.12654763460159302,
+      "rewards/margins": 0.3562249541282654,
+      "rewards/rejected": -0.4827726483345032,
+      "step": 5000
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5996214546371888e-06,
+      "logits/chosen": -2.5684890747070312,
+      "logits/rejected": -2.515471935272217,
+      "logps/chosen": -296.56500244140625,
+      "logps/rejected": -351.08599853515625,
+      "loss": 0.5236,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.08794933557510376,
+      "rewards/margins": 0.4533146917819977,
+      "rewards/rejected": -0.5412639379501343,
+      "step": 5010
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.588976631551697e-06,
+      "logits/chosen": -2.5542407035827637,
+      "logits/rejected": -2.405961751937866,
+      "logps/chosen": -317.7288818359375,
+      "logps/rejected": -312.1694641113281,
+      "loss": 0.5665,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.17019042372703552,
+      "rewards/margins": 0.36578369140625,
+      "rewards/rejected": -0.5359741449356079,
+      "step": 5020
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5783508205823412e-06,
+      "logits/chosen": -2.6269588470458984,
+      "logits/rejected": -2.5455031394958496,
+      "logps/chosen": -289.838623046875,
+      "logps/rejected": -311.3556213378906,
+      "loss": 0.556,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.13314411044120789,
+      "rewards/margins": 0.41248899698257446,
+      "rewards/rejected": -0.5456331372261047,
+      "step": 5030
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5677442434788143e-06,
+      "logits/chosen": -2.56490159034729,
+      "logits/rejected": -2.5372207164764404,
+      "logps/chosen": -337.6856384277344,
+      "logps/rejected": -302.85650634765625,
+      "loss": 0.62,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.1947760134935379,
+      "rewards/margins": 0.2283238172531128,
+      "rewards/rejected": -0.4230998456478119,
+      "step": 5040
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5571571215894181e-06,
+      "logits/chosen": -2.496464490890503,
+      "logits/rejected": -2.458303928375244,
+      "logps/chosen": -266.31622314453125,
+      "logps/rejected": -288.2103271484375,
+      "loss": 0.5567,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.14684097468852997,
+      "rewards/margins": 0.39641258120536804,
+      "rewards/rejected": -0.5432535409927368,
+      "step": 5050
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5465896758564452e-06,
+      "logits/chosen": -2.6084046363830566,
+      "logits/rejected": -2.558873414993286,
+      "logps/chosen": -300.2311096191406,
+      "logps/rejected": -301.22076416015625,
+      "loss": 0.6089,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.14224758744239807,
+      "rewards/margins": 0.290388286113739,
+      "rewards/rejected": -0.4326358735561371,
+      "step": 5060
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5360421268115653e-06,
+      "logits/chosen": -2.673206090927124,
+      "logits/rejected": -2.595384120941162,
+      "logps/chosen": -291.8248596191406,
+      "logps/rejected": -307.75738525390625,
+      "loss": 0.6002,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.19237418472766876,
+      "rewards/margins": 0.2755395770072937,
+      "rewards/rejected": -0.46791377663612366,
+      "step": 5070
+    },
+    {
+      "epoch": 0.66,
+      "learning_rate": 1.5255146945712267e-06,
+      "logits/chosen": -2.6262969970703125,
+      "logits/rejected": -2.6446533203125,
+      "logps/chosen": -265.8119201660156,
+      "logps/rejected": -287.59539794921875,
+      "loss": 0.5663,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.09747324883937836,
+      "rewards/margins": 0.3462264835834503,
+      "rewards/rejected": -0.44369974732398987,
+      "step": 5080
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.5150075988320594e-06,
+      "logits/chosen": -2.6456024646759033,
+      "logits/rejected": -2.522012233734131,
+      "logps/chosen": -271.15826416015625,
+      "logps/rejected": -264.8545837402344,
+      "loss": 0.5669,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.1582212597131729,
+      "rewards/margins": 0.3966771960258484,
+      "rewards/rejected": -0.5548985004425049,
+      "step": 5090
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.5045210588662929e-06,
+      "logits/chosen": -2.5904409885406494,
+      "logits/rejected": -2.586203098297119,
+      "logps/chosen": -332.86224365234375,
+      "logps/rejected": -337.92608642578125,
+      "loss": 0.6118,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.32937946915626526,
+      "rewards/margins": 0.2656363844871521,
+      "rewards/rejected": -0.595015823841095,
+      "step": 5100
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4940552935171781e-06,
+      "logits/chosen": -2.5327394008636475,
+      "logits/rejected": -2.482840061187744,
+      "logps/chosen": -302.0166015625,
+      "logps/rejected": -300.63214111328125,
+      "loss": 0.581,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.18903402984142303,
+      "rewards/margins": 0.36789780855178833,
+      "rewards/rejected": -0.5569317936897278,
+      "step": 5110
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.483610521194419e-06,
+      "logits/chosen": -2.536972999572754,
+      "logits/rejected": -2.491436719894409,
+      "logps/chosen": -250.9703826904297,
+      "logps/rejected": -276.5525817871094,
+      "loss": 0.5551,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.14237363636493683,
+      "rewards/margins": 0.4033837914466858,
+      "rewards/rejected": -0.5457574129104614,
+      "step": 5120
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4731869598696226e-06,
+      "logits/chosen": -2.6560111045837402,
+      "logits/rejected": -2.6021218299865723,
+      "logps/chosen": -344.84051513671875,
+      "logps/rejected": -338.4281921386719,
+      "loss": 0.5891,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.1760130524635315,
+      "rewards/margins": 0.3430495858192444,
+      "rewards/rejected": -0.5190626382827759,
+      "step": 5130
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4627848270717387e-06,
+      "logits/chosen": -2.6770198345184326,
+      "logits/rejected": -2.619194269180298,
+      "logps/chosen": -296.8146667480469,
+      "logps/rejected": -309.3045654296875,
+      "loss": 0.5704,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.14888279139995575,
+      "rewards/margins": 0.354114294052124,
+      "rewards/rejected": -0.502997100353241,
+      "step": 5140
+    },
+    {
+      "epoch": 0.67,
+      "learning_rate": 1.4524043398825277e-06,
+      "logits/chosen": -2.6568408012390137,
+      "logits/rejected": -2.4984958171844482,
+      "logps/chosen": -315.2223205566406,
+      "logps/rejected": -251.4141845703125,
+      "loss": 0.5596,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.09217970073223114,
+      "rewards/margins": 0.3866090774536133,
+      "rewards/rejected": -0.4787887930870056,
+      "step": 5150
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.4420457149320299e-06,
+      "logits/chosen": -2.625624418258667,
+      "logits/rejected": -2.5355777740478516,
+      "logps/chosen": -323.06890869140625,
+      "logps/rejected": -326.4612121582031,
+      "loss": 0.5693,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.16367566585540771,
+      "rewards/margins": 0.3551883101463318,
+      "rewards/rejected": -0.5188639760017395,
+      "step": 5160
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.431709168394042e-06,
+      "logits/chosen": -2.590869903564453,
+      "logits/rejected": -2.543855667114258,
+      "logps/chosen": -268.83270263671875,
+      "logps/rejected": -298.75592041015625,
+      "loss": 0.6124,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.12446723878383636,
+      "rewards/margins": 0.280962735414505,
+      "rewards/rejected": -0.4054299294948578,
+      "step": 5170
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.4213949159816059e-06,
+      "logits/chosen": -2.4375617504119873,
+      "logits/rejected": -2.4001147747039795,
+      "logps/chosen": -283.09033203125,
+      "logps/rejected": -331.08502197265625,
+      "loss": 0.5786,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.13570234179496765,
+      "rewards/margins": 0.3792361617088318,
+      "rewards/rejected": -0.514938473701477,
+      "step": 5180
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.4111031729425103e-06,
+      "logits/chosen": -2.5992751121520996,
+      "logits/rejected": -2.5536887645721436,
+      "logps/chosen": -303.00921630859375,
+      "logps/rejected": -328.16204833984375,
+      "loss": 0.5596,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.07691922038793564,
+      "rewards/margins": 0.38752609491348267,
+      "rewards/rejected": -0.4644453525543213,
+      "step": 5190
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.4008341540547965e-06,
+      "logits/chosen": -2.5871784687042236,
+      "logits/rejected": -2.557894229888916,
+      "logps/chosen": -271.25,
+      "logps/rejected": -255.15151977539062,
+      "loss": 0.5847,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.24902841448783875,
+      "rewards/margins": 0.3107166290283203,
+      "rewards/rejected": -0.5597450137138367,
+      "step": 5200
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.3905880736222737e-06,
+      "logits/chosen": -2.5511081218719482,
+      "logits/rejected": -2.504673719406128,
+      "logps/chosen": -268.77154541015625,
+      "logps/rejected": -278.37530517578125,
+      "loss": 0.581,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.21523432433605194,
+      "rewards/margins": 0.33661192655563354,
+      "rewards/rejected": -0.5518463253974915,
+      "step": 5210
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.3803651454700531e-06,
+      "logits/chosen": -2.619854688644409,
+      "logits/rejected": -2.5721137523651123,
+      "logps/chosen": -320.8299255371094,
+      "logps/rejected": -317.83563232421875,
+      "loss": 0.5612,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.17192170023918152,
+      "rewards/margins": 0.3536994457244873,
+      "rewards/rejected": -0.5256211161613464,
+      "step": 5220
+    },
+    {
+      "epoch": 0.68,
+      "learning_rate": 1.3701655829400773e-06,
+      "logits/chosen": -2.573817253112793,
+      "logits/rejected": -2.533365249633789,
+      "logps/chosen": -279.6182861328125,
+      "logps/rejected": -300.87213134765625,
+      "loss": 0.5956,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.17752423882484436,
+      "rewards/margins": 0.3138487637042999,
+      "rewards/rejected": -0.4913730025291443,
+      "step": 5230
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3599895988866756e-06,
+      "logits/chosen": -2.557749032974243,
+      "logits/rejected": -2.5533432960510254,
+      "logps/chosen": -264.18231201171875,
+      "logps/rejected": -313.55218505859375,
+      "loss": 0.5474,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.17697152495384216,
+      "rewards/margins": 0.4113278388977051,
+      "rewards/rejected": -0.5882993936538696,
+      "step": 5240
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3498374056721198e-06,
+      "logits/chosen": -2.553957939147949,
+      "logits/rejected": -2.481309652328491,
+      "logps/chosen": -338.162109375,
+      "logps/rejected": -301.181640625,
+      "loss": 0.5866,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.1477443426847458,
+      "rewards/margins": 0.3318925201892853,
+      "rewards/rejected": -0.4796368479728699,
+      "step": 5250
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3397092151621883e-06,
+      "logits/chosen": -2.5298349857330322,
+      "logits/rejected": -2.472118616104126,
+      "logps/chosen": -310.6473693847656,
+      "logps/rejected": -345.79522705078125,
+      "loss": 0.6043,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.32343584299087524,
+      "rewards/margins": 0.2905663549900055,
+      "rewards/rejected": -0.6140021085739136,
+      "step": 5260
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3296052387217484e-06,
+      "logits/chosen": -2.622016429901123,
+      "logits/rejected": -2.595959186553955,
+      "logps/chosen": -331.9591369628906,
+      "logps/rejected": -338.8408203125,
+      "loss": 0.5718,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.20151858031749725,
+      "rewards/margins": 0.36134520173072815,
+      "rewards/rejected": -0.5628637075424194,
+      "step": 5270
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3195256872103476e-06,
+      "logits/chosen": -2.6712913513183594,
+      "logits/rejected": -2.5495076179504395,
+      "logps/chosen": -365.97894287109375,
+      "logps/rejected": -299.7531433105469,
+      "loss": 0.5592,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.19577732682228088,
+      "rewards/margins": 0.3793767988681793,
+      "rewards/rejected": -0.5751541256904602,
+      "step": 5280
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.3094707709778068e-06,
+      "logits/chosen": -2.6107308864593506,
+      "logits/rejected": -2.5592257976531982,
+      "logps/chosen": -337.65728759765625,
+      "logps/rejected": -341.4754638671875,
+      "loss": 0.6154,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.3293580710887909,
+      "rewards/margins": 0.2625970244407654,
+      "rewards/rejected": -0.5919550657272339,
+      "step": 5290
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.2994406998598364e-06,
+      "logits/chosen": -2.6446738243103027,
+      "logits/rejected": -2.597778797149658,
+      "logps/chosen": -307.8355407714844,
+      "logps/rejected": -302.91131591796875,
+      "loss": 0.5938,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.1960253119468689,
+      "rewards/margins": 0.28771188855171204,
+      "rewards/rejected": -0.4837372899055481,
+      "step": 5300
+    },
+    {
+      "epoch": 0.69,
+      "learning_rate": 1.2894356831736558e-06,
+      "logits/chosen": -2.667022466659546,
+      "logits/rejected": -2.6112124919891357,
+      "logps/chosen": -341.14630126953125,
+      "logps/rejected": -328.5607604980469,
+      "loss": 0.5862,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.15319842100143433,
+      "rewards/margins": 0.3334108293056488,
+      "rewards/rejected": -0.4866093099117279,
+      "step": 5310
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2794559297136203e-06,
+      "logits/chosen": -2.5267646312713623,
+      "logits/rejected": -2.4591567516326904,
+      "logps/chosen": -325.8916931152344,
+      "logps/rejected": -345.9197082519531,
+      "loss": 0.6742,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -0.344811350107193,
+      "rewards/margins": 0.16068735718727112,
+      "rewards/rejected": -0.5054987668991089,
+      "step": 5320
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2695016477468724e-06,
+      "logits/chosen": -2.550507068634033,
+      "logits/rejected": -2.5594980716705322,
+      "logps/chosen": -271.2611999511719,
+      "logps/rejected": -327.543212890625,
+      "loss": 0.5415,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.1822739541530609,
+      "rewards/margins": 0.4413815438747406,
+      "rewards/rejected": -0.6236556172370911,
+      "step": 5330
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2595730450089874e-06,
+      "logits/chosen": -2.6637372970581055,
+      "logits/rejected": -2.6175498962402344,
+      "logps/chosen": -342.701416015625,
+      "logps/rejected": -333.0793762207031,
+      "loss": 0.5338,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.12556394934654236,
+      "rewards/margins": 0.4395695626735687,
+      "rewards/rejected": -0.5651335120201111,
+      "step": 5340
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2496703286996433e-06,
+      "logits/chosen": -2.6627414226531982,
+      "logits/rejected": -2.572467565536499,
+      "logps/chosen": -321.95806884765625,
+      "logps/rejected": -337.3905944824219,
+      "loss": 0.6194,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.24837830662727356,
+      "rewards/margins": 0.2546219825744629,
+      "rewards/rejected": -0.5030002593994141,
+      "step": 5350
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2397937054782961e-06,
+      "logits/chosen": -2.683563709259033,
+      "logits/rejected": -2.602421283721924,
+      "logps/chosen": -298.49737548828125,
+      "logps/rejected": -306.6959533691406,
+      "loss": 0.6369,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.2561039328575134,
+      "rewards/margins": 0.23970231413841248,
+      "rewards/rejected": -0.4958062767982483,
+      "step": 5360
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2299433814598635e-06,
+      "logits/chosen": -2.6439998149871826,
+      "logits/rejected": -2.6128690242767334,
+      "logps/chosen": -340.18988037109375,
+      "logps/rejected": -336.3358459472656,
+      "loss": 0.6406,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.271611750125885,
+      "rewards/margins": 0.1806417852640152,
+      "rewards/rejected": -0.452253520488739,
+      "step": 5370
+    },
+    {
+      "epoch": 0.7,
+      "learning_rate": 1.2201195622104265e-06,
+      "logits/chosen": -2.587672233581543,
+      "logits/rejected": -2.517838478088379,
+      "logps/chosen": -305.00048828125,
+      "logps/rejected": -326.0105895996094,
+      "loss": 0.559,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.18532529473304749,
+      "rewards/margins": 0.3918299973011017,
+      "rewards/rejected": -0.5771552324295044,
+      "step": 5380
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.2103224527429417e-06,
+      "logits/chosen": -2.6174709796905518,
+      "logits/rejected": -2.636540174484253,
+      "logps/chosen": -325.8754577636719,
+      "logps/rejected": -335.9193420410156,
+      "loss": 0.5868,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.18154212832450867,
+      "rewards/margins": 0.3239831328392029,
+      "rewards/rejected": -0.5055252313613892,
+      "step": 5390
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.2005522575129559e-06,
+      "logits/chosen": -2.678755521774292,
+      "logits/rejected": -2.6201252937316895,
+      "logps/chosen": -325.0459289550781,
+      "logps/rejected": -350.9663391113281,
+      "loss": 0.5472,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.14544162154197693,
+      "rewards/margins": 0.42693695425987244,
+      "rewards/rejected": -0.5723786950111389,
+      "step": 5400
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1908091804143469e-06,
+      "logits/chosen": -2.5132193565368652,
+      "logits/rejected": -2.4828622341156006,
+      "logps/chosen": -291.1443786621094,
+      "logps/rejected": -306.60711669921875,
+      "loss": 0.568,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.17812040448188782,
+      "rewards/margins": 0.3672374486923218,
+      "rewards/rejected": -0.5453578233718872,
+      "step": 5410
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1810934247750649e-06,
+      "logits/chosen": -2.5312628746032715,
+      "logits/rejected": -2.449885845184326,
+      "logps/chosen": -338.0185546875,
+      "logps/rejected": -310.1617126464844,
+      "loss": 0.6034,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.2280372828245163,
+      "rewards/margins": 0.30007949471473694,
+      "rewards/rejected": -0.5281167030334473,
+      "step": 5420
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.1714051933528881e-06,
+      "logits/chosen": -2.5902011394500732,
+      "logits/rejected": -2.5328450202941895,
+      "logps/chosen": -307.8944091796875,
+      "logps/rejected": -325.43408203125,
+      "loss": 0.5485,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.20920448005199432,
+      "rewards/margins": 0.4587844908237457,
+      "rewards/rejected": -0.6679890751838684,
+      "step": 5430
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.161744688331192e-06,
+      "logits/chosen": -2.532405138015747,
+      "logits/rejected": -2.4625115394592285,
+      "logps/chosen": -313.19378662109375,
+      "logps/rejected": -319.1415710449219,
+      "loss": 0.574,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.18393780291080475,
+      "rewards/margins": 0.34623345732688904,
+      "rewards/rejected": -0.530171275138855,
+      "step": 5440
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.152112111314733e-06,
+      "logits/chosen": -2.5994606018066406,
+      "logits/rejected": -2.6417124271392822,
+      "logps/chosen": -311.75872802734375,
+      "logps/rejected": -318.92266845703125,
+      "loss": 0.618,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.267758309841156,
+      "rewards/margins": 0.24568963050842285,
+      "rewards/rejected": -0.5134479999542236,
+      "step": 5450
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 1.142507663325439e-06,
+      "logits/chosen": -2.5682082176208496,
+      "logits/rejected": -2.5433108806610107,
+      "logps/chosen": -311.25933837890625,
+      "logps/rejected": -303.6724853515625,
+      "loss": 0.5545,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.1535353809595108,
+      "rewards/margins": 0.42988795042037964,
+      "rewards/rejected": -0.583423376083374,
+      "step": 5460
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.132931544798211e-06,
+      "logits/chosen": -2.6040048599243164,
+      "logits/rejected": -2.523660659790039,
+      "logps/chosen": -317.60015869140625,
+      "logps/rejected": -283.523193359375,
+      "loss": 0.5717,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.20379884541034698,
+      "rewards/margins": 0.3608744740486145,
+      "rewards/rejected": -0.5646733045578003,
+      "step": 5470
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.1233839555767482e-06,
+      "logits/chosen": -2.5608572959899902,
+      "logits/rejected": -2.5681984424591064,
+      "logps/chosen": -262.3978271484375,
+      "logps/rejected": -316.1744689941406,
+      "loss": 0.5685,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.20917022228240967,
+      "rewards/margins": 0.3712260127067566,
+      "rewards/rejected": -0.580396294593811,
+      "step": 5480
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.1138650949093668e-06,
+      "logits/chosen": -2.6220974922180176,
+      "logits/rejected": -2.594712495803833,
+      "logps/chosen": -274.40673828125,
+      "logps/rejected": -309.0545349121094,
+      "loss": 0.6097,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.20119595527648926,
+      "rewards/margins": 0.2615039348602295,
+      "rewards/rejected": -0.462699830532074,
+      "step": 5490
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.1043751614448543e-06,
+      "logits/chosen": -2.5410258769989014,
+      "logits/rejected": -2.5268661975860596,
+      "logps/chosen": -305.63446044921875,
+      "logps/rejected": -333.3314514160156,
+      "loss": 0.6291,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.287882536649704,
+      "rewards/margins": 0.2548755407333374,
+      "rewards/rejected": -0.5427581071853638,
+      "step": 5500
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.0949143532283107e-06,
+      "logits/chosen": -2.5882506370544434,
+      "logits/rejected": -2.498617172241211,
+      "logps/chosen": -287.95355224609375,
+      "logps/rejected": -294.55230712890625,
+      "loss": 0.5413,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.19452597200870514,
+      "rewards/margins": 0.44513052701950073,
+      "rewards/rejected": -0.6396564841270447,
+      "step": 5510
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.0854828676970275e-06,
+      "logits/chosen": -2.529478073120117,
+      "logits/rejected": -2.496150493621826,
+      "logps/chosen": -339.60247802734375,
+      "logps/rejected": -292.39886474609375,
+      "loss": 0.6112,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.2950858473777771,
+      "rewards/margins": 0.2816055119037628,
+      "rewards/rejected": -0.5766913294792175,
+      "step": 5520
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.076080901676361e-06,
+      "logits/chosen": -2.547044277191162,
+      "logits/rejected": -2.6276450157165527,
+      "logps/chosen": -299.1795349121094,
+      "logps/rejected": -356.1458740234375,
+      "loss": 0.6406,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.30218222737312317,
+      "rewards/margins": 0.19424603879451752,
+      "rewards/rejected": -0.49642831087112427,
+      "step": 5530
+    },
+    {
+      "epoch": 0.72,
+      "learning_rate": 1.0667086513756234e-06,
+      "logits/chosen": -2.5066745281219482,
+      "logits/rejected": -2.5055992603302,
+      "logps/chosen": -255.1097412109375,
+      "logps/rejected": -274.110107421875,
+      "loss": 0.5811,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.22848305106163025,
+      "rewards/margins": 0.324868381023407,
+      "rewards/rejected": -0.5533514618873596,
+      "step": 5540
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0573663123839912e-06,
+      "logits/chosen": -2.582517385482788,
+      "logits/rejected": -2.5165748596191406,
+      "logps/chosen": -249.65625,
+      "logps/rejected": -283.4542236328125,
+      "loss": 0.5694,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.23387250304222107,
+      "rewards/margins": 0.34701576828956604,
+      "rewards/rejected": -0.5808882117271423,
+      "step": 5550
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0480540796664251e-06,
+      "logits/chosen": -2.6427597999572754,
+      "logits/rejected": -2.598210573196411,
+      "logps/chosen": -353.24908447265625,
+      "logps/rejected": -361.3995666503906,
+      "loss": 0.6047,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.20503298938274384,
+      "rewards/margins": 0.302786648273468,
+      "rewards/rejected": -0.507819652557373,
+      "step": 5560
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0387721475595978e-06,
+      "logits/chosen": -2.6316816806793213,
+      "logits/rejected": -2.5892982482910156,
+      "logps/chosen": -342.3216552734375,
+      "logps/rejected": -330.4527282714844,
+      "loss": 0.6151,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.2715100049972534,
+      "rewards/margins": 0.2680637240409851,
+      "rewards/rejected": -0.5395737290382385,
+      "step": 5570
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0295207097678378e-06,
+      "logits/chosen": -2.635214328765869,
+      "logits/rejected": -2.522156238555908,
+      "logps/chosen": -302.3963623046875,
+      "logps/rejected": -285.1310119628906,
+      "loss": 0.6048,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.22624202072620392,
+      "rewards/margins": 0.2882040739059448,
+      "rewards/rejected": -0.5144461393356323,
+      "step": 5580
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0202999593590924e-06,
+      "logits/chosen": -2.5209271907806396,
+      "logits/rejected": -2.4779646396636963,
+      "logps/chosen": -272.66864013671875,
+      "logps/rejected": -251.2705078125,
+      "loss": 0.6097,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.20430497825145721,
+      "rewards/margins": 0.2564670443534851,
+      "rewards/rejected": -0.46077197790145874,
+      "step": 5590
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.011110088760891e-06,
+      "logits/chosen": -2.6857237815856934,
+      "logits/rejected": -2.5868523120880127,
+      "logps/chosen": -303.2496032714844,
+      "logps/rejected": -287.9459533691406,
+      "loss": 0.5711,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.14075031876564026,
+      "rewards/margins": 0.3630533814430237,
+      "rewards/rejected": -0.5038037300109863,
+      "step": 5600
+    },
+    {
+      "epoch": 0.73,
+      "learning_rate": 1.0019512897563347e-06,
+      "logits/chosen": -2.4832537174224854,
+      "logits/rejected": -2.4886534214019775,
+      "logps/chosen": -258.8486022949219,
+      "logps/rejected": -313.70648193359375,
+      "loss": 0.5195,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.11117883026599884,
+      "rewards/margins": 0.49211806058883667,
+      "rewards/rejected": -0.6032968759536743,
+      "step": 5610
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.928237534800935e-07,
+      "logits/chosen": -2.643057346343994,
+      "logits/rejected": -2.5183494091033936,
+      "logps/chosen": -311.29998779296875,
+      "logps/rejected": -300.5015563964844,
+      "loss": 0.5801,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.229877308011055,
+      "rewards/margins": 0.33290043473243713,
+      "rewards/rejected": -0.5627778172492981,
+      "step": 5620
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.837276704144174e-07,
+      "logits/chosen": -2.5406341552734375,
+      "logits/rejected": -2.5741326808929443,
+      "logps/chosen": -255.7300262451172,
+      "logps/rejected": -317.2926025390625,
+      "loss": 0.6382,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.2534419596195221,
+      "rewards/margins": 0.21565183997154236,
+      "rewards/rejected": -0.46909385919570923,
+      "step": 5630
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.746632303851569e-07,
+      "logits/chosen": -2.5920422077178955,
+      "logits/rejected": -2.5517804622650146,
+      "logps/chosen": -315.4493713378906,
+      "logps/rejected": -300.3988952636719,
+      "loss": 0.6074,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.12963609397411346,
+      "rewards/margins": 0.26822197437286377,
+      "rewards/rejected": -0.39785805344581604,
+      "step": 5640
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.65630622557809e-07,
+      "logits/chosen": -2.5037996768951416,
+      "logits/rejected": -2.50010347366333,
+      "logps/chosen": -277.56536865234375,
+      "logps/rejected": -300.10882568359375,
+      "loss": 0.5944,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.16001948714256287,
+      "rewards/margins": 0.3253334164619446,
+      "rewards/rejected": -0.48535284399986267,
+      "step": 5650
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.56630035433561e-07,
+      "logits/chosen": -2.6377015113830566,
+      "logits/rejected": -2.534681797027588,
+      "logps/chosen": -304.823486328125,
+      "logps/rejected": -307.82098388671875,
+      "loss": 0.5548,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.16076910495758057,
+      "rewards/margins": 0.38310015201568604,
+      "rewards/rejected": -0.5438693165779114,
+      "step": 5660
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.476616568453659e-07,
+      "logits/chosen": -2.5745491981506348,
+      "logits/rejected": -2.5050041675567627,
+      "logps/chosen": -299.784423828125,
+      "logps/rejected": -288.919189453125,
+      "loss": 0.624,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.23026995360851288,
+      "rewards/margins": 0.2685932219028473,
+      "rewards/rejected": -0.498863160610199,
+      "step": 5670
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.387256739540162e-07,
+      "logits/chosen": -2.539517879486084,
+      "logits/rejected": -2.4560577869415283,
+      "logps/chosen": -250.90402221679688,
+      "logps/rejected": -250.80258178710938,
+      "loss": 0.5879,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.12853455543518066,
+      "rewards/margins": 0.3143445551395416,
+      "rewards/rejected": -0.4428791105747223,
+      "step": 5680
+    },
+    {
+      "epoch": 0.74,
+      "learning_rate": 9.298222732442377e-07,
+      "logits/chosen": -2.603879928588867,
+      "logits/rejected": -2.5718045234680176,
+      "logps/chosen": -276.7682800292969,
+      "logps/rejected": -335.01904296875,
+      "loss": 0.5463,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.11522575467824936,
+      "rewards/margins": 0.43045562505722046,
+      "rewards/rejected": -0.5456814169883728,
+      "step": 5690
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 9.20951640520803e-07,
+      "logits/chosen": -2.705848455429077,
+      "logits/rejected": -2.631455898284912,
+      "logps/chosen": -325.058349609375,
+      "logps/rejected": -351.39471435546875,
+      "loss": 0.5691,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.11381908506155014,
+      "rewards/margins": 0.3463020622730255,
+      "rewards/rejected": -0.46012115478515625,
+      "step": 5700
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 9.121139609046484e-07,
+      "logits/chosen": -2.690091133117676,
+      "logits/rejected": -2.5844922065734863,
+      "logps/chosen": -287.170654296875,
+      "logps/rejected": -288.2826232910156,
+      "loss": 0.5985,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.1888628453016281,
+      "rewards/margins": 0.2854076623916626,
+      "rewards/rejected": -0.47427043318748474,
+      "step": 5710
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 9.033094188290121e-07,
+      "logits/chosen": -2.6597189903259277,
+      "logits/rejected": -2.6318678855895996,
+      "logps/chosen": -288.18145751953125,
+      "logps/rejected": -340.81793212890625,
+      "loss": 0.548,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.11018621921539307,
+      "rewards/margins": 0.44151702523231506,
+      "rewards/rejected": -0.551703155040741,
+      "step": 5720
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.945381980355889e-07,
+      "logits/chosen": -2.632452964782715,
+      "logits/rejected": -2.5865681171417236,
+      "logps/chosen": -303.8258361816406,
+      "logps/rejected": -318.6636047363281,
+      "loss": 0.64,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.1766664832830429,
+      "rewards/margins": 0.1963653266429901,
+      "rewards/rejected": -0.3730318546295166,
+      "step": 5730
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.858004815706919e-07,
+      "logits/chosen": -2.5732266902923584,
+      "logits/rejected": -2.576760768890381,
+      "logps/chosen": -251.0210723876953,
+      "logps/rejected": -290.65118408203125,
+      "loss": 0.5697,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.07114624232053757,
+      "rewards/margins": 0.37384241819381714,
+      "rewards/rejected": -0.4449886679649353,
+      "step": 5740
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.77096451781432e-07,
+      "logits/chosen": -2.506803512573242,
+      "logits/rejected": -2.512962818145752,
+      "logps/chosen": -291.24688720703125,
+      "logps/rejected": -293.74359130859375,
+      "loss": 0.5704,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.17455434799194336,
+      "rewards/margins": 0.35992860794067383,
+      "rewards/rejected": -0.5344828367233276,
+      "step": 5750
+    },
+    {
+      "epoch": 0.75,
+      "learning_rate": 8.684262903119165e-07,
+      "logits/chosen": -2.591378688812256,
+      "logits/rejected": -2.491837739944458,
+      "logps/chosen": -323.7724609375,
+      "logps/rejected": -289.60626220703125,
+      "loss": 0.5897,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.18045367300510406,
+      "rewards/margins": 0.3245195746421814,
+      "rewards/rejected": -0.504973292350769,
+      "step": 5760
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.597901780994525e-07,
+      "logits/chosen": -2.516756534576416,
+      "logits/rejected": -2.475712299346924,
+      "logps/chosen": -309.0379638671875,
+      "logps/rejected": -342.54522705078125,
+      "loss": 0.5631,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.10879604518413544,
+      "rewards/margins": 0.3767804801464081,
+      "rewards/rejected": -0.4855765402317047,
+      "step": 5770
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.511882953707773e-07,
+      "logits/chosen": -2.529524564743042,
+      "logits/rejected": -2.5715365409851074,
+      "logps/chosen": -296.8482360839844,
+      "logps/rejected": -326.4384460449219,
+      "loss": 0.5835,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.1463620364665985,
+      "rewards/margins": 0.35980477929115295,
+      "rewards/rejected": -0.5061668157577515,
+      "step": 5780
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.426208216382944e-07,
+      "logits/chosen": -2.4196619987487793,
+      "logits/rejected": -2.4203577041625977,
+      "logps/chosen": -242.55410766601562,
+      "logps/rejected": -301.5292663574219,
+      "loss": 0.601,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.13889290392398834,
+      "rewards/margins": 0.3272313177585602,
+      "rewards/rejected": -0.46612420678138733,
+      "step": 5790
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.340879356963245e-07,
+      "logits/chosen": -2.6108078956604004,
+      "logits/rejected": -2.5446953773498535,
+      "logps/chosen": -258.63873291015625,
+      "logps/rejected": -297.4355163574219,
+      "loss": 0.5885,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.14311614632606506,
+      "rewards/margins": 0.3221714198589325,
+      "rewards/rejected": -0.46528759598731995,
+      "step": 5800
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.255898156173777e-07,
+      "logits/chosen": -2.522681474685669,
+      "logits/rejected": -2.5140135288238525,
+      "logps/chosen": -352.51348876953125,
+      "logps/rejected": -326.9130859375,
+      "loss": 0.6383,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.2855757772922516,
+      "rewards/margins": 0.209019273519516,
+      "rewards/rejected": -0.49459511041641235,
+      "step": 5810
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.171266387484389e-07,
+      "logits/chosen": -2.555227518081665,
+      "logits/rejected": -2.5240612030029297,
+      "logps/chosen": -302.25958251953125,
+      "logps/rejected": -341.8251037597656,
+      "loss": 0.6121,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.19107194244861603,
+      "rewards/margins": 0.2737434506416321,
+      "rewards/rejected": -0.4648153781890869,
+      "step": 5820
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.086985817072604e-07,
+      "logits/chosen": -2.5600576400756836,
+      "logits/rejected": -2.5165228843688965,
+      "logps/chosen": -269.22540283203125,
+      "logps/rejected": -291.1938171386719,
+      "loss": 0.5308,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.09094101190567017,
+      "rewards/margins": 0.4695923328399658,
+      "rewards/rejected": -0.5605332851409912,
+      "step": 5830
+    },
+    {
+      "epoch": 0.76,
+      "learning_rate": 8.003058203786835e-07,
+      "logits/chosen": -2.605053663253784,
+      "logits/rejected": -2.524534225463867,
+      "logps/chosen": -286.05889892578125,
+      "logps/rejected": -272.46343994140625,
+      "loss": 0.6303,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.1592210829257965,
+      "rewards/margins": 0.21416108310222626,
+      "rewards/rejected": -0.3733821511268616,
+      "step": 5840
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.91948529910963e-07,
+      "logits/chosen": -2.653329849243164,
+      "logits/rejected": -2.5637519359588623,
+      "logps/chosen": -368.62127685546875,
+      "logps/rejected": -339.4736022949219,
+      "loss": 0.6069,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.2027558833360672,
+      "rewards/margins": 0.30033525824546814,
+      "rewards/rejected": -0.5030912160873413,
+      "step": 5850
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.836268847121126e-07,
+      "logits/chosen": -2.5435237884521484,
+      "logits/rejected": -2.476839542388916,
+      "logps/chosen": -311.3453674316406,
+      "logps/rejected": -327.08319091796875,
+      "loss": 0.5411,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.11651796102523804,
+      "rewards/margins": 0.4157235026359558,
+      "rewards/rejected": -0.5322414636611938,
+      "step": 5860
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.753410584462681e-07,
+      "logits/chosen": -2.576029062271118,
+      "logits/rejected": -2.5582990646362305,
+      "logps/chosen": -315.1184997558594,
+      "logps/rejected": -363.74554443359375,
+      "loss": 0.5573,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.23594622313976288,
+      "rewards/margins": 0.37643688917160034,
+      "rewards/rejected": -0.6123830676078796,
+      "step": 5870
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.670912240300596e-07,
+      "logits/chosen": -2.5990452766418457,
+      "logits/rejected": -2.5202131271362305,
+      "logps/chosen": -296.40325927734375,
+      "logps/rejected": -349.1062927246094,
+      "loss": 0.5995,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.19773322343826294,
+      "rewards/margins": 0.3164207339286804,
+      "rewards/rejected": -0.5141538381576538,
+      "step": 5880
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.588775536290035e-07,
+      "logits/chosen": -2.6614136695861816,
+      "logits/rejected": -2.616184949874878,
+      "logps/chosen": -309.0333251953125,
+      "logps/rejected": -315.36004638671875,
+      "loss": 0.613,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.17394475638866425,
+      "rewards/margins": 0.2682701647281647,
+      "rewards/rejected": -0.4422149062156677,
+      "step": 5890
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.507002186539147e-07,
+      "logits/chosen": -2.6625893115997314,
+      "logits/rejected": -2.6152846813201904,
+      "logps/chosen": -319.224853515625,
+      "logps/rejected": -331.1502685546875,
+      "loss": 0.5806,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.19561751186847687,
+      "rewards/margins": 0.35961249470710754,
+      "rewards/rejected": -0.5552300214767456,
+      "step": 5900
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.425593897573216e-07,
+      "logits/chosen": -2.5605714321136475,
+      "logits/rejected": -2.461122989654541,
+      "logps/chosen": -285.85455322265625,
+      "logps/rejected": -312.09686279296875,
+      "loss": 0.6015,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.18572023510932922,
+      "rewards/margins": 0.3016360402107239,
+      "rewards/rejected": -0.4873562753200531,
+      "step": 5910
+    },
+    {
+      "epoch": 0.77,
+      "learning_rate": 7.344552368299088e-07,
+      "logits/chosen": -2.5203638076782227,
+      "logits/rejected": -2.4832592010498047,
+      "logps/chosen": -274.9449157714844,
+      "logps/rejected": -308.3414306640625,
+      "loss": 0.5837,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.24539025127887726,
+      "rewards/margins": 0.31899529695510864,
+      "rewards/rejected": -0.5643855333328247,
+      "step": 5920
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 7.26387928996973e-07,
+      "logits/chosen": -2.534534454345703,
+      "logits/rejected": -2.4488308429718018,
+      "logps/chosen": -300.56829833984375,
+      "logps/rejected": -287.57135009765625,
+      "loss": 0.6311,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.22414541244506836,
+      "rewards/margins": 0.21638643741607666,
+      "rewards/rejected": -0.44053182005882263,
+      "step": 5930
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 7.183576346148899e-07,
+      "logits/chosen": -2.5961291790008545,
+      "logits/rejected": -2.5696041584014893,
+      "logps/chosen": -321.9046936035156,
+      "logps/rejected": -336.34478759765625,
+      "loss": 0.5581,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.14284758269786835,
+      "rewards/margins": 0.3697184920310974,
+      "rewards/rejected": -0.512566089630127,
+      "step": 5940
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 7.103645212676044e-07,
+      "logits/chosen": -2.607748508453369,
+      "logits/rejected": -2.5658648014068604,
+      "logps/chosen": -323.215087890625,
+      "logps/rejected": -324.53887939453125,
+      "loss": 0.5633,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.09434620290994644,
+      "rewards/margins": 0.3797812759876251,
+      "rewards/rejected": -0.4741274416446686,
+      "step": 5950
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 7.024087557631318e-07,
+      "logits/chosen": -2.6633009910583496,
+      "logits/rejected": -2.5637741088867188,
+      "logps/chosen": -304.86627197265625,
+      "logps/rejected": -316.8140563964844,
+      "loss": 0.576,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.20098192989826202,
+      "rewards/margins": 0.34813567996025085,
+      "rewards/rejected": -0.5491176247596741,
+      "step": 5960
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 6.944905041300739e-07,
+      "logits/chosen": -2.6593124866485596,
+      "logits/rejected": -2.5711112022399902,
+      "logps/chosen": -320.54888916015625,
+      "logps/rejected": -301.90277099609375,
+      "loss": 0.567,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.10521403700113297,
+      "rewards/margins": 0.3656689524650574,
+      "rewards/rejected": -0.4708830416202545,
+      "step": 5970
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 6.866099316141606e-07,
+      "logits/chosen": -2.541459083557129,
+      "logits/rejected": -2.5298831462860107,
+      "logps/chosen": -315.81048583984375,
+      "logps/rejected": -312.66412353515625,
+      "loss": 0.5833,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.14091964066028595,
+      "rewards/margins": 0.36008355021476746,
+      "rewards/rejected": -0.5010032057762146,
+      "step": 5980
+    },
+    {
+      "epoch": 0.78,
+      "learning_rate": 6.787672026747946e-07,
+      "logits/chosen": -2.522216320037842,
+      "logits/rejected": -2.4484128952026367,
+      "logps/chosen": -278.60675048828125,
+      "logps/rejected": -322.01544189453125,
+      "loss": 0.5326,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.05760330706834793,
+      "rewards/margins": 0.45164427161216736,
+      "rewards/rejected": -0.5092474818229675,
+      "step": 5990
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.709624809816223e-07,
+      "logits/chosen": -2.6187644004821777,
+      "logits/rejected": -2.5200893878936768,
+      "logps/chosen": -286.6143493652344,
+      "logps/rejected": -281.9729919433594,
+      "loss": 0.5839,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.1888885349035263,
+      "rewards/margins": 0.32074642181396484,
+      "rewards/rejected": -0.5096349716186523,
+      "step": 6000
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.6319592941112e-07,
+      "logits/chosen": -2.6047425270080566,
+      "logits/rejected": -2.5242390632629395,
+      "logps/chosen": -304.7151794433594,
+      "logps/rejected": -292.1768493652344,
+      "loss": 0.572,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.17885757982730865,
+      "rewards/margins": 0.3441098928451538,
+      "rewards/rejected": -0.5229673981666565,
+      "step": 6010
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.554677100431927e-07,
+      "logits/chosen": -2.590179443359375,
+      "logits/rejected": -2.5281975269317627,
+      "logps/chosen": -263.2532653808594,
+      "logps/rejected": -285.6487121582031,
+      "loss": 0.5354,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.10920965671539307,
+      "rewards/margins": 0.4652461111545563,
+      "rewards/rejected": -0.5744557976722717,
+      "step": 6020
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.4777798415779e-07,
+      "logits/chosen": -2.7149648666381836,
+      "logits/rejected": -2.6118805408477783,
+      "logps/chosen": -305.684814453125,
+      "logps/rejected": -302.0367126464844,
+      "loss": 0.5725,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.0624193474650383,
+      "rewards/margins": 0.35976505279541016,
+      "rewards/rejected": -0.42218437790870667,
+      "step": 6030
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.401269122315451e-07,
+      "logits/chosen": -2.577178955078125,
+      "logits/rejected": -2.5751290321350098,
+      "logps/chosen": -302.451171875,
+      "logps/rejected": -361.32611083984375,
+      "loss": 0.5799,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.16096945106983185,
+      "rewards/margins": 0.35466641187667847,
+      "rewards/rejected": -0.5156358480453491,
+      "step": 6040
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.325146539344196e-07,
+      "logits/chosen": -2.545144557952881,
+      "logits/rejected": -2.464059352874756,
+      "logps/chosen": -397.53546142578125,
+      "logps/rejected": -391.0182189941406,
+      "loss": 0.5695,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.21420156955718994,
+      "rewards/margins": 0.3888997435569763,
+      "rewards/rejected": -0.603101372718811,
+      "step": 6050
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.249413681263782e-07,
+      "logits/chosen": -2.5791893005371094,
+      "logits/rejected": -2.4865994453430176,
+      "logps/chosen": -237.5160675048828,
+      "logps/rejected": -324.67529296875,
+      "loss": 0.5774,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.1538625955581665,
+      "rewards/margins": 0.374470055103302,
+      "rewards/rejected": -0.5283327102661133,
+      "step": 6060
+    },
+    {
+      "epoch": 0.79,
+      "learning_rate": 6.174072128540686e-07,
+      "logits/chosen": -2.648714780807495,
+      "logits/rejected": -2.6040008068084717,
+      "logps/chosen": -324.8454284667969,
+      "logps/rejected": -296.79901123046875,
+      "loss": 0.6258,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.12855316698551178,
+      "rewards/margins": 0.24863454699516296,
+      "rewards/rejected": -0.37718766927719116,
+      "step": 6070
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 6.099123453475245e-07,
+      "logits/chosen": -2.590529441833496,
+      "logits/rejected": -2.4631597995758057,
+      "logps/chosen": -327.8921203613281,
+      "logps/rejected": -325.5595397949219,
+      "loss": 0.5882,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.20814700424671173,
+      "rewards/margins": 0.3109685182571411,
+      "rewards/rejected": -0.5191155076026917,
+      "step": 6080
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 6.024569220168836e-07,
+      "logits/chosen": -2.5863261222839355,
+      "logits/rejected": -2.4764561653137207,
+      "logps/chosen": -309.7364807128906,
+      "logps/rejected": -302.49139404296875,
+      "loss": 0.5302,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.03309829160571098,
+      "rewards/margins": 0.4651208519935608,
+      "rewards/rejected": -0.4982191026210785,
+      "step": 6090
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.950410984491268e-07,
+      "logits/chosen": -2.533797264099121,
+      "logits/rejected": -2.555459499359131,
+      "logps/chosen": -321.22027587890625,
+      "logps/rejected": -343.15863037109375,
+      "loss": 0.5704,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.13474434614181519,
+      "rewards/margins": 0.33791136741638184,
+      "rewards/rejected": -0.472655713558197,
+      "step": 6100
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.876650294048262e-07,
+      "logits/chosen": -2.604936361312866,
+      "logits/rejected": -2.5958733558654785,
+      "logps/chosen": -257.599853515625,
+      "logps/rejected": -290.68084716796875,
+      "loss": 0.6045,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.10048031806945801,
+      "rewards/margins": 0.2916949391365051,
+      "rewards/rejected": -0.39217525720596313,
+      "step": 6110
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.8032886881492e-07,
+      "logits/chosen": -2.5367000102996826,
+      "logits/rejected": -2.506078004837036,
+      "logps/chosen": -239.2827911376953,
+      "logps/rejected": -250.35690307617188,
+      "loss": 0.6124,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.14870566129684448,
+      "rewards/margins": 0.27639666199684143,
+      "rewards/rejected": -0.4251023232936859,
+      "step": 6120
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.730327697774988e-07,
+      "logits/chosen": -2.604822874069214,
+      "logits/rejected": -2.579761505126953,
+      "logps/chosen": -274.77471923828125,
+      "logps/rejected": -332.9385070800781,
+      "loss": 0.563,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.10271693766117096,
+      "rewards/margins": 0.374347060918808,
+      "rewards/rejected": -0.4770640432834625,
+      "step": 6130
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.657768845546068e-07,
+      "logits/chosen": -2.5598130226135254,
+      "logits/rejected": -2.498775005340576,
+      "logps/chosen": -230.98733520507812,
+      "logps/rejected": -259.582763671875,
+      "loss": 0.5956,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.13164815306663513,
+      "rewards/margins": 0.29687273502349854,
+      "rewards/rejected": -0.4285208582878113,
+      "step": 6140
+    },
+    {
+      "epoch": 0.8,
+      "learning_rate": 5.585613645690713e-07,
+      "logits/chosen": -2.577946901321411,
+      "logits/rejected": -2.504457950592041,
+      "logps/chosen": -275.3217468261719,
+      "logps/rejected": -318.43157958984375,
+      "loss": 0.5485,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -0.1482088714838028,
+      "rewards/margins": 0.4252738058567047,
+      "rewards/rejected": -0.5734826326370239,
+      "step": 6150
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.513863604013355e-07,
+      "logits/chosen": -2.637294292449951,
+      "logits/rejected": -2.5249412059783936,
+      "logps/chosen": -329.2545166015625,
+      "logps/rejected": -302.53997802734375,
+      "loss": 0.6297,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.23045971989631653,
+      "rewards/margins": 0.2257537841796875,
+      "rewards/rejected": -0.4562135338783264,
+      "step": 6160
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.442520217863215e-07,
+      "logits/chosen": -2.5901215076446533,
+      "logits/rejected": -2.541841983795166,
+      "logps/chosen": -278.38262939453125,
+      "logps/rejected": -288.7743225097656,
+      "loss": 0.6186,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -0.18791256844997406,
+      "rewards/margins": 0.26036161184310913,
+      "rewards/rejected": -0.4482742249965668,
+      "step": 6170
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.371584976103034e-07,
+      "logits/chosen": -2.607125759124756,
+      "logits/rejected": -2.5333781242370605,
+      "logps/chosen": -292.15948486328125,
+      "logps/rejected": -322.8818664550781,
+      "loss": 0.533,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.11747334897518158,
+      "rewards/margins": 0.4580891728401184,
+      "rewards/rejected": -0.5755625367164612,
+      "step": 6180
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.301059359077987e-07,
+      "logits/chosen": -2.5575060844421387,
+      "logits/rejected": -2.5281167030334473,
+      "logps/chosen": -260.3799743652344,
+      "logps/rejected": -285.0821228027344,
+      "loss": 0.6012,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.1609303206205368,
+      "rewards/margins": 0.2978779375553131,
+      "rewards/rejected": -0.4588082432746887,
+      "step": 6190
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.230944838584806e-07,
+      "logits/chosen": -2.614734172821045,
+      "logits/rejected": -2.6061034202575684,
+      "logps/chosen": -316.5515441894531,
+      "logps/rejected": -327.29705810546875,
+      "loss": 0.6087,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.12229619920253754,
+      "rewards/margins": 0.3081190884113312,
+      "rewards/rejected": -0.4304153025150299,
+      "step": 6200
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.161242877841083e-07,
+      "logits/chosen": -2.6642415523529053,
+      "logits/rejected": -2.5817365646362305,
+      "logps/chosen": -304.7231750488281,
+      "logps/rejected": -309.16595458984375,
+      "loss": 0.5662,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.18352526426315308,
+      "rewards/margins": 0.35207754373550415,
+      "rewards/rejected": -0.535602867603302,
+      "step": 6210
+    },
+    {
+      "epoch": 0.81,
+      "learning_rate": 5.091954931454682e-07,
+      "logits/chosen": -2.5234389305114746,
+      "logits/rejected": -2.480217218399048,
+      "logps/chosen": -289.94903564453125,
+      "logps/rejected": -311.3196716308594,
+      "loss": 0.5553,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.05380325764417648,
+      "rewards/margins": 0.38865208625793457,
+      "rewards/rejected": -0.44245535135269165,
+      "step": 6220
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 5.023082445393446e-07,
+      "logits/chosen": -2.572214126586914,
+      "logits/rejected": -2.5284531116485596,
+      "logps/chosen": -286.5644836425781,
+      "logps/rejected": -288.5205383300781,
+      "loss": 0.5191,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.07450740039348602,
+      "rewards/margins": 0.48594626784324646,
+      "rewards/rejected": -0.5604537129402161,
+      "step": 6230
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.95462685695498e-07,
+      "logits/chosen": -2.6354706287384033,
+      "logits/rejected": -2.5710415840148926,
+      "logps/chosen": -299.6126403808594,
+      "logps/rejected": -288.68365478515625,
+      "loss": 0.5966,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.17474600672721863,
+      "rewards/margins": 0.30064937472343445,
+      "rewards/rejected": -0.47539544105529785,
+      "step": 6240
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.88658959473666e-07,
+      "logits/chosen": -2.619030237197876,
+      "logits/rejected": -2.5606322288513184,
+      "logps/chosen": -308.16497802734375,
+      "logps/rejected": -294.54931640625,
+      "loss": 0.5597,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.11307351291179657,
+      "rewards/margins": 0.3922137916088104,
+      "rewards/rejected": -0.5052872896194458,
+      "step": 6250
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.818972078605821e-07,
+      "logits/chosen": -2.6023218631744385,
+      "logits/rejected": -2.508466958999634,
+      "logps/chosen": -353.3907775878906,
+      "logps/rejected": -297.12481689453125,
+      "loss": 0.5705,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.15918190777301788,
+      "rewards/margins": 0.37774786353111267,
+      "rewards/rejected": -0.536929726600647,
+      "step": 6260
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.7517757196701514e-07,
+      "logits/chosen": -2.653749942779541,
+      "logits/rejected": -2.564861297607422,
+      "logps/chosen": -263.03369140625,
+      "logps/rejected": -271.40423583984375,
+      "loss": 0.5583,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.13018305599689484,
+      "rewards/margins": 0.37058669328689575,
+      "rewards/rejected": -0.5007697343826294,
+      "step": 6270
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.6850019202482193e-07,
+      "logits/chosen": -2.601264238357544,
+      "logits/rejected": -2.482879638671875,
+      "logps/chosen": -262.0050354003906,
+      "logps/rejected": -275.4186096191406,
+      "loss": 0.5831,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.13502353429794312,
+      "rewards/margins": 0.33451586961746216,
+      "rewards/rejected": -0.4695393443107605,
+      "step": 6280
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.618652073840188e-07,
+      "logits/chosen": -2.6671946048736572,
+      "logits/rejected": -2.5722148418426514,
+      "logps/chosen": -322.63568115234375,
+      "logps/rejected": -312.2142028808594,
+      "loss": 0.5826,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.14907169342041016,
+      "rewards/margins": 0.3554302752017975,
+      "rewards/rejected": -0.50450199842453,
+      "step": 6290
+    },
+    {
+      "epoch": 0.82,
+      "learning_rate": 4.5527275650987965e-07,
+      "logits/chosen": -2.619171619415283,
+      "logits/rejected": -2.5671281814575195,
+      "logps/chosen": -295.90899658203125,
+      "logps/rejected": -290.890869140625,
+      "loss": 0.5953,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.15498390793800354,
+      "rewards/margins": 0.30924952030181885,
+      "rewards/rejected": -0.46423348784446716,
+      "step": 6300
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.487229769800394e-07,
+      "logits/chosen": -2.5786068439483643,
+      "logits/rejected": -2.503350257873535,
+      "logps/chosen": -251.93954467773438,
+      "logps/rejected": -258.75408935546875,
+      "loss": 0.6089,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.15024399757385254,
+      "rewards/margins": 0.2718487083911896,
+      "rewards/rejected": -0.4220927357673645,
+      "step": 6310
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.422160054816285e-07,
+      "logits/chosen": -2.517751693725586,
+      "logits/rejected": -2.452653169631958,
+      "logps/chosen": -332.330322265625,
+      "logps/rejected": -346.67120361328125,
+      "loss": 0.6218,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.19229301810264587,
+      "rewards/margins": 0.2461676299571991,
+      "rewards/rejected": -0.43846067786216736,
+      "step": 6320
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.35751977808416e-07,
+      "logits/chosen": -2.6088814735412598,
+      "logits/rejected": -2.547945737838745,
+      "logps/chosen": -292.50634765625,
+      "logps/rejected": -300.6086120605469,
+      "loss": 0.5403,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.10712258517742157,
+      "rewards/margins": 0.4287549555301666,
+      "rewards/rejected": -0.5358775854110718,
+      "step": 6330
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.293310288579794e-07,
+      "logits/chosen": -2.4563217163085938,
+      "logits/rejected": -2.470564842224121,
+      "logps/chosen": -280.95233154296875,
+      "logps/rejected": -300.3773193359375,
+      "loss": 0.6055,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.17948834598064423,
+      "rewards/margins": 0.2903778851032257,
+      "rewards/rejected": -0.4698662757873535,
+      "step": 6340
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.2295329262888733e-07,
+      "logits/chosen": -2.5638747215270996,
+      "logits/rejected": -2.493110179901123,
+      "logps/chosen": -282.89324951171875,
+      "logps/rejected": -281.40826416015625,
+      "loss": 0.5918,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.16501878201961517,
+      "rewards/margins": 0.32358044385910034,
+      "rewards/rejected": -0.4885992407798767,
+      "step": 6350
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.1661890221790316e-07,
+      "logits/chosen": -2.5607175827026367,
+      "logits/rejected": -2.5700955390930176,
+      "logps/chosen": -270.00054931640625,
+      "logps/rejected": -337.29718017578125,
+      "loss": 0.5352,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.06923635303974152,
+      "rewards/margins": 0.4399685859680176,
+      "rewards/rejected": -0.5092049837112427,
+      "step": 6360
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.103279898172072e-07,
+      "logits/chosen": -2.641157627105713,
+      "logits/rejected": -2.5395140647888184,
+      "logps/chosen": -348.9002990722656,
+      "logps/rejected": -317.4074401855469,
+      "loss": 0.6051,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.18896187841892242,
+      "rewards/margins": 0.2999545931816101,
+      "rewards/rejected": -0.48891645669937134,
+      "step": 6370
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 4.040806867116401e-07,
+      "logits/chosen": -2.57532000541687,
+      "logits/rejected": -2.551492214202881,
+      "logps/chosen": -311.6512145996094,
+      "logps/rejected": -309.5706481933594,
+      "loss": 0.6297,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.17313383519649506,
+      "rewards/margins": 0.21635062992572784,
+      "rewards/rejected": -0.3894844651222229,
+      "step": 6380
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.978771232759615e-07,
+      "logits/chosen": -2.573901653289795,
+      "logits/rejected": -2.5737104415893555,
+      "logps/chosen": -263.36859130859375,
+      "logps/rejected": -292.5953674316406,
+      "loss": 0.587,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.16891613602638245,
+      "rewards/margins": 0.30651307106018066,
+      "rewards/rejected": -0.4754292368888855,
+      "step": 6390
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.917174289721276e-07,
+      "logits/chosen": -2.6016876697540283,
+      "logits/rejected": -2.633063554763794,
+      "logps/chosen": -306.06585693359375,
+      "logps/rejected": -298.90216064453125,
+      "loss": 0.6224,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.18065698444843292,
+      "rewards/margins": 0.23400597274303436,
+      "rewards/rejected": -0.4146629869937897,
+      "step": 6400
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.856017323465938e-07,
+      "logits/chosen": -2.5487091541290283,
+      "logits/rejected": -2.498304843902588,
+      "logps/chosen": -265.94427490234375,
+      "logps/rejected": -285.1154479980469,
+      "loss": 0.6013,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.21739809215068817,
+      "rewards/margins": 0.2767632007598877,
+      "rewards/rejected": -0.49416130781173706,
+      "step": 6410
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.7953016102762695e-07,
+      "logits/chosen": -2.604661226272583,
+      "logits/rejected": -2.585566282272339,
+      "logps/chosen": -298.68817138671875,
+      "logps/rejected": -302.5409851074219,
+      "loss": 0.616,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -0.1614013910293579,
+      "rewards/margins": 0.27955660223960876,
+      "rewards/rejected": -0.44095802307128906,
+      "step": 6420
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.7350284172264493e-07,
+      "logits/chosen": -2.5873916149139404,
+      "logits/rejected": -2.5773706436157227,
+      "logps/chosen": -250.08053588867188,
+      "logps/rejected": -299.7464294433594,
+      "loss": 0.5906,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.1339772641658783,
+      "rewards/margins": 0.3003210425376892,
+      "rewards/rejected": -0.4342982769012451,
+      "step": 6430
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.67519900215573e-07,
+      "logits/chosen": -2.5705783367156982,
+      "logits/rejected": -2.52986216545105,
+      "logps/chosen": -282.07135009765625,
+      "logps/rejected": -286.83929443359375,
+      "loss": 0.5751,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.11118457466363907,
+      "rewards/margins": 0.3572140634059906,
+      "rewards/rejected": -0.4683986306190491,
+      "step": 6440
+    },
+    {
+      "epoch": 0.84,
+      "learning_rate": 3.615814613642174e-07,
+      "logits/chosen": -2.615342140197754,
+      "logits/rejected": -2.4929580688476562,
+      "logps/chosen": -332.75469970703125,
+      "logps/rejected": -300.51116943359375,
+      "loss": 0.5753,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.14485368132591248,
+      "rewards/margins": 0.3543422222137451,
+      "rewards/rejected": -0.4991958737373352,
+      "step": 6450
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.5568764909765795e-07,
+      "logits/chosen": -2.5775699615478516,
+      "logits/rejected": -2.4994049072265625,
+      "logps/chosen": -322.72064208984375,
+      "logps/rejected": -329.6020812988281,
+      "loss": 0.5617,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.12732650339603424,
+      "rewards/margins": 0.38075852394104004,
+      "rewards/rejected": -0.5080850124359131,
+      "step": 6460
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.498385864136672e-07,
+      "logits/chosen": -2.631946086883545,
+      "logits/rejected": -2.6130497455596924,
+      "logps/chosen": -297.0643310546875,
+      "logps/rejected": -316.00048828125,
+      "loss": 0.6191,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.21162505447864532,
+      "rewards/margins": 0.26513633131980896,
+      "rewards/rejected": -0.4767614006996155,
+      "step": 6470
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.440343953761363e-07,
+      "logits/chosen": -2.5831589698791504,
+      "logits/rejected": -2.509115695953369,
+      "logps/chosen": -286.7584533691406,
+      "logps/rejected": -284.6517639160156,
+      "loss": 0.6366,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.213495135307312,
+      "rewards/margins": 0.23171913623809814,
+      "rewards/rejected": -0.4452142119407654,
+      "step": 6480
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.382751971125345e-07,
+      "logits/chosen": -2.5791678428649902,
+      "logits/rejected": -2.5311591625213623,
+      "logps/chosen": -272.27752685546875,
+      "logps/rejected": -276.30584716796875,
+      "loss": 0.6143,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.1311412751674652,
+      "rewards/margins": 0.2584206461906433,
+      "rewards/rejected": -0.3895619213581085,
+      "step": 6490
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.3256111181137753e-07,
+      "logits/chosen": -2.6226067543029785,
+      "logits/rejected": -2.552586793899536,
+      "logps/chosen": -277.80364990234375,
+      "logps/rejected": -288.6063232421875,
+      "loss": 0.5723,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.14047552645206451,
+      "rewards/margins": 0.3488207757472992,
+      "rewards/rejected": -0.4892962574958801,
+      "step": 6500
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.2689225871971905e-07,
+      "logits/chosen": -2.5741589069366455,
+      "logits/rejected": -2.5458621978759766,
+      "logps/chosen": -299.5267333984375,
+      "logps/rejected": -335.2696533203125,
+      "loss": 0.5937,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.12385554611682892,
+      "rewards/margins": 0.34470731019973755,
+      "rewards/rejected": -0.4685627520084381,
+      "step": 6510
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.2126875614066523e-07,
+      "logits/chosen": -2.6185333728790283,
+      "logits/rejected": -2.5748603343963623,
+      "logps/chosen": -276.5582275390625,
+      "logps/rejected": -280.41461181640625,
+      "loss": 0.6082,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.14632555842399597,
+      "rewards/margins": 0.25596845149993896,
+      "rewards/rejected": -0.40229400992393494,
+      "step": 6520
+    },
+    {
+      "epoch": 0.85,
+      "learning_rate": 3.156907214309024e-07,
+      "logits/chosen": -2.530311107635498,
+      "logits/rejected": -2.488084316253662,
+      "logps/chosen": -267.0064392089844,
+      "logps/rejected": -301.04815673828125,
+      "loss": 0.5561,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.1583368480205536,
+      "rewards/margins": 0.3836449086666107,
+      "rewards/rejected": -0.5419816970825195,
+      "step": 6530
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 3.1015827099824923e-07,
+      "logits/chosen": -2.6760759353637695,
+      "logits/rejected": -2.5677216053009033,
+      "logps/chosen": -303.6805114746094,
+      "logps/rejected": -309.2142639160156,
+      "loss": 0.5598,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.1328735202550888,
+      "rewards/margins": 0.3772495687007904,
+      "rewards/rejected": -0.5101231336593628,
+      "step": 6540
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 3.0467152029922926e-07,
+      "logits/chosen": -2.532853603363037,
+      "logits/rejected": -2.5078790187835693,
+      "logps/chosen": -266.61627197265625,
+      "logps/rejected": -321.8629455566406,
+      "loss": 0.6359,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.17534369230270386,
+      "rewards/margins": 0.2076181173324585,
+      "rewards/rejected": -0.38296183943748474,
+      "step": 6550
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.992305838366591e-07,
+      "logits/chosen": -2.624075412750244,
+      "logits/rejected": -2.6009886264801025,
+      "logps/chosen": -349.44500732421875,
+      "logps/rejected": -336.8930969238281,
+      "loss": 0.5653,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.14597949385643005,
+      "rewards/margins": 0.38036808371543884,
+      "rewards/rejected": -0.5263476371765137,
+      "step": 6560
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.938355751572583e-07,
+      "logits/chosen": -2.571498394012451,
+      "logits/rejected": -2.497720718383789,
+      "logps/chosen": -296.1214294433594,
+      "logps/rejected": -287.1996154785156,
+      "loss": 0.6086,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.17738035321235657,
+      "rewards/margins": 0.26180464029312134,
+      "rewards/rejected": -0.4391849637031555,
+      "step": 6570
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.8848660684928307e-07,
+      "logits/chosen": -2.5616462230682373,
+      "logits/rejected": -2.523480176925659,
+      "logps/chosen": -318.0840759277344,
+      "logps/rejected": -308.376708984375,
+      "loss": 0.5997,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.19544881582260132,
+      "rewards/margins": 0.3023239076137543,
+      "rewards/rejected": -0.4977726936340332,
+      "step": 6580
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.8318379054017383e-07,
+      "logits/chosen": -2.581812620162964,
+      "logits/rejected": -2.525717258453369,
+      "logps/chosen": -307.4318542480469,
+      "logps/rejected": -289.84619140625,
+      "loss": 0.5411,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.09934178739786148,
+      "rewards/margins": 0.4410862326622009,
+      "rewards/rejected": -0.5404280424118042,
+      "step": 6590
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.779272368942246e-07,
+      "logits/chosen": -2.672194004058838,
+      "logits/rejected": -2.5740902423858643,
+      "logps/chosen": -313.4407958984375,
+      "logps/rejected": -333.7064208984375,
+      "loss": 0.5884,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.14735443890094757,
+      "rewards/margins": 0.34426793456077576,
+      "rewards/rejected": -0.4916223883628845,
+      "step": 6600
+    },
+    {
+      "epoch": 0.86,
+      "learning_rate": 2.7271705561027986e-07,
+      "logits/chosen": -2.620068073272705,
+      "logits/rejected": -2.5501561164855957,
+      "logps/chosen": -339.8446044921875,
+      "logps/rejected": -309.07049560546875,
+      "loss": 0.5674,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.12289301306009293,
+      "rewards/margins": 0.3857477605342865,
+      "rewards/rejected": -0.5086407661437988,
+      "step": 6610
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.6755335541943677e-07,
+      "logits/chosen": -2.650571823120117,
+      "logits/rejected": -2.6475863456726074,
+      "logps/chosen": -300.41986083984375,
+      "logps/rejected": -326.81365966796875,
+      "loss": 0.6318,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.2624179422855377,
+      "rewards/margins": 0.22734799981117249,
+      "rewards/rejected": -0.4897659420967102,
+      "step": 6620
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.62436244082781e-07,
+      "logits/chosen": -2.68837833404541,
+      "logits/rejected": -2.680441379547119,
+      "logps/chosen": -314.92852783203125,
+      "logps/rejected": -332.07720947265625,
+      "loss": 0.5167,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.10935411602258682,
+      "rewards/margins": 0.48752450942993164,
+      "rewards/rejected": -0.5968786478042603,
+      "step": 6630
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.5736582838913836e-07,
+      "logits/chosen": -2.6141715049743652,
+      "logits/rejected": -2.566948890686035,
+      "logps/chosen": -297.0909729003906,
+      "logps/rejected": -295.9375915527344,
+      "loss": 0.5328,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.08128849416971207,
+      "rewards/margins": 0.4745239317417145,
+      "rewards/rejected": -0.555812418460846,
+      "step": 6640
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.5234221415284363e-07,
+      "logits/chosen": -2.62785005569458,
+      "logits/rejected": -2.5692801475524902,
+      "logps/chosen": -305.32806396484375,
+      "logps/rejected": -345.72027587890625,
+      "loss": 0.5748,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.13727234303951263,
+      "rewards/margins": 0.34607818722724915,
+      "rewards/rejected": -0.4833505153656006,
+      "step": 6650
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.4736550621153375e-07,
+      "logits/chosen": -2.569892406463623,
+      "logits/rejected": -2.5169737339019775,
+      "logps/chosen": -270.4378356933594,
+      "logps/rejected": -292.44012451171875,
+      "loss": 0.5503,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.20987308025360107,
+      "rewards/margins": 0.42569002509117126,
+      "rewards/rejected": -0.6355631351470947,
+      "step": 6660
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.424358084239609e-07,
+      "logits/chosen": -2.625664234161377,
+      "logits/rejected": -2.634753942489624,
+      "logps/chosen": -287.9183654785156,
+      "logps/rejected": -306.96435546875,
+      "loss": 0.6504,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.16156315803527832,
+      "rewards/margins": 0.18512198328971863,
+      "rewards/rejected": -0.34668511152267456,
+      "step": 6670
+    },
+    {
+      "epoch": 0.87,
+      "learning_rate": 2.3755322366782158e-07,
+      "logits/chosen": -2.5567984580993652,
+      "logits/rejected": -2.5211338996887207,
+      "logps/chosen": -296.8184509277344,
+      "logps/rejected": -302.54949951171875,
+      "loss": 0.5522,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.06562675535678864,
+      "rewards/margins": 0.4156498312950134,
+      "rewards/rejected": -0.48127660155296326,
+      "step": 6680
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.3271785383761431e-07,
+      "logits/chosen": -2.6414246559143066,
+      "logits/rejected": -2.6082088947296143,
+      "logps/chosen": -294.6539306640625,
+      "logps/rejected": -294.13214111328125,
+      "loss": 0.6112,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.13177195191383362,
+      "rewards/margins": 0.2451949566602707,
+      "rewards/rejected": -0.3769668638706207,
+      "step": 6690
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.2792979984250978e-07,
+      "logits/chosen": -2.6344776153564453,
+      "logits/rejected": -2.561110258102417,
+      "logps/chosen": -314.31610107421875,
+      "logps/rejected": -294.9229736328125,
+      "loss": 0.5842,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.1052699089050293,
+      "rewards/margins": 0.315143883228302,
+      "rewards/rejected": -0.4204137921333313,
+      "step": 6700
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.231891616042453e-07,
+      "logits/chosen": -2.6803958415985107,
+      "logits/rejected": -2.641515016555786,
+      "logps/chosen": -306.5299987792969,
+      "logps/rejected": -337.0438537597656,
+      "loss": 0.5545,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.15858620405197144,
+      "rewards/margins": 0.41083598136901855,
+      "rewards/rejected": -0.56942218542099,
+      "step": 6710
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.1849603805504328e-07,
+      "logits/chosen": -2.6510190963745117,
+      "logits/rejected": -2.5821597576141357,
+      "logps/chosen": -311.9466857910156,
+      "logps/rejected": -322.2398681640625,
+      "loss": 0.5704,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.11363981664180756,
+      "rewards/margins": 0.37154248356819153,
+      "rewards/rejected": -0.4851822257041931,
+      "step": 6720
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.1385052713554066e-07,
+      "logits/chosen": -2.4852182865142822,
+      "logits/rejected": -2.4480271339416504,
+      "logps/chosen": -277.65606689453125,
+      "logps/rejected": -316.7997131347656,
+      "loss": 0.5777,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.16943712532520294,
+      "rewards/margins": 0.3445144593715668,
+      "rewards/rejected": -0.5139515995979309,
+      "step": 6730
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.0925272579274873e-07,
+      "logits/chosen": -2.5965895652770996,
+      "logits/rejected": -2.6418797969818115,
+      "logps/chosen": -288.65283203125,
+      "logps/rejected": -325.62347412109375,
+      "loss": 0.5474,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.10275232791900635,
+      "rewards/margins": 0.41379642486572266,
+      "rewards/rejected": -0.516548752784729,
+      "step": 6740
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.047027299780302e-07,
+      "logits/chosen": -2.6385979652404785,
+      "logits/rejected": -2.626549482345581,
+      "logps/chosen": -327.0989074707031,
+      "logps/rejected": -313.5267333984375,
+      "loss": 0.6042,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.13917601108551025,
+      "rewards/margins": 0.2780911922454834,
+      "rewards/rejected": -0.41726717352867126,
+      "step": 6750
+    },
+    {
+      "epoch": 0.88,
+      "learning_rate": 2.0020063464509492e-07,
+      "logits/chosen": -2.5647072792053223,
+      "logits/rejected": -2.5407252311706543,
+      "logps/chosen": -292.1898498535156,
+      "logps/rejected": -302.8434753417969,
+      "loss": 0.5572,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.11755616962909698,
+      "rewards/margins": 0.38299018144607544,
+      "rewards/rejected": -0.5005463361740112,
+      "step": 6760
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.957465337480191e-07,
+      "logits/chosen": -2.557737112045288,
+      "logits/rejected": -2.5099196434020996,
+      "logps/chosen": -334.20648193359375,
+      "logps/rejected": -340.0382080078125,
+      "loss": 0.5707,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.1331155002117157,
+      "rewards/margins": 0.37713751196861267,
+      "rewards/rejected": -0.5102530717849731,
+      "step": 6770
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.9134052023928622e-07,
+      "logits/chosen": -2.569523334503174,
+      "logits/rejected": -2.5409655570983887,
+      "logps/chosen": -279.5953369140625,
+      "logps/rejected": -310.7686767578125,
+      "loss": 0.5972,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.20895233750343323,
+      "rewards/margins": 0.29102927446365356,
+      "rewards/rejected": -0.499981552362442,
+      "step": 6780
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.8698268606784392e-07,
+      "logits/chosen": -2.5592174530029297,
+      "logits/rejected": -2.5852952003479004,
+      "logps/chosen": -267.05694580078125,
+      "logps/rejected": -301.8708801269531,
+      "loss": 0.5706,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -0.1693277657032013,
+      "rewards/margins": 0.40050187706947327,
+      "rewards/rejected": -0.5698296427726746,
+      "step": 6790
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.826731221771866e-07,
+      "logits/chosen": -2.63383412361145,
+      "logits/rejected": -2.5463290214538574,
+      "logps/chosen": -320.2628173828125,
+      "logps/rejected": -297.6620178222656,
+      "loss": 0.5778,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.16660620272159576,
+      "rewards/margins": 0.37828701734542847,
+      "rewards/rejected": -0.5448931455612183,
+      "step": 6800
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.7841191850345967e-07,
+      "logits/chosen": -2.520204782485962,
+      "logits/rejected": -2.527278184890747,
+      "logps/chosen": -305.18994140625,
+      "logps/rejected": -340.17022705078125,
+      "loss": 0.5437,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.10276700556278229,
+      "rewards/margins": 0.425798237323761,
+      "rewards/rejected": -0.5285652875900269,
+      "step": 6810
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.7419916397357905e-07,
+      "logits/chosen": -2.6337666511535645,
+      "logits/rejected": -2.5599422454833984,
+      "logps/chosen": -323.2042236328125,
+      "logps/rejected": -306.8524475097656,
+      "loss": 0.6187,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.2199595719575882,
+      "rewards/margins": 0.26666978001594543,
+      "rewards/rejected": -0.4866293966770172,
+      "step": 6820
+    },
+    {
+      "epoch": 0.89,
+      "learning_rate": 1.700349465033782e-07,
+      "logits/chosen": -2.56412672996521,
+      "logits/rejected": -2.5239360332489014,
+      "logps/chosen": -304.78631591796875,
+      "logps/rejected": -309.85491943359375,
+      "loss": 0.612,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.192495197057724,
+      "rewards/margins": 0.2633577287197113,
+      "rewards/rejected": -0.4558529257774353,
+      "step": 6830
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.6591935299577227e-07,
+      "logits/chosen": -2.5691449642181396,
+      "logits/rejected": -2.5329692363739014,
+      "logps/chosen": -288.4953918457031,
+      "logps/rejected": -267.4063415527344,
+      "loss": 0.5789,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.1454126238822937,
+      "rewards/margins": 0.3563196361064911,
+      "rewards/rejected": -0.5017322897911072,
+      "step": 6840
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.6185246933894338e-07,
+      "logits/chosen": -2.6313118934631348,
+      "logits/rejected": -2.6083617210388184,
+      "logps/chosen": -293.7181701660156,
+      "logps/rejected": -311.9383239746094,
+      "loss": 0.6014,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.15030762553215027,
+      "rewards/margins": 0.31505897641181946,
+      "rewards/rejected": -0.4653666019439697,
+      "step": 6850
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.5783438040455097e-07,
+      "logits/chosen": -2.6444010734558105,
+      "logits/rejected": -2.5574839115142822,
+      "logps/chosen": -317.16064453125,
+      "logps/rejected": -280.485107421875,
+      "loss": 0.5653,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.20444634556770325,
+      "rewards/margins": 0.3558157980442047,
+      "rewards/rejected": -0.5602622032165527,
+      "step": 6860
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.538651700459576e-07,
+      "logits/chosen": -2.5990586280822754,
+      "logits/rejected": -2.5246388912200928,
+      "logps/chosen": -324.9945983886719,
+      "logps/rejected": -402.9381408691406,
+      "loss": 0.5538,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.09169480204582214,
+      "rewards/margins": 0.39981403946876526,
+      "rewards/rejected": -0.4915088713169098,
+      "step": 6870
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.4994492109648151e-07,
+      "logits/chosen": -2.5804312229156494,
+      "logits/rejected": -2.524960517883301,
+      "logps/chosen": -266.78521728515625,
+      "logps/rejected": -275.1600646972656,
+      "loss": 0.5539,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.17357993125915527,
+      "rewards/margins": 0.40863847732543945,
+      "rewards/rejected": -0.5822184681892395,
+      "step": 6880
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.4607371536766695e-07,
+      "logits/chosen": -2.5991179943084717,
+      "logits/rejected": -2.562882900238037,
+      "logps/chosen": -254.6803741455078,
+      "logps/rejected": -276.7633056640625,
+      "loss": 0.5714,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.17787747085094452,
+      "rewards/margins": 0.3562357723712921,
+      "rewards/rejected": -0.5341132879257202,
+      "step": 6890
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.4225163364757655e-07,
+      "logits/chosen": -2.588153123855591,
+      "logits/rejected": -2.4963624477386475,
+      "logps/chosen": -309.7139587402344,
+      "logps/rejected": -312.49652099609375,
+      "loss": 0.6012,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.22634100914001465,
+      "rewards/margins": 0.27041059732437134,
+      "rewards/rejected": -0.496751606464386,
+      "step": 6900
+    },
+    {
+      "epoch": 0.9,
+      "learning_rate": 1.3847875569910462e-07,
+      "logits/chosen": -2.605311870574951,
+      "logits/rejected": -2.5205509662628174,
+      "logps/chosen": -293.03179931640625,
+      "logps/rejected": -304.44854736328125,
+      "loss": 0.5934,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.13108663260936737,
+      "rewards/margins": 0.3022516965866089,
+      "rewards/rejected": -0.43333831429481506,
+      "step": 6910
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.3475516025831552e-07,
+      "logits/chosen": -2.6266331672668457,
+      "logits/rejected": -2.623201847076416,
+      "logps/chosen": -289.32061767578125,
+      "logps/rejected": -320.59320068359375,
+      "loss": 0.576,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.14418147504329681,
+      "rewards/margins": 0.3588133454322815,
+      "rewards/rejected": -0.5029947757720947,
+      "step": 6920
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.310809250327974e-07,
+      "logits/chosen": -2.4527974128723145,
+      "logits/rejected": -2.4328925609588623,
+      "logps/chosen": -267.4524841308594,
+      "logps/rejected": -285.71441650390625,
+      "loss": 0.5434,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.0876695066690445,
+      "rewards/margins": 0.4406587481498718,
+      "rewards/rejected": -0.5283282399177551,
+      "step": 6930
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.2745612670004153e-07,
+      "logits/chosen": -2.5613865852355957,
+      "logits/rejected": -2.5249736309051514,
+      "logps/chosen": -280.7591857910156,
+      "logps/rejected": -376.81646728515625,
+      "loss": 0.5857,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.1627487689256668,
+      "rewards/margins": 0.3261997103691101,
+      "rewards/rejected": -0.48894843459129333,
+      "step": 6940
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.2388084090584395e-07,
+      "logits/chosen": -2.579641580581665,
+      "logits/rejected": -2.544524669647217,
+      "logps/chosen": -272.37603759765625,
+      "logps/rejected": -321.57647705078125,
+      "loss": 0.6029,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.23431356251239777,
+      "rewards/margins": 0.3198152780532837,
+      "rewards/rejected": -0.5541288256645203,
+      "step": 6950
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.2035514226272305e-07,
+      "logits/chosen": -2.554049015045166,
+      "logits/rejected": -2.4828734397888184,
+      "logps/chosen": -264.0655822753906,
+      "logps/rejected": -300.8733825683594,
+      "loss": 0.5688,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.13704833388328552,
+      "rewards/margins": 0.3907284140586853,
+      "rewards/rejected": -0.5277767777442932,
+      "step": 6960
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.1687910434836607e-07,
+      "logits/chosen": -2.557542324066162,
+      "logits/rejected": -2.5698935985565186,
+      "logps/chosen": -284.0539245605469,
+      "logps/rejected": -308.63739013671875,
+      "loss": 0.5949,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.18020404875278473,
+      "rewards/margins": 0.33270463347435,
+      "rewards/rejected": -0.5129086971282959,
+      "step": 6970
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.1345279970409128e-07,
+      "logits/chosen": -2.5548183917999268,
+      "logits/rejected": -2.5523457527160645,
+      "logps/chosen": -251.26644897460938,
+      "logps/rejected": -292.21954345703125,
+      "loss": 0.6444,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.17198213934898376,
+      "rewards/margins": 0.20152275264263153,
+      "rewards/rejected": -0.3735049068927765,
+      "step": 6980
+    },
+    {
+      "epoch": 0.91,
+      "learning_rate": 1.1007629983333629e-07,
+      "logits/chosen": -2.523085832595825,
+      "logits/rejected": -2.4751267433166504,
+      "logps/chosen": -280.7390441894531,
+      "logps/rejected": -293.4989929199219,
+      "loss": 0.5552,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -0.08755568414926529,
+      "rewards/margins": 0.387625515460968,
+      "rewards/rejected": -0.4751811921596527,
+      "step": 6990
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 1.067496752001626e-07,
+      "logits/chosen": -2.5587449073791504,
+      "logits/rejected": -2.5087859630584717,
+      "logps/chosen": -299.1822204589844,
+      "logps/rejected": -291.67156982421875,
+      "loss": 0.5703,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.18846185505390167,
+      "rewards/margins": 0.39327672123908997,
+      "rewards/rejected": -0.5817385911941528,
+      "step": 7000
+    },
+    {
+      "epoch": 0.92,
+      "eval_logits/chosen": -2.5765209197998047,
+      "eval_logits/rejected": -2.5235540866851807,
+      "eval_logps/chosen": -302.1863708496094,
+      "eval_logps/rejected": -314.6088562011719,
+      "eval_loss": 0.5834610462188721,
+      "eval_rewards/accuracies": 0.7139999866485596,
+      "eval_rewards/chosen": -0.1499684602022171,
+      "eval_rewards/margins": 0.33720675110816956,
+      "eval_rewards/rejected": -0.48717522621154785,
+      "eval_runtime": 1165.1452,
+      "eval_samples_per_second": 1.717,
+      "eval_steps_per_second": 0.215,
+      "step": 7000
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 1.0347299522778909e-07,
+      "logits/chosen": -2.617305040359497,
+      "logits/rejected": -2.574169397354126,
+      "logps/chosen": -323.5268249511719,
+      "logps/rejected": -327.29315185546875,
+      "loss": 0.5334,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.1206863671541214,
+      "rewards/margins": 0.45468536019325256,
+      "rewards/rejected": -0.5753717422485352,
+      "step": 7010
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 1.0024632829713971e-07,
+      "logits/chosen": -2.5649805068969727,
+      "logits/rejected": -2.6036572456359863,
+      "logps/chosen": -268.43011474609375,
+      "logps/rejected": -309.91680908203125,
+      "loss": 0.6149,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.16881686449050903,
+      "rewards/margins": 0.2559646964073181,
+      "rewards/rejected": -0.4247816205024719,
+      "step": 7020
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 9.706974174541889e-08,
+      "logits/chosen": -2.546327590942383,
+      "logits/rejected": -2.50756573677063,
+      "logps/chosen": -319.654296875,
+      "logps/rejected": -314.1730651855469,
+      "loss": 0.527,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.1125556007027626,
+      "rewards/margins": 0.47934332489967346,
+      "rewards/rejected": -0.5918988585472107,
+      "step": 7030
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 9.39433018647043e-08,
+      "logits/chosen": -2.7003421783447266,
+      "logits/rejected": -2.642490863800049,
+      "logps/chosen": -297.15557861328125,
+      "logps/rejected": -343.55560302734375,
+      "loss": 0.5569,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.10674717277288437,
+      "rewards/margins": 0.39574167132377625,
+      "rewards/rejected": -0.5024887919425964,
+      "step": 7040
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 9.086707390056543e-08,
+      "logits/chosen": -2.6433115005493164,
+      "logits/rejected": -2.591010332107544,
+      "logps/chosen": -306.50299072265625,
+      "logps/rejected": -310.269775390625,
+      "loss": 0.5718,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.10943205654621124,
+      "rewards/margins": 0.35870540142059326,
+      "rewards/rejected": -0.4681374430656433,
+      "step": 7050
+    },
+    {
+      "epoch": 0.92,
+      "learning_rate": 8.784112205070083e-08,
+      "logits/chosen": -2.5863049030303955,
+      "logits/rejected": -2.591738224029541,
+      "logps/chosen": -274.50067138671875,
+      "logps/rejected": -319.26702880859375,
+      "loss": 0.6188,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.19196417927742004,
+      "rewards/margins": 0.23430033028125763,
+      "rewards/rejected": -0.4262644648551941,
+      "step": 7060
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 8.486550946359779e-08,
+      "logits/chosen": -2.616931200027466,
+      "logits/rejected": -2.5391933917999268,
+      "logps/chosen": -315.7156677246094,
+      "logps/rejected": -326.2911071777344,
+      "loss": 0.6384,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -0.1374962329864502,
+      "rewards/margins": 0.19662030041217804,
+      "rewards/rejected": -0.33411651849746704,
+      "step": 7070
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 8.194029823721556e-08,
+      "logits/chosen": -2.5751264095306396,
+      "logits/rejected": -2.498807430267334,
+      "logps/chosen": -280.50042724609375,
+      "logps/rejected": -296.1541748046875,
+      "loss": 0.5619,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.09603309631347656,
+      "rewards/margins": 0.3662812411785126,
+      "rewards/rejected": -0.46231430768966675,
+      "step": 7080
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.906554941768896e-08,
+      "logits/chosen": -2.5691847801208496,
+      "logits/rejected": -2.555361747741699,
+      "logps/chosen": -269.3592224121094,
+      "logps/rejected": -290.31842041015625,
+      "loss": 0.6351,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.21483393013477325,
+      "rewards/margins": 0.22327573597431183,
+      "rewards/rejected": -0.4381096363067627,
+      "step": 7090
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.624132299805575e-08,
+      "logits/chosen": -2.585540294647217,
+      "logits/rejected": -2.6170363426208496,
+      "logps/chosen": -337.4608154296875,
+      "logps/rejected": -348.25177001953125,
+      "loss": 0.5645,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.16420039534568787,
+      "rewards/margins": 0.40035757422447205,
+      "rewards/rejected": -0.5645579099655151,
+      "step": 7100
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.346767791700127e-08,
+      "logits/chosen": -2.5605251789093018,
+      "logits/rejected": -2.5019607543945312,
+      "logps/chosen": -331.28118896484375,
+      "logps/rejected": -329.21124267578125,
+      "loss": 0.5124,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -0.06056060269474983,
+      "rewards/margins": 0.4939872622489929,
+      "rewards/rejected": -0.554547905921936,
+      "step": 7110
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 7.07446720576327e-08,
+      "logits/chosen": -2.568209171295166,
+      "logits/rejected": -2.522514820098877,
+      "logps/chosen": -269.71966552734375,
+      "logps/rejected": -284.6921691894531,
+      "loss": 0.6014,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -0.18392933905124664,
+      "rewards/margins": 0.3144926428794861,
+      "rewards/rejected": -0.4984219968318939,
+      "step": 7120
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.807236224626701e-08,
+      "logits/chosen": -2.601301670074463,
+      "logits/rejected": -2.567638397216797,
+      "logps/chosen": -306.0537109375,
+      "logps/rejected": -312.6717529296875,
+      "loss": 0.5962,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.1434520184993744,
+      "rewards/margins": 0.292459100484848,
+      "rewards/rejected": -0.4359111189842224,
+      "step": 7130
+    },
+    {
+      "epoch": 0.93,
+      "learning_rate": 6.545080425124888e-08,
+      "logits/chosen": -2.53633189201355,
+      "logits/rejected": -2.546085834503174,
+      "logps/chosen": -290.70062255859375,
+      "logps/rejected": -322.22491455078125,
+      "loss": 0.6012,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.15784256160259247,
+      "rewards/margins": 0.3317511975765228,
+      "rewards/rejected": -0.4895937442779541,
+      "step": 7140
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.288005278178382e-08,
+      "logits/chosen": -2.631765842437744,
+      "logits/rejected": -2.5946412086486816,
+      "logps/chosen": -303.0722961425781,
+      "logps/rejected": -310.3096618652344,
+      "loss": 0.5977,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.12588831782341003,
+      "rewards/margins": 0.3008343279361725,
+      "rewards/rejected": -0.4267226755619049,
+      "step": 7150
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 6.036016148679825e-08,
+      "logits/chosen": -2.6608455181121826,
+      "logits/rejected": -2.611614227294922,
+      "logps/chosen": -303.90869140625,
+      "logps/rejected": -275.47015380859375,
+      "loss": 0.5758,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.1558142751455307,
+      "rewards/margins": 0.3274328112602234,
+      "rewards/rejected": -0.4832470417022705,
+      "step": 7160
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.7891182953819235e-08,
+      "logits/chosen": -2.6389658451080322,
+      "logits/rejected": -2.5758321285247803,
+      "logps/chosen": -257.74822998046875,
+      "logps/rejected": -315.07537841796875,
+      "loss": 0.5716,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.1357962042093277,
+      "rewards/margins": 0.3668871223926544,
+      "rewards/rejected": -0.5026832818984985,
+      "step": 7170
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.547316870787689e-08,
+      "logits/chosen": -2.6923861503601074,
+      "logits/rejected": -2.574402332305908,
+      "logps/chosen": -315.2290954589844,
+      "logps/rejected": -313.4205627441406,
+      "loss": 0.5698,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.12659238278865814,
+      "rewards/margins": 0.4000171720981598,
+      "rewards/rejected": -0.5266095399856567,
+      "step": 7180
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.310616921042927e-08,
+      "logits/chosen": -2.656461238861084,
+      "logits/rejected": -2.539680242538452,
+      "logps/chosen": -339.12103271484375,
+      "logps/rejected": -324.2355041503906,
+      "loss": 0.5751,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.13634148240089417,
+      "rewards/margins": 0.37329429388046265,
+      "rewards/rejected": -0.5096357464790344,
+      "step": 7190
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 5.079023385830939e-08,
+      "logits/chosen": -2.522582769393921,
+      "logits/rejected": -2.4490981101989746,
+      "logps/chosen": -344.3766174316406,
+      "logps/rejected": -318.4351806640625,
+      "loss": 0.5749,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.21491388976573944,
+      "rewards/margins": 0.35927098989486694,
+      "rewards/rejected": -0.5741848945617676,
+      "step": 7200
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 4.8525410982695476e-08,
+      "logits/chosen": -2.6060354709625244,
+      "logits/rejected": -2.4787943363189697,
+      "logps/chosen": -333.44232177734375,
+      "logps/rejected": -343.4652404785156,
+      "loss": 0.5709,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.10581862926483154,
+      "rewards/margins": 0.3721676468849182,
+      "rewards/rejected": -0.47798624634742737,
+      "step": 7210
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 4.6311747848099e-08,
+      "logits/chosen": -2.6081249713897705,
+      "logits/rejected": -2.4986207485198975,
+      "logps/chosen": -272.32635498046875,
+      "logps/rejected": -273.2559509277344,
+      "loss": 0.5826,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.18592849373817444,
+      "rewards/margins": 0.3380093574523926,
+      "rewards/rejected": -0.5239378809928894,
+      "step": 7220
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.4149290651382405e-08,
+      "logits/chosen": -2.565302848815918,
+      "logits/rejected": -2.569431781768799,
+      "logps/chosen": -239.3040313720703,
+      "logps/rejected": -289.22265625,
+      "loss": 0.6427,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -0.18756955862045288,
+      "rewards/margins": 0.21723206341266632,
+      "rewards/rejected": -0.404801607131958,
+      "step": 7230
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 4.203808452079211e-08,
+      "logits/chosen": -2.5903501510620117,
+      "logits/rejected": -2.561399221420288,
+      "logps/chosen": -305.9120178222656,
+      "logps/rejected": -321.04937744140625,
+      "loss": 0.5645,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.08040084689855576,
+      "rewards/margins": 0.3619064688682556,
+      "rewards/rejected": -0.44230732321739197,
+      "step": 7240
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.9978173515018427e-08,
+      "logits/chosen": -2.7003531455993652,
+      "logits/rejected": -2.6039676666259766,
+      "logps/chosen": -349.1368103027344,
+      "logps/rejected": -326.9501037597656,
+      "loss": 0.5579,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.1501680165529251,
+      "rewards/margins": 0.4240794777870178,
+      "rewards/rejected": -0.5742474794387817,
+      "step": 7250
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.7969600622274614e-08,
+      "logits/chosen": -2.678081512451172,
+      "logits/rejected": -2.6611151695251465,
+      "logps/chosen": -350.559814453125,
+      "logps/rejected": -298.64349365234375,
+      "loss": 0.6023,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.18752655386924744,
+      "rewards/margins": 0.28474223613739014,
+      "rewards/rejected": -0.47226881980895996,
+      "step": 7260
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.601240775940151e-08,
+      "logits/chosen": -2.5666842460632324,
+      "logits/rejected": -2.5780835151672363,
+      "logps/chosen": -279.97528076171875,
+      "logps/rejected": -301.4402770996094,
+      "loss": 0.5986,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.20158791542053223,
+      "rewards/margins": 0.30927926301956177,
+      "rewards/rejected": -0.5108672380447388,
+      "step": 7270
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.410663577099071e-08,
+      "logits/chosen": -2.6622908115386963,
+      "logits/rejected": -2.593769073486328,
+      "logps/chosen": -358.89117431640625,
+      "logps/rejected": -348.1055603027344,
+      "loss": 0.5869,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.1967272311449051,
+      "rewards/margins": 0.34516477584838867,
+      "rewards/rejected": -0.5418919324874878,
+      "step": 7280
+    },
+    {
+      "epoch": 0.95,
+      "learning_rate": 3.2252324428534986e-08,
+      "logits/chosen": -2.5974385738372803,
+      "logits/rejected": -2.532322645187378,
+      "logps/chosen": -326.3222961425781,
+      "logps/rejected": -312.3917236328125,
+      "loss": 0.5643,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.040471017360687256,
+      "rewards/margins": 0.3828863203525543,
+      "rewards/rejected": -0.42335739731788635,
+      "step": 7290
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 3.0449512429594486e-08,
+      "logits/chosen": -2.533263683319092,
+      "logits/rejected": -2.4743704795837402,
+      "logps/chosen": -304.0922546386719,
+      "logps/rejected": -317.29888916015625,
+      "loss": 0.5371,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.09289166331291199,
+      "rewards/margins": 0.45378655195236206,
+      "rewards/rejected": -0.5466781854629517,
+      "step": 7300
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.8698237396992956e-08,
+      "logits/chosen": -2.6442668437957764,
+      "logits/rejected": -2.6561031341552734,
+      "logps/chosen": -298.43902587890625,
+      "logps/rejected": -327.131103515625,
+      "loss": 0.5687,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.18375542759895325,
+      "rewards/margins": 0.3725849688053131,
+      "rewards/rejected": -0.5563403367996216,
+      "step": 7310
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.6998535878030584e-08,
+      "logits/chosen": -2.585597038269043,
+      "logits/rejected": -2.501504421234131,
+      "logps/chosen": -233.4923553466797,
+      "logps/rejected": -222.73098754882812,
+      "loss": 0.5772,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.12340077012777328,
+      "rewards/margins": 0.3369767963886261,
+      "rewards/rejected": -0.4603775143623352,
+      "step": 7320
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.535044334372072e-08,
+      "logits/chosen": -2.578626871109009,
+      "logits/rejected": -2.5147461891174316,
+      "logps/chosen": -359.30267333984375,
+      "logps/rejected": -321.98699951171875,
+      "loss": 0.5836,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.13615083694458008,
+      "rewards/margins": 0.36145567893981934,
+      "rewards/rejected": -0.4976065754890442,
+      "step": 7330
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.3753994188051853e-08,
+      "logits/chosen": -2.6281094551086426,
+      "logits/rejected": -2.493013620376587,
+      "logps/chosen": -316.4076232910156,
+      "logps/rejected": -308.4248352050781,
+      "loss": 0.6008,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.197179913520813,
+      "rewards/margins": 0.30890092253685,
+      "rewards/rejected": -0.5060808062553406,
+      "step": 7340
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.220922172726764e-08,
+      "logits/chosen": -2.5653843879699707,
+      "logits/rejected": -2.5473217964172363,
+      "logps/chosen": -295.3752136230469,
+      "logps/rejected": -286.2983703613281,
+      "loss": 0.6057,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.1257595717906952,
+      "rewards/margins": 0.29185977578163147,
+      "rewards/rejected": -0.4176193177700043,
+      "step": 7350
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 2.071615819917244e-08,
+      "logits/chosen": -2.6426784992218018,
+      "logits/rejected": -2.6555488109588623,
+      "logps/chosen": -303.03424072265625,
+      "logps/rejected": -333.24969482421875,
+      "loss": 0.5823,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.13265112042427063,
+      "rewards/margins": 0.31431856751441956,
+      "rewards/rejected": -0.44696974754333496,
+      "step": 7360
+    },
+    {
+      "epoch": 0.96,
+      "learning_rate": 1.9274834762459393e-08,
+      "logits/chosen": -2.5824315547943115,
+      "logits/rejected": -2.51481556892395,
+      "logps/chosen": -318.1591491699219,
+      "logps/rejected": -294.57720947265625,
+      "loss": 0.5453,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.11575493961572647,
+      "rewards/margins": 0.42396584153175354,
+      "rewards/rejected": -0.5397207140922546,
+      "step": 7370
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.7885281496058947e-08,
+      "logits/chosen": -2.6852264404296875,
+      "logits/rejected": -2.622860908508301,
+      "logps/chosen": -317.85540771484375,
+      "logps/rejected": -376.3128967285156,
+      "loss": 0.5963,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.17231769859790802,
+      "rewards/margins": 0.29038047790527344,
+      "rewards/rejected": -0.46269816160202026,
+      "step": 7380
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.654752739851134e-08,
+      "logits/chosen": -2.6919147968292236,
+      "logits/rejected": -2.543328046798706,
+      "logps/chosen": -309.39300537109375,
+      "logps/rejected": -292.3887634277344,
+      "loss": 0.584,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -0.1373666524887085,
+      "rewards/margins": 0.35448721051216125,
+      "rewards/rejected": -0.49185386300086975,
+      "step": 7390
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.526160038736235e-08,
+      "logits/chosen": -2.6847984790802,
+      "logits/rejected": -2.569662570953369,
+      "logps/chosen": -355.0367736816406,
+      "logps/rejected": -321.8956298828125,
+      "loss": 0.5713,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -0.11090721935033798,
+      "rewards/margins": 0.38109397888183594,
+      "rewards/rejected": -0.49200114607810974,
+      "step": 7400
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.402752729857959e-08,
+      "logits/chosen": -2.567704677581787,
+      "logits/rejected": -2.546342134475708,
+      "logps/chosen": -279.26324462890625,
+      "logps/rejected": -332.239501953125,
+      "loss": 0.543,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -0.09983944147825241,
+      "rewards/margins": 0.42041102051734924,
+      "rewards/rejected": -0.5202504396438599,
+      "step": 7410
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.2845333885992683e-08,
+      "logits/chosen": -2.603126049041748,
+      "logits/rejected": -2.5585086345672607,
+      "logps/chosen": -304.98211669921875,
+      "logps/rejected": -294.26495361328125,
+      "loss": 0.5849,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.12184146791696548,
+      "rewards/margins": 0.34187227487564087,
+      "rewards/rejected": -0.46371373534202576,
+      "step": 7420
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.171504482075675e-08,
+      "logits/chosen": -2.668076992034912,
+      "logits/rejected": -2.593167304992676,
+      "logps/chosen": -339.0294494628906,
+      "logps/rejected": -360.12786865234375,
+      "loss": 0.544,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.09526848047971725,
+      "rewards/margins": 0.425849974155426,
+      "rewards/rejected": -0.5211184620857239,
+      "step": 7430
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 1.0636683690836147e-08,
+      "logits/chosen": -2.6599106788635254,
+      "logits/rejected": -2.602999210357666,
+      "logps/chosen": -287.73822021484375,
+      "logps/rejected": -298.683837890625,
+      "loss": 0.6038,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.14565321803092957,
+      "rewards/margins": 0.27900558710098267,
+      "rewards/rejected": -0.4246588349342346,
+      "step": 7440
+    },
+    {
+      "epoch": 0.97,
+      "learning_rate": 9.610273000513203e-09,
+      "logits/chosen": -2.6567225456237793,
+      "logits/rejected": -2.576296329498291,
+      "logps/chosen": -341.6380310058594,
+      "logps/rejected": -345.4418640136719,
+      "loss": 0.6406,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.24326744675636292,
+      "rewards/margins": 0.20419001579284668,
+      "rewards/rejected": -0.4474574625492096,
+      "step": 7450
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 8.635834169918312e-09,
+      "logits/chosen": -2.5776119232177734,
+      "logits/rejected": -2.5685813426971436,
+      "logps/chosen": -330.8794860839844,
+      "logps/rejected": -350.81524658203125,
+      "loss": 0.6064,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.15580378472805023,
+      "rewards/margins": 0.2754043638706207,
+      "rewards/rejected": -0.431208074092865,
+      "step": 7460
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 7.713387534582506e-09,
+      "logits/chosen": -2.54894757270813,
+      "logits/rejected": -2.4861245155334473,
+      "logps/chosen": -278.8340759277344,
+      "logps/rejected": -328.6170349121094,
+      "loss": 0.5613,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.11971104145050049,
+      "rewards/margins": 0.3778414726257324,
+      "rewards/rejected": -0.4975524842739105,
+      "step": 7470
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 6.84295234501392e-09,
+      "logits/chosen": -2.6073665618896484,
+      "logits/rejected": -2.5339808464050293,
+      "logps/chosen": -306.5630798339844,
+      "logps/rejected": -295.9570617675781,
+      "loss": 0.6194,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.19054028391838074,
+      "rewards/margins": 0.2690245509147644,
+      "rewards/rejected": -0.45956477522850037,
+      "step": 7480
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 6.024546766295325e-09,
+      "logits/chosen": -2.5746657848358154,
+      "logits/rejected": -2.4791531562805176,
+      "logps/chosen": -329.4606018066406,
+      "logps/rejected": -303.4569091796875,
+      "loss": 0.5372,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.11107777059078217,
+      "rewards/margins": 0.45022836327552795,
+      "rewards/rejected": -0.5613061189651489,
+      "step": 7490
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 5.2581878777049895e-09,
+      "logits/chosen": -2.6267223358154297,
+      "logits/rejected": -2.57027006149292,
+      "logps/chosen": -311.00286865234375,
+      "logps/rejected": -336.90234375,
+      "loss": 0.5359,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.10928332805633545,
+      "rewards/margins": 0.4430675506591797,
+      "rewards/rejected": -0.5523509383201599,
+      "step": 7500
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 4.543891672361411e-09,
+      "logits/chosen": -2.578234910964966,
+      "logits/rejected": -2.5378470420837402,
+      "logps/chosen": -308.0870666503906,
+      "logps/rejected": -326.76959228515625,
+      "loss": 0.5939,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -0.14650920033454895,
+      "rewards/margins": 0.3034004271030426,
+      "rewards/rejected": -0.44990959763526917,
+      "step": 7510
+    },
+    {
+      "epoch": 0.98,
+      "learning_rate": 3.881673056887747e-09,
+      "logits/chosen": -2.6423134803771973,
+      "logits/rejected": -2.620753049850464,
+      "logps/chosen": -321.03607177734375,
+      "logps/rejected": -317.3270263671875,
+      "loss": 0.6072,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.17036667466163635,
+      "rewards/margins": 0.29877328872680664,
+      "rewards/rejected": -0.469139963388443,
+      "step": 7520
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 3.2715458511023425e-09,
+      "logits/chosen": -2.5868382453918457,
+      "logits/rejected": -2.5425877571105957,
+      "logps/chosen": -305.48333740234375,
+      "logps/rejected": -334.73516845703125,
+      "loss": 0.6001,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -0.13013510406017303,
+      "rewards/margins": 0.3063547611236572,
+      "rewards/rejected": -0.4364898204803467,
+      "step": 7530
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 2.7135227877289617e-09,
+      "logits/chosen": -2.631929636001587,
+      "logits/rejected": -2.631361246109009,
+      "logps/chosen": -286.19512939453125,
+      "logps/rejected": -332.6454162597656,
+      "loss": 0.5868,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.10478571802377701,
+      "rewards/margins": 0.32329410314559937,
+      "rewards/rejected": -0.4280797839164734,
+      "step": 7540
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 2.2076155121328326e-09,
+      "logits/chosen": -2.4920830726623535,
+      "logits/rejected": -2.4238436222076416,
+      "logps/chosen": -273.57666015625,
+      "logps/rejected": -286.92120361328125,
+      "loss": 0.5474,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -0.10572569072246552,
+      "rewards/margins": 0.4078649878501892,
+      "rewards/rejected": -0.5135906934738159,
+      "step": 7550
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.7538345820755641e-09,
+      "logits/chosen": -2.634387493133545,
+      "logits/rejected": -2.5403928756713867,
+      "logps/chosen": -333.7857666015625,
+      "logps/rejected": -309.5990295410156,
+      "loss": 0.5528,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.10820519924163818,
+      "rewards/margins": 0.38095569610595703,
+      "rewards/rejected": -0.4891608655452728,
+      "step": 7560
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.3521894674961567e-09,
+      "logits/chosen": -2.5758776664733887,
+      "logits/rejected": -2.554539203643799,
+      "logps/chosen": -319.5722351074219,
+      "logps/rejected": -325.67425537109375,
+      "loss": 0.6005,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -0.19757722318172455,
+      "rewards/margins": 0.2903757691383362,
+      "rewards/rejected": -0.4879530072212219,
+      "step": 7570
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 1.0026885503131023e-09,
+      "logits/chosen": -2.6342082023620605,
+      "logits/rejected": -2.588468551635742,
+      "logps/chosen": -310.59149169921875,
+      "logps/rejected": -284.2586364746094,
+      "loss": 0.5373,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.1725652515888214,
+      "rewards/margins": 0.44714459776878357,
+      "rewards/rejected": -0.619709849357605,
+      "step": 7580
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 7.053391242492491e-10,
+      "logits/chosen": -2.492493152618408,
+      "logits/rejected": -2.444969654083252,
+      "logps/chosen": -311.7804260253906,
+      "logps/rejected": -319.5511474609375,
+      "loss": 0.6468,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -0.3409726023674011,
+      "rewards/margins": 0.2081405371427536,
+      "rewards/rejected": -0.5491131544113159,
+      "step": 7590
+    },
+    {
+      "epoch": 0.99,
+      "learning_rate": 4.6014739467997725e-10,
+      "logits/chosen": -2.579031229019165,
+      "logits/rejected": -2.5480751991271973,
+      "logps/chosen": -297.0292663574219,
+      "logps/rejected": -306.7109680175781,
+      "loss": 0.614,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.16445885598659515,
+      "rewards/margins": 0.2770572602748871,
+      "rewards/rejected": -0.44151610136032104,
+      "step": 7600
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 2.671184785033032e-10,
+      "logits/chosen": -2.705984592437744,
+      "logits/rejected": -2.6434762477874756,
+      "logps/chosen": -325.71832275390625,
+      "logps/rejected": -348.49005126953125,
+      "loss": 0.5703,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -0.1509009748697281,
+      "rewards/margins": 0.35966190695762634,
+      "rewards/rejected": -0.5105628371238708,
+      "step": 7610
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.2625640403302054e-10,
+      "logits/chosen": -2.623316526412964,
+      "logits/rejected": -2.5773911476135254,
+      "logps/chosen": -307.89971923828125,
+      "logps/rejected": -351.13067626953125,
+      "loss": 0.5465,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.10245436429977417,
+      "rewards/margins": 0.44011133909225464,
+      "rewards/rejected": -0.5425657629966736,
+      "step": 7620
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 3.756411091515588e-11,
+      "logits/chosen": -2.653872013092041,
+      "logits/rejected": -2.5944817066192627,
+      "logps/chosen": -344.91070556640625,
+      "logps/rejected": -346.4688415527344,
+      "loss": 0.598,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -0.14210054278373718,
+      "rewards/margins": 0.3198266923427582,
+      "rewards/rejected": -0.46192723512649536,
+      "step": 7630
+    },
+    {
+      "epoch": 1.0,
+      "learning_rate": 1.0434500657963143e-12,
+      "logits/chosen": -2.53847074508667,
+      "logits/rejected": -2.513888120651245,
+      "logps/chosen": -292.8641052246094,
+      "logps/rejected": -343.59576416015625,
+      "loss": 0.5584,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.16113708913326263,
+      "rewards/margins": 0.36939841508865356,
+      "rewards/rejected": -0.5305355191230774,
+      "step": 7640
+    },
+    {
+      "epoch": 1.0,
+      "step": 7642,
+      "total_flos": 0.0,
+      "train_loss": 0.6024045768410584,
+      "train_runtime": 59158.061,
+      "train_samples_per_second": 1.033,
+      "train_steps_per_second": 0.129
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 7642,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 1000,
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}