diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,17066 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.999297541394882,
+  "eval_steps": 400,
+  "global_step": 5604,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.002676032781401572,
+      "grad_norm": 2.561987821133718,
+      "learning_rate": 8.9126559714795e-09,
+      "logits/chosen": -0.0566539391875267,
+      "logits/rejected": 0.15201517939567566,
+      "logps/chosen": -1.716072678565979,
+      "logps/rejected": -1.8893629312515259,
+      "loss": 0.7205,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.716072678565979,
+      "rewards/margins": 0.17329008877277374,
+      "rewards/rejected": -1.8893629312515259,
+      "step": 5
+    },
+    {
+      "epoch": 0.005352065562803144,
+      "grad_norm": 1.299643711577144,
+      "learning_rate": 1.7825311942959e-08,
+      "logits/chosen": 0.015113027766346931,
+      "logits/rejected": 0.13499757647514343,
+      "logps/chosen": -1.8030494451522827,
+      "logps/rejected": -1.8467035293579102,
+      "loss": 0.7281,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.8030494451522827,
+      "rewards/margins": 0.04365422949194908,
+      "rewards/rejected": -1.8467035293579102,
+      "step": 10
+    },
+    {
+      "epoch": 0.008028098344204716,
+      "grad_norm": 1.311860971074175,
+      "learning_rate": 2.67379679144385e-08,
+      "logits/chosen": -0.034261275082826614,
+      "logits/rejected": 0.062092166393995285,
+      "logps/chosen": -1.6353956460952759,
+      "logps/rejected": -1.7657333612442017,
+      "loss": 0.7417,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -1.6353956460952759,
+      "rewards/margins": 0.13033755123615265,
+      "rewards/rejected": -1.7657333612442017,
+      "step": 15
+    },
+    {
+      "epoch": 0.010704131125606288,
+      "grad_norm": 1.7831348138003078,
+      "learning_rate": 3.5650623885918e-08,
+      "logits/chosen": -0.036803100258111954,
+      "logits/rejected": 0.0490817055106163,
+      "logps/chosen": -1.7248821258544922,
+      "logps/rejected": -1.8057520389556885,
+      "loss": 0.7423,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -1.7248821258544922,
+      "rewards/margins": 0.08086995780467987,
+      "rewards/rejected": -1.8057520389556885,
+      "step": 20
+    },
+    {
+      "epoch": 0.013380163907007862,
+      "grad_norm": 1.6416319814675253,
+      "learning_rate": 4.45632798573975e-08,
+      "logits/chosen": -0.04610865190625191,
+      "logits/rejected": 0.03766794502735138,
+      "logps/chosen": -1.8688602447509766,
+      "logps/rejected": -1.7797136306762695,
+      "loss": 0.7724,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -1.8688602447509766,
+      "rewards/margins": -0.08914665132761002,
+      "rewards/rejected": -1.7797136306762695,
+      "step": 25
+    },
+    {
+      "epoch": 0.016056196688409432,
+      "grad_norm": 1.160599024038853,
+      "learning_rate": 5.3475935828877e-08,
+      "logits/chosen": -0.08371341228485107,
+      "logits/rejected": 0.009293553419411182,
+      "logps/chosen": -1.9095375537872314,
+      "logps/rejected": -1.8327739238739014,
+      "loss": 0.7293,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -1.9095375537872314,
+      "rewards/margins": -0.07676338404417038,
+      "rewards/rejected": -1.8327739238739014,
+      "step": 30
+    },
+    {
+      "epoch": 0.018732229469811006,
+      "grad_norm": 1.6985652172348096,
+      "learning_rate": 6.23885918003565e-08,
+      "logits/chosen": -0.04432075470685959,
+      "logits/rejected": 0.11638696491718292,
+      "logps/chosen": -1.8466746807098389,
+      "logps/rejected": -1.9980194568634033,
+      "loss": 0.7454,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -1.8466746807098389,
+      "rewards/margins": 0.15134476125240326,
+      "rewards/rejected": -1.9980194568634033,
+      "step": 35
+    },
+    {
+      "epoch": 0.021408262251212576,
+      "grad_norm": 1.4942054474815603,
+      "learning_rate": 7.1301247771836e-08,
+      "logits/chosen": 0.05298803001642227,
+      "logits/rejected": 0.22868895530700684,
+      "logps/chosen": -1.8837831020355225,
+      "logps/rejected": -1.7443134784698486,
+      "loss": 0.7513,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -1.8837831020355225,
+      "rewards/margins": -0.1394696682691574,
+      "rewards/rejected": -1.7443134784698486,
+      "step": 40
+    },
+    {
+      "epoch": 0.02408429503261415,
+      "grad_norm": 1.582941730338934,
+      "learning_rate": 8.021390374331551e-08,
+      "logits/chosen": 0.008146551437675953,
+      "logits/rejected": 0.20127546787261963,
+      "logps/chosen": -1.8411849737167358,
+      "logps/rejected": -1.8757511377334595,
+      "loss": 0.7409,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -1.8411849737167358,
+      "rewards/margins": 0.03456615284085274,
+      "rewards/rejected": -1.8757511377334595,
+      "step": 45
+    },
+    {
+      "epoch": 0.026760327814015723,
+      "grad_norm": 1.345701519233919,
+      "learning_rate": 8.9126559714795e-08,
+      "logits/chosen": -0.046867601573467255,
+      "logits/rejected": 0.10230155289173126,
+      "logps/chosen": -1.9038970470428467,
+      "logps/rejected": -1.7828941345214844,
+      "loss": 0.7405,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.9038970470428467,
+      "rewards/margins": -0.12100280821323395,
+      "rewards/rejected": -1.7828941345214844,
+      "step": 50
+    },
+    {
+      "epoch": 0.029436360595417294,
+      "grad_norm": 1.4507924062695625,
+      "learning_rate": 9.80392156862745e-08,
+      "logits/chosen": -0.10860351473093033,
+      "logits/rejected": 0.10858392715454102,
+      "logps/chosen": -1.841615915298462,
+      "logps/rejected": -1.8762493133544922,
+      "loss": 0.7282,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.841615915298462,
+      "rewards/margins": 0.03463347628712654,
+      "rewards/rejected": -1.8762493133544922,
+      "step": 55
+    },
+    {
+      "epoch": 0.032112393376818864,
+      "grad_norm": 1.4381611443259015,
+      "learning_rate": 1.06951871657754e-07,
+      "logits/chosen": -0.09268250316381454,
+      "logits/rejected": 0.09467026591300964,
+      "logps/chosen": -1.8004745244979858,
+      "logps/rejected": -1.9061378240585327,
+      "loss": 0.7266,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.8004745244979858,
+      "rewards/margins": 0.10566333681344986,
+      "rewards/rejected": -1.9061378240585327,
+      "step": 60
+    },
+    {
+      "epoch": 0.03478842615822044,
+      "grad_norm": 1.3704020875904708,
+      "learning_rate": 1.158645276292335e-07,
+      "logits/chosen": -0.02606000378727913,
+      "logits/rejected": 0.1216067522764206,
+      "logps/chosen": -1.6499643325805664,
+      "logps/rejected": -1.7832790613174438,
+      "loss": 0.7315,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.6499643325805664,
+      "rewards/margins": 0.13331469893455505,
+      "rewards/rejected": -1.7832790613174438,
+      "step": 65
+    },
+    {
+      "epoch": 0.03746445893962201,
+      "grad_norm": 2.274478095942714,
+      "learning_rate": 1.24777183600713e-07,
+      "logits/chosen": -0.06817416101694107,
+      "logits/rejected": 0.0827322006225586,
+      "logps/chosen": -1.7819998264312744,
+      "logps/rejected": -1.8292831182479858,
+      "loss": 0.7445,
+      "rewards/accuracies": 0.42500001192092896,
+      "rewards/chosen": -1.7819998264312744,
+      "rewards/margins": 0.0472835935652256,
+      "rewards/rejected": -1.8292831182479858,
+      "step": 70
+    },
+    {
+      "epoch": 0.04014049172102358,
+      "grad_norm": 1.2387594262372192,
+      "learning_rate": 1.3368983957219251e-07,
+      "logits/chosen": -0.059679191559553146,
+      "logits/rejected": 0.12264664471149445,
+      "logps/chosen": -1.8020824193954468,
+      "logps/rejected": -2.0671308040618896,
+      "loss": 0.7296,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.8020824193954468,
+      "rewards/margins": 0.2650483250617981,
+      "rewards/rejected": -2.0671308040618896,
+      "step": 75
+    },
+    {
+      "epoch": 0.04281652450242515,
+      "grad_norm": 1.2942064439757537,
+      "learning_rate": 1.42602495543672e-07,
+      "logits/chosen": 0.015558160841464996,
+      "logits/rejected": 0.12188021838665009,
+      "logps/chosen": -1.7552436590194702,
+      "logps/rejected": -1.7870506048202515,
+      "loss": 0.7409,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.7552436590194702,
+      "rewards/margins": 0.03180689737200737,
+      "rewards/rejected": -1.7870506048202515,
+      "step": 80
+    },
+    {
+      "epoch": 0.04549255728382673,
+      "grad_norm": 1.4156691260310694,
+      "learning_rate": 1.5151515151515152e-07,
+      "logits/chosen": -0.13759846985340118,
+      "logits/rejected": 0.11128225177526474,
+      "logps/chosen": -1.836038589477539,
+      "logps/rejected": -2.0215656757354736,
+      "loss": 0.7419,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.836038589477539,
+      "rewards/margins": 0.1855269968509674,
+      "rewards/rejected": -2.0215656757354736,
+      "step": 85
+    },
+    {
+      "epoch": 0.0481685900652283,
+      "grad_norm": 1.0745731417541164,
+      "learning_rate": 1.6042780748663102e-07,
+      "logits/chosen": 0.08276908844709396,
+      "logits/rejected": 0.040061693638563156,
+      "logps/chosen": -1.806331992149353,
+      "logps/rejected": -1.8212474584579468,
+      "loss": 0.7432,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -1.806331992149353,
+      "rewards/margins": 0.014915299601852894,
+      "rewards/rejected": -1.8212474584579468,
+      "step": 90
+    },
+    {
+      "epoch": 0.05084462284662987,
+      "grad_norm": 1.2541689453664435,
+      "learning_rate": 1.693404634581105e-07,
+      "logits/chosen": -0.07889704406261444,
+      "logits/rejected": 0.06831653416156769,
+      "logps/chosen": -1.887216567993164,
+      "logps/rejected": -1.9737586975097656,
+      "loss": 0.7317,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.887216567993164,
+      "rewards/margins": 0.08654232323169708,
+      "rewards/rejected": -1.9737586975097656,
+      "step": 95
+    },
+    {
+      "epoch": 0.05352065562803145,
+      "grad_norm": 1.1175857772038114,
+      "learning_rate": 1.7825311942959e-07,
+      "logits/chosen": -0.025732051581144333,
+      "logits/rejected": 0.037325549870729446,
+      "logps/chosen": -1.753867506980896,
+      "logps/rejected": -1.8652225732803345,
+      "loss": 0.7291,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.753867506980896,
+      "rewards/margins": 0.11135502904653549,
+      "rewards/rejected": -1.8652225732803345,
+      "step": 100
+    },
+    {
+      "epoch": 0.05619668840943302,
+      "grad_norm": 1.1357769016911667,
+      "learning_rate": 1.8716577540106952e-07,
+      "logits/chosen": 0.06076180189847946,
+      "logits/rejected": 0.08529307693243027,
+      "logps/chosen": -1.714585542678833,
+      "logps/rejected": -1.878586769104004,
+      "loss": 0.7274,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.714585542678833,
+      "rewards/margins": 0.16400139033794403,
+      "rewards/rejected": -1.878586769104004,
+      "step": 105
+    },
+    {
+      "epoch": 0.05887272119083459,
+      "grad_norm": 1.2883389330850903,
+      "learning_rate": 1.96078431372549e-07,
+      "logits/chosen": 0.008630759082734585,
+      "logits/rejected": 0.10148169845342636,
+      "logps/chosen": -1.7859894037246704,
+      "logps/rejected": -1.8440386056900024,
+      "loss": 0.7412,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -1.7859894037246704,
+      "rewards/margins": 0.05804925411939621,
+      "rewards/rejected": -1.8440386056900024,
+      "step": 110
+    },
+    {
+      "epoch": 0.06154875397223616,
+      "grad_norm": 1.3897045250030802,
+      "learning_rate": 2.049910873440285e-07,
+      "logits/chosen": 0.02914786897599697,
+      "logits/rejected": 0.23738856613636017,
+      "logps/chosen": -1.7832180261611938,
+      "logps/rejected": -2.0988895893096924,
+      "loss": 0.7132,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -1.7832180261611938,
+      "rewards/margins": 0.31567174196243286,
+      "rewards/rejected": -2.0988895893096924,
+      "step": 115
+    },
+    {
+      "epoch": 0.06422478675363773,
+      "grad_norm": 0.9125913058770045,
+      "learning_rate": 2.13903743315508e-07,
+      "logits/chosen": -0.05630670115351677,
+      "logits/rejected": 0.12373454868793488,
+      "logps/chosen": -1.9146301746368408,
+      "logps/rejected": -2.0535943508148193,
+      "loss": 0.7228,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -1.9146301746368408,
+      "rewards/margins": 0.1389642208814621,
+      "rewards/rejected": -2.0535943508148193,
+      "step": 120
+    },
+    {
+      "epoch": 0.0669008195350393,
+      "grad_norm": 1.478222641045802,
+      "learning_rate": 2.2281639928698751e-07,
+      "logits/chosen": -0.04984898120164871,
+      "logits/rejected": 0.08430124819278717,
+      "logps/chosen": -1.827081322669983,
+      "logps/rejected": -1.7506812810897827,
+      "loss": 0.7454,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -1.827081322669983,
+      "rewards/margins": -0.07639992982149124,
+      "rewards/rejected": -1.7506812810897827,
+      "step": 125
+    },
+    {
+      "epoch": 0.06957685231644088,
+      "grad_norm": 1.8056353849479956,
+      "learning_rate": 2.31729055258467e-07,
+      "logits/chosen": 0.060035817325115204,
+      "logits/rejected": 0.19798612594604492,
+      "logps/chosen": -1.8903570175170898,
+      "logps/rejected": -2.0116584300994873,
+      "loss": 0.7311,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -1.8903570175170898,
+      "rewards/margins": 0.1213013306260109,
+      "rewards/rejected": -2.0116584300994873,
+      "step": 130
+    },
+    {
+      "epoch": 0.07225288509784245,
+      "grad_norm": 1.076903294641355,
+      "learning_rate": 2.406417112299465e-07,
+      "logits/chosen": -0.04373316094279289,
+      "logits/rejected": 0.077092744410038,
+      "logps/chosen": -1.9686565399169922,
+      "logps/rejected": -1.9508752822875977,
+      "loss": 0.7367,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.9686565399169922,
+      "rewards/margins": -0.0177813358604908,
+      "rewards/rejected": -1.9508752822875977,
+      "step": 135
+    },
+    {
+      "epoch": 0.07492891787924402,
+      "grad_norm": 1.7079240317702742,
+      "learning_rate": 2.49554367201426e-07,
+      "logits/chosen": -0.019400786608457565,
+      "logits/rejected": 0.15313062071800232,
+      "logps/chosen": -1.9299131631851196,
+      "logps/rejected": -2.1614887714385986,
+      "loss": 0.7219,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -1.9299131631851196,
+      "rewards/margins": 0.23157572746276855,
+      "rewards/rejected": -2.1614887714385986,
+      "step": 140
+    },
+    {
+      "epoch": 0.0776049506606456,
+      "grad_norm": 1.16635803841252,
+      "learning_rate": 2.5846702317290554e-07,
+      "logits/chosen": 0.007627617567777634,
+      "logits/rejected": 0.1680821180343628,
+      "logps/chosen": -1.8729937076568604,
+      "logps/rejected": -2.012742280960083,
+      "loss": 0.7265,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -1.8729937076568604,
+      "rewards/margins": 0.13974833488464355,
+      "rewards/rejected": -2.012742280960083,
+      "step": 145
+    },
+    {
+      "epoch": 0.08028098344204716,
+      "grad_norm": 1.604750608835527,
+      "learning_rate": 2.6737967914438503e-07,
+      "logits/chosen": -0.028501566499471664,
+      "logits/rejected": 0.14074988663196564,
+      "logps/chosen": -1.8379608392715454,
+      "logps/rejected": -1.839181900024414,
+      "loss": 0.7356,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.8379608392715454,
+      "rewards/margins": 0.0012212246656417847,
+      "rewards/rejected": -1.839181900024414,
+      "step": 150
+    },
+    {
+      "epoch": 0.08295701622344874,
+      "grad_norm": 1.41435946264522,
+      "learning_rate": 2.762923351158645e-07,
+      "logits/chosen": -0.03521673008799553,
+      "logits/rejected": 0.01141396351158619,
+      "logps/chosen": -1.9185978174209595,
+      "logps/rejected": -1.9794740676879883,
+      "loss": 0.7344,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -1.9185978174209595,
+      "rewards/margins": 0.06087625026702881,
+      "rewards/rejected": -1.9794740676879883,
+      "step": 155
+    },
+    {
+      "epoch": 0.0856330490048503,
+      "grad_norm": 1.1955118620684497,
+      "learning_rate": 2.85204991087344e-07,
+      "logits/chosen": -0.12313131988048553,
+      "logits/rejected": 0.02289755269885063,
+      "logps/chosen": -2.1053714752197266,
+      "logps/rejected": -2.0795540809631348,
+      "loss": 0.7408,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -2.1053714752197266,
+      "rewards/margins": -0.02581748738884926,
+      "rewards/rejected": -2.0795540809631348,
+      "step": 160
+    },
+    {
+      "epoch": 0.08830908178625188,
+      "grad_norm": 1.279488936540094,
+      "learning_rate": 2.941176470588235e-07,
+      "logits/chosen": -0.036632001399993896,
+      "logits/rejected": 0.14142459630966187,
+      "logps/chosen": -1.881126046180725,
+      "logps/rejected": -2.1454551219940186,
+      "loss": 0.7289,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.881126046180725,
+      "rewards/margins": 0.26432910561561584,
+      "rewards/rejected": -2.1454551219940186,
+      "step": 165
+    },
+    {
+      "epoch": 0.09098511456765346,
+      "grad_norm": 1.1705925093506078,
+      "learning_rate": 3.0303030303030305e-07,
+      "logits/chosen": -0.07056300342082977,
+      "logits/rejected": -0.018231799826025963,
+      "logps/chosen": -2.130246877670288,
+      "logps/rejected": -2.1267504692077637,
+      "loss": 0.726,
+      "rewards/accuracies": 0.45625001192092896,
+      "rewards/chosen": -2.130246877670288,
+      "rewards/margins": -0.0034964592196047306,
+      "rewards/rejected": -2.1267504692077637,
+      "step": 170
+    },
+    {
+      "epoch": 0.09366114734905502,
+      "grad_norm": 1.1383276623057228,
+      "learning_rate": 3.1194295900178254e-07,
+      "logits/chosen": 0.07122258096933365,
+      "logits/rejected": 0.06929449737071991,
+      "logps/chosen": -2.0167622566223145,
+      "logps/rejected": -2.0888168811798096,
+      "loss": 0.752,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -2.0167622566223145,
+      "rewards/margins": 0.0720541775226593,
+      "rewards/rejected": -2.0888168811798096,
+      "step": 175
+    },
+    {
+      "epoch": 0.0963371801304566,
+      "grad_norm": 0.9722124950207639,
+      "learning_rate": 3.2085561497326203e-07,
+      "logits/chosen": 0.038307756185531616,
+      "logits/rejected": 0.041559524834156036,
+      "logps/chosen": -2.096546173095703,
+      "logps/rejected": -2.104980945587158,
+      "loss": 0.7276,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -2.096546173095703,
+      "rewards/margins": 0.008434787392616272,
+      "rewards/rejected": -2.104980945587158,
+      "step": 180
+    },
+    {
+      "epoch": 0.09901321291185818,
+      "grad_norm": 1.3279428327122702,
+      "learning_rate": 3.297682709447415e-07,
+      "logits/chosen": -0.0909801796078682,
+      "logits/rejected": 0.0018308877479285002,
+      "logps/chosen": -2.037461757659912,
+      "logps/rejected": -2.1208338737487793,
+      "loss": 0.7391,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -2.037461757659912,
+      "rewards/margins": 0.08337188512086868,
+      "rewards/rejected": -2.1208338737487793,
+      "step": 185
+    },
+    {
+      "epoch": 0.10168924569325974,
+      "grad_norm": 1.6607452252892816,
+      "learning_rate": 3.38680926916221e-07,
+      "logits/chosen": 0.000672203314024955,
+      "logits/rejected": 0.13302960991859436,
+      "logps/chosen": -2.4371438026428223,
+      "logps/rejected": -2.3462295532226562,
+      "loss": 0.7285,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -2.4371438026428223,
+      "rewards/margins": -0.09091375768184662,
+      "rewards/rejected": -2.3462295532226562,
+      "step": 190
+    },
+    {
+      "epoch": 0.10436527847466132,
+      "grad_norm": 0.9398516145147321,
+      "learning_rate": 3.475935828877005e-07,
+      "logits/chosen": 0.05403388291597366,
+      "logits/rejected": 0.217643141746521,
+      "logps/chosen": -1.9241230487823486,
+      "logps/rejected": -2.008875608444214,
+      "loss": 0.7261,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -1.9241230487823486,
+      "rewards/margins": 0.08475302159786224,
+      "rewards/rejected": -2.008875608444214,
+      "step": 195
+    },
+    {
+      "epoch": 0.1070413112560629,
+      "grad_norm": 0.9592262498823383,
+      "learning_rate": 3.5650623885918e-07,
+      "logits/chosen": -0.02870246209204197,
+      "logits/rejected": 0.1202109083533287,
+      "logps/chosen": -2.27313232421875,
+      "logps/rejected": -2.080379009246826,
+      "loss": 0.7343,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -2.27313232421875,
+      "rewards/margins": -0.19275303184986115,
+      "rewards/rejected": -2.080379009246826,
+      "step": 200
+    },
+    {
+      "epoch": 0.10971734403746446,
+      "grad_norm": 1.1901075397677303,
+      "learning_rate": 3.654188948306595e-07,
+      "logits/chosen": -0.023173019289970398,
+      "logits/rejected": 0.1332111805677414,
+      "logps/chosen": -2.4323229789733887,
+      "logps/rejected": -2.2145020961761475,
+      "loss": 0.7298,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -2.4323229789733887,
+      "rewards/margins": -0.21782033145427704,
+      "rewards/rejected": -2.2145020961761475,
+      "step": 205
+    },
+    {
+      "epoch": 0.11239337681886603,
+      "grad_norm": 0.934264361633064,
+      "learning_rate": 3.7433155080213904e-07,
+      "logits/chosen": -0.12006914615631104,
+      "logits/rejected": 0.08099234104156494,
+      "logps/chosen": -2.2656447887420654,
+      "logps/rejected": -2.571408271789551,
+      "loss": 0.7088,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -2.2656447887420654,
+      "rewards/margins": 0.3057636022567749,
+      "rewards/rejected": -2.571408271789551,
+      "step": 210
+    },
+    {
+      "epoch": 0.1150694096002676,
+      "grad_norm": 0.8892242020155503,
+      "learning_rate": 3.8324420677361853e-07,
+      "logits/chosen": -0.1393505036830902,
+      "logits/rejected": 0.12259514629840851,
+      "logps/chosen": -2.086469888687134,
+      "logps/rejected": -2.188483953475952,
+      "loss": 0.7146,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -2.086469888687134,
+      "rewards/margins": 0.10201425850391388,
+      "rewards/rejected": -2.188483953475952,
+      "step": 215
+    },
+    {
+      "epoch": 0.11774544238166917,
+      "grad_norm": 0.9167467837822779,
+      "learning_rate": 3.92156862745098e-07,
+      "logits/chosen": 0.10229828208684921,
+      "logits/rejected": 0.2131464183330536,
+      "logps/chosen": -2.258795738220215,
+      "logps/rejected": -2.5814874172210693,
+      "loss": 0.7079,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -2.258795738220215,
+      "rewards/margins": 0.32269176840782166,
+      "rewards/rejected": -2.5814874172210693,
+      "step": 220
+    },
+    {
+      "epoch": 0.12042147516307075,
+      "grad_norm": 1.1688798231006134,
+      "learning_rate": 4.010695187165775e-07,
+      "logits/chosen": -0.0630713626742363,
+      "logits/rejected": 0.12097378820180893,
+      "logps/chosen": -2.1071224212646484,
+      "logps/rejected": -2.214156150817871,
+      "loss": 0.712,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -2.1071224212646484,
+      "rewards/margins": 0.10703366994857788,
+      "rewards/rejected": -2.214156150817871,
+      "step": 225
+    },
+    {
+      "epoch": 0.12309750794447231,
+      "grad_norm": 1.1116821512928248,
+      "learning_rate": 4.09982174688057e-07,
+      "logits/chosen": 0.01072123646736145,
+      "logits/rejected": 0.09426967799663544,
+      "logps/chosen": -2.3712756633758545,
+      "logps/rejected": -2.4545907974243164,
+      "loss": 0.7197,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -2.3712756633758545,
+      "rewards/margins": 0.08331520110368729,
+      "rewards/rejected": -2.4545907974243164,
+      "step": 230
+    },
+    {
+      "epoch": 0.1257735407258739,
+      "grad_norm": 0.9646148879876714,
+      "learning_rate": 4.188948306595365e-07,
+      "logits/chosen": 0.068171426653862,
+      "logits/rejected": 0.23123112320899963,
+      "logps/chosen": -2.189795970916748,
+      "logps/rejected": -2.425313711166382,
+      "loss": 0.7033,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -2.189795970916748,
+      "rewards/margins": 0.2355179488658905,
+      "rewards/rejected": -2.425313711166382,
+      "step": 235
+    },
+    {
+      "epoch": 0.12844957350727546,
+      "grad_norm": 1.0169844738562488,
+      "learning_rate": 4.27807486631016e-07,
+      "logits/chosen": -0.00020105205476284027,
+      "logits/rejected": 0.13562379777431488,
+      "logps/chosen": -2.274684190750122,
+      "logps/rejected": -2.4053611755371094,
+      "loss": 0.7283,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -2.274684190750122,
+      "rewards/margins": 0.13067689538002014,
+      "rewards/rejected": -2.4053611755371094,
+      "step": 240
+    },
+    {
+      "epoch": 0.13112560628867703,
+      "grad_norm": 0.9712835023626972,
+      "learning_rate": 4.3672014260249554e-07,
+      "logits/chosen": 0.06408463418483734,
+      "logits/rejected": 0.2000926285982132,
+      "logps/chosen": -2.164257049560547,
+      "logps/rejected": -2.474696636199951,
+      "loss": 0.7109,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -2.164257049560547,
+      "rewards/margins": 0.31043943762779236,
+      "rewards/rejected": -2.474696636199951,
+      "step": 245
+    },
+    {
+      "epoch": 0.1338016390700786,
+      "grad_norm": 1.1601817021660885,
+      "learning_rate": 4.4563279857397503e-07,
+      "logits/chosen": -0.021302152425050735,
+      "logits/rejected": 0.15869158506393433,
+      "logps/chosen": -2.6671276092529297,
+      "logps/rejected": -2.658985137939453,
+      "loss": 0.726,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -2.6671276092529297,
+      "rewards/margins": -0.008142667822539806,
+      "rewards/rejected": -2.658985137939453,
+      "step": 250
+    },
+    {
+      "epoch": 0.1364776718514802,
+      "grad_norm": 1.334046682945677,
+      "learning_rate": 4.545454545454545e-07,
+      "logits/chosen": -0.010843111202120781,
+      "logits/rejected": 0.1529301404953003,
+      "logps/chosen": -2.0579779148101807,
+      "logps/rejected": -2.3893814086914062,
+      "loss": 0.7126,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -2.0579779148101807,
+      "rewards/margins": 0.33140355348587036,
+      "rewards/rejected": -2.3893814086914062,
+      "step": 255
+    },
+    {
+      "epoch": 0.13915370463288176,
+      "grad_norm": 1.0509802076668213,
+      "learning_rate": 4.63458110516934e-07,
+      "logits/chosen": -0.19671954214572906,
+      "logits/rejected": -0.07614149898290634,
+      "logps/chosen": -2.57027268409729,
+      "logps/rejected": -2.412527084350586,
+      "loss": 0.706,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -2.57027268409729,
+      "rewards/margins": -0.1577456146478653,
+      "rewards/rejected": -2.412527084350586,
+      "step": 260
+    },
+    {
+      "epoch": 0.1418297374142833,
+      "grad_norm": 0.76126309316922,
+      "learning_rate": 4.723707664884135e-07,
+      "logits/chosen": -0.04485822468996048,
+      "logits/rejected": 0.04307831451296806,
+      "logps/chosen": -2.9655423164367676,
+      "logps/rejected": -2.6251704692840576,
+      "loss": 0.7195,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -2.9655423164367676,
+      "rewards/margins": -0.34037190675735474,
+      "rewards/rejected": -2.6251704692840576,
+      "step": 265
+    },
+    {
+      "epoch": 0.1445057701956849,
+      "grad_norm": 0.7580960949941329,
+      "learning_rate": 4.81283422459893e-07,
+      "logits/chosen": -0.0737549290060997,
+      "logits/rejected": 0.07632019370794296,
+      "logps/chosen": -2.589219570159912,
+      "logps/rejected": -2.770702600479126,
+      "loss": 0.7178,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -2.589219570159912,
+      "rewards/margins": 0.18148289620876312,
+      "rewards/rejected": -2.770702600479126,
+      "step": 270
+    },
+    {
+      "epoch": 0.14718180297708647,
+      "grad_norm": 1.1890637241254927,
+      "learning_rate": 4.901960784313725e-07,
+      "logits/chosen": -0.012269625440239906,
+      "logits/rejected": 0.08800043165683746,
+      "logps/chosen": -2.886566638946533,
+      "logps/rejected": -3.23748779296875,
+      "loss": 0.7241,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -2.886566638946533,
+      "rewards/margins": 0.35092100501060486,
+      "rewards/rejected": -3.23748779296875,
+      "step": 275
+    },
+    {
+      "epoch": 0.14985783575848804,
+      "grad_norm": 0.8257760540864373,
+      "learning_rate": 4.99108734402852e-07,
+      "logits/chosen": -0.08742717653512955,
+      "logits/rejected": 0.09241114556789398,
+      "logps/chosen": -3.3425731658935547,
+      "logps/rejected": -3.2566428184509277,
+      "loss": 0.7003,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -3.3425731658935547,
+      "rewards/margins": -0.0859302282333374,
+      "rewards/rejected": -3.2566428184509277,
+      "step": 280
+    },
+    {
+      "epoch": 0.15253386853988962,
+      "grad_norm": 1.0510774949585053,
+      "learning_rate": 5.080213903743315e-07,
+      "logits/chosen": -0.03808263689279556,
+      "logits/rejected": 0.11533351987600327,
+      "logps/chosen": -3.233295440673828,
+      "logps/rejected": -3.218484878540039,
+      "loss": 0.7195,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -3.233295440673828,
+      "rewards/margins": -0.014810264110565186,
+      "rewards/rejected": -3.218484878540039,
+      "step": 285
+    },
+    {
+      "epoch": 0.1552099013212912,
+      "grad_norm": 0.5014865928865729,
+      "learning_rate": 5.169340463458111e-07,
+      "logits/chosen": -0.08595093339681625,
+      "logits/rejected": 0.2390422374010086,
+      "logps/chosen": -2.8254647254943848,
+      "logps/rejected": -3.4851670265197754,
+      "loss": 0.6863,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -2.8254647254943848,
+      "rewards/margins": 0.6597023010253906,
+      "rewards/rejected": -3.4851670265197754,
+      "step": 290
+    },
+    {
+      "epoch": 0.15788593410269275,
+      "grad_norm": 0.6108614217074682,
+      "learning_rate": 5.258467023172905e-07,
+      "logits/chosen": -0.04723949357867241,
+      "logits/rejected": 0.012955295853316784,
+      "logps/chosen": -3.5980706214904785,
+      "logps/rejected": -3.3474907875061035,
+      "loss": 0.7087,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -3.5980706214904785,
+      "rewards/margins": -0.2505798637866974,
+      "rewards/rejected": -3.3474907875061035,
+      "step": 295
+    },
+    {
+      "epoch": 0.16056196688409433,
+      "grad_norm": 0.7496989555316811,
+      "learning_rate": 5.347593582887701e-07,
+      "logits/chosen": -0.046161286532878876,
+      "logits/rejected": 0.13373184204101562,
+      "logps/chosen": -3.8412575721740723,
+      "logps/rejected": -3.9758124351501465,
+      "loss": 0.7092,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -3.8412575721740723,
+      "rewards/margins": 0.13455531001091003,
+      "rewards/rejected": -3.9758124351501465,
+      "step": 300
+    },
+    {
+      "epoch": 0.1632379996654959,
+      "grad_norm": 0.8036005556001232,
+      "learning_rate": 5.436720142602496e-07,
+      "logits/chosen": -0.010632747784256935,
+      "logits/rejected": 0.06074709817767143,
+      "logps/chosen": -3.4550633430480957,
+      "logps/rejected": -3.4543659687042236,
+      "loss": 0.7118,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -3.4550633430480957,
+      "rewards/margins": -0.000697445881087333,
+      "rewards/rejected": -3.4543659687042236,
+      "step": 305
+    },
+    {
+      "epoch": 0.16591403244689748,
+      "grad_norm": 0.6340466989052684,
+      "learning_rate": 5.52584670231729e-07,
+      "logits/chosen": -0.14063246548175812,
+      "logits/rejected": -0.039143528789281845,
+      "logps/chosen": -3.986959457397461,
+      "logps/rejected": -4.087148189544678,
+      "loss": 0.6996,
+      "rewards/accuracies": 0.45625001192092896,
+      "rewards/chosen": -3.986959457397461,
+      "rewards/margins": 0.10018882900476456,
+      "rewards/rejected": -4.087148189544678,
+      "step": 310
+    },
+    {
+      "epoch": 0.16859006522829906,
+      "grad_norm": 0.5017572311779819,
+      "learning_rate": 5.614973262032086e-07,
+      "logits/chosen": 0.0376378670334816,
+      "logits/rejected": 0.21232381463050842,
+      "logps/chosen": -3.8969693183898926,
+      "logps/rejected": -3.9083220958709717,
+      "loss": 0.7036,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -3.8969693183898926,
+      "rewards/margins": 0.01135268248617649,
+      "rewards/rejected": -3.9083220958709717,
+      "step": 315
+    },
+    {
+      "epoch": 0.1712660980097006,
+      "grad_norm": 0.8757535089634223,
+      "learning_rate": 5.70409982174688e-07,
+      "logits/chosen": -0.0015147717203944921,
+      "logits/rejected": 0.13918903470039368,
+      "logps/chosen": -3.748753070831299,
+      "logps/rejected": -3.5725371837615967,
+      "loss": 0.7051,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -3.748753070831299,
+      "rewards/margins": -0.17621535062789917,
+      "rewards/rejected": -3.5725371837615967,
+      "step": 320
+    },
+    {
+      "epoch": 0.17394213079110218,
+      "grad_norm": 0.37360001632063977,
+      "learning_rate": 5.793226381461676e-07,
+      "logits/chosen": -0.060974158346652985,
+      "logits/rejected": 0.07585623860359192,
+      "logps/chosen": -4.702275276184082,
+      "logps/rejected": -4.991570472717285,
+      "loss": 0.6976,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -4.702275276184082,
+      "rewards/margins": 0.28929537534713745,
+      "rewards/rejected": -4.991570472717285,
+      "step": 325
+    },
+    {
+      "epoch": 0.17661816357250376,
+      "grad_norm": 0.47492586968623013,
+      "learning_rate": 5.88235294117647e-07,
+      "logits/chosen": 0.07355102896690369,
+      "logits/rejected": 0.24657635390758514,
+      "logps/chosen": -3.607623338699341,
+      "logps/rejected": -5.0255446434021,
+      "loss": 0.6984,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -3.607623338699341,
+      "rewards/margins": 1.4179216623306274,
+      "rewards/rejected": -5.0255446434021,
+      "step": 330
+    },
+    {
+      "epoch": 0.17929419635390534,
+      "grad_norm": 0.5752027470382951,
+      "learning_rate": 5.971479500891266e-07,
+      "logits/chosen": 0.1287570297718048,
+      "logits/rejected": 0.2576545774936676,
+      "logps/chosen": -5.235191345214844,
+      "logps/rejected": -5.1844329833984375,
+      "loss": 0.7011,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -5.235191345214844,
+      "rewards/margins": -0.05075795575976372,
+      "rewards/rejected": -5.1844329833984375,
+      "step": 335
+    },
+    {
+      "epoch": 0.18197022913530692,
+      "grad_norm": 0.36962599005297986,
+      "learning_rate": 6.060606060606061e-07,
+      "logits/chosen": 0.05724542587995529,
+      "logits/rejected": 0.22630390524864197,
+      "logps/chosen": -4.8453264236450195,
+      "logps/rejected": -5.203751564025879,
+      "loss": 0.6939,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -4.8453264236450195,
+      "rewards/margins": 0.35842466354370117,
+      "rewards/rejected": -5.203751564025879,
+      "step": 340
+    },
+    {
+      "epoch": 0.1846462619167085,
+      "grad_norm": 0.5644467963544338,
+      "learning_rate": 6.149732620320855e-07,
+      "logits/chosen": 0.1736188679933548,
+      "logits/rejected": 0.2111024409532547,
+      "logps/chosen": -5.405289173126221,
+      "logps/rejected": -5.271110534667969,
+      "loss": 0.7023,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -5.405289173126221,
+      "rewards/margins": -0.13417872786521912,
+      "rewards/rejected": -5.271110534667969,
+      "step": 345
+    },
+    {
+      "epoch": 0.18732229469811004,
+      "grad_norm": 0.3932769740609003,
+      "learning_rate": 6.238859180035651e-07,
+      "logits/chosen": 0.15090247988700867,
+      "logits/rejected": 0.26804447174072266,
+      "logps/chosen": -5.620543479919434,
+      "logps/rejected": -4.588951587677002,
+      "loss": 0.6976,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -5.620543479919434,
+      "rewards/margins": -1.0315921306610107,
+      "rewards/rejected": -4.588951587677002,
+      "step": 350
+    },
+    {
+      "epoch": 0.18999832747951162,
+      "grad_norm": 0.2959917852726394,
+      "learning_rate": 6.327985739750445e-07,
+      "logits/chosen": 0.07048813253641129,
+      "logits/rejected": 0.3342396318912506,
+      "logps/chosen": -5.611636638641357,
+      "logps/rejected": -5.329535007476807,
+      "loss": 0.6945,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -5.611636638641357,
+      "rewards/margins": -0.28210148215293884,
+      "rewards/rejected": -5.329535007476807,
+      "step": 355
+    },
+    {
+      "epoch": 0.1926743602609132,
+      "grad_norm": 0.577459832521352,
+      "learning_rate": 6.417112299465241e-07,
+      "logits/chosen": 0.11730299144983292,
+      "logits/rejected": 0.20945081114768982,
+      "logps/chosen": -6.094443321228027,
+      "logps/rejected": -6.117684364318848,
+      "loss": 0.6997,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -6.094443321228027,
+      "rewards/margins": 0.023241113871335983,
+      "rewards/rejected": -6.117684364318848,
+      "step": 360
+    },
+    {
+      "epoch": 0.19535039304231477,
+      "grad_norm": 0.5561506038537439,
+      "learning_rate": 6.506238859180035e-07,
+      "logits/chosen": 0.14349046349525452,
+      "logits/rejected": 0.24878618121147156,
+      "logps/chosen": -5.719912528991699,
+      "logps/rejected": -4.875842094421387,
+      "loss": 0.6966,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -5.719912528991699,
+      "rewards/margins": -0.8440699577331543,
+      "rewards/rejected": -4.875842094421387,
+      "step": 365
+    },
+    {
+      "epoch": 0.19802642582371635,
+      "grad_norm": 0.4232864080085214,
+      "learning_rate": 6.59536541889483e-07,
+      "logits/chosen": 0.14949120581150055,
+      "logits/rejected": 0.26837489008903503,
+      "logps/chosen": -5.026135444641113,
+      "logps/rejected": -4.876240253448486,
+      "loss": 0.7064,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -5.026135444641113,
+      "rewards/margins": -0.14989487826824188,
+      "rewards/rejected": -4.876240253448486,
+      "step": 370
+    },
+    {
+      "epoch": 0.2007024586051179,
+      "grad_norm": 0.4011330363194617,
+      "learning_rate": 6.684491978609626e-07,
+      "logits/chosen": 0.10750514268875122,
+      "logits/rejected": 0.2961978018283844,
+      "logps/chosen": -5.236952781677246,
+      "logps/rejected": -6.180326461791992,
+      "loss": 0.6942,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -5.236952781677246,
+      "rewards/margins": 0.9433735013008118,
+      "rewards/rejected": -6.180326461791992,
+      "step": 375
+    },
+    {
+      "epoch": 0.20337849138651948,
+      "grad_norm": 0.411790100781544,
+      "learning_rate": 6.77361853832442e-07,
+      "logits/chosen": 0.14636285603046417,
+      "logits/rejected": 0.2552714943885803,
+      "logps/chosen": -6.010157585144043,
+      "logps/rejected": -6.052901744842529,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -6.010157585144043,
+      "rewards/margins": 0.04274497181177139,
+      "rewards/rejected": -6.052901744842529,
+      "step": 380
+    },
+    {
+      "epoch": 0.20605452416792105,
+      "grad_norm": 0.33236525228212505,
+      "learning_rate": 6.862745098039216e-07,
+      "logits/chosen": 0.18907134234905243,
+      "logits/rejected": 0.2890511155128479,
+      "logps/chosen": -4.906566619873047,
+      "logps/rejected": -5.490983486175537,
+      "loss": 0.6952,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -4.906566619873047,
+      "rewards/margins": 0.584416389465332,
+      "rewards/rejected": -5.490983486175537,
+      "step": 385
+    },
+    {
+      "epoch": 0.20873055694932263,
+      "grad_norm": 0.32662686588540074,
+      "learning_rate": 6.95187165775401e-07,
+      "logits/chosen": 0.26170283555984497,
+      "logits/rejected": 0.4747176170349121,
+      "logps/chosen": -5.420689105987549,
+      "logps/rejected": -6.125526428222656,
+      "loss": 0.6945,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -5.420689105987549,
+      "rewards/margins": 0.7048367261886597,
+      "rewards/rejected": -6.125526428222656,
+      "step": 390
+    },
+    {
+      "epoch": 0.2114065897307242,
+      "grad_norm": 0.4330079976179824,
+      "learning_rate": 7.040998217468806e-07,
+      "logits/chosen": 0.09333498775959015,
+      "logits/rejected": 0.2983303666114807,
+      "logps/chosen": -4.823489189147949,
+      "logps/rejected": -5.299779415130615,
+      "loss": 0.6973,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -4.823489189147949,
+      "rewards/margins": 0.476290225982666,
+      "rewards/rejected": -5.299779415130615,
+      "step": 395
+    },
+    {
+      "epoch": 0.2140826225121258,
+      "grad_norm": 0.5113830471741896,
+      "learning_rate": 7.1301247771836e-07,
+      "logits/chosen": 0.18131273984909058,
+      "logits/rejected": 0.30238741636276245,
+      "logps/chosen": -5.721251487731934,
+      "logps/rejected": -5.549353122711182,
+      "loss": 0.6893,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -5.721251487731934,
+      "rewards/margins": -0.171898752450943,
+      "rewards/rejected": -5.549353122711182,
+      "step": 400
+    },
+    {
+      "epoch": 0.2140826225121258,
+      "eval_logits/chosen": 0.49702373147010803,
+      "eval_logits/rejected": 0.6073431968688965,
+      "eval_logps/chosen": -5.63992977142334,
+      "eval_logps/rejected": -5.651395797729492,
+      "eval_loss": 0.6975800395011902,
+      "eval_rewards/accuracies": 0.5133531093597412,
+      "eval_rewards/chosen": -5.63992977142334,
+      "eval_rewards/margins": 0.011466006748378277,
+      "eval_rewards/rejected": -5.651395797729492,
+      "eval_runtime": 41.775,
+      "eval_samples_per_second": 32.196,
+      "eval_steps_per_second": 8.067,
+      "step": 400
+    },
+    {
+      "epoch": 0.21675865529352734,
+      "grad_norm": 0.2938373595516762,
+      "learning_rate": 7.219251336898395e-07,
+      "logits/chosen": 0.19062530994415283,
+      "logits/rejected": 0.2938390374183655,
+      "logps/chosen": -5.894046783447266,
+      "logps/rejected": -6.2626166343688965,
+      "loss": 0.7011,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -5.894046783447266,
+      "rewards/margins": 0.36857062578201294,
+      "rewards/rejected": -6.2626166343688965,
+      "step": 405
+    },
+    {
+      "epoch": 0.2194346880749289,
+      "grad_norm": 0.37899077546530213,
+      "learning_rate": 7.30837789661319e-07,
+      "logits/chosen": 0.22660979628562927,
+      "logits/rejected": 0.38428425788879395,
+      "logps/chosen": -6.168007850646973,
+      "logps/rejected": -6.864206790924072,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -6.168007850646973,
+      "rewards/margins": 0.6961987614631653,
+      "rewards/rejected": -6.864206790924072,
+      "step": 410
+    },
+    {
+      "epoch": 0.2221107208563305,
+      "grad_norm": 0.2992812000924985,
+      "learning_rate": 7.397504456327985e-07,
+      "logits/chosen": 0.19611527025699615,
+      "logits/rejected": 0.25555574893951416,
+      "logps/chosen": -6.9365057945251465,
+      "logps/rejected": -5.654603481292725,
+      "loss": 0.7011,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -6.9365057945251465,
+      "rewards/margins": -1.2819023132324219,
+      "rewards/rejected": -5.654603481292725,
+      "step": 415
+    },
+    {
+      "epoch": 0.22478675363773207,
+      "grad_norm": 0.573938963684281,
+      "learning_rate": 7.486631016042781e-07,
+      "logits/chosen": 0.11325088888406754,
+      "logits/rejected": 0.3763885498046875,
+      "logps/chosen": -5.879007816314697,
+      "logps/rejected": -5.892158508300781,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -5.879007816314697,
+      "rewards/margins": 0.013150262646377087,
+      "rewards/rejected": -5.892158508300781,
+      "step": 420
+    },
+    {
+      "epoch": 0.22746278641913364,
+      "grad_norm": 0.4274640071050483,
+      "learning_rate": 7.575757575757575e-07,
+      "logits/chosen": 0.10492966324090958,
+      "logits/rejected": 0.34241434931755066,
+      "logps/chosen": -6.645754337310791,
+      "logps/rejected": -6.289525032043457,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -6.645754337310791,
+      "rewards/margins": -0.3562292456626892,
+      "rewards/rejected": -6.289525032043457,
+      "step": 425
+    },
+    {
+      "epoch": 0.2301388192005352,
+      "grad_norm": 0.42878301291549364,
+      "learning_rate": 7.664884135472371e-07,
+      "logits/chosen": 0.04065848886966705,
+      "logits/rejected": 0.28358420729637146,
+      "logps/chosen": -5.721266269683838,
+      "logps/rejected": -7.1136064529418945,
+      "loss": 0.694,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -5.721266269683838,
+      "rewards/margins": 1.3923394680023193,
+      "rewards/rejected": -7.1136064529418945,
+      "step": 430
+    },
+    {
+      "epoch": 0.23281485198193677,
+      "grad_norm": 0.6101795913386221,
+      "learning_rate": 7.754010695187165e-07,
+      "logits/chosen": 0.16764353215694427,
+      "logits/rejected": 0.27640286087989807,
+      "logps/chosen": -6.7787275314331055,
+      "logps/rejected": -5.879710674285889,
+      "loss": 0.6955,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -6.7787275314331055,
+      "rewards/margins": -0.899017333984375,
+      "rewards/rejected": -5.879710674285889,
+      "step": 435
+    },
+    {
+      "epoch": 0.23549088476333835,
+      "grad_norm": 0.4517132586098904,
+      "learning_rate": 7.84313725490196e-07,
+      "logits/chosen": 0.1396738737821579,
+      "logits/rejected": 0.26584815979003906,
+      "logps/chosen": -4.967270851135254,
+      "logps/rejected": -5.396585941314697,
+      "loss": 0.6965,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -4.967270851135254,
+      "rewards/margins": 0.42931586503982544,
+      "rewards/rejected": -5.396585941314697,
+      "step": 440
+    },
+    {
+      "epoch": 0.23816691754473993,
+      "grad_norm": 0.4317266440316381,
+      "learning_rate": 7.932263814616755e-07,
+      "logits/chosen": 0.10195654630661011,
+      "logits/rejected": 0.23662152886390686,
+      "logps/chosen": -6.003907203674316,
+      "logps/rejected": -6.293694972991943,
+      "loss": 0.6942,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -6.003907203674316,
+      "rewards/margins": 0.28978732228279114,
+      "rewards/rejected": -6.293694972991943,
+      "step": 445
+    },
+    {
+      "epoch": 0.2408429503261415,
+      "grad_norm": 0.2881950513632804,
+      "learning_rate": 8.02139037433155e-07,
+      "logits/chosen": 0.15644700825214386,
+      "logits/rejected": 0.3218119740486145,
+      "logps/chosen": -5.904261112213135,
+      "logps/rejected": -5.573202133178711,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -5.904261112213135,
+      "rewards/margins": -0.3310585618019104,
+      "rewards/rejected": -5.573202133178711,
+      "step": 450
+    },
+    {
+      "epoch": 0.24351898310754308,
+      "grad_norm": 0.36747125453567453,
+      "learning_rate": 8.110516934046346e-07,
+      "logits/chosen": 0.18731513619422913,
+      "logits/rejected": 0.29376420378685,
+      "logps/chosen": -6.2077131271362305,
+      "logps/rejected": -6.673652648925781,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -6.2077131271362305,
+      "rewards/margins": 0.46593934297561646,
+      "rewards/rejected": -6.673652648925781,
+      "step": 455
+    },
+    {
+      "epoch": 0.24619501588894463,
+      "grad_norm": 0.7992786388556145,
+      "learning_rate": 8.19964349376114e-07,
+      "logits/chosen": 0.011485517956316471,
+      "logits/rejected": 0.17833183705806732,
+      "logps/chosen": -6.326829433441162,
+      "logps/rejected": -6.177127838134766,
+      "loss": 0.6975,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -6.326829433441162,
+      "rewards/margins": -0.1497005671262741,
+      "rewards/rejected": -6.177127838134766,
+      "step": 460
+    },
+    {
+      "epoch": 0.2488710486703462,
+      "grad_norm": 0.5127063063441213,
+      "learning_rate": 8.288770053475936e-07,
+      "logits/chosen": 0.3017039895057678,
+      "logits/rejected": 0.3497074246406555,
+      "logps/chosen": -7.3247175216674805,
+      "logps/rejected": -7.367332458496094,
+      "loss": 0.691,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -7.3247175216674805,
+      "rewards/margins": 0.04261467605829239,
+      "rewards/rejected": -7.367332458496094,
+      "step": 465
+    },
+    {
+      "epoch": 0.2515470814517478,
+      "grad_norm": 8.266985266351112,
+      "learning_rate": 8.37789661319073e-07,
+      "logits/chosen": 0.248381569981575,
+      "logits/rejected": 0.14312608540058136,
+      "logps/chosen": -7.462276458740234,
+      "logps/rejected": -6.2620368003845215,
+      "loss": 0.7006,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -7.462276458740234,
+      "rewards/margins": -1.2002400159835815,
+      "rewards/rejected": -6.2620368003845215,
+      "step": 470
+    },
+    {
+      "epoch": 0.25422311423314936,
+      "grad_norm": 0.6140042352084357,
+      "learning_rate": 8.467023172905525e-07,
+      "logits/chosen": -0.05337027832865715,
+      "logits/rejected": 0.14719079434871674,
+      "logps/chosen": -6.329145908355713,
+      "logps/rejected": -8.072843551635742,
+      "loss": 0.6865,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -6.329145908355713,
+      "rewards/margins": 1.7436981201171875,
+      "rewards/rejected": -8.072843551635742,
+      "step": 475
+    },
+    {
+      "epoch": 0.2568991470145509,
+      "grad_norm": 0.3772880456309565,
+      "learning_rate": 8.55614973262032e-07,
+      "logits/chosen": 0.10138843208551407,
+      "logits/rejected": 0.33208367228507996,
+      "logps/chosen": -6.503623962402344,
+      "logps/rejected": -7.560891628265381,
+      "loss": 0.6934,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -6.503623962402344,
+      "rewards/margins": 1.057267189025879,
+      "rewards/rejected": -7.560891628265381,
+      "step": 480
+    },
+    {
+      "epoch": 0.2595751797959525,
+      "grad_norm": 1.5225786113089712,
+      "learning_rate": 8.645276292335115e-07,
+      "logits/chosen": 0.13484086096286774,
+      "logits/rejected": 0.1996881663799286,
+      "logps/chosen": -7.7192063331604,
+      "logps/rejected": -6.5416107177734375,
+      "loss": 0.7006,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -7.7192063331604,
+      "rewards/margins": -1.177594780921936,
+      "rewards/rejected": -6.5416107177734375,
+      "step": 485
+    },
+    {
+      "epoch": 0.26225121257735406,
+      "grad_norm": 0.7728326769292436,
+      "learning_rate": 8.734402852049911e-07,
+      "logits/chosen": 0.12758545577526093,
+      "logits/rejected": 0.195746511220932,
+      "logps/chosen": -7.4417314529418945,
+      "logps/rejected": -7.916699409484863,
+      "loss": 0.6993,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -7.4417314529418945,
+      "rewards/margins": 0.47496843338012695,
+      "rewards/rejected": -7.916699409484863,
+      "step": 490
+    },
+    {
+      "epoch": 0.26492724535875567,
+      "grad_norm": 0.5650802033378776,
+      "learning_rate": 8.823529411764705e-07,
+      "logits/chosen": 0.06879357248544693,
+      "logits/rejected": 0.07481779903173447,
+      "logps/chosen": -8.613202095031738,
+      "logps/rejected": -8.19923210144043,
+      "loss": 0.6938,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -8.613202095031738,
+      "rewards/margins": -0.4139712452888489,
+      "rewards/rejected": -8.19923210144043,
+      "step": 495
+    },
+    {
+      "epoch": 0.2676032781401572,
+      "grad_norm": 3.190002762958353,
+      "learning_rate": 8.912655971479501e-07,
+      "logits/chosen": 0.018533959984779358,
+      "logits/rejected": 0.13243678212165833,
+      "logps/chosen": -7.8292670249938965,
+      "logps/rejected": -8.657615661621094,
+      "loss": 0.6944,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -7.8292670249938965,
+      "rewards/margins": 0.8283494710922241,
+      "rewards/rejected": -8.657615661621094,
+      "step": 500
+    },
+    {
+      "epoch": 0.27027931092155877,
+      "grad_norm": 0.6607528659719145,
+      "learning_rate": 9.001782531194295e-07,
+      "logits/chosen": -0.048227228224277496,
+      "logits/rejected": 0.10579868406057358,
+      "logps/chosen": -7.318779945373535,
+      "logps/rejected": -6.701746463775635,
+      "loss": 0.6942,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -7.318779945373535,
+      "rewards/margins": -0.6170336008071899,
+      "rewards/rejected": -6.701746463775635,
+      "step": 505
+    },
+    {
+      "epoch": 0.2729553437029604,
+      "grad_norm": 2.4473329850704433,
+      "learning_rate": 9.09090909090909e-07,
+      "logits/chosen": 0.17527154088020325,
+      "logits/rejected": 0.23689062893390656,
+      "logps/chosen": -8.291070938110352,
+      "logps/rejected": -8.05940055847168,
+      "loss": 0.6931,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -8.291070938110352,
+      "rewards/margins": -0.2316698282957077,
+      "rewards/rejected": -8.05940055847168,
+      "step": 510
+    },
+    {
+      "epoch": 0.2756313764843619,
+      "grad_norm": 1.2765480160313272,
+      "learning_rate": 9.180035650623885e-07,
+      "logits/chosen": 0.012978377752006054,
+      "logits/rejected": 0.1495513916015625,
+      "logps/chosen": -8.148285865783691,
+      "logps/rejected": -6.6392011642456055,
+      "loss": 0.6869,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -8.148285865783691,
+      "rewards/margins": -1.5090851783752441,
+      "rewards/rejected": -6.6392011642456055,
+      "step": 515
+    },
+    {
+      "epoch": 0.27830740926576353,
+      "grad_norm": 1.316730150799058,
+      "learning_rate": 9.26916221033868e-07,
+      "logits/chosen": 0.01837373524904251,
+      "logits/rejected": 0.19669003784656525,
+      "logps/chosen": -8.109740257263184,
+      "logps/rejected": -7.669831275939941,
+      "loss": 0.6933,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -8.109740257263184,
+      "rewards/margins": -0.43990975618362427,
+      "rewards/rejected": -7.669831275939941,
+      "step": 520
+    },
+    {
+      "epoch": 0.2809834420471651,
+      "grad_norm": 0.5642580751191001,
+      "learning_rate": 9.358288770053476e-07,
+      "logits/chosen": 0.153331458568573,
+      "logits/rejected": 0.26244035363197327,
+      "logps/chosen": -8.862241744995117,
+      "logps/rejected": -8.683265686035156,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -8.862241744995117,
+      "rewards/margins": -0.17897605895996094,
+      "rewards/rejected": -8.683265686035156,
+      "step": 525
+    },
+    {
+      "epoch": 0.2836594748285666,
+      "grad_norm": 0.3751766849442403,
+      "learning_rate": 9.44741532976827e-07,
+      "logits/chosen": 0.10893809795379639,
+      "logits/rejected": 0.15763886272907257,
+      "logps/chosen": -7.532681465148926,
+      "logps/rejected": -7.462843894958496,
+      "loss": 0.6953,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -7.532681465148926,
+      "rewards/margins": -0.06983815133571625,
+      "rewards/rejected": -7.462843894958496,
+      "step": 530
+    },
+    {
+      "epoch": 0.28633550760996823,
+      "grad_norm": 0.6095563168120589,
+      "learning_rate": 9.536541889483066e-07,
+      "logits/chosen": 0.027346592396497726,
+      "logits/rejected": 0.35276252031326294,
+      "logps/chosen": -8.090580940246582,
+      "logps/rejected": -9.56582260131836,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -8.090580940246582,
+      "rewards/margins": 1.475242018699646,
+      "rewards/rejected": -9.56582260131836,
+      "step": 535
+    },
+    {
+      "epoch": 0.2890115403913698,
+      "grad_norm": 0.6502619755536633,
+      "learning_rate": 9.62566844919786e-07,
+      "logits/chosen": 0.10511992126703262,
+      "logits/rejected": 0.21119940280914307,
+      "logps/chosen": -6.845664024353027,
+      "logps/rejected": -6.653723239898682,
+      "loss": 0.6963,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -6.845664024353027,
+      "rewards/margins": -0.1919410526752472,
+      "rewards/rejected": -6.653723239898682,
+      "step": 540
+    },
+    {
+      "epoch": 0.2916875731727714,
+      "grad_norm": 0.4714067501148884,
+      "learning_rate": 9.714795008912655e-07,
+      "logits/chosen": 0.026676470413804054,
+      "logits/rejected": 0.27182188630104065,
+      "logps/chosen": -7.278785705566406,
+      "logps/rejected": -6.778079986572266,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -7.278785705566406,
+      "rewards/margins": -0.5007057189941406,
+      "rewards/rejected": -6.778079986572266,
+      "step": 545
+    },
+    {
+      "epoch": 0.29436360595417294,
+      "grad_norm": 0.7689420365836325,
+      "learning_rate": 9.80392156862745e-07,
+      "logits/chosen": 0.1788049340248108,
+      "logits/rejected": 0.2550205588340759,
+      "logps/chosen": -5.815304756164551,
+      "logps/rejected": -5.32241678237915,
+      "loss": 0.6941,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -5.815304756164551,
+      "rewards/margins": -0.49288803339004517,
+      "rewards/rejected": -5.32241678237915,
+      "step": 550
+    },
+    {
+      "epoch": 0.2970396387355745,
+      "grad_norm": 0.46580993483766825,
+      "learning_rate": 9.893048128342244e-07,
+      "logits/chosen": 0.08636007457971573,
+      "logits/rejected": 0.2525786757469177,
+      "logps/chosen": -7.308444023132324,
+      "logps/rejected": -7.2437639236450195,
+      "loss": 0.6944,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -7.308444023132324,
+      "rewards/margins": -0.06468057632446289,
+      "rewards/rejected": -7.2437639236450195,
+      "step": 555
+    },
+    {
+      "epoch": 0.2997156715169761,
+      "grad_norm": 0.5172865220625742,
+      "learning_rate": 9.98217468805704e-07,
+      "logits/chosen": 0.1951034516096115,
+      "logits/rejected": 0.22605688869953156,
+      "logps/chosen": -8.74112319946289,
+      "logps/rejected": -7.048243045806885,
+      "loss": 0.692,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -8.74112319946289,
+      "rewards/margins": -1.6928800344467163,
+      "rewards/rejected": -7.048243045806885,
+      "step": 560
+    },
+    {
+      "epoch": 0.30239170429837764,
+      "grad_norm": 1.309857784983351,
+      "learning_rate": 9.999984476788462e-07,
+      "logits/chosen": 0.2193707972764969,
+      "logits/rejected": 0.30722564458847046,
+      "logps/chosen": -7.271849632263184,
+      "logps/rejected": -7.647642612457275,
+      "loss": 0.6945,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -7.271849632263184,
+      "rewards/margins": 0.3757931888103485,
+      "rewards/rejected": -7.647642612457275,
+      "step": 565
+    },
+    {
+      "epoch": 0.30506773707977924,
+      "grad_norm": 0.5404655600845336,
+      "learning_rate": 9.999921413906797e-07,
+      "logits/chosen": 0.16190442442893982,
+      "logits/rejected": 0.42977142333984375,
+      "logps/chosen": -8.50904369354248,
+      "logps/rejected": -8.440361976623535,
+      "loss": 0.692,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -8.50904369354248,
+      "rewards/margins": -0.06868257373571396,
+      "rewards/rejected": -8.440361976623535,
+      "step": 570
+    },
+    {
+      "epoch": 0.3077437698611808,
+      "grad_norm": 0.4717241051840405,
+      "learning_rate": 9.999809841765644e-07,
+      "logits/chosen": 0.21549710631370544,
+      "logits/rejected": 0.22803433239459991,
+      "logps/chosen": -8.10963249206543,
+      "logps/rejected": -8.179125785827637,
+      "loss": 0.6927,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -8.10963249206543,
+      "rewards/margins": 0.06949272006750107,
+      "rewards/rejected": -8.179125785827637,
+      "step": 575
+    },
+    {
+      "epoch": 0.3104198026425824,
+      "grad_norm": 0.7765647739189981,
+      "learning_rate": 9.999649761447477e-07,
+      "logits/chosen": 0.29694879055023193,
+      "logits/rejected": 0.5250946879386902,
+      "logps/chosen": -7.878275394439697,
+      "logps/rejected": -9.05508041381836,
+      "loss": 0.6935,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -7.878275394439697,
+      "rewards/margins": 1.1768052577972412,
+      "rewards/rejected": -9.05508041381836,
+      "step": 580
+    },
+    {
+      "epoch": 0.31309583542398395,
+      "grad_norm": 0.7488690868442348,
+      "learning_rate": 9.999441174505398e-07,
+      "logits/chosen": 0.2125202715396881,
+      "logits/rejected": 0.31268957257270813,
+      "logps/chosen": -10.100735664367676,
+      "logps/rejected": -9.474138259887695,
+      "loss": 0.6932,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -10.100735664367676,
+      "rewards/margins": -0.6265988945960999,
+      "rewards/rejected": -9.474138259887695,
+      "step": 585
+    },
+    {
+      "epoch": 0.3157718682053855,
+      "grad_norm": 0.44037779595849996,
+      "learning_rate": 9.999184082963116e-07,
+      "logits/chosen": 0.1530287265777588,
+      "logits/rejected": 0.28570112586021423,
+      "logps/chosen": -7.592609405517578,
+      "logps/rejected": -7.852643013000488,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -7.592609405517578,
+      "rewards/margins": 0.26003432273864746,
+      "rewards/rejected": -7.852643013000488,
+      "step": 590
+    },
+    {
+      "epoch": 0.3184479009867871,
+      "grad_norm": 0.5650869587855691,
+      "learning_rate": 9.998878489314937e-07,
+      "logits/chosen": 0.2078695297241211,
+      "logits/rejected": 0.39020439982414246,
+      "logps/chosen": -7.55035400390625,
+      "logps/rejected": -9.387155532836914,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -7.55035400390625,
+      "rewards/margins": 1.836801528930664,
+      "rewards/rejected": -9.387155532836914,
+      "step": 595
+    },
+    {
+      "epoch": 0.32112393376818865,
+      "grad_norm": 1.4410275157013466,
+      "learning_rate": 9.99852439652573e-07,
+      "logits/chosen": 0.05508248880505562,
+      "logits/rejected": 0.24116215109825134,
+      "logps/chosen": -5.543477535247803,
+      "logps/rejected": -6.33013916015625,
+      "loss": 0.6946,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -5.543477535247803,
+      "rewards/margins": 0.7866613864898682,
+      "rewards/rejected": -6.33013916015625,
+      "step": 600
+    },
+    {
+      "epoch": 0.32379996654959026,
+      "grad_norm": 0.6197020830886648,
+      "learning_rate": 9.998121808030904e-07,
+      "logits/chosen": 0.1440209448337555,
+      "logits/rejected": 0.22981485724449158,
+      "logps/chosen": -8.053476333618164,
+      "logps/rejected": -7.331965446472168,
+      "loss": 0.6903,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -8.053476333618164,
+      "rewards/margins": -0.7215118408203125,
+      "rewards/rejected": -7.331965446472168,
+      "step": 605
+    },
+    {
+      "epoch": 0.3264759993309918,
+      "grad_norm": 1.2882667414877567,
+      "learning_rate": 9.997670727736379e-07,
+      "logits/chosen": 0.17184646427631378,
+      "logits/rejected": 0.4028119146823883,
+      "logps/chosen": -8.993307113647461,
+      "logps/rejected": -8.458648681640625,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -8.993307113647461,
+      "rewards/margins": -0.5346589088439941,
+      "rewards/rejected": -8.458648681640625,
+      "step": 610
+    },
+    {
+      "epoch": 0.32915203211239336,
+      "grad_norm": 0.6174606945329881,
+      "learning_rate": 9.99717116001853e-07,
+      "logits/chosen": 0.2044260948896408,
+      "logits/rejected": 0.31323227286338806,
+      "logps/chosen": -7.18270206451416,
+      "logps/rejected": -7.079034328460693,
+      "loss": 0.691,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -7.18270206451416,
+      "rewards/margins": -0.10366741567850113,
+      "rewards/rejected": -7.079034328460693,
+      "step": 615
+    },
+    {
+      "epoch": 0.33182806489379496,
+      "grad_norm": 0.534543772516486,
+      "learning_rate": 9.996623109724173e-07,
+      "logits/chosen": 0.2867094874382019,
+      "logits/rejected": 0.3787018656730652,
+      "logps/chosen": -7.248446464538574,
+      "logps/rejected": -6.194828987121582,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -7.248446464538574,
+      "rewards/margins": -1.0536184310913086,
+      "rewards/rejected": -6.194828987121582,
+      "step": 620
+    },
+    {
+      "epoch": 0.3345040976751965,
+      "grad_norm": 0.5512667428642255,
+      "learning_rate": 9.996026582170488e-07,
+      "logits/chosen": 0.22856783866882324,
+      "logits/rejected": 0.38975828886032104,
+      "logps/chosen": -8.347597122192383,
+      "logps/rejected": -10.185302734375,
+      "loss": 0.692,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -8.347597122192383,
+      "rewards/margins": 1.8377052545547485,
+      "rewards/rejected": -10.185302734375,
+      "step": 625
+    },
+    {
+      "epoch": 0.3371801304565981,
+      "grad_norm": 0.4858491245128917,
+      "learning_rate": 9.995381583144996e-07,
+      "logits/chosen": 0.15616248548030853,
+      "logits/rejected": 0.32802727818489075,
+      "logps/chosen": -8.323324203491211,
+      "logps/rejected": -7.5688886642456055,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -8.323324203491211,
+      "rewards/margins": -0.7544358968734741,
+      "rewards/rejected": -7.5688886642456055,
+      "step": 630
+    },
+    {
+      "epoch": 0.33985616323799966,
+      "grad_norm": 0.47967433065693993,
+      "learning_rate": 9.994688118905471e-07,
+      "logits/chosen": 0.24968722462654114,
+      "logits/rejected": 0.5516321659088135,
+      "logps/chosen": -7.0057783126831055,
+      "logps/rejected": -7.301898956298828,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -7.0057783126831055,
+      "rewards/margins": 0.2961201071739197,
+      "rewards/rejected": -7.301898956298828,
+      "step": 635
+    },
+    {
+      "epoch": 0.3425321960194012,
+      "grad_norm": 0.8805784528577906,
+      "learning_rate": 9.993946196179912e-07,
+      "logits/chosen": 0.14842477440834045,
+      "logits/rejected": 0.3936559557914734,
+      "logps/chosen": -9.815520286560059,
+      "logps/rejected": -9.4787015914917,
+      "loss": 0.6924,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -9.815520286560059,
+      "rewards/margins": -0.33681780099868774,
+      "rewards/rejected": -9.4787015914917,
+      "step": 640
+    },
+    {
+      "epoch": 0.3452082288008028,
+      "grad_norm": 0.8668778919129861,
+      "learning_rate": 9.993155822166455e-07,
+      "logits/chosen": 0.22226440906524658,
+      "logits/rejected": 0.2776411473751068,
+      "logps/chosen": -7.501551628112793,
+      "logps/rejected": -7.846782684326172,
+      "loss": 0.6918,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -7.501551628112793,
+      "rewards/margins": 0.34523048996925354,
+      "rewards/rejected": -7.846782684326172,
+      "step": 645
+    },
+    {
+      "epoch": 0.34788426158220437,
+      "grad_norm": 0.2939709916810599,
+      "learning_rate": 9.992317004533313e-07,
+      "logits/chosen": 0.2366681843996048,
+      "logits/rejected": 0.3466010093688965,
+      "logps/chosen": -7.205712795257568,
+      "logps/rejected": -8.257243156433105,
+      "loss": 0.6914,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -7.205712795257568,
+      "rewards/margins": 1.0515297651290894,
+      "rewards/rejected": -8.257243156433105,
+      "step": 650
+    },
+    {
+      "epoch": 0.350560294363606,
+      "grad_norm": 0.6944867489976411,
+      "learning_rate": 9.991429751418696e-07,
+      "logits/chosen": 0.30371397733688354,
+      "logits/rejected": 0.3411691188812256,
+      "logps/chosen": -9.049365043640137,
+      "logps/rejected": -8.24947738647461,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -9.049365043640137,
+      "rewards/margins": -0.7998883724212646,
+      "rewards/rejected": -8.24947738647461,
+      "step": 655
+    },
+    {
+      "epoch": 0.3532363271450075,
+      "grad_norm": 0.4099688946121479,
+      "learning_rate": 9.99049407143074e-07,
+      "logits/chosen": 0.3385065197944641,
+      "logits/rejected": 0.49500903487205505,
+      "logps/chosen": -9.800408363342285,
+      "logps/rejected": -9.511049270629883,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -9.800408363342285,
+      "rewards/margins": -0.2893592417240143,
+      "rewards/rejected": -9.511049270629883,
+      "step": 660
+    },
+    {
+      "epoch": 0.35591235992640907,
+      "grad_norm": 0.6656909224737024,
+      "learning_rate": 9.989509973647416e-07,
+      "logits/chosen": 0.35746780037879944,
+      "logits/rejected": 0.538483738899231,
+      "logps/chosen": -9.719289779663086,
+      "logps/rejected": -9.478790283203125,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -9.719289779663086,
+      "rewards/margins": -0.24049940705299377,
+      "rewards/rejected": -9.478790283203125,
+      "step": 665
+    },
+    {
+      "epoch": 0.3585883927078107,
+      "grad_norm": 0.7514398106676574,
+      "learning_rate": 9.988477467616445e-07,
+      "logits/chosen": 0.26860225200653076,
+      "logits/rejected": 0.5164141654968262,
+      "logps/chosen": -8.090912818908691,
+      "logps/rejected": -8.551098823547363,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -8.090912818908691,
+      "rewards/margins": 0.4601851999759674,
+      "rewards/rejected": -8.551098823547363,
+      "step": 670
+    },
+    {
+      "epoch": 0.3612644254892122,
+      "grad_norm": 0.7009288405395743,
+      "learning_rate": 9.987396563355205e-07,
+      "logits/chosen": 0.24938449263572693,
+      "logits/rejected": 0.3557703197002411,
+      "logps/chosen": -7.9355902671813965,
+      "logps/rejected": -8.981306076049805,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -7.9355902671813965,
+      "rewards/margins": 1.0457159280776978,
+      "rewards/rejected": -8.981306076049805,
+      "step": 675
+    },
+    {
+      "epoch": 0.36394045827061383,
+      "grad_norm": 0.7719247030876621,
+      "learning_rate": 9.986267271350631e-07,
+      "logits/chosen": 0.26580381393432617,
+      "logits/rejected": 0.43320298194885254,
+      "logps/chosen": -7.306980133056641,
+      "logps/rejected": -7.0948638916015625,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -7.306980133056641,
+      "rewards/margins": -0.21211600303649902,
+      "rewards/rejected": -7.0948638916015625,
+      "step": 680
+    },
+    {
+      "epoch": 0.3666164910520154,
+      "grad_norm": 1.5067262703048878,
+      "learning_rate": 9.985089602559123e-07,
+      "logits/chosen": 0.2592657506465912,
+      "logits/rejected": 0.46832719445228577,
+      "logps/chosen": -7.24490213394165,
+      "logps/rejected": -8.365945816040039,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -7.24490213394165,
+      "rewards/margins": 1.121044397354126,
+      "rewards/rejected": -8.365945816040039,
+      "step": 685
+    },
+    {
+      "epoch": 0.369292523833417,
+      "grad_norm": 0.9914152553441178,
+      "learning_rate": 9.983863568406428e-07,
+      "logits/chosen": 0.41557416319847107,
+      "logits/rejected": 0.44996967911720276,
+      "logps/chosen": -8.090818405151367,
+      "logps/rejected": -7.145881652832031,
+      "loss": 0.6928,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -8.090818405151367,
+      "rewards/margins": -0.9449364542961121,
+      "rewards/rejected": -7.145881652832031,
+      "step": 690
+    },
+    {
+      "epoch": 0.37196855661481854,
+      "grad_norm": 0.5538247084474434,
+      "learning_rate": 9.982589180787532e-07,
+      "logits/chosen": 0.4121316373348236,
+      "logits/rejected": 0.5353908538818359,
+      "logps/chosen": -9.570096969604492,
+      "logps/rejected": -9.62109375,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -9.570096969604492,
+      "rewards/margins": 0.05099555850028992,
+      "rewards/rejected": -9.62109375,
+      "step": 695
+    },
+    {
+      "epoch": 0.3746445893962201,
+      "grad_norm": 0.6342426512997031,
+      "learning_rate": 9.981266452066553e-07,
+      "logits/chosen": 0.42995786666870117,
+      "logits/rejected": 0.5626598000526428,
+      "logps/chosen": -8.36759090423584,
+      "logps/rejected": -9.217076301574707,
+      "loss": 0.6917,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -8.36759090423584,
+      "rewards/margins": 0.8494853973388672,
+      "rewards/rejected": -9.217076301574707,
+      "step": 700
+    },
+    {
+      "epoch": 0.3773206221776217,
+      "grad_norm": 1.1659020744311992,
+      "learning_rate": 9.979895395076608e-07,
+      "logits/chosen": 0.4946301579475403,
+      "logits/rejected": 0.7543737888336182,
+      "logps/chosen": -9.087930679321289,
+      "logps/rejected": -8.502263069152832,
+      "loss": 0.6871,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -9.087930679321289,
+      "rewards/margins": -0.5856682062149048,
+      "rewards/rejected": -8.502263069152832,
+      "step": 705
+    },
+    {
+      "epoch": 0.37999665495902324,
+      "grad_norm": 0.8499084521103942,
+      "learning_rate": 9.9784760231197e-07,
+      "logits/chosen": 0.47661757469177246,
+      "logits/rejected": 0.6000131368637085,
+      "logps/chosen": -7.572332859039307,
+      "logps/rejected": -7.875659942626953,
+      "loss": 0.6857,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -7.572332859039307,
+      "rewards/margins": 0.30332818627357483,
+      "rewards/rejected": -7.875659942626953,
+      "step": 710
+    },
+    {
+      "epoch": 0.38267268774042484,
+      "grad_norm": 0.831336356980509,
+      "learning_rate": 9.97700834996658e-07,
+      "logits/chosen": 0.40051689743995667,
+      "logits/rejected": 0.6121428608894348,
+      "logps/chosen": -8.804323196411133,
+      "logps/rejected": -8.28823184967041,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -8.804323196411133,
+      "rewards/margins": -0.5160931944847107,
+      "rewards/rejected": -8.28823184967041,
+      "step": 715
+    },
+    {
+      "epoch": 0.3853487205218264,
+      "grad_norm": 0.5187738853198139,
+      "learning_rate": 9.97549238985662e-07,
+      "logits/chosen": 0.5327664613723755,
+      "logits/rejected": 0.8064025640487671,
+      "logps/chosen": -9.374398231506348,
+      "logps/rejected": -9.66627311706543,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -9.374398231506348,
+      "rewards/margins": 0.2918764054775238,
+      "rewards/rejected": -9.66627311706543,
+      "step": 720
+    },
+    {
+      "epoch": 0.38802475330322794,
+      "grad_norm": 0.8477836527987385,
+      "learning_rate": 9.973928157497674e-07,
+      "logits/chosen": 0.514908492565155,
+      "logits/rejected": 0.6800402402877808,
+      "logps/chosen": -9.698270797729492,
+      "logps/rejected": -10.130010604858398,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -9.698270797729492,
+      "rewards/margins": 0.4317414164543152,
+      "rewards/rejected": -10.130010604858398,
+      "step": 725
+    },
+    {
+      "epoch": 0.39070078608462955,
+      "grad_norm": 0.6754661134194753,
+      "learning_rate": 9.972315668065927e-07,
+      "logits/chosen": 0.378134161233902,
+      "logits/rejected": 0.5673595070838928,
+      "logps/chosen": -9.575207710266113,
+      "logps/rejected": -10.066390991210938,
+      "loss": 0.69,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -9.575207710266113,
+      "rewards/margins": 0.49118533730506897,
+      "rewards/rejected": -10.066390991210938,
+      "step": 730
+    },
+    {
+      "epoch": 0.3933768188660311,
+      "grad_norm": 0.5687405174405994,
+      "learning_rate": 9.97065493720576e-07,
+      "logits/chosen": 0.38819068670272827,
+      "logits/rejected": 0.5158494710922241,
+      "logps/chosen": -7.9654035568237305,
+      "logps/rejected": -8.373147010803223,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -7.9654035568237305,
+      "rewards/margins": 0.40774279832839966,
+      "rewards/rejected": -8.373147010803223,
+      "step": 735
+    },
+    {
+      "epoch": 0.3960528516474327,
+      "grad_norm": 0.5462258253154052,
+      "learning_rate": 9.968945981029594e-07,
+      "logits/chosen": 0.2939468026161194,
+      "logits/rejected": 0.5399002432823181,
+      "logps/chosen": -7.994257926940918,
+      "logps/rejected": -8.254899024963379,
+      "loss": 0.6883,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -7.994257926940918,
+      "rewards/margins": 0.26064029335975647,
+      "rewards/rejected": -8.254899024963379,
+      "step": 740
+    },
+    {
+      "epoch": 0.39872888442883425,
+      "grad_norm": 0.9289446405338412,
+      "learning_rate": 9.967188816117726e-07,
+      "logits/chosen": 0.3909088373184204,
+      "logits/rejected": 0.5126386880874634,
+      "logps/chosen": -8.077153205871582,
+      "logps/rejected": -8.46070671081543,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -8.077153205871582,
+      "rewards/margins": 0.3835543692111969,
+      "rewards/rejected": -8.46070671081543,
+      "step": 745
+    },
+    {
+      "epoch": 0.4014049172102358,
+      "grad_norm": 0.7081773098848291,
+      "learning_rate": 9.965383459518179e-07,
+      "logits/chosen": 0.3393404483795166,
+      "logits/rejected": 0.5507976412773132,
+      "logps/chosen": -7.5438385009765625,
+      "logps/rejected": -7.2933220863342285,
+      "loss": 0.6854,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -7.5438385009765625,
+      "rewards/margins": -0.2505169212818146,
+      "rewards/rejected": -7.2933220863342285,
+      "step": 750
+    },
+    {
+      "epoch": 0.4040809499916374,
+      "grad_norm": 0.6079319994946737,
+      "learning_rate": 9.963529928746533e-07,
+      "logits/chosen": 0.4588445723056793,
+      "logits/rejected": 0.6263395547866821,
+      "logps/chosen": -7.540353298187256,
+      "logps/rejected": -8.465313911437988,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -7.540353298187256,
+      "rewards/margins": 0.9249606132507324,
+      "rewards/rejected": -8.465313911437988,
+      "step": 755
+    },
+    {
+      "epoch": 0.40675698277303896,
+      "grad_norm": 0.640162429117986,
+      "learning_rate": 9.961628241785746e-07,
+      "logits/chosen": 0.29960551857948303,
+      "logits/rejected": 0.4226643443107605,
+      "logps/chosen": -7.460961818695068,
+      "logps/rejected": -8.469831466674805,
+      "loss": 0.6919,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -7.460961818695068,
+      "rewards/margins": 1.0088703632354736,
+      "rewards/rejected": -8.469831466674805,
+      "step": 760
+    },
+    {
+      "epoch": 0.40943301555444056,
+      "grad_norm": 0.7872699260089134,
+      "learning_rate": 9.959678417085998e-07,
+      "logits/chosen": 0.5074780583381653,
+      "logits/rejected": 0.6349397897720337,
+      "logps/chosen": -9.510750770568848,
+      "logps/rejected": -10.259366989135742,
+      "loss": 0.6904,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -9.510750770568848,
+      "rewards/margins": 0.74861741065979,
+      "rewards/rejected": -10.259366989135742,
+      "step": 765
+    },
+    {
+      "epoch": 0.4121090483358421,
+      "grad_norm": 0.5824431420764663,
+      "learning_rate": 9.957680473564493e-07,
+      "logits/chosen": 0.77257239818573,
+      "logits/rejected": 1.0051229000091553,
+      "logps/chosen": -11.814358711242676,
+      "logps/rejected": -12.053152084350586,
+      "loss": 0.6907,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -11.814358711242676,
+      "rewards/margins": 0.23879511654376984,
+      "rewards/rejected": -12.053152084350586,
+      "step": 770
+    },
+    {
+      "epoch": 0.41478508111724366,
+      "grad_norm": 0.631318792534833,
+      "learning_rate": 9.95563443060529e-07,
+      "logits/chosen": 0.5852163434028625,
+      "logits/rejected": 0.8069238662719727,
+      "logps/chosen": -11.445940971374512,
+      "logps/rejected": -13.207618713378906,
+      "loss": 0.689,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -11.445940971374512,
+      "rewards/margins": 1.7616784572601318,
+      "rewards/rejected": -13.207618713378906,
+      "step": 775
+    },
+    {
+      "epoch": 0.41746111389864526,
+      "grad_norm": 0.9398151675007941,
+      "learning_rate": 9.95354030805911e-07,
+      "logits/chosen": 0.5266901254653931,
+      "logits/rejected": 0.7262865304946899,
+      "logps/chosen": -11.00364875793457,
+      "logps/rejected": -11.895853042602539,
+      "loss": 0.6898,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -11.00364875793457,
+      "rewards/margins": 0.8922064900398254,
+      "rewards/rejected": -11.895853042602539,
+      "step": 780
+    },
+    {
+      "epoch": 0.4201371466800468,
+      "grad_norm": 3.2907998951808803,
+      "learning_rate": 9.951398126243133e-07,
+      "logits/chosen": 0.6696627736091614,
+      "logits/rejected": 0.8333109617233276,
+      "logps/chosen": -12.024290084838867,
+      "logps/rejected": -11.336647033691406,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -12.024290084838867,
+      "rewards/margins": -0.6876426935195923,
+      "rewards/rejected": -11.336647033691406,
+      "step": 785
+    },
+    {
+      "epoch": 0.4228131794614484,
+      "grad_norm": 1.1214448589795827,
+      "learning_rate": 9.94920790594082e-07,
+      "logits/chosen": 0.6618527770042419,
+      "logits/rejected": 0.9209414720535278,
+      "logps/chosen": -10.175979614257812,
+      "logps/rejected": -11.981436729431152,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -10.175979614257812,
+      "rewards/margins": 1.8054561614990234,
+      "rewards/rejected": -11.981436729431152,
+      "step": 790
+    },
+    {
+      "epoch": 0.42548921224284997,
+      "grad_norm": 1.5459108163479394,
+      "learning_rate": 9.946969668401696e-07,
+      "logits/chosen": 0.3843339681625366,
+      "logits/rejected": 0.6263396143913269,
+      "logps/chosen": -8.760916709899902,
+      "logps/rejected": -9.778768539428711,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -8.760916709899902,
+      "rewards/margins": 1.0178515911102295,
+      "rewards/rejected": -9.778768539428711,
+      "step": 795
+    },
+    {
+      "epoch": 0.4281652450242516,
+      "grad_norm": 0.6251740483356498,
+      "learning_rate": 9.944683435341155e-07,
+      "logits/chosen": 0.657813549041748,
+      "logits/rejected": 0.7981842756271362,
+      "logps/chosen": -11.129118919372559,
+      "logps/rejected": -11.368085861206055,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -11.129118919372559,
+      "rewards/margins": 0.2389659583568573,
+      "rewards/rejected": -11.368085861206055,
+      "step": 800
+    },
+    {
+      "epoch": 0.4281652450242516,
+      "eval_logits/chosen": 0.7851170301437378,
+      "eval_logits/rejected": 0.9367091655731201,
+      "eval_logps/chosen": -9.594195365905762,
+      "eval_logps/rejected": -10.221723556518555,
+      "eval_loss": 0.6888346076011658,
+      "eval_rewards/accuracies": 0.577151358127594,
+      "eval_rewards/chosen": -9.594195365905762,
+      "eval_rewards/margins": 0.6275279521942139,
+      "eval_rewards/rejected": -10.221723556518555,
+      "eval_runtime": 40.6241,
+      "eval_samples_per_second": 33.108,
+      "eval_steps_per_second": 8.296,
+      "step": 800
+    },
+    {
+      "epoch": 0.4308412778056531,
+      "grad_norm": 1.0964375301467324,
+      "learning_rate": 9.942349228940236e-07,
+      "logits/chosen": 0.5348111987113953,
+      "logits/rejected": 0.7988623380661011,
+      "logps/chosen": -8.759607315063477,
+      "logps/rejected": -11.213102340698242,
+      "loss": 0.6845,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -8.759607315063477,
+      "rewards/margins": 2.4534950256347656,
+      "rewards/rejected": -11.213102340698242,
+      "step": 805
+    },
+    {
+      "epoch": 0.43351731058705467,
+      "grad_norm": 0.6556083778908031,
+      "learning_rate": 9.939967071845424e-07,
+      "logits/chosen": 0.5958086848258972,
+      "logits/rejected": 0.7012108564376831,
+      "logps/chosen": -8.914748191833496,
+      "logps/rejected": -9.694511413574219,
+      "loss": 0.6921,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -8.914748191833496,
+      "rewards/margins": 0.7797621488571167,
+      "rewards/rejected": -9.694511413574219,
+      "step": 810
+    },
+    {
+      "epoch": 0.4361933433684563,
+      "grad_norm": 0.7038030707445684,
+      "learning_rate": 9.937536987168413e-07,
+      "logits/chosen": 0.4898884892463684,
+      "logits/rejected": 0.6750887632369995,
+      "logps/chosen": -9.951103210449219,
+      "logps/rejected": -9.870951652526855,
+      "loss": 0.6853,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -9.951103210449219,
+      "rewards/margins": -0.08015060424804688,
+      "rewards/rejected": -9.870951652526855,
+      "step": 815
+    },
+    {
+      "epoch": 0.4388693761498578,
+      "grad_norm": 0.7308326474928027,
+      "learning_rate": 9.935058998485896e-07,
+      "logits/chosen": 0.5495724678039551,
+      "logits/rejected": 0.7073918581008911,
+      "logps/chosen": -9.490917205810547,
+      "logps/rejected": -8.659790992736816,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -9.490917205810547,
+      "rewards/margins": -0.8311271667480469,
+      "rewards/rejected": -8.659790992736816,
+      "step": 820
+    },
+    {
+      "epoch": 0.44154540893125943,
+      "grad_norm": 1.1173493982946345,
+      "learning_rate": 9.932533129839333e-07,
+      "logits/chosen": 0.5069432258605957,
+      "logits/rejected": 0.717812180519104,
+      "logps/chosen": -9.494039535522461,
+      "logps/rejected": -9.09168529510498,
+      "loss": 0.6916,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -9.494039535522461,
+      "rewards/margins": -0.40235501527786255,
+      "rewards/rejected": -9.09168529510498,
+      "step": 825
+    },
+    {
+      "epoch": 0.444221441712661,
+      "grad_norm": 0.7099011812557678,
+      "learning_rate": 9.929959405734711e-07,
+      "logits/chosen": 0.6144655346870422,
+      "logits/rejected": 0.910998523235321,
+      "logps/chosen": -7.788238525390625,
+      "logps/rejected": -8.91134262084961,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -7.788238525390625,
+      "rewards/margins": 1.1231048107147217,
+      "rewards/rejected": -8.91134262084961,
+      "step": 830
+    },
+    {
+      "epoch": 0.44689747449406253,
+      "grad_norm": 0.703968461873609,
+      "learning_rate": 9.927337851142314e-07,
+      "logits/chosen": 0.8873071670532227,
+      "logits/rejected": 1.0572532415390015,
+      "logps/chosen": -12.024337768554688,
+      "logps/rejected": -11.493799209594727,
+      "loss": 0.6905,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -12.024337768554688,
+      "rewards/margins": -0.5305396318435669,
+      "rewards/rejected": -11.493799209594727,
+      "step": 835
+    },
+    {
+      "epoch": 0.44957350727546413,
+      "grad_norm": 1.6695454202646887,
+      "learning_rate": 9.924668491496474e-07,
+      "logits/chosen": 0.9370532035827637,
+      "logits/rejected": 1.2157726287841797,
+      "logps/chosen": -11.991620063781738,
+      "logps/rejected": -13.540040969848633,
+      "loss": 0.691,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -11.991620063781738,
+      "rewards/margins": 1.548421859741211,
+      "rewards/rejected": -13.540040969848633,
+      "step": 840
+    },
+    {
+      "epoch": 0.4522495400568657,
+      "grad_norm": 0.968747178776573,
+      "learning_rate": 9.92195135269533e-07,
+      "logits/chosen": 0.8903911709785461,
+      "logits/rejected": 1.0320746898651123,
+      "logps/chosen": -11.49575138092041,
+      "logps/rejected": -12.298863410949707,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -11.49575138092041,
+      "rewards/margins": 0.8031136393547058,
+      "rewards/rejected": -12.298863410949707,
+      "step": 845
+    },
+    {
+      "epoch": 0.4549255728382673,
+      "grad_norm": 1.1621145956237275,
+      "learning_rate": 9.919186461100574e-07,
+      "logits/chosen": 0.7317888736724854,
+      "logits/rejected": 0.863473117351532,
+      "logps/chosen": -9.187520980834961,
+      "logps/rejected": -10.800423622131348,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -9.187520980834961,
+      "rewards/margins": 1.6129024028778076,
+      "rewards/rejected": -10.800423622131348,
+      "step": 850
+    },
+    {
+      "epoch": 0.45760160561966884,
+      "grad_norm": 13.326748428419855,
+      "learning_rate": 9.9163738435372e-07,
+      "logits/chosen": 0.5143827199935913,
+      "logits/rejected": 0.6835073232650757,
+      "logps/chosen": -8.556034088134766,
+      "logps/rejected": -9.341585159301758,
+      "loss": 0.6897,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -8.556034088134766,
+      "rewards/margins": 0.7855512499809265,
+      "rewards/rejected": -9.341585159301758,
+      "step": 855
+    },
+    {
+      "epoch": 0.4602776384010704,
+      "grad_norm": 0.7158372207018323,
+      "learning_rate": 9.913513527293234e-07,
+      "logits/chosen": 0.6176631450653076,
+      "logits/rejected": 0.8486563563346863,
+      "logps/chosen": -8.386320114135742,
+      "logps/rejected": -9.283613204956055,
+      "loss": 0.6857,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -8.386320114135742,
+      "rewards/margins": 0.8972939252853394,
+      "rewards/rejected": -9.283613204956055,
+      "step": 860
+    },
+    {
+      "epoch": 0.462953671182472,
+      "grad_norm": 0.7903348332529198,
+      "learning_rate": 9.910605540119474e-07,
+      "logits/chosen": 0.7250866889953613,
+      "logits/rejected": 0.9388289451599121,
+      "logps/chosen": -11.212921142578125,
+      "logps/rejected": -11.586729049682617,
+      "loss": 0.6911,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -11.212921142578125,
+      "rewards/margins": 0.37380653619766235,
+      "rewards/rejected": -11.586729049682617,
+      "step": 865
+    },
+    {
+      "epoch": 0.46562970396387354,
+      "grad_norm": 0.9466165346612166,
+      "learning_rate": 9.907649910229227e-07,
+      "logits/chosen": 0.483265221118927,
+      "logits/rejected": 0.7757227420806885,
+      "logps/chosen": -9.228546142578125,
+      "logps/rejected": -9.840031623840332,
+      "loss": 0.6865,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -9.228546142578125,
+      "rewards/margins": 0.6114859580993652,
+      "rewards/rejected": -9.840031623840332,
+      "step": 870
+    },
+    {
+      "epoch": 0.46830573674527515,
+      "grad_norm": 1.1993262700314389,
+      "learning_rate": 9.90464666629803e-07,
+      "logits/chosen": 0.5446566343307495,
+      "logits/rejected": 0.7473292350769043,
+      "logps/chosen": -7.957808017730713,
+      "logps/rejected": -9.380017280578613,
+      "loss": 0.689,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -7.957808017730713,
+      "rewards/margins": 1.4222087860107422,
+      "rewards/rejected": -9.380017280578613,
+      "step": 875
+    },
+    {
+      "epoch": 0.4709817695266767,
+      "grad_norm": 0.6297043393699787,
+      "learning_rate": 9.901595837463363e-07,
+      "logits/chosen": 0.3973233103752136,
+      "logits/rejected": 0.6832461357116699,
+      "logps/chosen": -6.691615104675293,
+      "logps/rejected": -7.4792304039001465,
+      "loss": 0.6861,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -6.691615104675293,
+      "rewards/margins": 0.7876148223876953,
+      "rewards/rejected": -7.4792304039001465,
+      "step": 880
+    },
+    {
+      "epoch": 0.47365780230807825,
+      "grad_norm": 0.4787651151006367,
+      "learning_rate": 9.898497453324384e-07,
+      "logits/chosen": 0.4575297236442566,
+      "logits/rejected": 0.6205445528030396,
+      "logps/chosen": -8.16806411743164,
+      "logps/rejected": -8.270221710205078,
+      "loss": 0.6868,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -8.16806411743164,
+      "rewards/margins": 0.10215761512517929,
+      "rewards/rejected": -8.270221710205078,
+      "step": 885
+    },
+    {
+      "epoch": 0.47633383508947985,
+      "grad_norm": 1.0655806487543196,
+      "learning_rate": 9.895351543941628e-07,
+      "logits/chosen": 0.299403578042984,
+      "logits/rejected": 0.49562111496925354,
+      "logps/chosen": -7.278923034667969,
+      "logps/rejected": -7.385705471038818,
+      "loss": 0.6909,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -7.278923034667969,
+      "rewards/margins": 0.10678346455097198,
+      "rewards/rejected": -7.385705471038818,
+      "step": 890
+    },
+    {
+      "epoch": 0.4790098678708814,
+      "grad_norm": 1.1279728148010517,
+      "learning_rate": 9.892158139836724e-07,
+      "logits/chosen": 0.456309974193573,
+      "logits/rejected": 0.5898152589797974,
+      "logps/chosen": -8.074499130249023,
+      "logps/rejected": -7.565207481384277,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -8.074499130249023,
+      "rewards/margins": -0.5092910528182983,
+      "rewards/rejected": -7.565207481384277,
+      "step": 895
+    },
+    {
+      "epoch": 0.481685900652283,
+      "grad_norm": 0.8526543243148195,
+      "learning_rate": 9.88891727199209e-07,
+      "logits/chosen": 0.4814811646938324,
+      "logits/rejected": 0.6509904265403748,
+      "logps/chosen": -9.073254585266113,
+      "logps/rejected": -9.399925231933594,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -9.073254585266113,
+      "rewards/margins": 0.3266717791557312,
+      "rewards/rejected": -9.399925231933594,
+      "step": 900
+    },
+    {
+      "epoch": 0.48436193343368455,
+      "grad_norm": 0.7191662763278387,
+      "learning_rate": 9.885628971850641e-07,
+      "logits/chosen": 0.5493252277374268,
+      "logits/rejected": 0.7795707583427429,
+      "logps/chosen": -9.049610137939453,
+      "logps/rejected": -9.759359359741211,
+      "loss": 0.6884,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -9.049610137939453,
+      "rewards/margins": 0.7097499370574951,
+      "rewards/rejected": -9.759359359741211,
+      "step": 905
+    },
+    {
+      "epoch": 0.48703796621508616,
+      "grad_norm": 0.6333246263347979,
+      "learning_rate": 9.882293271315481e-07,
+      "logits/chosen": 0.5227259397506714,
+      "logits/rejected": 0.6247232556343079,
+      "logps/chosen": -8.103181838989258,
+      "logps/rejected": -8.444541931152344,
+      "loss": 0.6913,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -8.103181838989258,
+      "rewards/margins": 0.34135836362838745,
+      "rewards/rejected": -8.444541931152344,
+      "step": 910
+    },
+    {
+      "epoch": 0.4897139989964877,
+      "grad_norm": 1.2520994259776028,
+      "learning_rate": 9.878910202749589e-07,
+      "logits/chosen": 0.5669925212860107,
+      "logits/rejected": 0.8243115544319153,
+      "logps/chosen": -7.751688480377197,
+      "logps/rejected": -8.991536140441895,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -7.751688480377197,
+      "rewards/margins": 1.23984694480896,
+      "rewards/rejected": -8.991536140441895,
+      "step": 915
+    },
+    {
+      "epoch": 0.49239003177788926,
+      "grad_norm": 1.9876824434187088,
+      "learning_rate": 9.875479798975512e-07,
+      "logits/chosen": 0.6848721504211426,
+      "logits/rejected": 0.8988482356071472,
+      "logps/chosen": -9.660905838012695,
+      "logps/rejected": -12.19223690032959,
+      "loss": 0.6863,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -9.660905838012695,
+      "rewards/margins": 2.53132963180542,
+      "rewards/rejected": -12.19223690032959,
+      "step": 920
+    },
+    {
+      "epoch": 0.49506606455929086,
+      "grad_norm": 0.6560577360894913,
+      "learning_rate": 9.87200209327504e-07,
+      "logits/chosen": 0.4612664580345154,
+      "logits/rejected": 0.6616185307502747,
+      "logps/chosen": -8.395280838012695,
+      "logps/rejected": -8.866620063781738,
+      "loss": 0.691,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -8.395280838012695,
+      "rewards/margins": 0.4713391661643982,
+      "rewards/rejected": -8.866620063781738,
+      "step": 925
+    },
+    {
+      "epoch": 0.4977420973406924,
+      "grad_norm": 0.6458927025797335,
+      "learning_rate": 9.868477119388894e-07,
+      "logits/chosen": 0.36156103014945984,
+      "logits/rejected": 0.5215521454811096,
+      "logps/chosen": -7.715545654296875,
+      "logps/rejected": -9.06513500213623,
+      "loss": 0.6908,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -7.715545654296875,
+      "rewards/margins": 1.3495886325836182,
+      "rewards/rejected": -9.06513500213623,
+      "step": 930
+    },
+    {
+      "epoch": 0.500418130122094,
+      "grad_norm": 1.460978152402183,
+      "learning_rate": 9.864904911516383e-07,
+      "logits/chosen": 0.4456873834133148,
+      "logits/rejected": 0.5572026371955872,
+      "logps/chosen": -7.822953701019287,
+      "logps/rejected": -8.640012741088867,
+      "loss": 0.6891,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -7.822953701019287,
+      "rewards/margins": 0.8170582056045532,
+      "rewards/rejected": -8.640012741088867,
+      "step": 935
+    },
+    {
+      "epoch": 0.5030941629034956,
+      "grad_norm": 0.5866002539054574,
+      "learning_rate": 9.861285504315084e-07,
+      "logits/chosen": 0.40688657760620117,
+      "logits/rejected": 0.5501396059989929,
+      "logps/chosen": -7.868184566497803,
+      "logps/rejected": -8.624244689941406,
+      "loss": 0.6873,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -7.868184566497803,
+      "rewards/margins": 0.7560598254203796,
+      "rewards/rejected": -8.624244689941406,
+      "step": 940
+    },
+    {
+      "epoch": 0.5057701956848971,
+      "grad_norm": 0.8022665124431038,
+      "learning_rate": 9.857618932900502e-07,
+      "logits/chosen": 0.3432442843914032,
+      "logits/rejected": 0.5239924192428589,
+      "logps/chosen": -6.6535139083862305,
+      "logps/rejected": -7.579787254333496,
+      "loss": 0.6865,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -6.6535139083862305,
+      "rewards/margins": 0.9262722134590149,
+      "rewards/rejected": -7.579787254333496,
+      "step": 945
+    },
+    {
+      "epoch": 0.5084462284662987,
+      "grad_norm": 1.4866780584212473,
+      "learning_rate": 9.853905232845727e-07,
+      "logits/chosen": 0.24382999539375305,
+      "logits/rejected": 0.4546380639076233,
+      "logps/chosen": -6.303766250610352,
+      "logps/rejected": -6.8016357421875,
+      "loss": 0.6926,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -6.303766250610352,
+      "rewards/margins": 0.4978693425655365,
+      "rewards/rejected": -6.8016357421875,
+      "step": 950
+    },
+    {
+      "epoch": 0.5111222612477003,
+      "grad_norm": 0.8225778084266941,
+      "learning_rate": 9.850144440181095e-07,
+      "logits/chosen": 0.3233112692832947,
+      "logits/rejected": 0.5592790842056274,
+      "logps/chosen": -6.758313179016113,
+      "logps/rejected": -7.018370151519775,
+      "loss": 0.6894,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -6.758313179016113,
+      "rewards/margins": 0.2600574791431427,
+      "rewards/rejected": -7.018370151519775,
+      "step": 955
+    },
+    {
+      "epoch": 0.5137982940291018,
+      "grad_norm": 1.0506630715856096,
+      "learning_rate": 9.846336591393832e-07,
+      "logits/chosen": 0.3547149896621704,
+      "logits/rejected": 0.5715459585189819,
+      "logps/chosen": -7.1310529708862305,
+      "logps/rejected": -8.017168045043945,
+      "loss": 0.6864,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -7.1310529708862305,
+      "rewards/margins": 0.8861141204833984,
+      "rewards/rejected": -8.017168045043945,
+      "step": 960
+    },
+    {
+      "epoch": 0.5164743268105034,
+      "grad_norm": 0.5947903359548206,
+      "learning_rate": 9.842481723427704e-07,
+      "logits/chosen": 0.4581148028373718,
+      "logits/rejected": 0.5078209042549133,
+      "logps/chosen": -6.965640068054199,
+      "logps/rejected": -7.689741611480713,
+      "loss": 0.6896,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -6.965640068054199,
+      "rewards/margins": 0.7241010069847107,
+      "rewards/rejected": -7.689741611480713,
+      "step": 965
+    },
+    {
+      "epoch": 0.519150359591905,
+      "grad_norm": 0.9956847608660038,
+      "learning_rate": 9.838579873682658e-07,
+      "logits/chosen": 0.4598515033721924,
+      "logits/rejected": 0.5059674382209778,
+      "logps/chosen": -7.184128761291504,
+      "logps/rejected": -8.299097061157227,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -7.184128761291504,
+      "rewards/margins": 1.1149678230285645,
+      "rewards/rejected": -8.299097061157227,
+      "step": 970
+    },
+    {
+      "epoch": 0.5218263923733065,
+      "grad_norm": 0.7899752660878872,
+      "learning_rate": 9.834631080014457e-07,
+      "logits/chosen": 0.3902721107006073,
+      "logits/rejected": 0.664018988609314,
+      "logps/chosen": -7.401497840881348,
+      "logps/rejected": -7.561249732971191,
+      "loss": 0.6846,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -7.401497840881348,
+      "rewards/margins": 0.15975186228752136,
+      "rewards/rejected": -7.561249732971191,
+      "step": 975
+    },
+    {
+      "epoch": 0.5245024251547081,
+      "grad_norm": 1.3111521876645702,
+      "learning_rate": 9.830635380734312e-07,
+      "logits/chosen": 0.24221524596214294,
+      "logits/rejected": 0.4658379554748535,
+      "logps/chosen": -6.323492050170898,
+      "logps/rejected": -7.141410827636719,
+      "loss": 0.688,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -6.323492050170898,
+      "rewards/margins": 0.8179187774658203,
+      "rewards/rejected": -7.141410827636719,
+      "step": 980
+    },
+    {
+      "epoch": 0.5271784579361097,
+      "grad_norm": 0.784634440541109,
+      "learning_rate": 9.826592814608517e-07,
+      "logits/chosen": 0.2853652238845825,
+      "logits/rejected": 0.49533382058143616,
+      "logps/chosen": -6.5780229568481445,
+      "logps/rejected": -6.212478160858154,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -6.5780229568481445,
+      "rewards/margins": -0.3655441701412201,
+      "rewards/rejected": -6.212478160858154,
+      "step": 985
+    },
+    {
+      "epoch": 0.5298544907175113,
+      "grad_norm": 0.8272046684232806,
+      "learning_rate": 9.822503420858067e-07,
+      "logits/chosen": 0.463323175907135,
+      "logits/rejected": 0.5229992270469666,
+      "logps/chosen": -6.830767631530762,
+      "logps/rejected": -6.339776039123535,
+      "loss": 0.688,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -6.830767631530762,
+      "rewards/margins": -0.49099087715148926,
+      "rewards/rejected": -6.339776039123535,
+      "step": 990
+    },
+    {
+      "epoch": 0.5325305234989128,
+      "grad_norm": 1.274918790151156,
+      "learning_rate": 9.818367239158277e-07,
+      "logits/chosen": 0.3963191509246826,
+      "logits/rejected": 0.5297439098358154,
+      "logps/chosen": -8.023408889770508,
+      "logps/rejected": -7.809914588928223,
+      "loss": 0.6878,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -8.023408889770508,
+      "rewards/margins": -0.21349303424358368,
+      "rewards/rejected": -7.809914588928223,
+      "step": 995
+    },
+    {
+      "epoch": 0.5352065562803144,
+      "grad_norm": 0.8084289932157476,
+      "learning_rate": 9.8141843096384e-07,
+      "logits/chosen": 0.5174039006233215,
+      "logits/rejected": 0.7029634714126587,
+      "logps/chosen": -7.266225337982178,
+      "logps/rejected": -8.875396728515625,
+      "loss": 0.6864,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -7.266225337982178,
+      "rewards/margins": 1.609170913696289,
+      "rewards/rejected": -8.875396728515625,
+      "step": 1000
+    },
+    {
+      "epoch": 0.537882589061716,
+      "grad_norm": 0.7727621790408593,
+      "learning_rate": 9.809954672881237e-07,
+      "logits/chosen": 0.5699678659439087,
+      "logits/rejected": 0.7671824097633362,
+      "logps/chosen": -8.62289047241211,
+      "logps/rejected": -9.944162368774414,
+      "loss": 0.69,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -8.62289047241211,
+      "rewards/margins": 1.3212722539901733,
+      "rewards/rejected": -9.944162368774414,
+      "step": 1005
+    },
+    {
+      "epoch": 0.5405586218431175,
+      "grad_norm": 1.1116597586179247,
+      "learning_rate": 9.80567836992274e-07,
+      "logits/chosen": 0.4644365906715393,
+      "logits/rejected": 0.6966451406478882,
+      "logps/chosen": -7.478371620178223,
+      "logps/rejected": -8.590936660766602,
+      "loss": 0.6869,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -7.478371620178223,
+      "rewards/margins": 1.112565040588379,
+      "rewards/rejected": -8.590936660766602,
+      "step": 1010
+    },
+    {
+      "epoch": 0.5432346546245191,
+      "grad_norm": 1.0659253416864887,
+      "learning_rate": 9.801355442251625e-07,
+      "logits/chosen": 0.3607712984085083,
+      "logits/rejected": 0.5436204671859741,
+      "logps/chosen": -6.935671806335449,
+      "logps/rejected": -7.413914680480957,
+      "loss": 0.6866,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -6.935671806335449,
+      "rewards/margins": 0.4782429337501526,
+      "rewards/rejected": -7.413914680480957,
+      "step": 1015
+    },
+    {
+      "epoch": 0.5459106874059207,
+      "grad_norm": 0.9206412248452015,
+      "learning_rate": 9.796985931808949e-07,
+      "logits/chosen": 0.4543460011482239,
+      "logits/rejected": 0.6163672208786011,
+      "logps/chosen": -8.67431640625,
+      "logps/rejected": -9.274568557739258,
+      "loss": 0.6872,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -8.67431640625,
+      "rewards/margins": 0.600252628326416,
+      "rewards/rejected": -9.274568557739258,
+      "step": 1020
+    },
+    {
+      "epoch": 0.5485867201873222,
+      "grad_norm": 1.1611940774037564,
+      "learning_rate": 9.792569880987724e-07,
+      "logits/chosen": 0.3250092566013336,
+      "logits/rejected": 0.4857628345489502,
+      "logps/chosen": -7.421984672546387,
+      "logps/rejected": -8.421952247619629,
+      "loss": 0.6856,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -7.421984672546387,
+      "rewards/margins": 0.9999664425849915,
+      "rewards/rejected": -8.421952247619629,
+      "step": 1025
+    },
+    {
+      "epoch": 0.5512627529687238,
+      "grad_norm": 2.6149958655768364,
+      "learning_rate": 9.788107332632493e-07,
+      "logits/chosen": 0.2606380581855774,
+      "logits/rejected": 0.385797381401062,
+      "logps/chosen": -7.015688896179199,
+      "logps/rejected": -6.666955471038818,
+      "loss": 0.6929,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -7.015688896179199,
+      "rewards/margins": -0.3487338721752167,
+      "rewards/rejected": -6.666955471038818,
+      "step": 1030
+    },
+    {
+      "epoch": 0.5539387857501255,
+      "grad_norm": 0.6479328494409596,
+      "learning_rate": 9.783598330038924e-07,
+      "logits/chosen": 0.22008688747882843,
+      "logits/rejected": 0.4026494026184082,
+      "logps/chosen": -6.391833305358887,
+      "logps/rejected": -7.006650447845459,
+      "loss": 0.6865,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -6.391833305358887,
+      "rewards/margins": 0.6148167848587036,
+      "rewards/rejected": -7.006650447845459,
+      "step": 1035
+    },
+    {
+      "epoch": 0.5566148185315271,
+      "grad_norm": 1.1005744336950136,
+      "learning_rate": 9.779042916953376e-07,
+      "logits/chosen": 0.17987090349197388,
+      "logits/rejected": 0.40716320276260376,
+      "logps/chosen": -6.337095737457275,
+      "logps/rejected": -7.5586700439453125,
+      "loss": 0.6863,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -6.337095737457275,
+      "rewards/margins": 1.2215741872787476,
+      "rewards/rejected": -7.5586700439453125,
+      "step": 1040
+    },
+    {
+      "epoch": 0.5592908513129285,
+      "grad_norm": 1.027124696454523,
+      "learning_rate": 9.774441137572487e-07,
+      "logits/chosen": 0.18706539273262024,
+      "logits/rejected": 0.41073161363601685,
+      "logps/chosen": -7.25717306137085,
+      "logps/rejected": -8.315431594848633,
+      "loss": 0.6836,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -7.25717306137085,
+      "rewards/margins": 1.058258295059204,
+      "rewards/rejected": -8.315431594848633,
+      "step": 1045
+    },
+    {
+      "epoch": 0.5619668840943302,
+      "grad_norm": 0.989309447797015,
+      "learning_rate": 9.76979303654274e-07,
+      "logits/chosen": 0.2203431874513626,
+      "logits/rejected": 0.34982797503471375,
+      "logps/chosen": -6.5714430809021,
+      "logps/rejected": -7.884464263916016,
+      "loss": 0.6875,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -6.5714430809021,
+      "rewards/margins": 1.313021183013916,
+      "rewards/rejected": -7.884464263916016,
+      "step": 1050
+    },
+    {
+      "epoch": 0.5646429168757318,
+      "grad_norm": 1.4670404971363504,
+      "learning_rate": 9.765098658960035e-07,
+      "logits/chosen": 0.32880187034606934,
+      "logits/rejected": 0.5556719899177551,
+      "logps/chosen": -6.695840358734131,
+      "logps/rejected": -8.2882080078125,
+      "loss": 0.6828,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -6.695840358734131,
+      "rewards/margins": 1.5923668146133423,
+      "rewards/rejected": -8.2882080078125,
+      "step": 1055
+    },
+    {
+      "epoch": 0.5673189496571333,
+      "grad_norm": 2.6142429423458777,
+      "learning_rate": 9.76035805036924e-07,
+      "logits/chosen": 0.2474546879529953,
+      "logits/rejected": 0.5012051463127136,
+      "logps/chosen": -5.418895244598389,
+      "logps/rejected": -6.452478885650635,
+      "loss": 0.6863,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -5.418895244598389,
+      "rewards/margins": 1.033583402633667,
+      "rewards/rejected": -6.452478885650635,
+      "step": 1060
+    },
+    {
+      "epoch": 0.5699949824385349,
+      "grad_norm": 1.113539498004411,
+      "learning_rate": 9.755571256763764e-07,
+      "logits/chosen": 0.20273438096046448,
+      "logits/rejected": 0.38245826959609985,
+      "logps/chosen": -5.019228935241699,
+      "logps/rejected": -6.003324508666992,
+      "loss": 0.6848,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -5.019228935241699,
+      "rewards/margins": 0.984095573425293,
+      "rewards/rejected": -6.003324508666992,
+      "step": 1065
+    },
+    {
+      "epoch": 0.5726710152199365,
+      "grad_norm": 1.85779448190842,
+      "learning_rate": 9.750738324585097e-07,
+      "logits/chosen": 0.005217778496444225,
+      "logits/rejected": 0.2433578073978424,
+      "logps/chosen": -4.005131721496582,
+      "logps/rejected": -4.415915489196777,
+      "loss": 0.6843,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.005131721496582,
+      "rewards/margins": 0.41078391671180725,
+      "rewards/rejected": -4.415915489196777,
+      "step": 1070
+    },
+    {
+      "epoch": 0.5753470480013381,
+      "grad_norm": 0.7170412384256368,
+      "learning_rate": 9.74585930072237e-07,
+      "logits/chosen": 0.004342047963291407,
+      "logits/rejected": 0.11906015872955322,
+      "logps/chosen": -4.188816070556641,
+      "logps/rejected": -4.867667198181152,
+      "loss": 0.6871,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.188816070556641,
+      "rewards/margins": 0.6788507699966431,
+      "rewards/rejected": -4.867667198181152,
+      "step": 1075
+    },
+    {
+      "epoch": 0.5780230807827396,
+      "grad_norm": 1.4661461473251098,
+      "learning_rate": 9.740934232511892e-07,
+      "logits/chosen": -0.08273928612470627,
+      "logits/rejected": 0.007201015949249268,
+      "logps/chosen": -4.747479438781738,
+      "logps/rejected": -4.7678632736206055,
+      "loss": 0.6854,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -4.747479438781738,
+      "rewards/margins": 0.02038402482867241,
+      "rewards/rejected": -4.7678632736206055,
+      "step": 1080
+    },
+    {
+      "epoch": 0.5806991135641412,
+      "grad_norm": 0.9203340446933426,
+      "learning_rate": 9.735963167736698e-07,
+      "logits/chosen": 0.05226155370473862,
+      "logits/rejected": 0.21045759320259094,
+      "logps/chosen": -4.777052879333496,
+      "logps/rejected": -5.4679083824157715,
+      "loss": 0.6872,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.777052879333496,
+      "rewards/margins": 0.690855860710144,
+      "rewards/rejected": -5.4679083824157715,
+      "step": 1085
+    },
+    {
+      "epoch": 0.5833751463455428,
+      "grad_norm": 1.4709370899000804,
+      "learning_rate": 9.730946154626078e-07,
+      "logits/chosen": 0.11681755632162094,
+      "logits/rejected": 0.31135350465774536,
+      "logps/chosen": -5.49279260635376,
+      "logps/rejected": -6.539482116699219,
+      "loss": 0.6865,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -5.49279260635376,
+      "rewards/margins": 1.0466889142990112,
+      "rewards/rejected": -6.539482116699219,
+      "step": 1090
+    },
+    {
+      "epoch": 0.5860511791269443,
+      "grad_norm": 0.989500559730571,
+      "learning_rate": 9.725883241855117e-07,
+      "logits/chosen": 0.17682883143424988,
+      "logits/rejected": 0.33254188299179077,
+      "logps/chosen": -6.3105387687683105,
+      "logps/rejected": -7.248024940490723,
+      "loss": 0.6841,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -6.3105387687683105,
+      "rewards/margins": 0.9374858140945435,
+      "rewards/rejected": -7.248024940490723,
+      "step": 1095
+    },
+    {
+      "epoch": 0.5887272119083459,
+      "grad_norm": 1.6240022739404205,
+      "learning_rate": 9.720774478544218e-07,
+      "logits/chosen": 0.28323718905448914,
+      "logits/rejected": 0.4969344139099121,
+      "logps/chosen": -6.534448146820068,
+      "logps/rejected": -7.5301618576049805,
+      "loss": 0.6875,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -6.534448146820068,
+      "rewards/margins": 0.995714008808136,
+      "rewards/rejected": -7.5301618576049805,
+      "step": 1100
+    },
+    {
+      "epoch": 0.5914032446897475,
+      "grad_norm": 1.2120061226477608,
+      "learning_rate": 9.715619914258624e-07,
+      "logits/chosen": 0.11322431266307831,
+      "logits/rejected": 0.2758231461048126,
+      "logps/chosen": -5.705962181091309,
+      "logps/rejected": -6.200955867767334,
+      "loss": 0.6838,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -5.705962181091309,
+      "rewards/margins": 0.4949936866760254,
+      "rewards/rejected": -6.200955867767334,
+      "step": 1105
+    },
+    {
+      "epoch": 0.594079277471149,
+      "grad_norm": 1.2169672682248156,
+      "learning_rate": 9.710419599007937e-07,
+      "logits/chosen": 0.18717986345291138,
+      "logits/rejected": 0.37118101119995117,
+      "logps/chosen": -5.0706915855407715,
+      "logps/rejected": -5.850651741027832,
+      "loss": 0.6863,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -5.0706915855407715,
+      "rewards/margins": 0.7799601554870605,
+      "rewards/rejected": -5.850651741027832,
+      "step": 1110
+    },
+    {
+      "epoch": 0.5967553102525506,
+      "grad_norm": 1.0446817820791257,
+      "learning_rate": 9.705173583245643e-07,
+      "logits/chosen": 0.1748601347208023,
+      "logits/rejected": 0.3534494936466217,
+      "logps/chosen": -5.417031288146973,
+      "logps/rejected": -6.638477325439453,
+      "loss": 0.6889,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -5.417031288146973,
+      "rewards/margins": 1.2214462757110596,
+      "rewards/rejected": -6.638477325439453,
+      "step": 1115
+    },
+    {
+      "epoch": 0.5994313430339522,
+      "grad_norm": 0.9930501835214839,
+      "learning_rate": 9.699881917868609e-07,
+      "logits/chosen": 0.07010771334171295,
+      "logits/rejected": 0.2474389523267746,
+      "logps/chosen": -5.306896209716797,
+      "logps/rejected": -5.633444786071777,
+      "loss": 0.6802,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -5.306896209716797,
+      "rewards/margins": 0.32654890418052673,
+      "rewards/rejected": -5.633444786071777,
+      "step": 1120
+    },
+    {
+      "epoch": 0.6021073758153538,
+      "grad_norm": 1.0798638588296121,
+      "learning_rate": 9.694544654216594e-07,
+      "logits/chosen": 0.05427680164575577,
+      "logits/rejected": 0.25671952962875366,
+      "logps/chosen": -4.983899116516113,
+      "logps/rejected": -5.704018592834473,
+      "loss": 0.6831,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.983899116516113,
+      "rewards/margins": 0.7201195955276489,
+      "rewards/rejected": -5.704018592834473,
+      "step": 1125
+    },
+    {
+      "epoch": 0.6047834085967553,
+      "grad_norm": 1.128380426479024,
+      "learning_rate": 9.689161844071755e-07,
+      "logits/chosen": 0.19646504521369934,
+      "logits/rejected": 0.3040979206562042,
+      "logps/chosen": -4.413432598114014,
+      "logps/rejected": -5.456568717956543,
+      "loss": 0.6899,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -4.413432598114014,
+      "rewards/margins": 1.043136715888977,
+      "rewards/rejected": -5.456568717956543,
+      "step": 1130
+    },
+    {
+      "epoch": 0.6074594413781569,
+      "grad_norm": 1.691010750922381,
+      "learning_rate": 9.683733539658138e-07,
+      "logits/chosen": 0.0669851079583168,
+      "logits/rejected": 0.31118959188461304,
+      "logps/chosen": -4.251648426055908,
+      "logps/rejected": -5.503758430480957,
+      "loss": 0.6817,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.251648426055908,
+      "rewards/margins": 1.2521098852157593,
+      "rewards/rejected": -5.503758430480957,
+      "step": 1135
+    },
+    {
+      "epoch": 0.6101354741595585,
+      "grad_norm": 1.2443020302751402,
+      "learning_rate": 9.678259793641178e-07,
+      "logits/chosen": 0.06555162370204926,
+      "logits/rejected": 0.1594078540802002,
+      "logps/chosen": -4.308938980102539,
+      "logps/rejected": -4.8594465255737305,
+      "loss": 0.6915,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -4.308938980102539,
+      "rewards/margins": 0.5505070686340332,
+      "rewards/rejected": -4.8594465255737305,
+      "step": 1140
+    },
+    {
+      "epoch": 0.61281150694096,
+      "grad_norm": 0.9671119538870153,
+      "learning_rate": 9.672740659127183e-07,
+      "logits/chosen": -0.04723416641354561,
+      "logits/rejected": 0.08484413474798203,
+      "logps/chosen": -4.558487415313721,
+      "logps/rejected": -5.207844257354736,
+      "loss": 0.6846,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.558487415313721,
+      "rewards/margins": 0.6493566036224365,
+      "rewards/rejected": -5.207844257354736,
+      "step": 1145
+    },
+    {
+      "epoch": 0.6154875397223616,
+      "grad_norm": 1.278567697921431,
+      "learning_rate": 9.667176189662818e-07,
+      "logits/chosen": 0.05078287795186043,
+      "logits/rejected": 0.2716083526611328,
+      "logps/chosen": -4.88139533996582,
+      "logps/rejected": -5.627421855926514,
+      "loss": 0.6818,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.88139533996582,
+      "rewards/margins": 0.746026337146759,
+      "rewards/rejected": -5.627421855926514,
+      "step": 1150
+    },
+    {
+      "epoch": 0.6181635725037632,
+      "grad_norm": 0.8481603756624189,
+      "learning_rate": 9.661566439234592e-07,
+      "logits/chosen": 0.11545266211032867,
+      "logits/rejected": 0.30309993028640747,
+      "logps/chosen": -4.713515758514404,
+      "logps/rejected": -5.333220481872559,
+      "loss": 0.6831,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.713515758514404,
+      "rewards/margins": 0.6197041273117065,
+      "rewards/rejected": -5.333220481872559,
+      "step": 1155
+    },
+    {
+      "epoch": 0.6208396052851648,
+      "grad_norm": 1.3202255322787648,
+      "learning_rate": 9.655911462268327e-07,
+      "logits/chosen": 0.07598944753408432,
+      "logits/rejected": 0.24061048030853271,
+      "logps/chosen": -4.049023151397705,
+      "logps/rejected": -5.073098659515381,
+      "loss": 0.6845,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.049023151397705,
+      "rewards/margins": 1.0240756273269653,
+      "rewards/rejected": -5.073098659515381,
+      "step": 1160
+    },
+    {
+      "epoch": 0.6235156380665663,
+      "grad_norm": 1.3955269003692383,
+      "learning_rate": 9.650211313628636e-07,
+      "logits/chosen": 0.03517208248376846,
+      "logits/rejected": 0.13691256940364838,
+      "logps/chosen": -3.6470649242401123,
+      "logps/rejected": -3.8894906044006348,
+      "loss": 0.6885,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -3.6470649242401123,
+      "rewards/margins": 0.24242529273033142,
+      "rewards/rejected": -3.8894906044006348,
+      "step": 1165
+    },
+    {
+      "epoch": 0.6261916708479679,
+      "grad_norm": 1.6181717711324135,
+      "learning_rate": 9.644466048618386e-07,
+      "logits/chosen": 0.020070726051926613,
+      "logits/rejected": 0.16812816262245178,
+      "logps/chosen": -4.0630974769592285,
+      "logps/rejected": -4.24812126159668,
+      "loss": 0.6886,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -4.0630974769592285,
+      "rewards/margins": 0.18502388894557953,
+      "rewards/rejected": -4.24812126159668,
+      "step": 1170
+    },
+    {
+      "epoch": 0.6288677036293695,
+      "grad_norm": 1.3609777589389427,
+      "learning_rate": 9.63867572297816e-07,
+      "logits/chosen": 0.035177040845155716,
+      "logits/rejected": 0.25093525648117065,
+      "logps/chosen": -3.8069138526916504,
+      "logps/rejected": -4.506682395935059,
+      "loss": 0.6788,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -3.8069138526916504,
+      "rewards/margins": 0.6997678875923157,
+      "rewards/rejected": -4.506682395935059,
+      "step": 1175
+    },
+    {
+      "epoch": 0.631543736410771,
+      "grad_norm": 1.1625354251100837,
+      "learning_rate": 9.632840392885727e-07,
+      "logits/chosen": 0.05817805603146553,
+      "logits/rejected": 0.23943626880645752,
+      "logps/chosen": -4.008138656616211,
+      "logps/rejected": -5.119635581970215,
+      "loss": 0.6842,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.008138656616211,
+      "rewards/margins": 1.1114972829818726,
+      "rewards/rejected": -5.119635581970215,
+      "step": 1180
+    },
+    {
+      "epoch": 0.6342197691921726,
+      "grad_norm": 1.9104078974107173,
+      "learning_rate": 9.626960114955483e-07,
+      "logits/chosen": 0.0819910541176796,
+      "logits/rejected": 0.291669636964798,
+      "logps/chosen": -3.69677996635437,
+      "logps/rejected": -5.077053546905518,
+      "loss": 0.6853,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -3.69677996635437,
+      "rewards/margins": 1.3802732229232788,
+      "rewards/rejected": -5.077053546905518,
+      "step": 1185
+    },
+    {
+      "epoch": 0.6368958019735742,
+      "grad_norm": 1.3096057581351552,
+      "learning_rate": 9.621034946237909e-07,
+      "logits/chosen": 0.03582355007529259,
+      "logits/rejected": 0.23636093735694885,
+      "logps/chosen": -3.798649311065674,
+      "logps/rejected": -4.913937091827393,
+      "loss": 0.6807,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -3.798649311065674,
+      "rewards/margins": 1.1152878999710083,
+      "rewards/rejected": -4.913937091827393,
+      "step": 1190
+    },
+    {
+      "epoch": 0.6395718347549757,
+      "grad_norm": 1.1345089167009932,
+      "learning_rate": 9.615064944219021e-07,
+      "logits/chosen": 0.03378887474536896,
+      "logits/rejected": 0.16949906945228577,
+      "logps/chosen": -3.581099033355713,
+      "logps/rejected": -4.300440311431885,
+      "loss": 0.6824,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -3.581099033355713,
+      "rewards/margins": 0.7193417549133301,
+      "rewards/rejected": -4.300440311431885,
+      "step": 1195
+    },
+    {
+      "epoch": 0.6422478675363773,
+      "grad_norm": 2.562659011877274,
+      "learning_rate": 9.609050166819803e-07,
+      "logits/chosen": -0.0218222513794899,
+      "logits/rejected": 0.02612129971385002,
+      "logps/chosen": -3.7166907787323,
+      "logps/rejected": -4.157186031341553,
+      "loss": 0.6827,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -3.7166907787323,
+      "rewards/margins": 0.44049492478370667,
+      "rewards/rejected": -4.157186031341553,
+      "step": 1200
+    },
+    {
+      "epoch": 0.6422478675363773,
+      "eval_logits/chosen": 0.3099616467952728,
+      "eval_logits/rejected": 0.46279844641685486,
+      "eval_logps/chosen": -3.7037312984466553,
+      "eval_logps/rejected": -4.683093070983887,
+      "eval_loss": 0.6809219121932983,
+      "eval_rewards/accuracies": 0.6416913866996765,
+      "eval_rewards/chosen": -3.7037312984466553,
+      "eval_rewards/margins": 0.9793617725372314,
+      "eval_rewards/rejected": -4.683093070983887,
+      "eval_runtime": 40.7188,
+      "eval_samples_per_second": 33.031,
+      "eval_steps_per_second": 8.276,
+      "step": 1200
+    },
+    {
+      "epoch": 0.6449239003177789,
+      "grad_norm": 1.9943666601828354,
+      "learning_rate": 9.602990672395653e-07,
+      "logits/chosen": -0.07859648764133453,
+      "logits/rejected": 0.13176962733268738,
+      "logps/chosen": -4.054784774780273,
+      "logps/rejected": -4.883113384246826,
+      "loss": 0.6801,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.054784774780273,
+      "rewards/margins": 0.8283289670944214,
+      "rewards/rejected": -4.883113384246826,
+      "step": 1205
+    },
+    {
+      "epoch": 0.6475999330991805,
+      "grad_norm": 2.4445064352721984,
+      "learning_rate": 9.59688651973581e-07,
+      "logits/chosen": 0.03421860560774803,
+      "logits/rejected": 0.31082746386528015,
+      "logps/chosen": -4.471826076507568,
+      "logps/rejected": -5.141530990600586,
+      "loss": 0.6858,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -4.471826076507568,
+      "rewards/margins": 0.6697045564651489,
+      "rewards/rejected": -5.141530990600586,
+      "step": 1210
+    },
+    {
+      "epoch": 0.650275965880582,
+      "grad_norm": 1.9472518557695,
+      "learning_rate": 9.590737768062792e-07,
+      "logits/chosen": 0.011806154623627663,
+      "logits/rejected": 0.16049370169639587,
+      "logps/chosen": -4.6412434577941895,
+      "logps/rejected": -5.313283443450928,
+      "loss": 0.6871,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -4.6412434577941895,
+      "rewards/margins": 0.6720403432846069,
+      "rewards/rejected": -5.313283443450928,
+      "step": 1215
+    },
+    {
+      "epoch": 0.6529519986619836,
+      "grad_norm": 1.8861447805246092,
+      "learning_rate": 9.584544477031816e-07,
+      "logits/chosen": 0.19159173965454102,
+      "logits/rejected": 0.4076632559299469,
+      "logps/chosen": -4.294541835784912,
+      "logps/rejected": -5.601269721984863,
+      "loss": 0.6805,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.294541835784912,
+      "rewards/margins": 1.306727409362793,
+      "rewards/rejected": -5.601269721984863,
+      "step": 1220
+    },
+    {
+      "epoch": 0.6556280314433852,
+      "grad_norm": 2.0172690798172255,
+      "learning_rate": 9.578306706730215e-07,
+      "logits/chosen": 0.012652593664824963,
+      "logits/rejected": 0.25653165578842163,
+      "logps/chosen": -4.189724922180176,
+      "logps/rejected": -4.691739559173584,
+      "loss": 0.6881,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -4.189724922180176,
+      "rewards/margins": 0.5020139813423157,
+      "rewards/rejected": -4.691739559173584,
+      "step": 1225
+    },
+    {
+      "epoch": 0.6583040642247867,
+      "grad_norm": 1.3826694031943363,
+      "learning_rate": 9.572024517676865e-07,
+      "logits/chosen": 0.24912109971046448,
+      "logits/rejected": 0.36395543813705444,
+      "logps/chosen": -4.854912757873535,
+      "logps/rejected": -5.603348731994629,
+      "loss": 0.6831,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -4.854912757873535,
+      "rewards/margins": 0.7484362721443176,
+      "rewards/rejected": -5.603348731994629,
+      "step": 1230
+    },
+    {
+      "epoch": 0.6609800970061883,
+      "grad_norm": 1.8874855519948077,
+      "learning_rate": 9.565697970821593e-07,
+      "logits/chosen": 0.3492785394191742,
+      "logits/rejected": 0.6070412397384644,
+      "logps/chosen": -5.5090651512146,
+      "logps/rejected": -7.2579450607299805,
+      "loss": 0.6858,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -5.5090651512146,
+      "rewards/margins": 1.74888014793396,
+      "rewards/rejected": -7.2579450607299805,
+      "step": 1235
+    },
+    {
+      "epoch": 0.6636561297875899,
+      "grad_norm": 1.5414520148202555,
+      "learning_rate": 9.559327127544585e-07,
+      "logits/chosen": 0.2557428479194641,
+      "logits/rejected": 0.5399208068847656,
+      "logps/chosen": -5.617169380187988,
+      "logps/rejected": -7.224648952484131,
+      "loss": 0.6854,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -5.617169380187988,
+      "rewards/margins": 1.6074802875518799,
+      "rewards/rejected": -7.224648952484131,
+      "step": 1240
+    },
+    {
+      "epoch": 0.6663321625689914,
+      "grad_norm": 1.6194261947371658,
+      "learning_rate": 9.552912049655789e-07,
+      "logits/chosen": 0.21549543738365173,
+      "logits/rejected": 0.4484475255012512,
+      "logps/chosen": -4.899280071258545,
+      "logps/rejected": -6.114068031311035,
+      "loss": 0.6824,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.899280071258545,
+      "rewards/margins": 1.2147880792617798,
+      "rewards/rejected": -6.114068031311035,
+      "step": 1245
+    },
+    {
+      "epoch": 0.669008195350393,
+      "grad_norm": 1.8816863806850932,
+      "learning_rate": 9.546452799394315e-07,
+      "logits/chosen": 0.1818639189004898,
+      "logits/rejected": 0.45496663451194763,
+      "logps/chosen": -4.317521095275879,
+      "logps/rejected": -5.623841285705566,
+      "loss": 0.6866,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -4.317521095275879,
+      "rewards/margins": 1.3063205480575562,
+      "rewards/rejected": -5.623841285705566,
+      "step": 1250
+    },
+    {
+      "epoch": 0.6716842281317946,
+      "grad_norm": 1.6616185611661263,
+      "learning_rate": 9.539949439427846e-07,
+      "logits/chosen": 0.14488592743873596,
+      "logits/rejected": 0.3099380433559418,
+      "logps/chosen": -4.23885440826416,
+      "logps/rejected": -5.151686191558838,
+      "loss": 0.6876,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.23885440826416,
+      "rewards/margins": 0.9128311276435852,
+      "rewards/rejected": -5.151686191558838,
+      "step": 1255
+    },
+    {
+      "epoch": 0.6743602609131962,
+      "grad_norm": 1.509484822770816,
+      "learning_rate": 9.533402032852002e-07,
+      "logits/chosen": 0.19197924435138702,
+      "logits/rejected": 0.40496253967285156,
+      "logps/chosen": -4.485262870788574,
+      "logps/rejected": -6.233352184295654,
+      "loss": 0.6816,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.485262870788574,
+      "rewards/margins": 1.7480895519256592,
+      "rewards/rejected": -6.233352184295654,
+      "step": 1260
+    },
+    {
+      "epoch": 0.6770362936945977,
+      "grad_norm": 2.2511179316218457,
+      "learning_rate": 9.526810643189754e-07,
+      "logits/chosen": 0.19483545422554016,
+      "logits/rejected": 0.41464361548423767,
+      "logps/chosen": -4.3624587059021,
+      "logps/rejected": -5.556997776031494,
+      "loss": 0.6803,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.3624587059021,
+      "rewards/margins": 1.194538950920105,
+      "rewards/rejected": -5.556997776031494,
+      "step": 1265
+    },
+    {
+      "epoch": 0.6797123264759993,
+      "grad_norm": 2.383589504637693,
+      "learning_rate": 9.52017533439079e-07,
+      "logits/chosen": 0.12329081445932388,
+      "logits/rejected": 0.2999495565891266,
+      "logps/chosen": -4.19939661026001,
+      "logps/rejected": -5.149897575378418,
+      "loss": 0.6839,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.19939661026001,
+      "rewards/margins": 0.95050048828125,
+      "rewards/rejected": -5.149897575378418,
+      "step": 1270
+    },
+    {
+      "epoch": 0.6823883592574009,
+      "grad_norm": 1.9118065060839307,
+      "learning_rate": 9.513496170830909e-07,
+      "logits/chosen": 0.26234468817710876,
+      "logits/rejected": 0.5008620619773865,
+      "logps/chosen": -4.823470592498779,
+      "logps/rejected": -6.653124809265137,
+      "loss": 0.6795,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.823470592498779,
+      "rewards/margins": 1.8296546936035156,
+      "rewards/rejected": -6.653124809265137,
+      "step": 1275
+    },
+    {
+      "epoch": 0.6850643920388024,
+      "grad_norm": 1.151249846509719,
+      "learning_rate": 9.506773217311382e-07,
+      "logits/chosen": 0.21844908595085144,
+      "logits/rejected": 0.4722215533256531,
+      "logps/chosen": -5.233174800872803,
+      "logps/rejected": -6.469101905822754,
+      "loss": 0.6827,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -5.233174800872803,
+      "rewards/margins": 1.235926866531372,
+      "rewards/rejected": -6.469101905822754,
+      "step": 1280
+    },
+    {
+      "epoch": 0.687740424820204,
+      "grad_norm": 2.2648824886071797,
+      "learning_rate": 9.500006539058334e-07,
+      "logits/chosen": 0.23313912749290466,
+      "logits/rejected": 0.41881927847862244,
+      "logps/chosen": -4.401029586791992,
+      "logps/rejected": -5.274280548095703,
+      "loss": 0.683,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.401029586791992,
+      "rewards/margins": 0.8732506036758423,
+      "rewards/rejected": -5.274280548095703,
+      "step": 1285
+    },
+    {
+      "epoch": 0.6904164576016056,
+      "grad_norm": 1.6308905393233772,
+      "learning_rate": 9.493196201722109e-07,
+      "logits/chosen": 0.08242791891098022,
+      "logits/rejected": 0.27069029211997986,
+      "logps/chosen": -4.305448055267334,
+      "logps/rejected": -4.996705532073975,
+      "loss": 0.6755,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.305448055267334,
+      "rewards/margins": 0.6912575960159302,
+      "rewards/rejected": -4.996705532073975,
+      "step": 1290
+    },
+    {
+      "epoch": 0.6930924903830072,
+      "grad_norm": 3.1823375740319464,
+      "learning_rate": 9.486342271376628e-07,
+      "logits/chosen": 0.19029052555561066,
+      "logits/rejected": 0.44329729676246643,
+      "logps/chosen": -4.334671974182129,
+      "logps/rejected": -6.3908843994140625,
+      "loss": 0.6791,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.334671974182129,
+      "rewards/margins": 2.0562126636505127,
+      "rewards/rejected": -6.3908843994140625,
+      "step": 1295
+    },
+    {
+      "epoch": 0.6957685231644087,
+      "grad_norm": 2.951144940091108,
+      "learning_rate": 9.479444814518755e-07,
+      "logits/chosen": 0.17311161756515503,
+      "logits/rejected": 0.49754804372787476,
+      "logps/chosen": -3.837864637374878,
+      "logps/rejected": -5.268630027770996,
+      "loss": 0.6757,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -3.837864637374878,
+      "rewards/margins": 1.4307657480239868,
+      "rewards/rejected": -5.268630027770996,
+      "step": 1300
+    },
+    {
+      "epoch": 0.6984445559458103,
+      "grad_norm": 2.0816109529945943,
+      "learning_rate": 9.472503898067645e-07,
+      "logits/chosen": 0.22857145965099335,
+      "logits/rejected": 0.36378926038742065,
+      "logps/chosen": -3.861457347869873,
+      "logps/rejected": -4.8487067222595215,
+      "loss": 0.6879,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -3.861457347869873,
+      "rewards/margins": 0.9872493743896484,
+      "rewards/rejected": -4.8487067222595215,
+      "step": 1305
+    },
+    {
+      "epoch": 0.701120588727212,
+      "grad_norm": 2.560374742972752,
+      "learning_rate": 9.465519589364099e-07,
+      "logits/chosen": 0.31125545501708984,
+      "logits/rejected": 0.5338835716247559,
+      "logps/chosen": -4.220249652862549,
+      "logps/rejected": -6.33150577545166,
+      "loss": 0.6787,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.220249652862549,
+      "rewards/margins": 2.1112558841705322,
+      "rewards/rejected": -6.33150577545166,
+      "step": 1310
+    },
+    {
+      "epoch": 0.7037966215086134,
+      "grad_norm": 2.5429520468152518,
+      "learning_rate": 9.458491956169914e-07,
+      "logits/chosen": 0.28559333086013794,
+      "logits/rejected": 0.4688785672187805,
+      "logps/chosen": -4.426626682281494,
+      "logps/rejected": -6.0352373123168945,
+      "loss": 0.6726,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.426626682281494,
+      "rewards/margins": 1.6086101531982422,
+      "rewards/rejected": -6.0352373123168945,
+      "step": 1315
+    },
+    {
+      "epoch": 0.706472654290015,
+      "grad_norm": 2.133076444327326,
+      "learning_rate": 9.451421066667215e-07,
+      "logits/chosen": 0.2543080449104309,
+      "logits/rejected": 0.49480241537094116,
+      "logps/chosen": -4.615199565887451,
+      "logps/rejected": -5.8069000244140625,
+      "loss": 0.676,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.615199565887451,
+      "rewards/margins": 1.1917006969451904,
+      "rewards/rejected": -5.8069000244140625,
+      "step": 1320
+    },
+    {
+      "epoch": 0.7091486870714167,
+      "grad_norm": 2.8883422014792024,
+      "learning_rate": 9.444306989457805e-07,
+      "logits/chosen": 0.2816043496131897,
+      "logits/rejected": 0.5633341073989868,
+      "logps/chosen": -3.937284469604492,
+      "logps/rejected": -5.835335731506348,
+      "loss": 0.6755,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -3.937284469604492,
+      "rewards/margins": 1.8980509042739868,
+      "rewards/rejected": -5.835335731506348,
+      "step": 1325
+    },
+    {
+      "epoch": 0.7118247198528181,
+      "grad_norm": 3.2002016318583126,
+      "learning_rate": 9.437149793562489e-07,
+      "logits/chosen": 0.3006569743156433,
+      "logits/rejected": 0.47477731108665466,
+      "logps/chosen": -4.165520668029785,
+      "logps/rejected": -5.0348896980285645,
+      "loss": 0.6813,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.165520668029785,
+      "rewards/margins": 0.869369387626648,
+      "rewards/rejected": -5.0348896980285645,
+      "step": 1330
+    },
+    {
+      "epoch": 0.7145007526342197,
+      "grad_norm": 2.4471423364255793,
+      "learning_rate": 9.429949548420417e-07,
+      "logits/chosen": 0.21002614498138428,
+      "logits/rejected": 0.4002617299556732,
+      "logps/chosen": -3.7680156230926514,
+      "logps/rejected": -4.7523088455200195,
+      "loss": 0.6769,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -3.7680156230926514,
+      "rewards/margins": 0.9842929840087891,
+      "rewards/rejected": -4.7523088455200195,
+      "step": 1335
+    },
+    {
+      "epoch": 0.7171767854156214,
+      "grad_norm": 2.7764257620070096,
+      "learning_rate": 9.422706323888396e-07,
+      "logits/chosen": 0.20405197143554688,
+      "logits/rejected": 0.4651819169521332,
+      "logps/chosen": -2.9019267559051514,
+      "logps/rejected": -4.646694660186768,
+      "loss": 0.6713,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -2.9019267559051514,
+      "rewards/margins": 1.7447679042816162,
+      "rewards/rejected": -4.646694660186768,
+      "step": 1340
+    },
+    {
+      "epoch": 0.719852818197023,
+      "grad_norm": 3.5442280784290197,
+      "learning_rate": 9.415420190240225e-07,
+      "logits/chosen": 0.1991603821516037,
+      "logits/rejected": 0.5677786469459534,
+      "logps/chosen": -2.8502278327941895,
+      "logps/rejected": -4.43939208984375,
+      "loss": 0.672,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -2.8502278327941895,
+      "rewards/margins": 1.58916437625885,
+      "rewards/rejected": -4.43939208984375,
+      "step": 1345
+    },
+    {
+      "epoch": 0.7225288509784245,
+      "grad_norm": 4.446484837007382,
+      "learning_rate": 9.408091218166002e-07,
+      "logits/chosen": 0.21837444603443146,
+      "logits/rejected": 0.35738322138786316,
+      "logps/chosen": -3.295255184173584,
+      "logps/rejected": -3.969597339630127,
+      "loss": 0.6854,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -3.295255184173584,
+      "rewards/margins": 0.6743419766426086,
+      "rewards/rejected": -3.969597339630127,
+      "step": 1350
+    },
+    {
+      "epoch": 0.7252048837598261,
+      "grad_norm": 3.1253194103572457,
+      "learning_rate": 9.400719478771449e-07,
+      "logits/chosen": 0.4615071713924408,
+      "logits/rejected": 0.8090218305587769,
+      "logps/chosen": -5.208754539489746,
+      "logps/rejected": -6.785752296447754,
+      "loss": 0.6794,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -5.208754539489746,
+      "rewards/margins": 1.576997995376587,
+      "rewards/rejected": -6.785752296447754,
+      "step": 1355
+    },
+    {
+      "epoch": 0.7278809165412277,
+      "grad_norm": 4.029557699850957,
+      "learning_rate": 9.393305043577209e-07,
+      "logits/chosen": 0.31698986887931824,
+      "logits/rejected": 0.6424908638000488,
+      "logps/chosen": -4.471649646759033,
+      "logps/rejected": -6.317807197570801,
+      "loss": 0.6752,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.471649646759033,
+      "rewards/margins": 1.8461577892303467,
+      "rewards/rejected": -6.317807197570801,
+      "step": 1360
+    },
+    {
+      "epoch": 0.7305569493226292,
+      "grad_norm": 2.788047866424518,
+      "learning_rate": 9.38584798451817e-07,
+      "logits/chosen": 0.41330796480178833,
+      "logits/rejected": 0.6663674712181091,
+      "logps/chosen": -4.383449077606201,
+      "logps/rejected": -5.618782043457031,
+      "loss": 0.677,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.383449077606201,
+      "rewards/margins": 1.2353328466415405,
+      "rewards/rejected": -5.618782043457031,
+      "step": 1365
+    },
+    {
+      "epoch": 0.7332329821040308,
+      "grad_norm": 2.650728539405042,
+      "learning_rate": 9.37834837394275e-07,
+      "logits/chosen": 0.4001213610172272,
+      "logits/rejected": 0.6442126035690308,
+      "logps/chosen": -4.852554798126221,
+      "logps/rejected": -6.548978328704834,
+      "loss": 0.6777,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.852554798126221,
+      "rewards/margins": 1.696423888206482,
+      "rewards/rejected": -6.548978328704834,
+      "step": 1370
+    },
+    {
+      "epoch": 0.7359090148854324,
+      "grad_norm": 2.7944900965882407,
+      "learning_rate": 9.370806284612203e-07,
+      "logits/chosen": 0.3938954770565033,
+      "logits/rejected": 0.711518406867981,
+      "logps/chosen": -4.804046154022217,
+      "logps/rejected": -7.044665336608887,
+      "loss": 0.6877,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.804046154022217,
+      "rewards/margins": 2.240619659423828,
+      "rewards/rejected": -7.044665336608887,
+      "step": 1375
+    },
+    {
+      "epoch": 0.738585047666834,
+      "grad_norm": 1.4942263310986759,
+      "learning_rate": 9.363221789699912e-07,
+      "logits/chosen": 0.5034587383270264,
+      "logits/rejected": 0.6983398199081421,
+      "logps/chosen": -5.572027683258057,
+      "logps/rejected": -6.90764856338501,
+      "loss": 0.6922,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -5.572027683258057,
+      "rewards/margins": 1.3356218338012695,
+      "rewards/rejected": -6.90764856338501,
+      "step": 1380
+    },
+    {
+      "epoch": 0.7412610804482355,
+      "grad_norm": 2.554062052737816,
+      "learning_rate": 9.355594962790682e-07,
+      "logits/chosen": 0.5503712892532349,
+      "logits/rejected": 0.960155189037323,
+      "logps/chosen": -6.330416679382324,
+      "logps/rejected": -9.104744911193848,
+      "loss": 0.6803,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -6.330416679382324,
+      "rewards/margins": 2.7743279933929443,
+      "rewards/rejected": -9.104744911193848,
+      "step": 1385
+    },
+    {
+      "epoch": 0.7439371132296371,
+      "grad_norm": 2.0188126604610943,
+      "learning_rate": 9.34792587788002e-07,
+      "logits/chosen": 0.5474240183830261,
+      "logits/rejected": 0.8618350028991699,
+      "logps/chosen": -5.962157249450684,
+      "logps/rejected": -7.729863166809082,
+      "loss": 0.6852,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -5.962157249450684,
+      "rewards/margins": 1.7677053213119507,
+      "rewards/rejected": -7.729863166809082,
+      "step": 1390
+    },
+    {
+      "epoch": 0.7466131460110387,
+      "grad_norm": 3.762030799958685,
+      "learning_rate": 9.34021460937342e-07,
+      "logits/chosen": 0.45572027564048767,
+      "logits/rejected": 0.6209717392921448,
+      "logps/chosen": -4.9505815505981445,
+      "logps/rejected": -5.666743755340576,
+      "loss": 0.688,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.9505815505981445,
+      "rewards/margins": 0.7161625027656555,
+      "rewards/rejected": -5.666743755340576,
+      "step": 1395
+    },
+    {
+      "epoch": 0.7492891787924402,
+      "grad_norm": 1.8095439178578054,
+      "learning_rate": 9.332461232085646e-07,
+      "logits/chosen": 0.16211746633052826,
+      "logits/rejected": 0.3920448124408722,
+      "logps/chosen": -4.878157615661621,
+      "logps/rejected": -6.265026092529297,
+      "loss": 0.6847,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.878157615661621,
+      "rewards/margins": 1.3868690729141235,
+      "rewards/rejected": -6.265026092529297,
+      "step": 1400
+    },
+    {
+      "epoch": 0.7519652115738418,
+      "grad_norm": 2.379908364952983,
+      "learning_rate": 9.324665821239998e-07,
+      "logits/chosen": 0.2290266454219818,
+      "logits/rejected": 0.40708255767822266,
+      "logps/chosen": -5.399727821350098,
+      "logps/rejected": -6.423524379730225,
+      "loss": 0.6901,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -5.399727821350098,
+      "rewards/margins": 1.0237969160079956,
+      "rewards/rejected": -6.423524379730225,
+      "step": 1405
+    },
+    {
+      "epoch": 0.7546412443552434,
+      "grad_norm": 2.159399674085278,
+      "learning_rate": 9.316828452467583e-07,
+      "logits/chosen": 0.26150617003440857,
+      "logits/rejected": 0.5792301893234253,
+      "logps/chosen": -5.049007415771484,
+      "logps/rejected": -6.9095611572265625,
+      "loss": 0.6802,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -5.049007415771484,
+      "rewards/margins": 1.8605533838272095,
+      "rewards/rejected": -6.9095611572265625,
+      "step": 1410
+    },
+    {
+      "epoch": 0.7573172771366449,
+      "grad_norm": 1.628556776458848,
+      "learning_rate": 9.30894920180659e-07,
+      "logits/chosen": 0.26271912455558777,
+      "logits/rejected": 0.5467618107795715,
+      "logps/chosen": -5.053939342498779,
+      "logps/rejected": -6.055159568786621,
+      "loss": 0.6804,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -5.053939342498779,
+      "rewards/margins": 1.001220464706421,
+      "rewards/rejected": -6.055159568786621,
+      "step": 1415
+    },
+    {
+      "epoch": 0.7599933099180465,
+      "grad_norm": 2.4813254645672047,
+      "learning_rate": 9.301028145701543e-07,
+      "logits/chosen": 0.23461830615997314,
+      "logits/rejected": 0.5434097051620483,
+      "logps/chosen": -4.378081321716309,
+      "logps/rejected": -6.3747124671936035,
+      "loss": 0.6745,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.378081321716309,
+      "rewards/margins": 1.9966309070587158,
+      "rewards/rejected": -6.3747124671936035,
+      "step": 1420
+    },
+    {
+      "epoch": 0.7626693426994481,
+      "grad_norm": 2.3284364091539453,
+      "learning_rate": 9.293065361002563e-07,
+      "logits/chosen": 0.09624779224395752,
+      "logits/rejected": 0.2894967794418335,
+      "logps/chosen": -3.396289110183716,
+      "logps/rejected": -4.365817070007324,
+      "loss": 0.6786,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -3.396289110183716,
+      "rewards/margins": 0.9695287942886353,
+      "rewards/rejected": -4.365817070007324,
+      "step": 1425
+    },
+    {
+      "epoch": 0.7653453754808497,
+      "grad_norm": 1.7209605788262352,
+      "learning_rate": 9.285060924964622e-07,
+      "logits/chosen": 0.030794482678174973,
+      "logits/rejected": 0.1494387686252594,
+      "logps/chosen": -3.104480028152466,
+      "logps/rejected": -3.6465091705322266,
+      "loss": 0.6851,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -3.104480028152466,
+      "rewards/margins": 0.5420293211936951,
+      "rewards/rejected": -3.6465091705322266,
+      "step": 1430
+    },
+    {
+      "epoch": 0.7680214082622512,
+      "grad_norm": 2.8560557282593817,
+      "learning_rate": 9.277014915246792e-07,
+      "logits/chosen": 0.20121511816978455,
+      "logits/rejected": 0.35372254252433777,
+      "logps/chosen": -3.5141708850860596,
+      "logps/rejected": -4.849537372589111,
+      "loss": 0.6745,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -3.5141708850860596,
+      "rewards/margins": 1.3353660106658936,
+      "rewards/rejected": -4.849537372589111,
+      "step": 1435
+    },
+    {
+      "epoch": 0.7706974410436528,
+      "grad_norm": 1.8941960241610374,
+      "learning_rate": 9.268927409911498e-07,
+      "logits/chosen": 0.16448000073432922,
+      "logits/rejected": 0.405501127243042,
+      "logps/chosen": -4.081229209899902,
+      "logps/rejected": -5.4828338623046875,
+      "loss": 0.6773,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.081229209899902,
+      "rewards/margins": 1.401605248451233,
+      "rewards/rejected": -5.4828338623046875,
+      "step": 1440
+    },
+    {
+      "epoch": 0.7733734738250544,
+      "grad_norm": 1.7722744375248851,
+      "learning_rate": 9.260798487423749e-07,
+      "logits/chosen": 0.35363513231277466,
+      "logits/rejected": 0.5803099870681763,
+      "logps/chosen": -5.028221130371094,
+      "logps/rejected": -6.09029483795166,
+      "loss": 0.6768,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -5.028221130371094,
+      "rewards/margins": 1.0620737075805664,
+      "rewards/rejected": -6.09029483795166,
+      "step": 1445
+    },
+    {
+      "epoch": 0.7760495066064559,
+      "grad_norm": 4.016196328810283,
+      "learning_rate": 9.252628226650389e-07,
+      "logits/chosen": 0.3388669788837433,
+      "logits/rejected": 0.5988542437553406,
+      "logps/chosen": -4.656680107116699,
+      "logps/rejected": -6.297695159912109,
+      "loss": 0.6769,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -4.656680107116699,
+      "rewards/margins": 1.6410150527954102,
+      "rewards/rejected": -6.297695159912109,
+      "step": 1450
+    },
+    {
+      "epoch": 0.7787255393878575,
+      "grad_norm": 2.11600229406371,
+      "learning_rate": 9.244416706859321e-07,
+      "logits/chosen": 0.30541691184043884,
+      "logits/rejected": 0.5631024241447449,
+      "logps/chosen": -4.344799041748047,
+      "logps/rejected": -5.342108249664307,
+      "loss": 0.6696,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.344799041748047,
+      "rewards/margins": 0.9973089098930359,
+      "rewards/rejected": -5.342108249664307,
+      "step": 1455
+    },
+    {
+      "epoch": 0.7814015721692591,
+      "grad_norm": 2.6817897601228693,
+      "learning_rate": 9.23616400771875e-07,
+      "logits/chosen": 0.3519609272480011,
+      "logits/rejected": 0.5697275400161743,
+      "logps/chosen": -4.527349948883057,
+      "logps/rejected": -5.709645748138428,
+      "loss": 0.6762,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.527349948883057,
+      "rewards/margins": 1.1822961568832397,
+      "rewards/rejected": -5.709645748138428,
+      "step": 1460
+    },
+    {
+      "epoch": 0.7840776049506607,
+      "grad_norm": 1.9231656342756416,
+      "learning_rate": 9.227870209296395e-07,
+      "logits/chosen": 0.3797231614589691,
+      "logits/rejected": 0.6502147912979126,
+      "logps/chosen": -4.592616081237793,
+      "logps/rejected": -6.053444862365723,
+      "loss": 0.6848,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.592616081237793,
+      "rewards/margins": 1.460828423500061,
+      "rewards/rejected": -6.053444862365723,
+      "step": 1465
+    },
+    {
+      "epoch": 0.7867536377320622,
+      "grad_norm": 2.1100826300248783,
+      "learning_rate": 9.219535392058728e-07,
+      "logits/chosen": 0.48647409677505493,
+      "logits/rejected": 0.67566978931427,
+      "logps/chosen": -5.700984477996826,
+      "logps/rejected": -7.679039001464844,
+      "loss": 0.6863,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -5.700984477996826,
+      "rewards/margins": 1.9780542850494385,
+      "rewards/rejected": -7.679039001464844,
+      "step": 1470
+    },
+    {
+      "epoch": 0.7894296705134638,
+      "grad_norm": 2.3794432359165603,
+      "learning_rate": 9.211159636870181e-07,
+      "logits/chosen": 0.651904821395874,
+      "logits/rejected": 0.9659870266914368,
+      "logps/chosen": -6.6016998291015625,
+      "logps/rejected": -8.631077766418457,
+      "loss": 0.6853,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -6.6016998291015625,
+      "rewards/margins": 2.0293779373168945,
+      "rewards/rejected": -8.631077766418457,
+      "step": 1475
+    },
+    {
+      "epoch": 0.7921057032948654,
+      "grad_norm": 3.0978704944965685,
+      "learning_rate": 9.202743024992367e-07,
+      "logits/chosen": 0.6191867589950562,
+      "logits/rejected": 0.9965786933898926,
+      "logps/chosen": -5.472815036773682,
+      "logps/rejected": -8.243182182312012,
+      "loss": 0.6801,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -5.472815036773682,
+      "rewards/margins": 2.77036714553833,
+      "rewards/rejected": -8.243182182312012,
+      "step": 1480
+    },
+    {
+      "epoch": 0.7947817360762669,
+      "grad_norm": 3.72624701582259,
+      "learning_rate": 9.194285638083293e-07,
+      "logits/chosen": 0.4426315724849701,
+      "logits/rejected": 0.7513459920883179,
+      "logps/chosen": -5.12876558303833,
+      "logps/rejected": -7.412458896636963,
+      "loss": 0.6743,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -5.12876558303833,
+      "rewards/margins": 2.2836925983428955,
+      "rewards/rejected": -7.412458896636963,
+      "step": 1485
+    },
+    {
+      "epoch": 0.7974577688576685,
+      "grad_norm": 3.0644632137263614,
+      "learning_rate": 9.185787558196562e-07,
+      "logits/chosen": 0.19062156975269318,
+      "logits/rejected": 0.5349089503288269,
+      "logps/chosen": -3.910240888595581,
+      "logps/rejected": -6.0176801681518555,
+      "loss": 0.6799,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -3.910240888595581,
+      "rewards/margins": 2.107438802719116,
+      "rewards/rejected": -6.0176801681518555,
+      "step": 1490
+    },
+    {
+      "epoch": 0.8001338016390701,
+      "grad_norm": 2.9458724489113917,
+      "learning_rate": 9.177248867780583e-07,
+      "logits/chosen": 0.17142339050769806,
+      "logits/rejected": 0.29531553387641907,
+      "logps/chosen": -4.237878799438477,
+      "logps/rejected": -5.01108455657959,
+      "loss": 0.6851,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -4.237878799438477,
+      "rewards/margins": 0.7732059955596924,
+      "rewards/rejected": -5.01108455657959,
+      "step": 1495
+    },
+    {
+      "epoch": 0.8028098344204716,
+      "grad_norm": 3.385725849938397,
+      "learning_rate": 9.168669649677769e-07,
+      "logits/chosen": 0.10742871463298798,
+      "logits/rejected": 0.1865159571170807,
+      "logps/chosen": -4.096005439758301,
+      "logps/rejected": -4.3491716384887695,
+      "loss": 0.6955,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -4.096005439758301,
+      "rewards/margins": 0.2531662583351135,
+      "rewards/rejected": -4.3491716384887695,
+      "step": 1500
+    },
+    {
+      "epoch": 0.8054858672018732,
+      "grad_norm": 3.62376150404819,
+      "learning_rate": 9.16004998712373e-07,
+      "logits/chosen": 0.5506114959716797,
+      "logits/rejected": 0.6847094297409058,
+      "logps/chosen": -5.9898881912231445,
+      "logps/rejected": -7.789364814758301,
+      "loss": 0.6698,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -5.9898881912231445,
+      "rewards/margins": 1.7994760274887085,
+      "rewards/rejected": -7.789364814758301,
+      "step": 1505
+    },
+    {
+      "epoch": 0.8081618999832748,
+      "grad_norm": 7.167939840983412,
+      "learning_rate": 9.151389963746472e-07,
+      "logits/chosen": 0.6102725267410278,
+      "logits/rejected": 0.9851118922233582,
+      "logps/chosen": -6.726051330566406,
+      "logps/rejected": -9.516088485717773,
+      "loss": 0.681,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -6.726051330566406,
+      "rewards/margins": 2.79003643989563,
+      "rewards/rejected": -9.516088485717773,
+      "step": 1510
+    },
+    {
+      "epoch": 0.8108379327646764,
+      "grad_norm": 3.6656314750890795,
+      "learning_rate": 9.142689663565577e-07,
+      "logits/chosen": 0.45727449655532837,
+      "logits/rejected": 0.7893425822257996,
+      "logps/chosen": -5.245751857757568,
+      "logps/rejected": -7.5106682777404785,
+      "loss": 0.6766,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -5.245751857757568,
+      "rewards/margins": 2.2649166584014893,
+      "rewards/rejected": -7.5106682777404785,
+      "step": 1515
+    },
+    {
+      "epoch": 0.8135139655460779,
+      "grad_norm": 4.037766346859406,
+      "learning_rate": 9.133949170991397e-07,
+      "logits/chosen": 0.4285614490509033,
+      "logits/rejected": 0.6595946550369263,
+      "logps/chosen": -4.5931010246276855,
+      "logps/rejected": -6.711119651794434,
+      "loss": 0.6863,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.5931010246276855,
+      "rewards/margins": 2.11801815032959,
+      "rewards/rejected": -6.711119651794434,
+      "step": 1520
+    },
+    {
+      "epoch": 0.8161899983274795,
+      "grad_norm": 3.6251356464414926,
+      "learning_rate": 9.125168570824231e-07,
+      "logits/chosen": 0.27577635645866394,
+      "logits/rejected": 0.5190774202346802,
+      "logps/chosen": -4.799222946166992,
+      "logps/rejected": -6.3917717933654785,
+      "loss": 0.6755,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.799222946166992,
+      "rewards/margins": 1.5925493240356445,
+      "rewards/rejected": -6.3917717933654785,
+      "step": 1525
+    },
+    {
+      "epoch": 0.8188660311088811,
+      "grad_norm": 3.054864189609471,
+      "learning_rate": 9.116347948253496e-07,
+      "logits/chosen": 0.21017956733703613,
+      "logits/rejected": 0.3882458209991455,
+      "logps/chosen": -4.486644268035889,
+      "logps/rejected": -5.9372358322143555,
+      "loss": 0.682,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.486644268035889,
+      "rewards/margins": 1.45059072971344,
+      "rewards/rejected": -5.9372358322143555,
+      "step": 1530
+    },
+    {
+      "epoch": 0.8215420638902826,
+      "grad_norm": 1.926492080391662,
+      "learning_rate": 9.107487388856916e-07,
+      "logits/chosen": 0.052388858050107956,
+      "logits/rejected": 0.2861334979534149,
+      "logps/chosen": -3.328448534011841,
+      "logps/rejected": -4.572847843170166,
+      "loss": 0.6733,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -3.328448534011841,
+      "rewards/margins": 1.2443994283676147,
+      "rewards/rejected": -4.572847843170166,
+      "step": 1535
+    },
+    {
+      "epoch": 0.8242180966716842,
+      "grad_norm": 1.5805107405982715,
+      "learning_rate": 9.098586978599673e-07,
+      "logits/chosen": 0.08017917722463608,
+      "logits/rejected": 0.30607134103775024,
+      "logps/chosen": -3.3920071125030518,
+      "logps/rejected": -4.5173211097717285,
+      "loss": 0.6701,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -3.3920071125030518,
+      "rewards/margins": 1.1253139972686768,
+      "rewards/rejected": -4.5173211097717285,
+      "step": 1540
+    },
+    {
+      "epoch": 0.8268941294530858,
+      "grad_norm": 2.007965595348576,
+      "learning_rate": 9.089646803833588e-07,
+      "logits/chosen": 0.17131750285625458,
+      "logits/rejected": 0.4483724534511566,
+      "logps/chosen": -3.4954745769500732,
+      "logps/rejected": -4.936315536499023,
+      "loss": 0.6781,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -3.4954745769500732,
+      "rewards/margins": 1.440840482711792,
+      "rewards/rejected": -4.936315536499023,
+      "step": 1545
+    },
+    {
+      "epoch": 0.8295701622344873,
+      "grad_norm": 1.775163013335356,
+      "learning_rate": 9.080666951296276e-07,
+      "logits/chosen": 0.014221330173313618,
+      "logits/rejected": 0.4280180335044861,
+      "logps/chosen": -3.7445449829101562,
+      "logps/rejected": -5.700022220611572,
+      "loss": 0.6707,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -3.7445449829101562,
+      "rewards/margins": 1.9554773569107056,
+      "rewards/rejected": -5.700022220611572,
+      "step": 1550
+    },
+    {
+      "epoch": 0.8322461950158889,
+      "grad_norm": 1.8038357285043716,
+      "learning_rate": 9.071647508110305e-07,
+      "logits/chosen": 0.0979781374335289,
+      "logits/rejected": 0.3504747450351715,
+      "logps/chosen": -3.754175901412964,
+      "logps/rejected": -5.672229766845703,
+      "loss": 0.672,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -3.754175901412964,
+      "rewards/margins": 1.918054223060608,
+      "rewards/rejected": -5.672229766845703,
+      "step": 1555
+    },
+    {
+      "epoch": 0.8349222277972905,
+      "grad_norm": 2.3276507362783163,
+      "learning_rate": 9.062588561782354e-07,
+      "logits/chosen": 0.3685339391231537,
+      "logits/rejected": 0.4094419479370117,
+      "logps/chosen": -4.768077850341797,
+      "logps/rejected": -5.758409023284912,
+      "loss": 0.6801,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.768077850341797,
+      "rewards/margins": 0.9903308153152466,
+      "rewards/rejected": -5.758409023284912,
+      "step": 1560
+    },
+    {
+      "epoch": 0.8375982605786921,
+      "grad_norm": 4.413141523579861,
+      "learning_rate": 9.053490200202358e-07,
+      "logits/chosen": 0.29415151476860046,
+      "logits/rejected": 0.5640724897384644,
+      "logps/chosen": -4.070713043212891,
+      "logps/rejected": -5.525948524475098,
+      "loss": 0.6756,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.070713043212891,
+      "rewards/margins": 1.4552361965179443,
+      "rewards/rejected": -5.525948524475098,
+      "step": 1565
+    },
+    {
+      "epoch": 0.8402742933600936,
+      "grad_norm": 3.676014643078602,
+      "learning_rate": 9.044352511642661e-07,
+      "logits/chosen": 0.5452852845191956,
+      "logits/rejected": 0.6077378988265991,
+      "logps/chosen": -5.612000465393066,
+      "logps/rejected": -6.2150702476501465,
+      "loss": 0.6859,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -5.612000465393066,
+      "rewards/margins": 0.6030700206756592,
+      "rewards/rejected": -6.2150702476501465,
+      "step": 1570
+    },
+    {
+      "epoch": 0.8429503261414952,
+      "grad_norm": 4.648412718509774,
+      "learning_rate": 9.03517558475716e-07,
+      "logits/chosen": 0.3769141733646393,
+      "logits/rejected": 0.7209679484367371,
+      "logps/chosen": -4.792849540710449,
+      "logps/rejected": -6.5275678634643555,
+      "loss": 0.6895,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.792849540710449,
+      "rewards/margins": 1.7347180843353271,
+      "rewards/rejected": -6.5275678634643555,
+      "step": 1575
+    },
+    {
+      "epoch": 0.8456263589228968,
+      "grad_norm": 4.883292365325657,
+      "learning_rate": 9.025959508580436e-07,
+      "logits/chosen": 0.36398422718048096,
+      "logits/rejected": 0.6989147067070007,
+      "logps/chosen": -4.414804935455322,
+      "logps/rejected": -5.621647834777832,
+      "loss": 0.6755,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.414804935455322,
+      "rewards/margins": 1.2068426609039307,
+      "rewards/rejected": -5.621647834777832,
+      "step": 1580
+    },
+    {
+      "epoch": 0.8483023917042983,
+      "grad_norm": 2.2842327588657323,
+      "learning_rate": 9.016704372526905e-07,
+      "logits/chosen": 0.3770214915275574,
+      "logits/rejected": 0.6714043021202087,
+      "logps/chosen": -4.138848304748535,
+      "logps/rejected": -5.544915199279785,
+      "loss": 0.6763,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.138848304748535,
+      "rewards/margins": 1.406067132949829,
+      "rewards/rejected": -5.544915199279785,
+      "step": 1585
+    },
+    {
+      "epoch": 0.8509784244856999,
+      "grad_norm": 2.1346444638536943,
+      "learning_rate": 9.007410266389934e-07,
+      "logits/chosen": 0.43157273530960083,
+      "logits/rejected": 0.7710301280021667,
+      "logps/chosen": -4.541441440582275,
+      "logps/rejected": -6.05191707611084,
+      "loss": 0.6788,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.541441440582275,
+      "rewards/margins": 1.51047682762146,
+      "rewards/rejected": -6.05191707611084,
+      "step": 1590
+    },
+    {
+      "epoch": 0.8536544572671015,
+      "grad_norm": 1.684703872478619,
+      "learning_rate": 8.998077280340981e-07,
+      "logits/chosen": 0.5937745571136475,
+      "logits/rejected": 0.9113284349441528,
+      "logps/chosen": -5.015867710113525,
+      "logps/rejected": -6.773443698883057,
+      "loss": 0.6777,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -5.015867710113525,
+      "rewards/margins": 1.757576584815979,
+      "rewards/rejected": -6.773443698883057,
+      "step": 1595
+    },
+    {
+      "epoch": 0.8563304900485031,
+      "grad_norm": 2.1456268242450927,
+      "learning_rate": 8.988705504928722e-07,
+      "logits/chosen": 0.4995730519294739,
+      "logits/rejected": 1.0070823431015015,
+      "logps/chosen": -4.284285068511963,
+      "logps/rejected": -7.341456413269043,
+      "loss": 0.665,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.284285068511963,
+      "rewards/margins": 3.0571720600128174,
+      "rewards/rejected": -7.341456413269043,
+      "step": 1600
+    },
+    {
+      "epoch": 0.8563304900485031,
+      "eval_logits/chosen": 0.6451614499092102,
+      "eval_logits/rejected": 0.9087243676185608,
+      "eval_logps/chosen": -4.159726142883301,
+      "eval_logps/rejected": -6.301677227020264,
+      "eval_loss": 0.6736590266227722,
+      "eval_rewards/accuracies": 0.6587536931037903,
+      "eval_rewards/chosen": -4.159726142883301,
+      "eval_rewards/margins": 2.141951560974121,
+      "eval_rewards/rejected": -6.301677227020264,
+      "eval_runtime": 40.6806,
+      "eval_samples_per_second": 33.062,
+      "eval_steps_per_second": 8.284,
+      "step": 1600
+    },
+    {
+      "epoch": 0.8590065228299046,
+      "grad_norm": 2.1672042563847884,
+      "learning_rate": 8.979295031078157e-07,
+      "logits/chosen": 0.4299837648868561,
+      "logits/rejected": 0.8064993619918823,
+      "logps/chosen": -4.245932579040527,
+      "logps/rejected": -5.819076061248779,
+      "loss": 0.6672,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.245932579040527,
+      "rewards/margins": 1.5731432437896729,
+      "rewards/rejected": -5.819076061248779,
+      "step": 1605
+    },
+    {
+      "epoch": 0.8616825556113062,
+      "grad_norm": 3.164137777833587,
+      "learning_rate": 8.969845950089751e-07,
+      "logits/chosen": 0.30427563190460205,
+      "logits/rejected": 0.6237763166427612,
+      "logps/chosen": -3.97059965133667,
+      "logps/rejected": -5.864052772521973,
+      "loss": 0.6773,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -3.97059965133667,
+      "rewards/margins": 1.8934532403945923,
+      "rewards/rejected": -5.864052772521973,
+      "step": 1610
+    },
+    {
+      "epoch": 0.8643585883927078,
+      "grad_norm": 3.1769143891740086,
+      "learning_rate": 8.960358353638526e-07,
+      "logits/chosen": 0.5140082240104675,
+      "logits/rejected": 0.6344442963600159,
+      "logps/chosen": -4.733736991882324,
+      "logps/rejected": -5.612375259399414,
+      "loss": 0.6759,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.733736991882324,
+      "rewards/margins": 0.8786379098892212,
+      "rewards/rejected": -5.612375259399414,
+      "step": 1615
+    },
+    {
+      "epoch": 0.8670346211741093,
+      "grad_norm": 3.527507038515628,
+      "learning_rate": 8.950832333773184e-07,
+      "logits/chosen": 0.6247620582580566,
+      "logits/rejected": 0.6795405149459839,
+      "logps/chosen": -5.149880409240723,
+      "logps/rejected": -5.742555618286133,
+      "loss": 0.68,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -5.149880409240723,
+      "rewards/margins": 0.5926753282546997,
+      "rewards/rejected": -5.742555618286133,
+      "step": 1620
+    },
+    {
+      "epoch": 0.869710653955511,
+      "grad_norm": 1.6877210686095039,
+      "learning_rate": 8.941267982915213e-07,
+      "logits/chosen": 0.8207963705062866,
+      "logits/rejected": 0.9572089910507202,
+      "logps/chosen": -6.4401655197143555,
+      "logps/rejected": -6.761213779449463,
+      "loss": 0.6906,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -6.4401655197143555,
+      "rewards/margins": 0.3210480213165283,
+      "rewards/rejected": -6.761213779449463,
+      "step": 1625
+    },
+    {
+      "epoch": 0.8723866867369126,
+      "grad_norm": 1.4753544487970713,
+      "learning_rate": 8.931665393857983e-07,
+      "logits/chosen": 0.8165861368179321,
+      "logits/rejected": 1.2086104154586792,
+      "logps/chosen": -6.06455135345459,
+      "logps/rejected": -8.737874031066895,
+      "loss": 0.679,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -6.06455135345459,
+      "rewards/margins": 2.6733224391937256,
+      "rewards/rejected": -8.737874031066895,
+      "step": 1630
+    },
+    {
+      "epoch": 0.875062719518314,
+      "grad_norm": 1.9424375645056051,
+      "learning_rate": 8.922024659765861e-07,
+      "logits/chosen": 0.6980465054512024,
+      "logits/rejected": 0.9099535942077637,
+      "logps/chosen": -5.780291557312012,
+      "logps/rejected": -7.549350738525391,
+      "loss": 0.6819,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -5.780291557312012,
+      "rewards/margins": 1.7690595388412476,
+      "rewards/rejected": -7.549350738525391,
+      "step": 1635
+    },
+    {
+      "epoch": 0.8777387522997157,
+      "grad_norm": 2.6977717325123907,
+      "learning_rate": 8.912345874173288e-07,
+      "logits/chosen": 0.553291916847229,
+      "logits/rejected": 0.9475140571594238,
+      "logps/chosen": -5.061959266662598,
+      "logps/rejected": -7.661961555480957,
+      "loss": 0.6821,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -5.061959266662598,
+      "rewards/margins": 2.6000025272369385,
+      "rewards/rejected": -7.661961555480957,
+      "step": 1640
+    },
+    {
+      "epoch": 0.8804147850811173,
+      "grad_norm": 1.8701912910591647,
+      "learning_rate": 8.902629130983885e-07,
+      "logits/chosen": 0.6150132417678833,
+      "logits/rejected": 0.7088582515716553,
+      "logps/chosen": -5.749318599700928,
+      "logps/rejected": -6.950533390045166,
+      "loss": 0.6874,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -5.749318599700928,
+      "rewards/margins": 1.20121431350708,
+      "rewards/rejected": -6.950533390045166,
+      "step": 1645
+    },
+    {
+      "epoch": 0.8830908178625189,
+      "grad_norm": 2.536840503678579,
+      "learning_rate": 8.892874524469537e-07,
+      "logits/chosen": 0.4409259855747223,
+      "logits/rejected": 0.7562704682350159,
+      "logps/chosen": -4.153137683868408,
+      "logps/rejected": -6.283951759338379,
+      "loss": 0.6739,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.153137683868408,
+      "rewards/margins": 2.1308135986328125,
+      "rewards/rejected": -6.283951759338379,
+      "step": 1650
+    },
+    {
+      "epoch": 0.8857668506439204,
+      "grad_norm": 1.8952592843663274,
+      "learning_rate": 8.883082149269478e-07,
+      "logits/chosen": 0.2752839922904968,
+      "logits/rejected": 0.5582292675971985,
+      "logps/chosen": -4.031248092651367,
+      "logps/rejected": -5.7244672775268555,
+      "loss": 0.6727,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.031248092651367,
+      "rewards/margins": 1.6932191848754883,
+      "rewards/rejected": -5.7244672775268555,
+      "step": 1655
+    },
+    {
+      "epoch": 0.888442883425322,
+      "grad_norm": 2.5804948339359552,
+      "learning_rate": 8.873252100389377e-07,
+      "logits/chosen": 0.2102358341217041,
+      "logits/rejected": 0.3867553770542145,
+      "logps/chosen": -3.473963499069214,
+      "logps/rejected": -5.092469215393066,
+      "loss": 0.6749,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -3.473963499069214,
+      "rewards/margins": 1.6185052394866943,
+      "rewards/rejected": -5.092469215393066,
+      "step": 1660
+    },
+    {
+      "epoch": 0.8911189162067236,
+      "grad_norm": 2.6528808587274786,
+      "learning_rate": 8.863384473200411e-07,
+      "logits/chosen": 0.21295198798179626,
+      "logits/rejected": 0.3272942006587982,
+      "logps/chosen": -3.5999653339385986,
+      "logps/rejected": -4.715779781341553,
+      "loss": 0.6805,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -3.5999653339385986,
+      "rewards/margins": 1.1158145666122437,
+      "rewards/rejected": -4.715779781341553,
+      "step": 1665
+    },
+    {
+      "epoch": 0.8937949489881251,
+      "grad_norm": 1.185223505118452,
+      "learning_rate": 8.853479363438342e-07,
+      "logits/chosen": 0.4361394941806793,
+      "logits/rejected": 0.5936468839645386,
+      "logps/chosen": -4.365379810333252,
+      "logps/rejected": -5.042609691619873,
+      "loss": 0.6855,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -4.365379810333252,
+      "rewards/margins": 0.6772298812866211,
+      "rewards/rejected": -5.042609691619873,
+      "step": 1670
+    },
+    {
+      "epoch": 0.8964709817695267,
+      "grad_norm": 1.913649046697622,
+      "learning_rate": 8.843536867202588e-07,
+      "logits/chosen": 0.3654400408267975,
+      "logits/rejected": 0.7011191248893738,
+      "logps/chosen": -4.858678817749023,
+      "logps/rejected": -5.749300479888916,
+      "loss": 0.6784,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.858678817749023,
+      "rewards/margins": 0.8906217813491821,
+      "rewards/rejected": -5.749300479888916,
+      "step": 1675
+    },
+    {
+      "epoch": 0.8991470145509283,
+      "grad_norm": 2.83893215204242,
+      "learning_rate": 8.833557080955292e-07,
+      "logits/chosen": 0.5713294744491577,
+      "logits/rejected": 0.7341232299804688,
+      "logps/chosen": -5.41585636138916,
+      "logps/rejected": -6.79622745513916,
+      "loss": 0.679,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -5.41585636138916,
+      "rewards/margins": 1.380371332168579,
+      "rewards/rejected": -6.79622745513916,
+      "step": 1680
+    },
+    {
+      "epoch": 0.9018230473323299,
+      "grad_norm": 1.8733846242263823,
+      "learning_rate": 8.823540101520381e-07,
+      "logits/chosen": 0.6719552874565125,
+      "logits/rejected": 0.9449108242988586,
+      "logps/chosen": -5.541573524475098,
+      "logps/rejected": -7.155600547790527,
+      "loss": 0.6805,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -5.541573524475098,
+      "rewards/margins": 1.614027738571167,
+      "rewards/rejected": -7.155600547790527,
+      "step": 1685
+    },
+    {
+      "epoch": 0.9044990801137314,
+      "grad_norm": 2.814786118130039,
+      "learning_rate": 8.813486026082637e-07,
+      "logits/chosen": 0.6039652824401855,
+      "logits/rejected": 1.0825363397598267,
+      "logps/chosen": -5.284527778625488,
+      "logps/rejected": -8.020164489746094,
+      "loss": 0.6722,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -5.284527778625488,
+      "rewards/margins": 2.7356362342834473,
+      "rewards/rejected": -8.020164489746094,
+      "step": 1690
+    },
+    {
+      "epoch": 0.907175112895133,
+      "grad_norm": 3.9794999077858306,
+      "learning_rate": 8.803394952186742e-07,
+      "logits/chosen": 0.4436036944389343,
+      "logits/rejected": 0.8945782780647278,
+      "logps/chosen": -4.91702127456665,
+      "logps/rejected": -7.885725498199463,
+      "loss": 0.6796,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.91702127456665,
+      "rewards/margins": 2.9687042236328125,
+      "rewards/rejected": -7.885725498199463,
+      "step": 1695
+    },
+    {
+      "epoch": 0.9098511456765346,
+      "grad_norm": 3.5578749914493284,
+      "learning_rate": 8.793266977736342e-07,
+      "logits/chosen": 0.642309308052063,
+      "logits/rejected": 0.742534339427948,
+      "logps/chosen": -5.432114601135254,
+      "logps/rejected": -6.873453617095947,
+      "loss": 0.6837,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -5.432114601135254,
+      "rewards/margins": 1.4413385391235352,
+      "rewards/rejected": -6.873453617095947,
+      "step": 1700
+    },
+    {
+      "epoch": 0.9125271784579361,
+      "grad_norm": 4.340749438972021,
+      "learning_rate": 8.783102200993085e-07,
+      "logits/chosen": 0.4326944351196289,
+      "logits/rejected": 0.7939949035644531,
+      "logps/chosen": -4.601889610290527,
+      "logps/rejected": -7.065014839172363,
+      "loss": 0.6751,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.601889610290527,
+      "rewards/margins": 2.463125228881836,
+      "rewards/rejected": -7.065014839172363,
+      "step": 1705
+    },
+    {
+      "epoch": 0.9152032112393377,
+      "grad_norm": 3.4946572909734845,
+      "learning_rate": 8.772900720575683e-07,
+      "logits/chosen": 0.5794827342033386,
+      "logits/rejected": 0.857323944568634,
+      "logps/chosen": -5.830109596252441,
+      "logps/rejected": -7.824678897857666,
+      "loss": 0.6827,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -5.830109596252441,
+      "rewards/margins": 1.9945694208145142,
+      "rewards/rejected": -7.824678897857666,
+      "step": 1710
+    },
+    {
+      "epoch": 0.9178792440207393,
+      "grad_norm": 2.477870218020931,
+      "learning_rate": 8.762662635458944e-07,
+      "logits/chosen": 0.4651104509830475,
+      "logits/rejected": 0.7560599446296692,
+      "logps/chosen": -5.590277194976807,
+      "logps/rejected": -7.24977970123291,
+      "loss": 0.6809,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -5.590277194976807,
+      "rewards/margins": 1.659502625465393,
+      "rewards/rejected": -7.24977970123291,
+      "step": 1715
+    },
+    {
+      "epoch": 0.9205552768021408,
+      "grad_norm": 4.825634562596867,
+      "learning_rate": 8.752388044972811e-07,
+      "logits/chosen": 0.4105263650417328,
+      "logits/rejected": 0.6490617394447327,
+      "logps/chosen": -4.525534629821777,
+      "logps/rejected": -6.6069183349609375,
+      "loss": 0.6699,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.525534629821777,
+      "rewards/margins": 2.0813841819763184,
+      "rewards/rejected": -6.6069183349609375,
+      "step": 1720
+    },
+    {
+      "epoch": 0.9232313095835424,
+      "grad_norm": 2.3874583538343868,
+      "learning_rate": 8.74207704880141e-07,
+      "logits/chosen": 0.3307218551635742,
+      "logits/rejected": 0.7823120355606079,
+      "logps/chosen": -4.1847662925720215,
+      "logps/rejected": -6.996400356292725,
+      "loss": 0.6682,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.1847662925720215,
+      "rewards/margins": 2.8116350173950195,
+      "rewards/rejected": -6.996400356292725,
+      "step": 1725
+    },
+    {
+      "epoch": 0.925907342364944,
+      "grad_norm": 2.4328265367125343,
+      "learning_rate": 8.731729746982068e-07,
+      "logits/chosen": 0.3716750741004944,
+      "logits/rejected": 0.5443974137306213,
+      "logps/chosen": -4.2114033699035645,
+      "logps/rejected": -6.034554481506348,
+      "loss": 0.6757,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.2114033699035645,
+      "rewards/margins": 1.8231515884399414,
+      "rewards/rejected": -6.034554481506348,
+      "step": 1730
+    },
+    {
+      "epoch": 0.9285833751463456,
+      "grad_norm": 2.1870342896224657,
+      "learning_rate": 8.721346239904355e-07,
+      "logits/chosen": 0.26164019107818604,
+      "logits/rejected": 0.4228759706020355,
+      "logps/chosen": -4.127760887145996,
+      "logps/rejected": -5.478959560394287,
+      "loss": 0.6801,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.127760887145996,
+      "rewards/margins": 1.3511989116668701,
+      "rewards/rejected": -5.478959560394287,
+      "step": 1735
+    },
+    {
+      "epoch": 0.9312594079277471,
+      "grad_norm": 1.5994411912352227,
+      "learning_rate": 8.710926628309101e-07,
+      "logits/chosen": 0.38809365034103394,
+      "logits/rejected": 0.5797819495201111,
+      "logps/chosen": -4.224000453948975,
+      "logps/rejected": -5.704766750335693,
+      "loss": 0.6739,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.224000453948975,
+      "rewards/margins": 1.4807662963867188,
+      "rewards/rejected": -5.704766750335693,
+      "step": 1740
+    },
+    {
+      "epoch": 0.9339354407091487,
+      "grad_norm": 2.490024584832531,
+      "learning_rate": 8.700471013287424e-07,
+      "logits/chosen": 0.34820038080215454,
+      "logits/rejected": 0.4968600869178772,
+      "logps/chosen": -3.84651517868042,
+      "logps/rejected": -5.4693708419799805,
+      "loss": 0.6776,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -3.84651517868042,
+      "rewards/margins": 1.62285578250885,
+      "rewards/rejected": -5.4693708419799805,
+      "step": 1745
+    },
+    {
+      "epoch": 0.9366114734905503,
+      "grad_norm": 2.4903414131077835,
+      "learning_rate": 8.689979496279746e-07,
+      "logits/chosen": 0.5010639429092407,
+      "logits/rejected": 0.6454726457595825,
+      "logps/chosen": -5.1135783195495605,
+      "logps/rejected": -5.85701322555542,
+      "loss": 0.6855,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -5.1135783195495605,
+      "rewards/margins": 0.7434352040290833,
+      "rewards/rejected": -5.85701322555542,
+      "step": 1750
+    },
+    {
+      "epoch": 0.9392875062719518,
+      "grad_norm": 3.654900440440794,
+      "learning_rate": 8.679452179074811e-07,
+      "logits/chosen": 0.6590859889984131,
+      "logits/rejected": 1.0399192571640015,
+      "logps/chosen": -5.415560245513916,
+      "logps/rejected": -7.6439924240112305,
+      "loss": 0.6745,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -5.415560245513916,
+      "rewards/margins": 2.22843337059021,
+      "rewards/rejected": -7.6439924240112305,
+      "step": 1755
+    },
+    {
+      "epoch": 0.9419635390533534,
+      "grad_norm": 2.586049090460199,
+      "learning_rate": 8.668889163808698e-07,
+      "logits/chosen": 0.631452739238739,
+      "logits/rejected": 0.8882015943527222,
+      "logps/chosen": -5.132933616638184,
+      "logps/rejected": -7.239762783050537,
+      "loss": 0.6788,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -5.132933616638184,
+      "rewards/margins": 2.1068291664123535,
+      "rewards/rejected": -7.239762783050537,
+      "step": 1760
+    },
+    {
+      "epoch": 0.944639571834755,
+      "grad_norm": 3.1367403104134097,
+      "learning_rate": 8.658290552963827e-07,
+      "logits/chosen": 0.6119955778121948,
+      "logits/rejected": 0.6981198787689209,
+      "logps/chosen": -4.9246368408203125,
+      "logps/rejected": -6.53448486328125,
+      "loss": 0.683,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.9246368408203125,
+      "rewards/margins": 1.6098486185073853,
+      "rewards/rejected": -6.53448486328125,
+      "step": 1765
+    },
+    {
+      "epoch": 0.9473156046161565,
+      "grad_norm": 3.6583270542210387,
+      "learning_rate": 8.647656449367966e-07,
+      "logits/chosen": 0.6457494497299194,
+      "logits/rejected": 0.8314827680587769,
+      "logps/chosen": -4.517415523529053,
+      "logps/rejected": -5.976507186889648,
+      "loss": 0.6723,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -4.517415523529053,
+      "rewards/margins": 1.4590915441513062,
+      "rewards/rejected": -5.976507186889648,
+      "step": 1770
+    },
+    {
+      "epoch": 0.9499916373975581,
+      "grad_norm": 5.555111813277425,
+      "learning_rate": 8.636986956193235e-07,
+      "logits/chosen": 0.542106032371521,
+      "logits/rejected": 0.8763257265090942,
+      "logps/chosen": -4.959171772003174,
+      "logps/rejected": -7.132624626159668,
+      "loss": 0.673,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.959171772003174,
+      "rewards/margins": 2.1734538078308105,
+      "rewards/rejected": -7.132624626159668,
+      "step": 1775
+    },
+    {
+      "epoch": 0.9526676701789597,
+      "grad_norm": 4.089177300349932,
+      "learning_rate": 8.626282176955104e-07,
+      "logits/chosen": 0.8160732388496399,
+      "logits/rejected": 1.0516173839569092,
+      "logps/chosen": -5.714818000793457,
+      "logps/rejected": -7.349417209625244,
+      "loss": 0.6682,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -5.714818000793457,
+      "rewards/margins": 1.6345994472503662,
+      "rewards/rejected": -7.349417209625244,
+      "step": 1780
+    },
+    {
+      "epoch": 0.9553437029603613,
+      "grad_norm": 4.504756533629742,
+      "learning_rate": 8.615542215511389e-07,
+      "logits/chosen": 0.760348379611969,
+      "logits/rejected": 1.0033414363861084,
+      "logps/chosen": -5.582521915435791,
+      "logps/rejected": -7.369784355163574,
+      "loss": 0.6876,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -5.582521915435791,
+      "rewards/margins": 1.7872631549835205,
+      "rewards/rejected": -7.369784355163574,
+      "step": 1785
+    },
+    {
+      "epoch": 0.9580197357417628,
+      "grad_norm": 3.438391573536831,
+      "learning_rate": 8.604767176061241e-07,
+      "logits/chosen": 0.6965848803520203,
+      "logits/rejected": 0.9568136930465698,
+      "logps/chosen": -5.3412184715271,
+      "logps/rejected": -7.442730903625488,
+      "loss": 0.6834,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -5.3412184715271,
+      "rewards/margins": 2.101513147354126,
+      "rewards/rejected": -7.442730903625488,
+      "step": 1790
+    },
+    {
+      "epoch": 0.9606957685231644,
+      "grad_norm": 3.852899566163153,
+      "learning_rate": 8.593957163144141e-07,
+      "logits/chosen": 0.5057392716407776,
+      "logits/rejected": 0.913040280342102,
+      "logps/chosen": -4.750461101531982,
+      "logps/rejected": -7.840095520019531,
+      "loss": 0.6708,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.750461101531982,
+      "rewards/margins": 3.089634418487549,
+      "rewards/rejected": -7.840095520019531,
+      "step": 1795
+    },
+    {
+      "epoch": 0.963371801304566,
+      "grad_norm": 2.990275641021405,
+      "learning_rate": 8.58311228163888e-07,
+      "logits/chosen": 0.3929196894168854,
+      "logits/rejected": 0.729416012763977,
+      "logps/chosen": -4.192706108093262,
+      "logps/rejected": -6.446908473968506,
+      "loss": 0.6715,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.192706108093262,
+      "rewards/margins": 2.2542026042938232,
+      "rewards/rejected": -6.446908473968506,
+      "step": 1800
+    },
+    {
+      "epoch": 0.9660478340859675,
+      "grad_norm": 3.9437406496651914,
+      "learning_rate": 8.57223263676255e-07,
+      "logits/chosen": 0.13186553120613098,
+      "logits/rejected": 0.29414862394332886,
+      "logps/chosen": -3.0937082767486572,
+      "logps/rejected": -4.410549163818359,
+      "loss": 0.674,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -3.0937082767486572,
+      "rewards/margins": 1.3168408870697021,
+      "rewards/rejected": -4.410549163818359,
+      "step": 1805
+    },
+    {
+      "epoch": 0.9687238668673691,
+      "grad_norm": 3.306332137205538,
+      "learning_rate": 8.561318334069511e-07,
+      "logits/chosen": 0.09502754360437393,
+      "logits/rejected": 0.34869879484176636,
+      "logps/chosen": -2.726714611053467,
+      "logps/rejected": -4.068415641784668,
+      "loss": 0.674,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -2.726714611053467,
+      "rewards/margins": 1.3417021036148071,
+      "rewards/rejected": -4.068415641784668,
+      "step": 1810
+    },
+    {
+      "epoch": 0.9713998996487707,
+      "grad_norm": 2.85911396166109,
+      "learning_rate": 8.550369479450375e-07,
+      "logits/chosen": 0.1982755810022354,
+      "logits/rejected": 0.401765912771225,
+      "logps/chosen": -3.1529507637023926,
+      "logps/rejected": -4.6295623779296875,
+      "loss": 0.6716,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -3.1529507637023926,
+      "rewards/margins": 1.476611852645874,
+      "rewards/rejected": -4.6295623779296875,
+      "step": 1815
+    },
+    {
+      "epoch": 0.9740759324301723,
+      "grad_norm": 1.8560820317737405,
+      "learning_rate": 8.539386179130977e-07,
+      "logits/chosen": 0.19269558787345886,
+      "logits/rejected": 0.4720892012119293,
+      "logps/chosen": -2.9689865112304688,
+      "logps/rejected": -4.92160701751709,
+      "loss": 0.6729,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -2.9689865112304688,
+      "rewards/margins": 1.952620267868042,
+      "rewards/rejected": -4.92160701751709,
+      "step": 1820
+    },
+    {
+      "epoch": 0.9767519652115738,
+      "grad_norm": 3.307025594033135,
+      "learning_rate": 8.528368539671347e-07,
+      "logits/chosen": 0.2530207633972168,
+      "logits/rejected": 0.5844982266426086,
+      "logps/chosen": -3.6567111015319824,
+      "logps/rejected": -5.602196216583252,
+      "loss": 0.6678,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -3.6567111015319824,
+      "rewards/margins": 1.9454851150512695,
+      "rewards/rejected": -5.602196216583252,
+      "step": 1825
+    },
+    {
+      "epoch": 0.9794279979929754,
+      "grad_norm": 3.4550963961147017,
+      "learning_rate": 8.51731666796467e-07,
+      "logits/chosen": 0.360609233379364,
+      "logits/rejected": 0.5353338718414307,
+      "logps/chosen": -3.5833802223205566,
+      "logps/rejected": -4.733412742614746,
+      "loss": 0.687,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -3.5833802223205566,
+      "rewards/margins": 1.1500332355499268,
+      "rewards/rejected": -4.733412742614746,
+      "step": 1830
+    },
+    {
+      "epoch": 0.982104030774377,
+      "grad_norm": 2.9073164116504477,
+      "learning_rate": 8.506230671236254e-07,
+      "logits/chosen": 0.42179107666015625,
+      "logits/rejected": 0.6643646955490112,
+      "logps/chosen": -4.180876731872559,
+      "logps/rejected": -5.765685558319092,
+      "loss": 0.6789,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.180876731872559,
+      "rewards/margins": 1.584808588027954,
+      "rewards/rejected": -5.765685558319092,
+      "step": 1835
+    },
+    {
+      "epoch": 0.9847800635557785,
+      "grad_norm": 1.3901382845437062,
+      "learning_rate": 8.495110657042488e-07,
+      "logits/chosen": 0.6023179292678833,
+      "logits/rejected": 0.8929504156112671,
+      "logps/chosen": -5.053563117980957,
+      "logps/rejected": -6.87359094619751,
+      "loss": 0.6818,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -5.053563117980957,
+      "rewards/margins": 1.8200279474258423,
+      "rewards/rejected": -6.87359094619751,
+      "step": 1840
+    },
+    {
+      "epoch": 0.9874560963371801,
+      "grad_norm": 3.3814811059576653,
+      "learning_rate": 8.483956733269799e-07,
+      "logits/chosen": 0.5107431411743164,
+      "logits/rejected": 0.9178862571716309,
+      "logps/chosen": -4.5175065994262695,
+      "logps/rejected": -7.444479465484619,
+      "loss": 0.6783,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.5175065994262695,
+      "rewards/margins": 2.9269728660583496,
+      "rewards/rejected": -7.444479465484619,
+      "step": 1845
+    },
+    {
+      "epoch": 0.9901321291185817,
+      "grad_norm": 2.2107855250999604,
+      "learning_rate": 8.472769008133602e-07,
+      "logits/chosen": 0.4330655634403229,
+      "logits/rejected": 1.0128575563430786,
+      "logps/chosen": -4.818704605102539,
+      "logps/rejected": -7.983605861663818,
+      "loss": 0.6789,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.818704605102539,
+      "rewards/margins": 3.1649012565612793,
+      "rewards/rejected": -7.983605861663818,
+      "step": 1850
+    },
+    {
+      "epoch": 0.9928081618999832,
+      "grad_norm": 2.5391660448654463,
+      "learning_rate": 8.461547590177259e-07,
+      "logits/chosen": 0.4547053277492523,
+      "logits/rejected": 0.8804700970649719,
+      "logps/chosen": -4.6510009765625,
+      "logps/rejected": -7.597373962402344,
+      "loss": 0.669,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.6510009765625,
+      "rewards/margins": 2.9463725090026855,
+      "rewards/rejected": -7.597373962402344,
+      "step": 1855
+    },
+    {
+      "epoch": 0.9954841946813848,
+      "grad_norm": 2.2640206969356256,
+      "learning_rate": 8.450292588271014e-07,
+      "logits/chosen": 0.48401227593421936,
+      "logits/rejected": 0.7992264032363892,
+      "logps/chosen": -4.951890468597412,
+      "logps/rejected": -7.060677528381348,
+      "loss": 0.6701,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.951890468597412,
+      "rewards/margins": 2.1087868213653564,
+      "rewards/rejected": -7.060677528381348,
+      "step": 1860
+    },
+    {
+      "epoch": 0.9981602274627864,
+      "grad_norm": 3.198758584638352,
+      "learning_rate": 8.439004111610945e-07,
+      "logits/chosen": 0.33749428391456604,
+      "logits/rejected": 0.6293452978134155,
+      "logps/chosen": -3.8577282428741455,
+      "logps/rejected": -6.149928569793701,
+      "loss": 0.6597,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -3.8577282428741455,
+      "rewards/margins": 2.2922005653381348,
+      "rewards/rejected": -6.149928569793701,
+      "step": 1865
+    },
+    {
+      "epoch": 1.000836260244188,
+      "grad_norm": 2.120306588431797,
+      "learning_rate": 8.427682269717901e-07,
+      "logits/chosen": 0.21416667103767395,
+      "logits/rejected": 0.5628907084465027,
+      "logps/chosen": -3.9595913887023926,
+      "logps/rejected": -6.089322090148926,
+      "loss": 0.6838,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -3.9595913887023926,
+      "rewards/margins": 2.129730701446533,
+      "rewards/rejected": -6.089322090148926,
+      "step": 1870
+    },
+    {
+      "epoch": 1.0035122930255895,
+      "grad_norm": 1.9279164116646375,
+      "learning_rate": 8.416327172436446e-07,
+      "logits/chosen": 0.11998569965362549,
+      "logits/rejected": 0.33841758966445923,
+      "logps/chosen": -3.7738845348358154,
+      "logps/rejected": -4.8447065353393555,
+      "loss": 0.6826,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -3.7738845348358154,
+      "rewards/margins": 1.0708215236663818,
+      "rewards/rejected": -4.8447065353393555,
+      "step": 1875
+    },
+    {
+      "epoch": 1.0061883258069912,
+      "grad_norm": 4.657165753874262,
+      "learning_rate": 8.404938929933778e-07,
+      "logits/chosen": 0.20362138748168945,
+      "logits/rejected": 0.4377966523170471,
+      "logps/chosen": -3.7101776599884033,
+      "logps/rejected": -5.3315935134887695,
+      "loss": 0.6782,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -3.7101776599884033,
+      "rewards/margins": 1.6214158535003662,
+      "rewards/rejected": -5.3315935134887695,
+      "step": 1880
+    },
+    {
+      "epoch": 1.0088643585883927,
+      "grad_norm": 1.7441161713114814,
+      "learning_rate": 8.39351765269868e-07,
+      "logits/chosen": 0.07928471267223358,
+      "logits/rejected": 0.20676879584789276,
+      "logps/chosen": -3.372706651687622,
+      "logps/rejected": -4.516925811767578,
+      "loss": 0.6678,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -3.372706651687622,
+      "rewards/margins": 1.1442193984985352,
+      "rewards/rejected": -4.516925811767578,
+      "step": 1885
+    },
+    {
+      "epoch": 1.0115403913697942,
+      "grad_norm": 2.3040252795027025,
+      "learning_rate": 8.382063451540431e-07,
+      "logits/chosen": 0.25992661714553833,
+      "logits/rejected": 0.5881274342536926,
+      "logps/chosen": -4.005249500274658,
+      "logps/rejected": -5.77401065826416,
+      "loss": 0.6801,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.005249500274658,
+      "rewards/margins": 1.768761396408081,
+      "rewards/rejected": -5.77401065826416,
+      "step": 1890
+    },
+    {
+      "epoch": 1.014216424151196,
+      "grad_norm": 3.5174207571814438,
+      "learning_rate": 8.370576437587742e-07,
+      "logits/chosen": 0.2310439646244049,
+      "logits/rejected": 0.40340757369995117,
+      "logps/chosen": -4.28546142578125,
+      "logps/rejected": -6.008872985839844,
+      "loss": 0.6787,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.28546142578125,
+      "rewards/margins": 1.7234115600585938,
+      "rewards/rejected": -6.008872985839844,
+      "step": 1895
+    },
+    {
+      "epoch": 1.0168924569325974,
+      "grad_norm": 4.645380459470395,
+      "learning_rate": 8.359056722287674e-07,
+      "logits/chosen": 0.2886231243610382,
+      "logits/rejected": 0.5925121307373047,
+      "logps/chosen": -4.684205055236816,
+      "logps/rejected": -6.716443061828613,
+      "loss": 0.6669,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.684205055236816,
+      "rewards/margins": 2.0322372913360596,
+      "rewards/rejected": -6.716443061828613,
+      "step": 1900
+    },
+    {
+      "epoch": 1.019568489713999,
+      "grad_norm": 2.785697352342903,
+      "learning_rate": 8.347504417404553e-07,
+      "logits/chosen": 0.34027448296546936,
+      "logits/rejected": 0.6563127040863037,
+      "logps/chosen": -4.894976615905762,
+      "logps/rejected": -6.546079158782959,
+      "loss": 0.6789,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.894976615905762,
+      "rewards/margins": 1.6511027812957764,
+      "rewards/rejected": -6.546079158782959,
+      "step": 1905
+    },
+    {
+      "epoch": 1.0222445224954007,
+      "grad_norm": 3.0249495779364852,
+      "learning_rate": 8.335919635018893e-07,
+      "logits/chosen": 0.2734437584877014,
+      "logits/rejected": 0.45771917700767517,
+      "logps/chosen": -4.894295692443848,
+      "logps/rejected": -6.424278259277344,
+      "loss": 0.6774,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -4.894295692443848,
+      "rewards/margins": 1.5299819707870483,
+      "rewards/rejected": -6.424278259277344,
+      "step": 1910
+    },
+    {
+      "epoch": 1.0249205552768021,
+      "grad_norm": 3.75939304487704,
+      "learning_rate": 8.324302487526303e-07,
+      "logits/chosen": 0.36358386278152466,
+      "logits/rejected": 0.6832642555236816,
+      "logps/chosen": -5.106237888336182,
+      "logps/rejected": -6.83355712890625,
+      "loss": 0.6628,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -5.106237888336182,
+      "rewards/margins": 1.727318525314331,
+      "rewards/rejected": -6.83355712890625,
+      "step": 1915
+    },
+    {
+      "epoch": 1.0275965880582036,
+      "grad_norm": 2.9591365020888323,
+      "learning_rate": 8.312653087636398e-07,
+      "logits/chosen": 0.4620983600616455,
+      "logits/rejected": 0.6924127340316772,
+      "logps/chosen": -4.5988054275512695,
+      "logps/rejected": -6.7045578956604,
+      "loss": 0.6606,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.5988054275512695,
+      "rewards/margins": 2.10575270652771,
+      "rewards/rejected": -6.7045578956604,
+      "step": 1920
+    },
+    {
+      "epoch": 1.0302726208396054,
+      "grad_norm": 3.346830277740256,
+      "learning_rate": 8.300971548371711e-07,
+      "logits/chosen": 0.2602004408836365,
+      "logits/rejected": 0.560912013053894,
+      "logps/chosen": -4.502633571624756,
+      "logps/rejected": -6.552506923675537,
+      "loss": 0.6668,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.502633571624756,
+      "rewards/margins": 2.0498735904693604,
+      "rewards/rejected": -6.552506923675537,
+      "step": 1925
+    },
+    {
+      "epoch": 1.0329486536210069,
+      "grad_norm": 3.7881168674047614,
+      "learning_rate": 8.289257983066582e-07,
+      "logits/chosen": 0.3042537569999695,
+      "logits/rejected": 0.6516977548599243,
+      "logps/chosen": -4.367084503173828,
+      "logps/rejected": -6.415268898010254,
+      "loss": 0.6626,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.367084503173828,
+      "rewards/margins": 2.0481839179992676,
+      "rewards/rejected": -6.415268898010254,
+      "step": 1930
+    },
+    {
+      "epoch": 1.0356246864024083,
+      "grad_norm": 6.24155415282323,
+      "learning_rate": 8.277512505366077e-07,
+      "logits/chosen": 0.3136747479438782,
+      "logits/rejected": 0.5976271033287048,
+      "logps/chosen": -4.181825160980225,
+      "logps/rejected": -5.768950462341309,
+      "loss": 0.686,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.181825160980225,
+      "rewards/margins": 1.5871254205703735,
+      "rewards/rejected": -5.768950462341309,
+      "step": 1935
+    },
+    {
+      "epoch": 1.03830071918381,
+      "grad_norm": 2.3349055018503355,
+      "learning_rate": 8.265735229224868e-07,
+      "logits/chosen": 0.32152053713798523,
+      "logits/rejected": 0.7044113874435425,
+      "logps/chosen": -4.205860614776611,
+      "logps/rejected": -7.000582695007324,
+      "loss": 0.6661,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.205860614776611,
+      "rewards/margins": 2.79472279548645,
+      "rewards/rejected": -7.000582695007324,
+      "step": 1940
+    },
+    {
+      "epoch": 1.0409767519652116,
+      "grad_norm": 3.6574764236190536,
+      "learning_rate": 8.253926268906144e-07,
+      "logits/chosen": 0.4326678216457367,
+      "logits/rejected": 0.7422645092010498,
+      "logps/chosen": -5.186060905456543,
+      "logps/rejected": -7.2144012451171875,
+      "loss": 0.6701,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -5.186060905456543,
+      "rewards/margins": 2.0283405780792236,
+      "rewards/rejected": -7.2144012451171875,
+      "step": 1945
+    },
+    {
+      "epoch": 1.043652784746613,
+      "grad_norm": 1.8119206308983251,
+      "learning_rate": 8.242085738980487e-07,
+      "logits/chosen": 0.5380016565322876,
+      "logits/rejected": 0.8691626787185669,
+      "logps/chosen": -4.85878849029541,
+      "logps/rejected": -7.186851501464844,
+      "loss": 0.6745,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.85878849029541,
+      "rewards/margins": 2.3280622959136963,
+      "rewards/rejected": -7.186851501464844,
+      "step": 1950
+    },
+    {
+      "epoch": 1.0463288175280148,
+      "grad_norm": 2.813373523067881,
+      "learning_rate": 8.230213754324772e-07,
+      "logits/chosen": 0.43225279450416565,
+      "logits/rejected": 0.6026257872581482,
+      "logps/chosen": -4.8994951248168945,
+      "logps/rejected": -6.683709621429443,
+      "loss": 0.6653,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.8994951248168945,
+      "rewards/margins": 1.784214735031128,
+      "rewards/rejected": -6.683709621429443,
+      "step": 1955
+    },
+    {
+      "epoch": 1.0490048503094163,
+      "grad_norm": 2.5448737993058876,
+      "learning_rate": 8.218310430121045e-07,
+      "logits/chosen": 0.3250805139541626,
+      "logits/rejected": 0.4980701804161072,
+      "logps/chosen": -4.2781596183776855,
+      "logps/rejected": -5.934880256652832,
+      "loss": 0.6799,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -4.2781596183776855,
+      "rewards/margins": 1.656720757484436,
+      "rewards/rejected": -5.934880256652832,
+      "step": 1960
+    },
+    {
+      "epoch": 1.051680883090818,
+      "grad_norm": 3.8486626834127904,
+      "learning_rate": 8.20637588185541e-07,
+      "logits/chosen": 0.4386932849884033,
+      "logits/rejected": 0.8594549298286438,
+      "logps/chosen": -4.541189670562744,
+      "logps/rejected": -8.014142036437988,
+      "loss": 0.6574,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.541189670562744,
+      "rewards/margins": 3.472951889038086,
+      "rewards/rejected": -8.014142036437988,
+      "step": 1965
+    },
+    {
+      "epoch": 1.0543569158722195,
+      "grad_norm": 3.5490808202002704,
+      "learning_rate": 8.194410225316906e-07,
+      "logits/chosen": 0.43444743752479553,
+      "logits/rejected": 0.6980060935020447,
+      "logps/chosen": -4.293482303619385,
+      "logps/rejected": -6.182492733001709,
+      "loss": 0.6702,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.293482303619385,
+      "rewards/margins": 1.8890106678009033,
+      "rewards/rejected": -6.182492733001709,
+      "step": 1970
+    },
+    {
+      "epoch": 1.057032948653621,
+      "grad_norm": 2.7144739831642486,
+      "learning_rate": 8.182413576596385e-07,
+      "logits/chosen": 0.6322952508926392,
+      "logits/rejected": 0.9108446836471558,
+      "logps/chosen": -4.948765754699707,
+      "logps/rejected": -7.312441349029541,
+      "loss": 0.6706,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.948765754699707,
+      "rewards/margins": 2.363675594329834,
+      "rewards/rejected": -7.312441349029541,
+      "step": 1975
+    },
+    {
+      "epoch": 1.0597089814350227,
+      "grad_norm": 3.631353017859342,
+      "learning_rate": 8.170386052085389e-07,
+      "logits/chosen": 0.6367691159248352,
+      "logits/rejected": 0.8656429052352905,
+      "logps/chosen": -5.552123069763184,
+      "logps/rejected": -7.269471645355225,
+      "loss": 0.6824,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -5.552123069763184,
+      "rewards/margins": 1.7173490524291992,
+      "rewards/rejected": -7.269471645355225,
+      "step": 1980
+    },
+    {
+      "epoch": 1.0623850142164242,
+      "grad_norm": 3.176873121347835,
+      "learning_rate": 8.158327768475008e-07,
+      "logits/chosen": 0.6303120851516724,
+      "logits/rejected": 0.852099597454071,
+      "logps/chosen": -5.4813642501831055,
+      "logps/rejected": -7.3996686935424805,
+      "loss": 0.6767,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -5.4813642501831055,
+      "rewards/margins": 1.918304443359375,
+      "rewards/rejected": -7.3996686935424805,
+      "step": 1985
+    },
+    {
+      "epoch": 1.0650610469978257,
+      "grad_norm": 4.856709214669437,
+      "learning_rate": 8.146238842754767e-07,
+      "logits/chosen": 0.6783785820007324,
+      "logits/rejected": 0.8435104489326477,
+      "logps/chosen": -6.136765480041504,
+      "logps/rejected": -6.695761680603027,
+      "loss": 0.6812,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -6.136765480041504,
+      "rewards/margins": 0.5589962005615234,
+      "rewards/rejected": -6.695761680603027,
+      "step": 1990
+    },
+    {
+      "epoch": 1.0677370797792274,
+      "grad_norm": 2.038751241284821,
+      "learning_rate": 8.134119392211476e-07,
+      "logits/chosen": 0.8075531721115112,
+      "logits/rejected": 1.1609325408935547,
+      "logps/chosen": -6.011834144592285,
+      "logps/rejected": -8.752424240112305,
+      "loss": 0.6718,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -6.011834144592285,
+      "rewards/margins": 2.7405900955200195,
+      "rewards/rejected": -8.752424240112305,
+      "step": 1995
+    },
+    {
+      "epoch": 1.0704131125606289,
+      "grad_norm": 3.006944822188376,
+      "learning_rate": 8.121969534428094e-07,
+      "logits/chosen": 0.5149596333503723,
+      "logits/rejected": 0.9659347534179688,
+      "logps/chosen": -5.3081536293029785,
+      "logps/rejected": -8.017338752746582,
+      "loss": 0.674,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -5.3081536293029785,
+      "rewards/margins": 2.7091853618621826,
+      "rewards/rejected": -8.017338752746582,
+      "step": 2000
+    },
+    {
+      "epoch": 1.0704131125606289,
+      "eval_logits/chosen": 0.707153856754303,
+      "eval_logits/rejected": 1.0243008136749268,
+      "eval_logps/chosen": -4.709293842315674,
+      "eval_logps/rejected": -7.459430694580078,
+      "eval_loss": 0.6701847910881042,
+      "eval_rewards/accuracies": 0.6676557660102844,
+      "eval_rewards/chosen": -4.709293842315674,
+      "eval_rewards/margins": 2.750135898590088,
+      "eval_rewards/rejected": -7.459430694580078,
+      "eval_runtime": 40.7107,
+      "eval_samples_per_second": 33.038,
+      "eval_steps_per_second": 8.278,
+      "step": 2000
+    },
+    {
+      "epoch": 1.0730891453420304,
+      "grad_norm": 3.1187991894101468,
+      "learning_rate": 8.109789387282599e-07,
+      "logits/chosen": 0.5026798248291016,
+      "logits/rejected": 0.9169502258300781,
+      "logps/chosen": -4.838769435882568,
+      "logps/rejected": -7.446521759033203,
+      "loss": 0.6715,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.838769435882568,
+      "rewards/margins": 2.6077518463134766,
+      "rewards/rejected": -7.446521759033203,
+      "step": 2005
+    },
+    {
+      "epoch": 1.075765178123432,
+      "grad_norm": 2.263968951282661,
+      "learning_rate": 8.097579068946827e-07,
+      "logits/chosen": 0.3796899914741516,
+      "logits/rejected": 0.6517937779426575,
+      "logps/chosen": -3.8600246906280518,
+      "logps/rejected": -5.725969314575195,
+      "loss": 0.6614,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -3.8600246906280518,
+      "rewards/margins": 1.8659451007843018,
+      "rewards/rejected": -5.725969314575195,
+      "step": 2010
+    },
+    {
+      "epoch": 1.0784412109048336,
+      "grad_norm": 2.050273013455402,
+      "learning_rate": 8.085338697885344e-07,
+      "logits/chosen": 0.3802728056907654,
+      "logits/rejected": 0.6778868436813354,
+      "logps/chosen": -4.133148670196533,
+      "logps/rejected": -6.376175403594971,
+      "loss": 0.6721,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.133148670196533,
+      "rewards/margins": 2.2430269718170166,
+      "rewards/rejected": -6.376175403594971,
+      "step": 2015
+    },
+    {
+      "epoch": 1.081117243686235,
+      "grad_norm": 2.2246808347505875,
+      "learning_rate": 8.073068392854282e-07,
+      "logits/chosen": 0.3356051445007324,
+      "logits/rejected": 0.7089319825172424,
+      "logps/chosen": -4.4659318923950195,
+      "logps/rejected": -6.992592811584473,
+      "loss": 0.6695,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.4659318923950195,
+      "rewards/margins": 2.5266611576080322,
+      "rewards/rejected": -6.992592811584473,
+      "step": 2020
+    },
+    {
+      "epoch": 1.0837932764676368,
+      "grad_norm": 4.369549204693455,
+      "learning_rate": 8.060768272900193e-07,
+      "logits/chosen": 0.4004606306552887,
+      "logits/rejected": 0.6923831701278687,
+      "logps/chosen": -4.223063945770264,
+      "logps/rejected": -5.992419242858887,
+      "loss": 0.6719,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.223063945770264,
+      "rewards/margins": 1.7693555355072021,
+      "rewards/rejected": -5.992419242858887,
+      "step": 2025
+    },
+    {
+      "epoch": 1.0864693092490383,
+      "grad_norm": 4.086634083225792,
+      "learning_rate": 8.0484384573589e-07,
+      "logits/chosen": 0.44715365767478943,
+      "logits/rejected": 0.6176769137382507,
+      "logps/chosen": -4.541064262390137,
+      "logps/rejected": -6.3306989669799805,
+      "loss": 0.6739,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.541064262390137,
+      "rewards/margins": 1.789634108543396,
+      "rewards/rejected": -6.3306989669799805,
+      "step": 2030
+    },
+    {
+      "epoch": 1.0891453420304398,
+      "grad_norm": 3.074944026776072,
+      "learning_rate": 8.03607906585432e-07,
+      "logits/chosen": 0.6410759687423706,
+      "logits/rejected": 0.996433436870575,
+      "logps/chosen": -5.565412998199463,
+      "logps/rejected": -7.586836814880371,
+      "loss": 0.6702,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -5.565412998199463,
+      "rewards/margins": 2.02142333984375,
+      "rewards/rejected": -7.586836814880371,
+      "step": 2035
+    },
+    {
+      "epoch": 1.0918213748118415,
+      "grad_norm": 3.692542256242918,
+      "learning_rate": 8.023690218297329e-07,
+      "logits/chosen": 0.48594146966934204,
+      "logits/rejected": 0.8640475273132324,
+      "logps/chosen": -5.3375372886657715,
+      "logps/rejected": -8.071279525756836,
+      "loss": 0.6667,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -5.3375372886657715,
+      "rewards/margins": 2.733743667602539,
+      "rewards/rejected": -8.071279525756836,
+      "step": 2040
+    },
+    {
+      "epoch": 1.094497407593243,
+      "grad_norm": 5.489292370034196,
+      "learning_rate": 8.01127203488458e-07,
+      "logits/chosen": 0.7145429253578186,
+      "logits/rejected": 1.0490604639053345,
+      "logps/chosen": -5.746392250061035,
+      "logps/rejected": -8.661168098449707,
+      "loss": 0.67,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -5.746392250061035,
+      "rewards/margins": 2.9147753715515137,
+      "rewards/rejected": -8.661168098449707,
+      "step": 2045
+    },
+    {
+      "epoch": 1.0971734403746445,
+      "grad_norm": 3.7535205579774287,
+      "learning_rate": 7.998824636097339e-07,
+      "logits/chosen": 0.530070424079895,
+      "logits/rejected": 1.0845911502838135,
+      "logps/chosen": -5.277037143707275,
+      "logps/rejected": -8.197465896606445,
+      "loss": 0.6794,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -5.277037143707275,
+      "rewards/margins": 2.92042875289917,
+      "rewards/rejected": -8.197465896606445,
+      "step": 2050
+    },
+    {
+      "epoch": 1.0998494731560462,
+      "grad_norm": 3.789332465221397,
+      "learning_rate": 7.986348142700328e-07,
+      "logits/chosen": 0.5201945900917053,
+      "logits/rejected": 1.0137704610824585,
+      "logps/chosen": -5.003693580627441,
+      "logps/rejected": -8.586662292480469,
+      "loss": 0.6695,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -5.003693580627441,
+      "rewards/margins": 3.5829689502716064,
+      "rewards/rejected": -8.586662292480469,
+      "step": 2055
+    },
+    {
+      "epoch": 1.1025255059374477,
+      "grad_norm": 2.664180611479063,
+      "learning_rate": 7.973842675740539e-07,
+      "logits/chosen": 0.3481920659542084,
+      "logits/rejected": 0.8373286128044128,
+      "logps/chosen": -4.2427263259887695,
+      "logps/rejected": -7.2804155349731445,
+      "loss": 0.6688,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.2427263259887695,
+      "rewards/margins": 3.0376884937286377,
+      "rewards/rejected": -7.2804155349731445,
+      "step": 2060
+    },
+    {
+      "epoch": 1.1052015387188494,
+      "grad_norm": 2.850577456480681,
+      "learning_rate": 7.961308356546066e-07,
+      "logits/chosen": 0.3327382206916809,
+      "logits/rejected": 0.7441484332084656,
+      "logps/chosen": -4.521571636199951,
+      "logps/rejected": -6.82846212387085,
+      "loss": 0.667,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.521571636199951,
+      "rewards/margins": 2.3068900108337402,
+      "rewards/rejected": -6.82846212387085,
+      "step": 2065
+    },
+    {
+      "epoch": 1.107877571500251,
+      "grad_norm": 2.8012453602778096,
+      "learning_rate": 7.948745306724931e-07,
+      "logits/chosen": 0.32197409868240356,
+      "logits/rejected": 0.8083009719848633,
+      "logps/chosen": -4.034804344177246,
+      "logps/rejected": -6.844886779785156,
+      "loss": 0.6624,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.034804344177246,
+      "rewards/margins": 2.81008243560791,
+      "rewards/rejected": -6.844886779785156,
+      "step": 2070
+    },
+    {
+      "epoch": 1.1105536042816524,
+      "grad_norm": 3.9507407024090355,
+      "learning_rate": 7.936153648163897e-07,
+      "logits/chosen": 0.45707374811172485,
+      "logits/rejected": 0.6401292085647583,
+      "logps/chosen": -5.078164577484131,
+      "logps/rejected": -6.948702335357666,
+      "loss": 0.6794,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -5.078164577484131,
+      "rewards/margins": 1.870537519454956,
+      "rewards/rejected": -6.948702335357666,
+      "step": 2075
+    },
+    {
+      "epoch": 1.1132296370630541,
+      "grad_norm": 3.6278255247981375,
+      "learning_rate": 7.92353350302729e-07,
+      "logits/chosen": 0.368817538022995,
+      "logits/rejected": 0.6687429547309875,
+      "logps/chosen": -4.680308818817139,
+      "logps/rejected": -6.6659255027771,
+      "loss": 0.6617,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.680308818817139,
+      "rewards/margins": 1.9856170415878296,
+      "rewards/rejected": -6.6659255027771,
+      "step": 2080
+    },
+    {
+      "epoch": 1.1159056698444556,
+      "grad_norm": 7.484031175829903,
+      "learning_rate": 7.910884993755816e-07,
+      "logits/chosen": 0.5547356009483337,
+      "logits/rejected": 0.9300824999809265,
+      "logps/chosen": -5.195932388305664,
+      "logps/rejected": -7.922307014465332,
+      "loss": 0.6746,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -5.195932388305664,
+      "rewards/margins": 2.7263741493225098,
+      "rewards/rejected": -7.922307014465332,
+      "step": 2085
+    },
+    {
+      "epoch": 1.118581702625857,
+      "grad_norm": 3.897698553080582,
+      "learning_rate": 7.898208243065367e-07,
+      "logits/chosen": 0.6290103793144226,
+      "logits/rejected": 0.7474266886711121,
+      "logps/chosen": -5.442200660705566,
+      "logps/rejected": -6.870667457580566,
+      "loss": 0.6844,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -5.442200660705566,
+      "rewards/margins": 1.4284669160842896,
+      "rewards/rejected": -6.870667457580566,
+      "step": 2090
+    },
+    {
+      "epoch": 1.1212577354072588,
+      "grad_norm": 3.0328355609353563,
+      "learning_rate": 7.88550337394583e-07,
+      "logits/chosen": 0.623863160610199,
+      "logits/rejected": 1.0263431072235107,
+      "logps/chosen": -6.167305946350098,
+      "logps/rejected": -8.475918769836426,
+      "loss": 0.6764,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -6.167305946350098,
+      "rewards/margins": 2.3086118698120117,
+      "rewards/rejected": -8.475918769836426,
+      "step": 2095
+    },
+    {
+      "epoch": 1.1239337681886603,
+      "grad_norm": 2.5324912898414533,
+      "learning_rate": 7.872770509659905e-07,
+      "logits/chosen": 0.5885211229324341,
+      "logits/rejected": 1.0754683017730713,
+      "logps/chosen": -5.830451488494873,
+      "logps/rejected": -9.073602676391602,
+      "loss": 0.6717,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -5.830451488494873,
+      "rewards/margins": 3.243152141571045,
+      "rewards/rejected": -9.073602676391602,
+      "step": 2100
+    },
+    {
+      "epoch": 1.1266098009700618,
+      "grad_norm": 2.7474213364980606,
+      "learning_rate": 7.860009773741896e-07,
+      "logits/chosen": 0.7916486263275146,
+      "logits/rejected": 1.0321776866912842,
+      "logps/chosen": -6.291813850402832,
+      "logps/rejected": -8.643472671508789,
+      "loss": 0.677,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -6.291813850402832,
+      "rewards/margins": 2.351658821105957,
+      "rewards/rejected": -8.643472671508789,
+      "step": 2105
+    },
+    {
+      "epoch": 1.1292858337514635,
+      "grad_norm": 3.1912211007722764,
+      "learning_rate": 7.84722128999652e-07,
+      "logits/chosen": 0.5304238200187683,
+      "logits/rejected": 0.8591171503067017,
+      "logps/chosen": -5.217892646789551,
+      "logps/rejected": -7.710452079772949,
+      "loss": 0.681,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -5.217892646789551,
+      "rewards/margins": 2.4925589561462402,
+      "rewards/rejected": -7.710452079772949,
+      "step": 2110
+    },
+    {
+      "epoch": 1.131961866532865,
+      "grad_norm": 5.518620543300996,
+      "learning_rate": 7.834405182497699e-07,
+      "logits/chosen": 0.49657925963401794,
+      "logits/rejected": 0.8159845471382141,
+      "logps/chosen": -5.38375997543335,
+      "logps/rejected": -7.525177001953125,
+      "loss": 0.6787,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -5.38375997543335,
+      "rewards/margins": 2.141416549682617,
+      "rewards/rejected": -7.525177001953125,
+      "step": 2115
+    },
+    {
+      "epoch": 1.1346378993142665,
+      "grad_norm": 3.6373006630901434,
+      "learning_rate": 7.821561575587368e-07,
+      "logits/chosen": 0.29933446645736694,
+      "logits/rejected": 0.6298454999923706,
+      "logps/chosen": -3.9840996265411377,
+      "logps/rejected": -6.197592735290527,
+      "loss": 0.6726,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -3.9840996265411377,
+      "rewards/margins": 2.2134931087493896,
+      "rewards/rejected": -6.197592735290527,
+      "step": 2120
+    },
+    {
+      "epoch": 1.1373139320956682,
+      "grad_norm": 2.726521742186479,
+      "learning_rate": 7.808690593874254e-07,
+      "logits/chosen": 0.29463350772857666,
+      "logits/rejected": 0.6226871609687805,
+      "logps/chosen": -3.9465365409851074,
+      "logps/rejected": -6.220671653747559,
+      "loss": 0.6691,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -3.9465365409851074,
+      "rewards/margins": 2.2741355895996094,
+      "rewards/rejected": -6.220671653747559,
+      "step": 2125
+    },
+    {
+      "epoch": 1.1399899648770697,
+      "grad_norm": 4.6083808386948295,
+      "learning_rate": 7.79579236223268e-07,
+      "logits/chosen": 0.2340255230665207,
+      "logits/rejected": 0.574184238910675,
+      "logps/chosen": -3.2350964546203613,
+      "logps/rejected": -5.4751973152160645,
+      "loss": 0.672,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -3.2350964546203613,
+      "rewards/margins": 2.240100860595703,
+      "rewards/rejected": -5.4751973152160645,
+      "step": 2130
+    },
+    {
+      "epoch": 1.1426659976584714,
+      "grad_norm": 3.2364912075567527,
+      "learning_rate": 7.782867005801346e-07,
+      "logits/chosen": 0.3084893226623535,
+      "logits/rejected": 0.7913509607315063,
+      "logps/chosen": -3.5943686962127686,
+      "logps/rejected": -6.299056529998779,
+      "loss": 0.6656,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -3.5943686962127686,
+      "rewards/margins": 2.704688310623169,
+      "rewards/rejected": -6.299056529998779,
+      "step": 2135
+    },
+    {
+      "epoch": 1.145342030439873,
+      "grad_norm": 4.214824920120603,
+      "learning_rate": 7.769914649982117e-07,
+      "logits/chosen": 0.32396435737609863,
+      "logits/rejected": 0.6387280225753784,
+      "logps/chosen": -4.008212089538574,
+      "logps/rejected": -6.1198883056640625,
+      "loss": 0.6656,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.008212089538574,
+      "rewards/margins": 2.111675977706909,
+      "rewards/rejected": -6.1198883056640625,
+      "step": 2140
+    },
+    {
+      "epoch": 1.1480180632212744,
+      "grad_norm": 3.4662701970711605,
+      "learning_rate": 7.756935420438803e-07,
+      "logits/chosen": 0.3958703875541687,
+      "logits/rejected": 0.7393504977226257,
+      "logps/chosen": -4.412589073181152,
+      "logps/rejected": -7.179342746734619,
+      "loss": 0.6574,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.412589073181152,
+      "rewards/margins": 2.766754150390625,
+      "rewards/rejected": -7.179342746734619,
+      "step": 2145
+    },
+    {
+      "epoch": 1.1506940960026761,
+      "grad_norm": 2.2750041287761706,
+      "learning_rate": 7.743929443095951e-07,
+      "logits/chosen": 0.31420978903770447,
+      "logits/rejected": 0.5971919298171997,
+      "logps/chosen": -4.221810817718506,
+      "logps/rejected": -6.001790523529053,
+      "loss": 0.6789,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.221810817718506,
+      "rewards/margins": 1.7799797058105469,
+      "rewards/rejected": -6.001790523529053,
+      "step": 2150
+    },
+    {
+      "epoch": 1.1533701287840776,
+      "grad_norm": 2.3876400334489034,
+      "learning_rate": 7.730896844137609e-07,
+      "logits/chosen": 0.4558553695678711,
+      "logits/rejected": 0.8150135278701782,
+      "logps/chosen": -5.003996849060059,
+      "logps/rejected": -7.3831939697265625,
+      "loss": 0.6791,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -5.003996849060059,
+      "rewards/margins": 2.379197359085083,
+      "rewards/rejected": -7.3831939697265625,
+      "step": 2155
+    },
+    {
+      "epoch": 1.1560461615654791,
+      "grad_norm": 2.2869975785498613,
+      "learning_rate": 7.717837750006106e-07,
+      "logits/chosen": 0.5201845765113831,
+      "logits/rejected": 0.927703857421875,
+      "logps/chosen": -5.149142265319824,
+      "logps/rejected": -7.569343566894531,
+      "loss": 0.6649,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -5.149142265319824,
+      "rewards/margins": 2.420201539993286,
+      "rewards/rejected": -7.569343566894531,
+      "step": 2160
+    },
+    {
+      "epoch": 1.1587221943468808,
+      "grad_norm": 2.3896098988171794,
+      "learning_rate": 7.704752287400832e-07,
+      "logits/chosen": 0.6755658388137817,
+      "logits/rejected": 0.936316967010498,
+      "logps/chosen": -5.516927242279053,
+      "logps/rejected": -7.924630165100098,
+      "loss": 0.6731,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -5.516927242279053,
+      "rewards/margins": 2.407702922821045,
+      "rewards/rejected": -7.924630165100098,
+      "step": 2165
+    },
+    {
+      "epoch": 1.1613982271282823,
+      "grad_norm": 3.122007608587924,
+      "learning_rate": 7.691640583277004e-07,
+      "logits/chosen": 0.5770570635795593,
+      "logits/rejected": 0.9612566232681274,
+      "logps/chosen": -5.069675922393799,
+      "logps/rejected": -7.859865665435791,
+      "loss": 0.6658,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -5.069675922393799,
+      "rewards/margins": 2.790189743041992,
+      "rewards/rejected": -7.859865665435791,
+      "step": 2170
+    },
+    {
+      "epoch": 1.1640742599096838,
+      "grad_norm": 3.1736205395091934,
+      "learning_rate": 7.678502764844433e-07,
+      "logits/chosen": 0.6274005770683289,
+      "logits/rejected": 0.9279302358627319,
+      "logps/chosen": -5.164000511169434,
+      "logps/rejected": -7.407866477966309,
+      "loss": 0.6719,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -5.164000511169434,
+      "rewards/margins": 2.2438671588897705,
+      "rewards/rejected": -7.407866477966309,
+      "step": 2175
+    },
+    {
+      "epoch": 1.1667502926910855,
+      "grad_norm": 3.6052487754028086,
+      "learning_rate": 7.665338959566288e-07,
+      "logits/chosen": 0.5967945456504822,
+      "logits/rejected": 1.0632880926132202,
+      "logps/chosen": -5.617501735687256,
+      "logps/rejected": -8.772265434265137,
+      "loss": 0.6521,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -5.617501735687256,
+      "rewards/margins": 3.154763698577881,
+      "rewards/rejected": -8.772265434265137,
+      "step": 2180
+    },
+    {
+      "epoch": 1.169426325472487,
+      "grad_norm": 4.388442900058716,
+      "learning_rate": 7.652149295157868e-07,
+      "logits/chosen": 0.5812999606132507,
+      "logits/rejected": 0.8652387857437134,
+      "logps/chosen": -5.01140022277832,
+      "logps/rejected": -6.997218132019043,
+      "loss": 0.6686,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -5.01140022277832,
+      "rewards/margins": 1.9858182668685913,
+      "rewards/rejected": -6.997218132019043,
+      "step": 2185
+    },
+    {
+      "epoch": 1.1721023582538885,
+      "grad_norm": 4.12637179281544,
+      "learning_rate": 7.638933899585354e-07,
+      "logits/chosen": 0.6278091669082642,
+      "logits/rejected": 0.9094529151916504,
+      "logps/chosen": -4.41973876953125,
+      "logps/rejected": -6.670314788818359,
+      "loss": 0.6697,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.41973876953125,
+      "rewards/margins": 2.250575542449951,
+      "rewards/rejected": -6.670314788818359,
+      "step": 2190
+    },
+    {
+      "epoch": 1.1747783910352902,
+      "grad_norm": 9.981541294672875,
+      "learning_rate": 7.625692901064573e-07,
+      "logits/chosen": 0.47594016790390015,
+      "logits/rejected": 0.7671922445297241,
+      "logps/chosen": -4.542372703552246,
+      "logps/rejected": -6.517141819000244,
+      "loss": 0.6723,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.542372703552246,
+      "rewards/margins": 1.9747698307037354,
+      "rewards/rejected": -6.517141819000244,
+      "step": 2195
+    },
+    {
+      "epoch": 1.1774544238166917,
+      "grad_norm": 4.27811905716112,
+      "learning_rate": 7.61242642805975e-07,
+      "logits/chosen": 0.5905254483222961,
+      "logits/rejected": 0.6978011131286621,
+      "logps/chosen": -4.915173053741455,
+      "logps/rejected": -6.42767333984375,
+      "loss": 0.6665,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.915173053741455,
+      "rewards/margins": 1.5125004053115845,
+      "rewards/rejected": -6.42767333984375,
+      "step": 2200
+    },
+    {
+      "epoch": 1.1801304565980932,
+      "grad_norm": 2.822292581777567,
+      "learning_rate": 7.599134609282266e-07,
+      "logits/chosen": 0.6500779390335083,
+      "logits/rejected": 0.8667106628417969,
+      "logps/chosen": -5.3390374183654785,
+      "logps/rejected": -7.004266262054443,
+      "loss": 0.6716,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -5.3390374183654785,
+      "rewards/margins": 1.6652286052703857,
+      "rewards/rejected": -7.004266262054443,
+      "step": 2205
+    },
+    {
+      "epoch": 1.182806489379495,
+      "grad_norm": 8.789131987852608,
+      "learning_rate": 7.585817573689402e-07,
+      "logits/chosen": 0.5284963846206665,
+      "logits/rejected": 0.9361312985420227,
+      "logps/chosen": -4.48715877532959,
+      "logps/rejected": -7.245203495025635,
+      "loss": 0.6539,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.48715877532959,
+      "rewards/margins": 2.758044481277466,
+      "rewards/rejected": -7.245203495025635,
+      "step": 2210
+    },
+    {
+      "epoch": 1.1854825221608964,
+      "grad_norm": 4.135249902016511,
+      "learning_rate": 7.572475450483098e-07,
+      "logits/chosen": 0.4511447846889496,
+      "logits/rejected": 0.7571781873703003,
+      "logps/chosen": -4.566274642944336,
+      "logps/rejected": -6.5473480224609375,
+      "loss": 0.6666,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.566274642944336,
+      "rewards/margins": 1.9810739755630493,
+      "rewards/rejected": -6.5473480224609375,
+      "step": 2215
+    },
+    {
+      "epoch": 1.188158554942298,
+      "grad_norm": 3.215300957207208,
+      "learning_rate": 7.559108369108689e-07,
+      "logits/chosen": 0.6096934080123901,
+      "logits/rejected": 0.9583576321601868,
+      "logps/chosen": -4.873701572418213,
+      "logps/rejected": -7.237061500549316,
+      "loss": 0.6792,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.873701572418213,
+      "rewards/margins": 2.363358974456787,
+      "rewards/rejected": -7.237061500549316,
+      "step": 2220
+    },
+    {
+      "epoch": 1.1908345877236997,
+      "grad_norm": 3.1326151952358354,
+      "learning_rate": 7.54571645925366e-07,
+      "logits/chosen": 0.6661874055862427,
+      "logits/rejected": 1.1282289028167725,
+      "logps/chosen": -5.191311359405518,
+      "logps/rejected": -8.1383056640625,
+      "loss": 0.6714,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -5.191311359405518,
+      "rewards/margins": 2.9469945430755615,
+      "rewards/rejected": -8.1383056640625,
+      "step": 2225
+    },
+    {
+      "epoch": 1.1935106205051011,
+      "grad_norm": 5.0125789110119765,
+      "learning_rate": 7.532299850846378e-07,
+      "logits/chosen": 0.5655343532562256,
+      "logits/rejected": 0.9725745320320129,
+      "logps/chosen": -4.40358829498291,
+      "logps/rejected": -7.195400238037109,
+      "loss": 0.6701,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.40358829498291,
+      "rewards/margins": 2.791810989379883,
+      "rewards/rejected": -7.195400238037109,
+      "step": 2230
+    },
+    {
+      "epoch": 1.1961866532865026,
+      "grad_norm": 3.8285394325902353,
+      "learning_rate": 7.518858674054838e-07,
+      "logits/chosen": 0.4737967550754547,
+      "logits/rejected": 0.8088616132736206,
+      "logps/chosen": -4.2924909591674805,
+      "logps/rejected": -6.332769870758057,
+      "loss": 0.6674,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.2924909591674805,
+      "rewards/margins": 2.040278673171997,
+      "rewards/rejected": -6.332769870758057,
+      "step": 2235
+    },
+    {
+      "epoch": 1.1988626860679044,
+      "grad_norm": 3.8237668112981207,
+      "learning_rate": 7.505393059285394e-07,
+      "logits/chosen": 0.4756793975830078,
+      "logits/rejected": 0.7973596453666687,
+      "logps/chosen": -4.371269702911377,
+      "logps/rejected": -6.5424604415893555,
+      "loss": 0.6639,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.371269702911377,
+      "rewards/margins": 2.1711907386779785,
+      "rewards/rejected": -6.5424604415893555,
+      "step": 2240
+    },
+    {
+      "epoch": 1.2015387188493059,
+      "grad_norm": 3.2411637221643996,
+      "learning_rate": 7.491903137181501e-07,
+      "logits/chosen": 0.3967435657978058,
+      "logits/rejected": 0.6559614539146423,
+      "logps/chosen": -4.101269245147705,
+      "logps/rejected": -5.868131160736084,
+      "loss": 0.6627,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.101269245147705,
+      "rewards/margins": 1.7668616771697998,
+      "rewards/rejected": -5.868131160736084,
+      "step": 2245
+    },
+    {
+      "epoch": 1.2042147516307076,
+      "grad_norm": 3.6742852083581976,
+      "learning_rate": 7.478389038622441e-07,
+      "logits/chosen": 0.4639032483100891,
+      "logits/rejected": 0.7510157823562622,
+      "logps/chosen": -4.193720817565918,
+      "logps/rejected": -6.445101737976074,
+      "loss": 0.6554,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.193720817565918,
+      "rewards/margins": 2.2513809204101562,
+      "rewards/rejected": -6.445101737976074,
+      "step": 2250
+    },
+    {
+      "epoch": 1.206890784412109,
+      "grad_norm": 3.4776557450575614,
+      "learning_rate": 7.46485089472206e-07,
+      "logits/chosen": 0.41793927550315857,
+      "logits/rejected": 0.8235371708869934,
+      "logps/chosen": -4.607667446136475,
+      "logps/rejected": -7.169938087463379,
+      "loss": 0.6714,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.607667446136475,
+      "rewards/margins": 2.5622706413269043,
+      "rewards/rejected": -7.169938087463379,
+      "step": 2255
+    },
+    {
+      "epoch": 1.2095668171935106,
+      "grad_norm": 3.5758692120868187,
+      "learning_rate": 7.451288836827487e-07,
+      "logits/chosen": 0.6270647644996643,
+      "logits/rejected": 0.8618199229240417,
+      "logps/chosen": -4.955227851867676,
+      "logps/rejected": -6.990008354187012,
+      "loss": 0.6709,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -4.955227851867676,
+      "rewards/margins": 2.034780263900757,
+      "rewards/rejected": -6.990008354187012,
+      "step": 2260
+    },
+    {
+      "epoch": 1.2122428499749123,
+      "grad_norm": 4.139171831555638,
+      "learning_rate": 7.437702996517869e-07,
+      "logits/chosen": 0.7277342081069946,
+      "logits/rejected": 1.089735984802246,
+      "logps/chosen": -5.653992176055908,
+      "logps/rejected": -8.529203414916992,
+      "loss": 0.6703,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -5.653992176055908,
+      "rewards/margins": 2.875211238861084,
+      "rewards/rejected": -8.529203414916992,
+      "step": 2265
+    },
+    {
+      "epoch": 1.2149188827563138,
+      "grad_norm": 4.7325604231017975,
+      "learning_rate": 7.424093505603087e-07,
+      "logits/chosen": 0.5270437002182007,
+      "logits/rejected": 1.0848582983016968,
+      "logps/chosen": -5.331143379211426,
+      "logps/rejected": -8.906461715698242,
+      "loss": 0.656,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -5.331143379211426,
+      "rewards/margins": 3.575317859649658,
+      "rewards/rejected": -8.906461715698242,
+      "step": 2270
+    },
+    {
+      "epoch": 1.2175949155377153,
+      "grad_norm": 4.358295311095726,
+      "learning_rate": 7.410460496122482e-07,
+      "logits/chosen": 0.36506104469299316,
+      "logits/rejected": 0.7621029615402222,
+      "logps/chosen": -4.464076042175293,
+      "logps/rejected": -6.9969587326049805,
+      "loss": 0.6545,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.464076042175293,
+      "rewards/margins": 2.5328826904296875,
+      "rewards/rejected": -6.9969587326049805,
+      "step": 2275
+    },
+    {
+      "epoch": 1.220270948319117,
+      "grad_norm": 8.507197091661196,
+      "learning_rate": 7.396804100343572e-07,
+      "logits/chosen": 0.38669511675834656,
+      "logits/rejected": 0.6955000162124634,
+      "logps/chosen": -4.456143379211426,
+      "logps/rejected": -6.454135894775391,
+      "loss": 0.6691,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.456143379211426,
+      "rewards/margins": 1.997992753982544,
+      "rewards/rejected": -6.454135894775391,
+      "step": 2280
+    },
+    {
+      "epoch": 1.2229469811005185,
+      "grad_norm": 3.216748305698501,
+      "learning_rate": 7.383124450760768e-07,
+      "logits/chosen": 0.43976813554763794,
+      "logits/rejected": 0.8502799868583679,
+      "logps/chosen": -4.593364715576172,
+      "logps/rejected": -6.6187028884887695,
+      "loss": 0.6611,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.593364715576172,
+      "rewards/margins": 2.0253379344940186,
+      "rewards/rejected": -6.6187028884887695,
+      "step": 2285
+    },
+    {
+      "epoch": 1.22562301388192,
+      "grad_norm": 7.149385162351271,
+      "learning_rate": 7.369421680094091e-07,
+      "logits/chosen": 0.3715898394584656,
+      "logits/rejected": 0.6424241662025452,
+      "logps/chosen": -4.013710975646973,
+      "logps/rejected": -6.060699462890625,
+      "loss": 0.6764,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.013710975646973,
+      "rewards/margins": 2.0469892024993896,
+      "rewards/rejected": -6.060699462890625,
+      "step": 2290
+    },
+    {
+      "epoch": 1.2282990466633217,
+      "grad_norm": 3.000089497847203,
+      "learning_rate": 7.355695921287881e-07,
+      "logits/chosen": 0.34754472970962524,
+      "logits/rejected": 0.4658147394657135,
+      "logps/chosen": -4.514842987060547,
+      "logps/rejected": -5.168885231018066,
+      "loss": 0.6812,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.514842987060547,
+      "rewards/margins": 0.6540419459342957,
+      "rewards/rejected": -5.168885231018066,
+      "step": 2295
+    },
+    {
+      "epoch": 1.2309750794447232,
+      "grad_norm": 2.8129392778699778,
+      "learning_rate": 7.341947307509513e-07,
+      "logits/chosen": 0.31978780031204224,
+      "logits/rejected": 0.7085596919059753,
+      "logps/chosen": -3.697920560836792,
+      "logps/rejected": -5.894768238067627,
+      "loss": 0.6727,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -3.697920560836792,
+      "rewards/margins": 2.196847438812256,
+      "rewards/rejected": -5.894768238067627,
+      "step": 2300
+    },
+    {
+      "epoch": 1.233651112226125,
+      "grad_norm": 3.0988206086327557,
+      "learning_rate": 7.328175972148094e-07,
+      "logits/chosen": 0.35212650895118713,
+      "logits/rejected": 0.6313988566398621,
+      "logps/chosen": -4.558783054351807,
+      "logps/rejected": -6.122622966766357,
+      "loss": 0.6753,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.558783054351807,
+      "rewards/margins": 1.5638400316238403,
+      "rewards/rejected": -6.122622966766357,
+      "step": 2305
+    },
+    {
+      "epoch": 1.2363271450075264,
+      "grad_norm": 2.9974139865752902,
+      "learning_rate": 7.314382048813185e-07,
+      "logits/chosen": 0.33032432198524475,
+      "logits/rejected": 0.7997673153877258,
+      "logps/chosen": -3.8773193359375,
+      "logps/rejected": -6.711348533630371,
+      "loss": 0.6633,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -3.8773193359375,
+      "rewards/margins": 2.834029197692871,
+      "rewards/rejected": -6.711348533630371,
+      "step": 2310
+    },
+    {
+      "epoch": 1.2390031777889279,
+      "grad_norm": 3.0862494199860087,
+      "learning_rate": 7.300565671333486e-07,
+      "logits/chosen": 0.36690792441368103,
+      "logits/rejected": 0.7783200144767761,
+      "logps/chosen": -4.427572727203369,
+      "logps/rejected": -6.465891361236572,
+      "loss": 0.6611,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.427572727203369,
+      "rewards/margins": 2.038318634033203,
+      "rewards/rejected": -6.465891361236572,
+      "step": 2315
+    },
+    {
+      "epoch": 1.2416792105703296,
+      "grad_norm": 3.4985123218007432,
+      "learning_rate": 7.286726973755554e-07,
+      "logits/chosen": 0.4915011525154114,
+      "logits/rejected": 0.7907226085662842,
+      "logps/chosen": -4.345424175262451,
+      "logps/rejected": -6.781322479248047,
+      "loss": 0.6703,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.345424175262451,
+      "rewards/margins": 2.4358978271484375,
+      "rewards/rejected": -6.781322479248047,
+      "step": 2320
+    },
+    {
+      "epoch": 1.244355243351731,
+      "grad_norm": 2.8804285787937225,
+      "learning_rate": 7.272866090342493e-07,
+      "logits/chosen": 0.48382940888404846,
+      "logits/rejected": 0.8013609051704407,
+      "logps/chosen": -4.143895149230957,
+      "logps/rejected": -6.293688774108887,
+      "loss": 0.6671,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.143895149230957,
+      "rewards/margins": 2.1497936248779297,
+      "rewards/rejected": -6.293688774108887,
+      "step": 2325
+    },
+    {
+      "epoch": 1.2470312761331326,
+      "grad_norm": 3.4925388581623174,
+      "learning_rate": 7.258983155572656e-07,
+      "logits/chosen": 0.32790225744247437,
+      "logits/rejected": 0.6737746596336365,
+      "logps/chosen": -4.402360439300537,
+      "logps/rejected": -6.9154767990112305,
+      "loss": 0.6573,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.402360439300537,
+      "rewards/margins": 2.5131161212921143,
+      "rewards/rejected": -6.9154767990112305,
+      "step": 2330
+    },
+    {
+      "epoch": 1.2497073089145343,
+      "grad_norm": 3.9853844095821924,
+      "learning_rate": 7.245078304138335e-07,
+      "logits/chosen": 0.4474734663963318,
+      "logits/rejected": 0.8368266224861145,
+      "logps/chosen": -4.347743034362793,
+      "logps/rejected": -7.297874450683594,
+      "loss": 0.6619,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.347743034362793,
+      "rewards/margins": 2.9501311779022217,
+      "rewards/rejected": -7.297874450683594,
+      "step": 2335
+    },
+    {
+      "epoch": 1.2523833416959358,
+      "grad_norm": 3.7849168080573063,
+      "learning_rate": 7.231151670944462e-07,
+      "logits/chosen": 0.297460675239563,
+      "logits/rejected": 0.5676824450492859,
+      "logps/chosen": -4.668747425079346,
+      "logps/rejected": -6.069169044494629,
+      "loss": 0.6693,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.668747425079346,
+      "rewards/margins": 1.4004216194152832,
+      "rewards/rejected": -6.069169044494629,
+      "step": 2340
+    },
+    {
+      "epoch": 1.2550593744773373,
+      "grad_norm": 3.849378610423955,
+      "learning_rate": 7.217203391107291e-07,
+      "logits/chosen": 0.3212379217147827,
+      "logits/rejected": 0.746213436126709,
+      "logps/chosen": -4.067599773406982,
+      "logps/rejected": -6.66454553604126,
+      "loss": 0.6598,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.067599773406982,
+      "rewards/margins": 2.5969462394714355,
+      "rewards/rejected": -6.66454553604126,
+      "step": 2345
+    },
+    {
+      "epoch": 1.257735407258739,
+      "grad_norm": 2.9242381319315682,
+      "learning_rate": 7.203233599953096e-07,
+      "logits/chosen": 0.4937950074672699,
+      "logits/rejected": 0.8575268983840942,
+      "logps/chosen": -4.850513935089111,
+      "logps/rejected": -6.9589080810546875,
+      "loss": 0.6734,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.850513935089111,
+      "rewards/margins": 2.108393669128418,
+      "rewards/rejected": -6.9589080810546875,
+      "step": 2350
+    },
+    {
+      "epoch": 1.2604114400401405,
+      "grad_norm": 3.884923362851379,
+      "learning_rate": 7.189242433016852e-07,
+      "logits/chosen": 0.6191721558570862,
+      "logits/rejected": 0.9723224639892578,
+      "logps/chosen": -5.288998603820801,
+      "logps/rejected": -7.989449501037598,
+      "loss": 0.6665,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -5.288998603820801,
+      "rewards/margins": 2.7004494667053223,
+      "rewards/rejected": -7.989449501037598,
+      "step": 2355
+    },
+    {
+      "epoch": 1.263087472821542,
+      "grad_norm": 3.3659364332369925,
+      "learning_rate": 7.17523002604092e-07,
+      "logits/chosen": 0.7426954507827759,
+      "logits/rejected": 1.0522527694702148,
+      "logps/chosen": -6.39832878112793,
+      "logps/rejected": -9.263955116271973,
+      "loss": 0.6688,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -6.39832878112793,
+      "rewards/margins": 2.865626573562622,
+      "rewards/rejected": -9.263955116271973,
+      "step": 2360
+    },
+    {
+      "epoch": 1.2657635056029437,
+      "grad_norm": 4.6945846157099735,
+      "learning_rate": 7.161196514973734e-07,
+      "logits/chosen": 0.4228861927986145,
+      "logits/rejected": 0.841906726360321,
+      "logps/chosen": -5.178776264190674,
+      "logps/rejected": -8.35342788696289,
+      "loss": 0.6703,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -5.178776264190674,
+      "rewards/margins": 3.174652576446533,
+      "rewards/rejected": -8.35342788696289,
+      "step": 2365
+    },
+    {
+      "epoch": 1.2684395383843452,
+      "grad_norm": 3.2010865553705243,
+      "learning_rate": 7.147142035968483e-07,
+      "logits/chosen": 0.6555970311164856,
+      "logits/rejected": 1.1197798252105713,
+      "logps/chosen": -5.881132125854492,
+      "logps/rejected": -8.795707702636719,
+      "loss": 0.6694,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -5.881132125854492,
+      "rewards/margins": 2.91457462310791,
+      "rewards/rejected": -8.795707702636719,
+      "step": 2370
+    },
+    {
+      "epoch": 1.2711155711657467,
+      "grad_norm": 4.486843948800588,
+      "learning_rate": 7.133066725381781e-07,
+      "logits/chosen": 0.41754016280174255,
+      "logits/rejected": 0.8772650957107544,
+      "logps/chosen": -5.28355073928833,
+      "logps/rejected": -7.86264705657959,
+      "loss": 0.6671,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -5.28355073928833,
+      "rewards/margins": 2.5790963172912598,
+      "rewards/rejected": -7.86264705657959,
+      "step": 2375
+    },
+    {
+      "epoch": 1.2737916039471484,
+      "grad_norm": 3.5859394071649535,
+      "learning_rate": 7.118970719772354e-07,
+      "logits/chosen": 0.44439998269081116,
+      "logits/rejected": 0.7736958265304565,
+      "logps/chosen": -5.297356605529785,
+      "logps/rejected": -7.381107330322266,
+      "loss": 0.6667,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -5.297356605529785,
+      "rewards/margins": 2.0837507247924805,
+      "rewards/rejected": -7.381107330322266,
+      "step": 2380
+    },
+    {
+      "epoch": 1.27646763672855,
+      "grad_norm": 3.8204509837508525,
+      "learning_rate": 7.104854155899711e-07,
+      "logits/chosen": 0.6192377209663391,
+      "logits/rejected": 1.0053602457046509,
+      "logps/chosen": -5.065618991851807,
+      "logps/rejected": -7.727991580963135,
+      "loss": 0.6593,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -5.065618991851807,
+      "rewards/margins": 2.66237211227417,
+      "rewards/rejected": -7.727991580963135,
+      "step": 2385
+    },
+    {
+      "epoch": 1.2791436695099514,
+      "grad_norm": 3.8458271471073657,
+      "learning_rate": 7.090717170722817e-07,
+      "logits/chosen": 0.3417230546474457,
+      "logits/rejected": 0.8998492956161499,
+      "logps/chosen": -4.296016216278076,
+      "logps/rejected": -7.809932708740234,
+      "loss": 0.6529,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.296016216278076,
+      "rewards/margins": 3.513916492462158,
+      "rewards/rejected": -7.809932708740234,
+      "step": 2390
+    },
+    {
+      "epoch": 1.2818197022913531,
+      "grad_norm": 8.720949682098174,
+      "learning_rate": 7.076559901398762e-07,
+      "logits/chosen": 0.45270270109176636,
+      "logits/rejected": 0.6055335402488708,
+      "logps/chosen": -4.55720329284668,
+      "logps/rejected": -5.764530658721924,
+      "loss": 0.7026,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -4.55720329284668,
+      "rewards/margins": 1.207327127456665,
+      "rewards/rejected": -5.764530658721924,
+      "step": 2395
+    },
+    {
+      "epoch": 1.2844957350727546,
+      "grad_norm": 5.104344216846636,
+      "learning_rate": 7.062382485281436e-07,
+      "logits/chosen": 0.3514675498008728,
+      "logits/rejected": 0.5760213136672974,
+      "logps/chosen": -4.06247615814209,
+      "logps/rejected": -5.610538482666016,
+      "loss": 0.6648,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.06247615814209,
+      "rewards/margins": 1.5480626821517944,
+      "rewards/rejected": -5.610538482666016,
+      "step": 2400
+    },
+    {
+      "epoch": 1.2844957350727546,
+      "eval_logits/chosen": 0.6519383788108826,
+      "eval_logits/rejected": 0.9759930968284607,
+      "eval_logps/chosen": -4.232706069946289,
+      "eval_logps/rejected": -7.026699542999268,
+      "eval_loss": 0.6650967597961426,
+      "eval_rewards/accuracies": 0.6654302477836609,
+      "eval_rewards/chosen": -4.232706069946289,
+      "eval_rewards/margins": 2.7939934730529785,
+      "eval_rewards/rejected": -7.026699542999268,
+      "eval_runtime": 40.7394,
+      "eval_samples_per_second": 33.015,
+      "eval_steps_per_second": 8.272,
+      "step": 2400
+    },
+    {
+      "epoch": 1.287171767854156,
+      "grad_norm": 5.139844618868565,
+      "learning_rate": 7.048185059920193e-07,
+      "logits/chosen": 0.4573967456817627,
+      "logits/rejected": 0.8660893440246582,
+      "logps/chosen": -4.175953388214111,
+      "logps/rejected": -7.783020973205566,
+      "loss": 0.6622,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.175953388214111,
+      "rewards/margins": 3.607067108154297,
+      "rewards/rejected": -7.783020973205566,
+      "step": 2405
+    },
+    {
+      "epoch": 1.2898478006355578,
+      "grad_norm": 2.433922933330385,
+      "learning_rate": 7.033967763058516e-07,
+      "logits/chosen": 0.47058114409446716,
+      "logits/rejected": 0.9444220662117004,
+      "logps/chosen": -4.8154706954956055,
+      "logps/rejected": -7.790162086486816,
+      "loss": 0.6749,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.8154706954956055,
+      "rewards/margins": 2.9746909141540527,
+      "rewards/rejected": -7.790162086486816,
+      "step": 2410
+    },
+    {
+      "epoch": 1.2925238334169593,
+      "grad_norm": 6.953175112890486,
+      "learning_rate": 7.019730732632681e-07,
+      "logits/chosen": 0.646740734577179,
+      "logits/rejected": 1.0486104488372803,
+      "logps/chosen": -4.917275905609131,
+      "logps/rejected": -7.771488189697266,
+      "loss": 0.6638,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.917275905609131,
+      "rewards/margins": 2.8542122840881348,
+      "rewards/rejected": -7.771488189697266,
+      "step": 2415
+    },
+    {
+      "epoch": 1.2951998661983608,
+      "grad_norm": 5.188750890287502,
+      "learning_rate": 7.005474106770418e-07,
+      "logits/chosen": 0.438167005777359,
+      "logits/rejected": 0.8725101351737976,
+      "logps/chosen": -4.67425012588501,
+      "logps/rejected": -7.736947059631348,
+      "loss": 0.6655,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.67425012588501,
+      "rewards/margins": 3.062696695327759,
+      "rewards/rejected": -7.736947059631348,
+      "step": 2420
+    },
+    {
+      "epoch": 1.2978758989797625,
+      "grad_norm": 4.825055711323919,
+      "learning_rate": 6.991198023789577e-07,
+      "logits/chosen": 0.42146021127700806,
+      "logits/rejected": 0.8264020085334778,
+      "logps/chosen": -4.125103950500488,
+      "logps/rejected": -7.05697774887085,
+      "loss": 0.6491,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.125103950500488,
+      "rewards/margins": 2.931873321533203,
+      "rewards/rejected": -7.05697774887085,
+      "step": 2425
+    },
+    {
+      "epoch": 1.300551931761164,
+      "grad_norm": 2.096626683247078,
+      "learning_rate": 6.976902622196776e-07,
+      "logits/chosen": 0.4445365071296692,
+      "logits/rejected": 0.751555323600769,
+      "logps/chosen": -4.667377948760986,
+      "logps/rejected": -7.111600399017334,
+      "loss": 0.6763,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.667377948760986,
+      "rewards/margins": 2.4442224502563477,
+      "rewards/rejected": -7.111600399017334,
+      "step": 2430
+    },
+    {
+      "epoch": 1.3032279645425655,
+      "grad_norm": 5.319634965343273,
+      "learning_rate": 6.962588040686064e-07,
+      "logits/chosen": 0.7315508127212524,
+      "logits/rejected": 0.8564912676811218,
+      "logps/chosen": -5.60080623626709,
+      "logps/rejected": -7.095902919769287,
+      "loss": 0.6773,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -5.60080623626709,
+      "rewards/margins": 1.495096206665039,
+      "rewards/rejected": -7.095902919769287,
+      "step": 2435
+    },
+    {
+      "epoch": 1.3059039973239672,
+      "grad_norm": 6.423245664050399,
+      "learning_rate": 6.948254418137573e-07,
+      "logits/chosen": 0.4204215407371521,
+      "logits/rejected": 0.7555961608886719,
+      "logps/chosen": -5.21366024017334,
+      "logps/rejected": -7.252695560455322,
+      "loss": 0.6747,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -5.21366024017334,
+      "rewards/margins": 2.039034605026245,
+      "rewards/rejected": -7.252695560455322,
+      "step": 2440
+    },
+    {
+      "epoch": 1.3085800301053687,
+      "grad_norm": 3.4194070443671696,
+      "learning_rate": 6.933901893616174e-07,
+      "logits/chosen": 0.487474262714386,
+      "logits/rejected": 0.8958366513252258,
+      "logps/chosen": -4.732874393463135,
+      "logps/rejected": -7.547441005706787,
+      "loss": 0.6663,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.732874393463135,
+      "rewards/margins": 2.8145663738250732,
+      "rewards/rejected": -7.547441005706787,
+      "step": 2445
+    },
+    {
+      "epoch": 1.3112560628867704,
+      "grad_norm": 2.6838599417214803,
+      "learning_rate": 6.919530606370121e-07,
+      "logits/chosen": 0.561201274394989,
+      "logits/rejected": 0.8994849920272827,
+      "logps/chosen": -4.793757438659668,
+      "logps/rejected": -6.8656158447265625,
+      "loss": 0.6587,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.793757438659668,
+      "rewards/margins": 2.0718588829040527,
+      "rewards/rejected": -6.8656158447265625,
+      "step": 2450
+    },
+    {
+      "epoch": 1.313932095668172,
+      "grad_norm": 2.0805630011927896,
+      "learning_rate": 6.905140695829706e-07,
+      "logits/chosen": 0.4208841323852539,
+      "logits/rejected": 0.8574946522712708,
+      "logps/chosen": -4.449456214904785,
+      "logps/rejected": -6.771750450134277,
+      "loss": 0.6598,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.449456214904785,
+      "rewards/margins": 2.322293996810913,
+      "rewards/rejected": -6.771750450134277,
+      "step": 2455
+    },
+    {
+      "epoch": 1.3166081284495736,
+      "grad_norm": 2.8751135195349065,
+      "learning_rate": 6.890732301605904e-07,
+      "logits/chosen": 0.384815514087677,
+      "logits/rejected": 0.8088987469673157,
+      "logps/chosen": -4.053538799285889,
+      "logps/rejected": -6.509243965148926,
+      "loss": 0.6656,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.053538799285889,
+      "rewards/margins": 2.4557044506073,
+      "rewards/rejected": -6.509243965148926,
+      "step": 2460
+    },
+    {
+      "epoch": 1.3192841612309751,
+      "grad_norm": 4.543539567017563,
+      "learning_rate": 6.876305563489021e-07,
+      "logits/chosen": 0.5102087259292603,
+      "logits/rejected": 1.108045220375061,
+      "logps/chosen": -4.282928466796875,
+      "logps/rejected": -7.440434455871582,
+      "loss": 0.6492,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.282928466796875,
+      "rewards/margins": 3.157505512237549,
+      "rewards/rejected": -7.440434455871582,
+      "step": 2465
+    },
+    {
+      "epoch": 1.3219601940123766,
+      "grad_norm": 7.746186487701945,
+      "learning_rate": 6.861860621447331e-07,
+      "logits/chosen": 0.7031388282775879,
+      "logits/rejected": 0.8465927839279175,
+      "logps/chosen": -5.390783786773682,
+      "logps/rejected": -7.428811550140381,
+      "loss": 0.6753,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -5.390783786773682,
+      "rewards/margins": 2.038029193878174,
+      "rewards/rejected": -7.428811550140381,
+      "step": 2470
+    },
+    {
+      "epoch": 1.3246362267937783,
+      "grad_norm": 6.275282264844987,
+      "learning_rate": 6.847397615625725e-07,
+      "logits/chosen": 0.5497499108314514,
+      "logits/rejected": 0.907917857170105,
+      "logps/chosen": -5.13415002822876,
+      "logps/rejected": -7.049164772033691,
+      "loss": 0.6802,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -5.13415002822876,
+      "rewards/margins": 1.915014624595642,
+      "rewards/rejected": -7.049164772033691,
+      "step": 2475
+    },
+    {
+      "epoch": 1.3273122595751798,
+      "grad_norm": 3.623598830644055,
+      "learning_rate": 6.83291668634435e-07,
+      "logits/chosen": 0.5866016149520874,
+      "logits/rejected": 0.9558430910110474,
+      "logps/chosen": -5.2170729637146,
+      "logps/rejected": -8.290575981140137,
+      "loss": 0.6689,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -5.2170729637146,
+      "rewards/margins": 3.0735034942626953,
+      "rewards/rejected": -8.290575981140137,
+      "step": 2480
+    },
+    {
+      "epoch": 1.3299882923565813,
+      "grad_norm": 5.161952024326086,
+      "learning_rate": 6.818417974097246e-07,
+      "logits/chosen": 0.5654295682907104,
+      "logits/rejected": 1.1329233646392822,
+      "logps/chosen": -4.838503837585449,
+      "logps/rejected": -7.938961982727051,
+      "loss": 0.6659,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.838503837585449,
+      "rewards/margins": 3.1004574298858643,
+      "rewards/rejected": -7.938961982727051,
+      "step": 2485
+    },
+    {
+      "epoch": 1.332664325137983,
+      "grad_norm": 3.242263939482135,
+      "learning_rate": 6.803901619550981e-07,
+      "logits/chosen": 0.41636770963668823,
+      "logits/rejected": 0.7300804257392883,
+      "logps/chosen": -4.774802207946777,
+      "logps/rejected": -7.003907680511475,
+      "loss": 0.6606,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.774802207946777,
+      "rewards/margins": 2.2291057109832764,
+      "rewards/rejected": -7.003907680511475,
+      "step": 2490
+    },
+    {
+      "epoch": 1.3353403579193845,
+      "grad_norm": 3.7101699406766375,
+      "learning_rate": 6.789367763543292e-07,
+      "logits/chosen": 0.5458256006240845,
+      "logits/rejected": 0.7408765554428101,
+      "logps/chosen": -5.198576927185059,
+      "logps/rejected": -6.262999534606934,
+      "loss": 0.6733,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -5.198576927185059,
+      "rewards/margins": 1.0644224882125854,
+      "rewards/rejected": -6.262999534606934,
+      "step": 2495
+    },
+    {
+      "epoch": 1.338016390700786,
+      "grad_norm": 3.4752524463601984,
+      "learning_rate": 6.774816547081714e-07,
+      "logits/chosen": 0.6001649498939514,
+      "logits/rejected": 0.8408709764480591,
+      "logps/chosen": -4.854853630065918,
+      "logps/rejected": -7.022569179534912,
+      "loss": 0.6686,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.854853630065918,
+      "rewards/margins": 2.167715311050415,
+      "rewards/rejected": -7.022569179534912,
+      "step": 2500
+    },
+    {
+      "epoch": 1.3406924234821878,
+      "grad_norm": 3.3919481742075774,
+      "learning_rate": 6.760248111342211e-07,
+      "logits/chosen": 0.5284663438796997,
+      "logits/rejected": 1.0002573728561401,
+      "logps/chosen": -4.572020053863525,
+      "logps/rejected": -7.63519811630249,
+      "loss": 0.66,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.572020053863525,
+      "rewards/margins": 3.063178300857544,
+      "rewards/rejected": -7.63519811630249,
+      "step": 2505
+    },
+    {
+      "epoch": 1.3433684562635893,
+      "grad_norm": 3.3490942067905913,
+      "learning_rate": 6.745662597667813e-07,
+      "logits/chosen": 0.37775829434394836,
+      "logits/rejected": 0.9038238525390625,
+      "logps/chosen": -4.064273834228516,
+      "logps/rejected": -7.407649040222168,
+      "loss": 0.662,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.064273834228516,
+      "rewards/margins": 3.3433756828308105,
+      "rewards/rejected": -7.407649040222168,
+      "step": 2510
+    },
+    {
+      "epoch": 1.3460444890449907,
+      "grad_norm": 3.172267678024082,
+      "learning_rate": 6.731060147567236e-07,
+      "logits/chosen": 0.3750378489494324,
+      "logits/rejected": 0.8202258944511414,
+      "logps/chosen": -4.014922142028809,
+      "logps/rejected": -6.935701847076416,
+      "loss": 0.6678,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.014922142028809,
+      "rewards/margins": 2.9207794666290283,
+      "rewards/rejected": -6.935701847076416,
+      "step": 2515
+    },
+    {
+      "epoch": 1.3487205218263925,
+      "grad_norm": 5.310619687033622,
+      "learning_rate": 6.716440902713515e-07,
+      "logits/chosen": 0.40596944093704224,
+      "logits/rejected": 0.6146842837333679,
+      "logps/chosen": -3.8049635887145996,
+      "logps/rejected": -5.905032157897949,
+      "loss": 0.6695,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -3.8049635887145996,
+      "rewards/margins": 2.1000683307647705,
+      "rewards/rejected": -5.905032157897949,
+      "step": 2520
+    },
+    {
+      "epoch": 1.351396554607794,
+      "grad_norm": 4.1591955152011355,
+      "learning_rate": 6.701805004942627e-07,
+      "logits/chosen": 0.42841362953186035,
+      "logits/rejected": 0.7592076063156128,
+      "logps/chosen": -4.147752285003662,
+      "logps/rejected": -6.479126930236816,
+      "loss": 0.6639,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.147752285003662,
+      "rewards/margins": 2.331373929977417,
+      "rewards/rejected": -6.479126930236816,
+      "step": 2525
+    },
+    {
+      "epoch": 1.3540725873891954,
+      "grad_norm": 5.96000219868739,
+      "learning_rate": 6.687152596252119e-07,
+      "logits/chosen": 0.693516194820404,
+      "logits/rejected": 0.9172321557998657,
+      "logps/chosen": -5.3929548263549805,
+      "logps/rejected": -7.416684627532959,
+      "loss": 0.6837,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -5.3929548263549805,
+      "rewards/margins": 2.023730754852295,
+      "rewards/rejected": -7.416684627532959,
+      "step": 2530
+    },
+    {
+      "epoch": 1.3567486201705972,
+      "grad_norm": 4.842794158473402,
+      "learning_rate": 6.672483818799722e-07,
+      "logits/chosen": 0.6958993077278137,
+      "logits/rejected": 1.0786511898040771,
+      "logps/chosen": -5.7353515625,
+      "logps/rejected": -8.012418746948242,
+      "loss": 0.679,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -5.7353515625,
+      "rewards/margins": 2.2770657539367676,
+      "rewards/rejected": -8.012418746948242,
+      "step": 2535
+    },
+    {
+      "epoch": 1.3594246529519987,
+      "grad_norm": 3.8282636572979065,
+      "learning_rate": 6.657798814901978e-07,
+      "logits/chosen": 0.6943500638008118,
+      "logits/rejected": 1.0813144445419312,
+      "logps/chosen": -5.330437660217285,
+      "logps/rejected": -7.413270473480225,
+      "loss": 0.6734,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -5.330437660217285,
+      "rewards/margins": 2.0828328132629395,
+      "rewards/rejected": -7.413270473480225,
+      "step": 2540
+    },
+    {
+      "epoch": 1.3621006857334002,
+      "grad_norm": 3.524890307790773,
+      "learning_rate": 6.643097727032863e-07,
+      "logits/chosen": 0.670290470123291,
+      "logits/rejected": 1.2046759128570557,
+      "logps/chosen": -5.353348731994629,
+      "logps/rejected": -8.623632431030273,
+      "loss": 0.6687,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -5.353348731994629,
+      "rewards/margins": 3.2702834606170654,
+      "rewards/rejected": -8.623632431030273,
+      "step": 2545
+    },
+    {
+      "epoch": 1.3647767185148019,
+      "grad_norm": 3.357694395423991,
+      "learning_rate": 6.628380697822392e-07,
+      "logits/chosen": 0.642590343952179,
+      "logits/rejected": 1.0837582349777222,
+      "logps/chosen": -5.133480072021484,
+      "logps/rejected": -7.863539695739746,
+      "loss": 0.6772,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -5.133480072021484,
+      "rewards/margins": 2.7300591468811035,
+      "rewards/rejected": -7.863539695739746,
+      "step": 2550
+    },
+    {
+      "epoch": 1.3674527512962034,
+      "grad_norm": 4.713274463815641,
+      "learning_rate": 6.61364787005525e-07,
+      "logits/chosen": 0.6590396165847778,
+      "logits/rejected": 0.999890923500061,
+      "logps/chosen": -5.111676216125488,
+      "logps/rejected": -7.323256492614746,
+      "loss": 0.6633,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -5.111676216125488,
+      "rewards/margins": 2.211580753326416,
+      "rewards/rejected": -7.323256492614746,
+      "step": 2555
+    },
+    {
+      "epoch": 1.3701287840776049,
+      "grad_norm": 3.6790976145693217,
+      "learning_rate": 6.598899386669395e-07,
+      "logits/chosen": 0.5116773843765259,
+      "logits/rejected": 0.8613109588623047,
+      "logps/chosen": -4.45967960357666,
+      "logps/rejected": -6.599764347076416,
+      "loss": 0.6644,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.45967960357666,
+      "rewards/margins": 2.140085220336914,
+      "rewards/rejected": -6.599764347076416,
+      "step": 2560
+    },
+    {
+      "epoch": 1.3728048168590066,
+      "grad_norm": 5.2535403470133994,
+      "learning_rate": 6.584135390754679e-07,
+      "logits/chosen": 0.5104745626449585,
+      "logits/rejected": 0.8871710896492004,
+      "logps/chosen": -4.5939555168151855,
+      "logps/rejected": -7.136366367340088,
+      "loss": 0.6639,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.5939555168151855,
+      "rewards/margins": 2.542410135269165,
+      "rewards/rejected": -7.136366367340088,
+      "step": 2565
+    },
+    {
+      "epoch": 1.375480849640408,
+      "grad_norm": 3.228196331191132,
+      "learning_rate": 6.569356025551454e-07,
+      "logits/chosen": 0.5835349559783936,
+      "logits/rejected": 0.857711911201477,
+      "logps/chosen": -4.619875907897949,
+      "logps/rejected": -6.761366367340088,
+      "loss": 0.6721,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.619875907897949,
+      "rewards/margins": 2.1414904594421387,
+      "rewards/rejected": -6.761366367340088,
+      "step": 2570
+    },
+    {
+      "epoch": 1.3781568824218096,
+      "grad_norm": 5.154155706622387,
+      "learning_rate": 6.554561434449186e-07,
+      "logits/chosen": 0.5288004875183105,
+      "logits/rejected": 0.8088958859443665,
+      "logps/chosen": -4.746927261352539,
+      "logps/rejected": -6.9343461990356445,
+      "loss": 0.674,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.746927261352539,
+      "rewards/margins": 2.1874184608459473,
+      "rewards/rejected": -6.9343461990356445,
+      "step": 2575
+    },
+    {
+      "epoch": 1.3808329152032113,
+      "grad_norm": 3.8285448570315292,
+      "learning_rate": 6.539751760985063e-07,
+      "logits/chosen": 0.4972612261772156,
+      "logits/rejected": 0.6562612056732178,
+      "logps/chosen": -4.325499057769775,
+      "logps/rejected": -5.6072211265563965,
+      "loss": 0.6729,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -4.325499057769775,
+      "rewards/margins": 1.2817223072052002,
+      "rewards/rejected": -5.6072211265563965,
+      "step": 2580
+    },
+    {
+      "epoch": 1.3835089479846128,
+      "grad_norm": 5.498965181788379,
+      "learning_rate": 6.524927148842602e-07,
+      "logits/chosen": 0.4025653004646301,
+      "logits/rejected": 0.716884434223175,
+      "logps/chosen": -3.3739089965820312,
+      "logps/rejected": -5.501266956329346,
+      "loss": 0.6591,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -3.3739089965820312,
+      "rewards/margins": 2.1273586750030518,
+      "rewards/rejected": -5.501266956329346,
+      "step": 2585
+    },
+    {
+      "epoch": 1.3861849807660143,
+      "grad_norm": 3.5080523213297106,
+      "learning_rate": 6.510087741850254e-07,
+      "logits/chosen": 0.22842660546302795,
+      "logits/rejected": 0.48549166321754456,
+      "logps/chosen": -3.0935218334198,
+      "logps/rejected": -4.274221897125244,
+      "loss": 0.6656,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -3.0935218334198,
+      "rewards/margins": 1.1806997060775757,
+      "rewards/rejected": -4.274221897125244,
+      "step": 2590
+    },
+    {
+      "epoch": 1.388861013547416,
+      "grad_norm": 4.777562073811817,
+      "learning_rate": 6.495233683980012e-07,
+      "logits/chosen": 0.3675912022590637,
+      "logits/rejected": 0.6793826818466187,
+      "logps/chosen": -3.818723678588867,
+      "logps/rejected": -5.552756309509277,
+      "loss": 0.6674,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -3.818723678588867,
+      "rewards/margins": 1.7340322732925415,
+      "rewards/rejected": -5.552756309509277,
+      "step": 2595
+    },
+    {
+      "epoch": 1.3915370463288175,
+      "grad_norm": 4.08138679499348,
+      "learning_rate": 6.480365119346011e-07,
+      "logits/chosen": 0.40656375885009766,
+      "logits/rejected": 0.8608638048171997,
+      "logps/chosen": -3.760906219482422,
+      "logps/rejected": -6.2037153244018555,
+      "loss": 0.6559,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -3.760906219482422,
+      "rewards/margins": 2.4428086280822754,
+      "rewards/rejected": -6.2037153244018555,
+      "step": 2600
+    },
+    {
+      "epoch": 1.394213079110219,
+      "grad_norm": 2.7130349210627713,
+      "learning_rate": 6.465482192203129e-07,
+      "logits/chosen": 0.5149949789047241,
+      "logits/rejected": 0.6578909158706665,
+      "logps/chosen": -3.861821413040161,
+      "logps/rejected": -5.551662921905518,
+      "loss": 0.6706,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -3.861821413040161,
+      "rewards/margins": 1.6898412704467773,
+      "rewards/rejected": -5.551662921905518,
+      "step": 2605
+    },
+    {
+      "epoch": 1.3968891118916207,
+      "grad_norm": 3.4030397719441616,
+      "learning_rate": 6.45058504694559e-07,
+      "logits/chosen": 0.562764048576355,
+      "logits/rejected": 0.8396113514900208,
+      "logps/chosen": -4.49348258972168,
+      "logps/rejected": -6.357996463775635,
+      "loss": 0.6563,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.49348258972168,
+      "rewards/margins": 1.8645137548446655,
+      "rewards/rejected": -6.357996463775635,
+      "step": 2610
+    },
+    {
+      "epoch": 1.3995651446730222,
+      "grad_norm": 3.7015471547173453,
+      "learning_rate": 6.435673828105564e-07,
+      "logits/chosen": 0.7428125143051147,
+      "logits/rejected": 0.9405514001846313,
+      "logps/chosen": -5.256471633911133,
+      "logps/rejected": -7.135949611663818,
+      "loss": 0.6719,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -5.256471633911133,
+      "rewards/margins": 1.8794782161712646,
+      "rewards/rejected": -7.135949611663818,
+      "step": 2615
+    },
+    {
+      "epoch": 1.402241177454424,
+      "grad_norm": 5.020335094073277,
+      "learning_rate": 6.420748680351763e-07,
+      "logits/chosen": 0.7665892243385315,
+      "logits/rejected": 0.9790259599685669,
+      "logps/chosen": -5.0907440185546875,
+      "logps/rejected": -6.731441497802734,
+      "loss": 0.6783,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -5.0907440185546875,
+      "rewards/margins": 1.6406971216201782,
+      "rewards/rejected": -6.731441497802734,
+      "step": 2620
+    },
+    {
+      "epoch": 1.4049172102358254,
+      "grad_norm": 2.307762632293372,
+      "learning_rate": 6.405809748488032e-07,
+      "logits/chosen": 0.9623994827270508,
+      "logits/rejected": 1.343601942062378,
+      "logps/chosen": -6.306636810302734,
+      "logps/rejected": -9.191667556762695,
+      "loss": 0.6614,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -6.306636810302734,
+      "rewards/margins": 2.8850297927856445,
+      "rewards/rejected": -9.191667556762695,
+      "step": 2625
+    },
+    {
+      "epoch": 1.4075932430172269,
+      "grad_norm": 3.1751136908091633,
+      "learning_rate": 6.390857177451956e-07,
+      "logits/chosen": 0.7286649942398071,
+      "logits/rejected": 1.2087997198104858,
+      "logps/chosen": -5.723919868469238,
+      "logps/rejected": -8.941774368286133,
+      "loss": 0.6779,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -5.723919868469238,
+      "rewards/margins": 3.2178549766540527,
+      "rewards/rejected": -8.941774368286133,
+      "step": 2630
+    },
+    {
+      "epoch": 1.4102692757986286,
+      "grad_norm": 3.1260241648989617,
+      "learning_rate": 6.375891112313445e-07,
+      "logits/chosen": 0.7707761526107788,
+      "logits/rejected": 1.2005568742752075,
+      "logps/chosen": -6.175969123840332,
+      "logps/rejected": -9.113056182861328,
+      "loss": 0.6648,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -6.175969123840332,
+      "rewards/margins": 2.9370875358581543,
+      "rewards/rejected": -9.113056182861328,
+      "step": 2635
+    },
+    {
+      "epoch": 1.41294530858003,
+      "grad_norm": 4.252997516760918,
+      "learning_rate": 6.360911698273326e-07,
+      "logits/chosen": 0.7802666425704956,
+      "logits/rejected": 1.139528512954712,
+      "logps/chosen": -5.612812519073486,
+      "logps/rejected": -8.081510543823242,
+      "loss": 0.6719,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -5.612812519073486,
+      "rewards/margins": 2.4686970710754395,
+      "rewards/rejected": -8.081510543823242,
+      "step": 2640
+    },
+    {
+      "epoch": 1.4156213413614318,
+      "grad_norm": 3.6773089432541393,
+      "learning_rate": 6.345919080661944e-07,
+      "logits/chosen": 0.7430061101913452,
+      "logits/rejected": 1.1362390518188477,
+      "logps/chosen": -5.091499328613281,
+      "logps/rejected": -7.981128692626953,
+      "loss": 0.6532,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -5.091499328613281,
+      "rewards/margins": 2.8896288871765137,
+      "rewards/rejected": -7.981128692626953,
+      "step": 2645
+    },
+    {
+      "epoch": 1.4182973741428333,
+      "grad_norm": 3.617587770562684,
+      "learning_rate": 6.330913404937737e-07,
+      "logits/chosen": 0.5812464356422424,
+      "logits/rejected": 0.9259502291679382,
+      "logps/chosen": -4.658377647399902,
+      "logps/rejected": -6.326845645904541,
+      "loss": 0.6825,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.658377647399902,
+      "rewards/margins": 1.6684678792953491,
+      "rewards/rejected": -6.326845645904541,
+      "step": 2650
+    },
+    {
+      "epoch": 1.4209734069242348,
+      "grad_norm": 4.582649287347201,
+      "learning_rate": 6.315894816685838e-07,
+      "logits/chosen": 0.6558564305305481,
+      "logits/rejected": 1.0309983491897583,
+      "logps/chosen": -4.705728530883789,
+      "logps/rejected": -7.165279388427734,
+      "loss": 0.6705,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.705728530883789,
+      "rewards/margins": 2.4595508575439453,
+      "rewards/rejected": -7.165279388427734,
+      "step": 2655
+    },
+    {
+      "epoch": 1.4236494397056365,
+      "grad_norm": 5.296303362552367,
+      "learning_rate": 6.300863461616657e-07,
+      "logits/chosen": 0.5549914240837097,
+      "logits/rejected": 0.7927250862121582,
+      "logps/chosen": -4.596503257751465,
+      "logps/rejected": -6.781968593597412,
+      "loss": 0.6634,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.596503257751465,
+      "rewards/margins": 2.1854655742645264,
+      "rewards/rejected": -6.781968593597412,
+      "step": 2660
+    },
+    {
+      "epoch": 1.426325472487038,
+      "grad_norm": 2.5601629350014736,
+      "learning_rate": 6.285819485564465e-07,
+      "logits/chosen": 0.5071433186531067,
+      "logits/rejected": 0.9669731259346008,
+      "logps/chosen": -4.409031867980957,
+      "logps/rejected": -7.584481716156006,
+      "loss": 0.6608,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.409031867980957,
+      "rewards/margins": 3.1754491329193115,
+      "rewards/rejected": -7.584481716156006,
+      "step": 2665
+    },
+    {
+      "epoch": 1.4290015052684395,
+      "grad_norm": 3.9326541757593194,
+      "learning_rate": 6.270763034485986e-07,
+      "logits/chosen": 0.48495572805404663,
+      "logits/rejected": 0.9546284675598145,
+      "logps/chosen": -4.049344539642334,
+      "logps/rejected": -6.975147247314453,
+      "loss": 0.6646,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.049344539642334,
+      "rewards/margins": 2.92580246925354,
+      "rewards/rejected": -6.975147247314453,
+      "step": 2670
+    },
+    {
+      "epoch": 1.4316775380498412,
+      "grad_norm": 5.061056602860982,
+      "learning_rate": 6.255694254458972e-07,
+      "logits/chosen": 0.5427643060684204,
+      "logits/rejected": 1.0339653491973877,
+      "logps/chosen": -4.416680335998535,
+      "logps/rejected": -7.046632289886475,
+      "loss": 0.6735,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.416680335998535,
+      "rewards/margins": 2.6299519538879395,
+      "rewards/rejected": -7.046632289886475,
+      "step": 2675
+    },
+    {
+      "epoch": 1.4343535708312427,
+      "grad_norm": 5.880155235367304,
+      "learning_rate": 6.240613291680795e-07,
+      "logits/chosen": 0.4117976129055023,
+      "logits/rejected": 0.7500210404396057,
+      "logps/chosen": -3.8321433067321777,
+      "logps/rejected": -5.634713172912598,
+      "loss": 0.6769,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -3.8321433067321777,
+      "rewards/margins": 1.8025699853897095,
+      "rewards/rejected": -5.634713172912598,
+      "step": 2680
+    },
+    {
+      "epoch": 1.4370296036126442,
+      "grad_norm": 6.944204849018753,
+      "learning_rate": 6.225520292467021e-07,
+      "logits/chosen": 0.5048757791519165,
+      "logits/rejected": 1.0274089574813843,
+      "logps/chosen": -4.2515130043029785,
+      "logps/rejected": -6.869149684906006,
+      "loss": 0.6617,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.2515130043029785,
+      "rewards/margins": 2.6176376342773438,
+      "rewards/rejected": -6.869149684906006,
+      "step": 2685
+    },
+    {
+      "epoch": 1.439705636394046,
+      "grad_norm": 2.8325093316694976,
+      "learning_rate": 6.210415403249993e-07,
+      "logits/chosen": 0.38317713141441345,
+      "logits/rejected": 0.7674576044082642,
+      "logps/chosen": -3.9982428550720215,
+      "logps/rejected": -6.592028617858887,
+      "loss": 0.6674,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -3.9982428550720215,
+      "rewards/margins": 2.5937857627868652,
+      "rewards/rejected": -6.592028617858887,
+      "step": 2690
+    },
+    {
+      "epoch": 1.4423816691754474,
+      "grad_norm": 3.5394583848492522,
+      "learning_rate": 6.195298770577415e-07,
+      "logits/chosen": 0.7970117330551147,
+      "logits/rejected": 1.0172641277313232,
+      "logps/chosen": -4.791231155395508,
+      "logps/rejected": -6.977169036865234,
+      "loss": 0.6673,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.791231155395508,
+      "rewards/margins": 2.1859383583068848,
+      "rewards/rejected": -6.977169036865234,
+      "step": 2695
+    },
+    {
+      "epoch": 1.445057701956849,
+      "grad_norm": 2.3298744346765723,
+      "learning_rate": 6.180170541110923e-07,
+      "logits/chosen": 0.7475436329841614,
+      "logits/rejected": 1.1175967454910278,
+      "logps/chosen": -5.429424285888672,
+      "logps/rejected": -7.141626834869385,
+      "loss": 0.6766,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -5.429424285888672,
+      "rewards/margins": 1.7122020721435547,
+      "rewards/rejected": -7.141626834869385,
+      "step": 2700
+    },
+    {
+      "epoch": 1.4477337347382506,
+      "grad_norm": 2.2858207967904964,
+      "learning_rate": 6.165030861624663e-07,
+      "logits/chosen": 0.7445830702781677,
+      "logits/rejected": 1.105346441268921,
+      "logps/chosen": -5.1168107986450195,
+      "logps/rejected": -8.131561279296875,
+      "loss": 0.654,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -5.1168107986450195,
+      "rewards/margins": 3.0147507190704346,
+      "rewards/rejected": -8.131561279296875,
+      "step": 2705
+    },
+    {
+      "epoch": 1.4504097675196521,
+      "grad_norm": 4.887682413985806,
+      "learning_rate": 6.149879879003876e-07,
+      "logits/chosen": 0.7380735278129578,
+      "logits/rejected": 1.1758512258529663,
+      "logps/chosen": -4.771734714508057,
+      "logps/rejected": -7.881124019622803,
+      "loss": 0.6516,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.771734714508057,
+      "rewards/margins": 3.1093904972076416,
+      "rewards/rejected": -7.881124019622803,
+      "step": 2710
+    },
+    {
+      "epoch": 1.4530858003010536,
+      "grad_norm": 4.817501749823713,
+      "learning_rate": 6.13471774024346e-07,
+      "logits/chosen": 0.6967607140541077,
+      "logits/rejected": 1.0630918741226196,
+      "logps/chosen": -4.715682506561279,
+      "logps/rejected": -7.72445011138916,
+      "loss": 0.6557,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.715682506561279,
+      "rewards/margins": 3.0087668895721436,
+      "rewards/rejected": -7.72445011138916,
+      "step": 2715
+    },
+    {
+      "epoch": 1.4557618330824553,
+      "grad_norm": 6.5702982746847205,
+      "learning_rate": 6.119544592446551e-07,
+      "logits/chosen": 0.49954885244369507,
+      "logits/rejected": 1.1746160984039307,
+      "logps/chosen": -3.993020534515381,
+      "logps/rejected": -7.630607604980469,
+      "loss": 0.6751,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -3.993020534515381,
+      "rewards/margins": 3.637587070465088,
+      "rewards/rejected": -7.630607604980469,
+      "step": 2720
+    },
+    {
+      "epoch": 1.4584378658638568,
+      "grad_norm": 11.707486116649003,
+      "learning_rate": 6.104360582823096e-07,
+      "logits/chosen": 0.608126699924469,
+      "logits/rejected": 0.9271984100341797,
+      "logps/chosen": -4.131842613220215,
+      "logps/rejected": -6.6137824058532715,
+      "loss": 0.6793,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.131842613220215,
+      "rewards/margins": 2.4819400310516357,
+      "rewards/rejected": -6.6137824058532715,
+      "step": 2725
+    },
+    {
+      "epoch": 1.4611138986452583,
+      "grad_norm": 5.450751821499734,
+      "learning_rate": 6.089165858688423e-07,
+      "logits/chosen": 0.5605559945106506,
+      "logits/rejected": 1.0179908275604248,
+      "logps/chosen": -3.9799797534942627,
+      "logps/rejected": -6.7788238525390625,
+      "loss": 0.6648,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -3.9799797534942627,
+      "rewards/margins": 2.7988440990448,
+      "rewards/rejected": -6.7788238525390625,
+      "step": 2730
+    },
+    {
+      "epoch": 1.46378993142666,
+      "grad_norm": 3.6901674845153987,
+      "learning_rate": 6.073960567461811e-07,
+      "logits/chosen": 0.6559443473815918,
+      "logits/rejected": 0.9530010223388672,
+      "logps/chosen": -4.276468753814697,
+      "logps/rejected": -6.329841613769531,
+      "loss": 0.6652,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.276468753814697,
+      "rewards/margins": 2.053372621536255,
+      "rewards/rejected": -6.329841613769531,
+      "step": 2735
+    },
+    {
+      "epoch": 1.4664659642080615,
+      "grad_norm": 5.667035615334597,
+      "learning_rate": 6.058744856665065e-07,
+      "logits/chosen": 0.6873584389686584,
+      "logits/rejected": 1.0295472145080566,
+      "logps/chosen": -5.08344030380249,
+      "logps/rejected": -7.5580596923828125,
+      "loss": 0.653,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -5.08344030380249,
+      "rewards/margins": 2.4746193885803223,
+      "rewards/rejected": -7.5580596923828125,
+      "step": 2740
+    },
+    {
+      "epoch": 1.469141996989463,
+      "grad_norm": 3.3733309246368863,
+      "learning_rate": 6.043518873921074e-07,
+      "logits/chosen": 0.5497598648071289,
+      "logits/rejected": 1.1676640510559082,
+      "logps/chosen": -3.9681172370910645,
+      "logps/rejected": -7.348383903503418,
+      "loss": 0.6615,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -3.9681172370910645,
+      "rewards/margins": 3.380268096923828,
+      "rewards/rejected": -7.348383903503418,
+      "step": 2745
+    },
+    {
+      "epoch": 1.4718180297708647,
+      "grad_norm": 4.488233644098116,
+      "learning_rate": 6.028282766952393e-07,
+      "logits/chosen": 0.5088897943496704,
+      "logits/rejected": 0.8235710263252258,
+      "logps/chosen": -4.411531925201416,
+      "logps/rejected": -6.3468804359436035,
+      "loss": 0.6592,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.411531925201416,
+      "rewards/margins": 1.9353485107421875,
+      "rewards/rejected": -6.3468804359436035,
+      "step": 2750
+    },
+    {
+      "epoch": 1.4744940625522662,
+      "grad_norm": 6.725212672564509,
+      "learning_rate": 6.013036683579798e-07,
+      "logits/chosen": 0.5577887892723083,
+      "logits/rejected": 0.954415500164032,
+      "logps/chosen": -3.592855930328369,
+      "logps/rejected": -6.556273460388184,
+      "loss": 0.6488,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -3.592855930328369,
+      "rewards/margins": 2.9634182453155518,
+      "rewards/rejected": -6.556273460388184,
+      "step": 2755
+    },
+    {
+      "epoch": 1.4771700953336677,
+      "grad_norm": 4.04704196541214,
+      "learning_rate": 5.997780771720854e-07,
+      "logits/chosen": 0.6331942677497864,
+      "logits/rejected": 0.9101142883300781,
+      "logps/chosen": -4.561062812805176,
+      "logps/rejected": -7.0290703773498535,
+      "loss": 0.6682,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.561062812805176,
+      "rewards/margins": 2.4680073261260986,
+      "rewards/rejected": -7.0290703773498535,
+      "step": 2760
+    },
+    {
+      "epoch": 1.4798461281150694,
+      "grad_norm": 6.841462797551864,
+      "learning_rate": 5.982515179388486e-07,
+      "logits/chosen": 0.7748144865036011,
+      "logits/rejected": 1.0951751470565796,
+      "logps/chosen": -4.867706298828125,
+      "logps/rejected": -7.462525367736816,
+      "loss": 0.6635,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.867706298828125,
+      "rewards/margins": 2.5948195457458496,
+      "rewards/rejected": -7.462525367736816,
+      "step": 2765
+    },
+    {
+      "epoch": 1.482522160896471,
+      "grad_norm": 4.7776758238039205,
+      "learning_rate": 5.967240054689541e-07,
+      "logits/chosen": 0.6590372323989868,
+      "logits/rejected": 1.0864486694335938,
+      "logps/chosen": -4.4977335929870605,
+      "logps/rejected": -7.7382636070251465,
+      "loss": 0.6603,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.4977335929870605,
+      "rewards/margins": 3.240530014038086,
+      "rewards/rejected": -7.7382636070251465,
+      "step": 2770
+    },
+    {
+      "epoch": 1.4851981936778724,
+      "grad_norm": 2.891187645438344,
+      "learning_rate": 5.951955545823342e-07,
+      "logits/chosen": 0.7124058604240417,
+      "logits/rejected": 1.1387099027633667,
+      "logps/chosen": -4.754769325256348,
+      "logps/rejected": -7.531829833984375,
+      "loss": 0.6755,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.754769325256348,
+      "rewards/margins": 2.777061939239502,
+      "rewards/rejected": -7.531829833984375,
+      "step": 2775
+    },
+    {
+      "epoch": 1.4878742264592741,
+      "grad_norm": 3.1594889209266444,
+      "learning_rate": 5.936661801080263e-07,
+      "logits/chosen": 0.5079807043075562,
+      "logits/rejected": 0.8246339559555054,
+      "logps/chosen": -4.063873767852783,
+      "logps/rejected": -5.839451789855957,
+      "loss": 0.6811,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.063873767852783,
+      "rewards/margins": 1.7755788564682007,
+      "rewards/rejected": -5.839451789855957,
+      "step": 2780
+    },
+    {
+      "epoch": 1.4905502592406756,
+      "grad_norm": 3.5767357977163097,
+      "learning_rate": 5.92135896884028e-07,
+      "logits/chosen": 0.6222492456436157,
+      "logits/rejected": 1.0677109956741333,
+      "logps/chosen": -4.634024143218994,
+      "logps/rejected": -7.222857475280762,
+      "loss": 0.6677,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.634024143218994,
+      "rewards/margins": 2.5888333320617676,
+      "rewards/rejected": -7.222857475280762,
+      "step": 2785
+    },
+    {
+      "epoch": 1.4932262920220774,
+      "grad_norm": 3.767334244441721,
+      "learning_rate": 5.906047197571541e-07,
+      "logits/chosen": 0.5892722010612488,
+      "logits/rejected": 0.8755434155464172,
+      "logps/chosen": -4.512574195861816,
+      "logps/rejected": -6.953259468078613,
+      "loss": 0.6787,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.512574195861816,
+      "rewards/margins": 2.4406847953796387,
+      "rewards/rejected": -6.953259468078613,
+      "step": 2790
+    },
+    {
+      "epoch": 1.4959023248034788,
+      "grad_norm": 4.158335828492619,
+      "learning_rate": 5.890726635828919e-07,
+      "logits/chosen": 0.650938093662262,
+      "logits/rejected": 0.85948646068573,
+      "logps/chosen": -4.21682596206665,
+      "logps/rejected": -6.367035865783691,
+      "loss": 0.6711,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.21682596206665,
+      "rewards/margins": 2.15021014213562,
+      "rewards/rejected": -6.367035865783691,
+      "step": 2795
+    },
+    {
+      "epoch": 1.4985783575848803,
+      "grad_norm": 5.571097117668622,
+      "learning_rate": 5.875397432252569e-07,
+      "logits/chosen": 0.4255022406578064,
+      "logits/rejected": 0.807613730430603,
+      "logps/chosen": -4.348812103271484,
+      "logps/rejected": -6.774697780609131,
+      "loss": 0.6665,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.348812103271484,
+      "rewards/margins": 2.4258856773376465,
+      "rewards/rejected": -6.774697780609131,
+      "step": 2800
+    },
+    {
+      "epoch": 1.4985783575848803,
+      "eval_logits/chosen": 0.72390216588974,
+      "eval_logits/rejected": 1.0821127891540527,
+      "eval_logps/chosen": -4.636701583862305,
+      "eval_logps/rejected": -7.6606764793396,
+      "eval_loss": 0.6653955578804016,
+      "eval_rewards/accuracies": 0.6706231236457825,
+      "eval_rewards/chosen": -4.636701583862305,
+      "eval_rewards/margins": 3.023974657058716,
+      "eval_rewards/rejected": -7.6606764793396,
+      "eval_runtime": 40.74,
+      "eval_samples_per_second": 33.014,
+      "eval_steps_per_second": 8.272,
+      "step": 2800
+    },
+    {
+      "epoch": 1.5012543903662818,
+      "grad_norm": 3.8997510263818724,
+      "learning_rate": 5.860059735566491e-07,
+      "logits/chosen": 0.44197386503219604,
+      "logits/rejected": 0.7775350213050842,
+      "logps/chosen": -4.304364204406738,
+      "logps/rejected": -6.959008693695068,
+      "loss": 0.6658,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.304364204406738,
+      "rewards/margins": 2.6546449661254883,
+      "rewards/rejected": -6.959008693695068,
+      "step": 2805
+    },
+    {
+      "epoch": 1.5039304231476835,
+      "grad_norm": 7.9587656869292545,
+      "learning_rate": 5.844713694577087e-07,
+      "logits/chosen": 0.5558545589447021,
+      "logits/rejected": 0.9379401206970215,
+      "logps/chosen": -4.740943908691406,
+      "logps/rejected": -7.499945640563965,
+      "loss": 0.6634,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.740943908691406,
+      "rewards/margins": 2.7590019702911377,
+      "rewards/rejected": -7.499945640563965,
+      "step": 2810
+    },
+    {
+      "epoch": 1.5066064559290853,
+      "grad_norm": 2.9442021615298164,
+      "learning_rate": 5.829359458171714e-07,
+      "logits/chosen": 0.5833736062049866,
+      "logits/rejected": 1.2951548099517822,
+      "logps/chosen": -5.335548400878906,
+      "logps/rejected": -9.607291221618652,
+      "loss": 0.659,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -5.335548400878906,
+      "rewards/margins": 4.271743297576904,
+      "rewards/rejected": -9.607291221618652,
+      "step": 2815
+    },
+    {
+      "epoch": 1.5092824887104868,
+      "grad_norm": 4.976424927605054,
+      "learning_rate": 5.81399717531724e-07,
+      "logits/chosen": 0.6099574565887451,
+      "logits/rejected": 1.0339701175689697,
+      "logps/chosen": -5.7498369216918945,
+      "logps/rejected": -8.352712631225586,
+      "loss": 0.6726,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -5.7498369216918945,
+      "rewards/margins": 2.6028757095336914,
+      "rewards/rejected": -8.352712631225586,
+      "step": 2820
+    },
+    {
+      "epoch": 1.5119585214918883,
+      "grad_norm": 3.515045409821351,
+      "learning_rate": 5.798626995058602e-07,
+      "logits/chosen": 0.4972403049468994,
+      "logits/rejected": 0.9842252731323242,
+      "logps/chosen": -5.314245223999023,
+      "logps/rejected": -8.69935131072998,
+      "loss": 0.6755,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -5.314245223999023,
+      "rewards/margins": 3.385105848312378,
+      "rewards/rejected": -8.69935131072998,
+      "step": 2825
+    },
+    {
+      "epoch": 1.51463455427329,
+      "grad_norm": 6.687985043671093,
+      "learning_rate": 5.783249066517354e-07,
+      "logits/chosen": 0.29380613565444946,
+      "logits/rejected": 0.8744326829910278,
+      "logps/chosen": -4.190525054931641,
+      "logps/rejected": -7.6448974609375,
+      "loss": 0.6741,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.190525054931641,
+      "rewards/margins": 3.4543724060058594,
+      "rewards/rejected": -7.6448974609375,
+      "step": 2830
+    },
+    {
+      "epoch": 1.5173105870546915,
+      "grad_norm": 4.189179730221065,
+      "learning_rate": 5.767863538890228e-07,
+      "logits/chosen": 0.6117783784866333,
+      "logits/rejected": 1.0786592960357666,
+      "logps/chosen": -5.2834672927856445,
+      "logps/rejected": -8.661787986755371,
+      "loss": 0.6678,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -5.2834672927856445,
+      "rewards/margins": 3.378321886062622,
+      "rewards/rejected": -8.661787986755371,
+      "step": 2835
+    },
+    {
+      "epoch": 1.519986619836093,
+      "grad_norm": 4.854029715216408,
+      "learning_rate": 5.75247056144768e-07,
+      "logits/chosen": 0.5171109437942505,
+      "logits/rejected": 0.8347162008285522,
+      "logps/chosen": -5.083793640136719,
+      "logps/rejected": -7.057982444763184,
+      "loss": 0.6645,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -5.083793640136719,
+      "rewards/margins": 1.9741894006729126,
+      "rewards/rejected": -7.057982444763184,
+      "step": 2840
+    },
+    {
+      "epoch": 1.5226626526174947,
+      "grad_norm": 3.646291567732392,
+      "learning_rate": 5.737070283532444e-07,
+      "logits/chosen": 0.8628705739974976,
+      "logits/rejected": 0.9581040143966675,
+      "logps/chosen": -6.294342041015625,
+      "logps/rejected": -7.216023921966553,
+      "loss": 0.6775,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -6.294342041015625,
+      "rewards/margins": 0.9216817617416382,
+      "rewards/rejected": -7.216023921966553,
+      "step": 2845
+    },
+    {
+      "epoch": 1.5253386853988962,
+      "grad_norm": 5.828046286650547,
+      "learning_rate": 5.721662854558084e-07,
+      "logits/chosen": 0.5216276049613953,
+      "logits/rejected": 1.0996596813201904,
+      "logps/chosen": -4.933318138122559,
+      "logps/rejected": -8.309542655944824,
+      "loss": 0.6604,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.933318138122559,
+      "rewards/margins": 3.376224994659424,
+      "rewards/rejected": -8.309542655944824,
+      "step": 2850
+    },
+    {
+      "epoch": 1.5280147181802977,
+      "grad_norm": 2.9963666064936816,
+      "learning_rate": 5.706248424007545e-07,
+      "logits/chosen": 0.48331108689308167,
+      "logits/rejected": 0.8985007405281067,
+      "logps/chosen": -4.947200298309326,
+      "logps/rejected": -8.003686904907227,
+      "loss": 0.6689,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.947200298309326,
+      "rewards/margins": 3.0564868450164795,
+      "rewards/rejected": -8.003686904907227,
+      "step": 2855
+    },
+    {
+      "epoch": 1.5306907509616994,
+      "grad_norm": 4.628744380870746,
+      "learning_rate": 5.690827141431699e-07,
+      "logits/chosen": 0.5681836009025574,
+      "logits/rejected": 0.9687778353691101,
+      "logps/chosen": -5.50781774520874,
+      "logps/rejected": -7.940520286560059,
+      "loss": 0.6727,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -5.50781774520874,
+      "rewards/margins": 2.4327032566070557,
+      "rewards/rejected": -7.940520286560059,
+      "step": 2860
+    },
+    {
+      "epoch": 1.5333667837431009,
+      "grad_norm": 5.3388393802211285,
+      "learning_rate": 5.675399156447897e-07,
+      "logits/chosen": 0.5265989303588867,
+      "logits/rejected": 0.817452609539032,
+      "logps/chosen": -5.186074733734131,
+      "logps/rejected": -7.289494514465332,
+      "loss": 0.6748,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -5.186074733734131,
+      "rewards/margins": 2.103419780731201,
+      "rewards/rejected": -7.289494514465332,
+      "step": 2865
+    },
+    {
+      "epoch": 1.5360428165245024,
+      "grad_norm": 4.935512747556083,
+      "learning_rate": 5.659964618738515e-07,
+      "logits/chosen": 0.44832196831703186,
+      "logits/rejected": 0.8838248252868652,
+      "logps/chosen": -4.709874153137207,
+      "logps/rejected": -7.236085414886475,
+      "loss": 0.6773,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.709874153137207,
+      "rewards/margins": 2.526211738586426,
+      "rewards/rejected": -7.236085414886475,
+      "step": 2870
+    },
+    {
+      "epoch": 1.538718849305904,
+      "grad_norm": 8.529505816619963,
+      "learning_rate": 5.644523678049509e-07,
+      "logits/chosen": 0.3706728518009186,
+      "logits/rejected": 0.9383756518363953,
+      "logps/chosen": -4.440794467926025,
+      "logps/rejected": -7.599753379821777,
+      "loss": 0.6617,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.440794467926025,
+      "rewards/margins": 3.1589579582214355,
+      "rewards/rejected": -7.599753379821777,
+      "step": 2875
+    },
+    {
+      "epoch": 1.5413948820873056,
+      "grad_norm": 3.5327064600504805,
+      "learning_rate": 5.629076484188952e-07,
+      "logits/chosen": 0.5346053242683411,
+      "logits/rejected": 0.9430686831474304,
+      "logps/chosen": -4.765469074249268,
+      "logps/rejected": -7.415452003479004,
+      "loss": 0.6598,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.765469074249268,
+      "rewards/margins": 2.6499829292297363,
+      "rewards/rejected": -7.415452003479004,
+      "step": 2880
+    },
+    {
+      "epoch": 1.544070914868707,
+      "grad_norm": 4.656245544112861,
+      "learning_rate": 5.613623187025587e-07,
+      "logits/chosen": 0.49984684586524963,
+      "logits/rejected": 1.0821008682250977,
+      "logps/chosen": -4.983664035797119,
+      "logps/rejected": -8.032984733581543,
+      "loss": 0.6686,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.983664035797119,
+      "rewards/margins": 3.049321413040161,
+      "rewards/rejected": -8.032984733581543,
+      "step": 2885
+    },
+    {
+      "epoch": 1.5467469476501088,
+      "grad_norm": 2.3110552238828577,
+      "learning_rate": 5.598163936487369e-07,
+      "logits/chosen": 0.5765513777732849,
+      "logits/rejected": 1.0297950506210327,
+      "logps/chosen": -5.634735584259033,
+      "logps/rejected": -8.715818405151367,
+      "loss": 0.6726,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -5.634735584259033,
+      "rewards/margins": 3.0810840129852295,
+      "rewards/rejected": -8.715818405151367,
+      "step": 2890
+    },
+    {
+      "epoch": 1.5494229804315103,
+      "grad_norm": 7.277162877079002,
+      "learning_rate": 5.582698882560017e-07,
+      "logits/chosen": 0.7015002369880676,
+      "logits/rejected": 1.0361802577972412,
+      "logps/chosen": -5.544536113739014,
+      "logps/rejected": -7.984232425689697,
+      "loss": 0.6688,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -5.544536113739014,
+      "rewards/margins": 2.4396963119506836,
+      "rewards/rejected": -7.984232425689697,
+      "step": 2895
+    },
+    {
+      "epoch": 1.5520990132129118,
+      "grad_norm": 5.148241086230649,
+      "learning_rate": 5.567228175285549e-07,
+      "logits/chosen": 0.406666100025177,
+      "logits/rejected": 1.078575611114502,
+      "logps/chosen": -4.360080242156982,
+      "logps/rejected": -8.947362899780273,
+      "loss": 0.6664,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.360080242156982,
+      "rewards/margins": 4.587283611297607,
+      "rewards/rejected": -8.947362899780273,
+      "step": 2900
+    },
+    {
+      "epoch": 1.5547750459943135,
+      "grad_norm": 4.514969063589977,
+      "learning_rate": 5.551751964760838e-07,
+      "logits/chosen": 0.627621054649353,
+      "logits/rejected": 1.1549687385559082,
+      "logps/chosen": -5.157894134521484,
+      "logps/rejected": -8.584880828857422,
+      "loss": 0.6607,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -5.157894134521484,
+      "rewards/margins": 3.426985502243042,
+      "rewards/rejected": -8.584880828857422,
+      "step": 2905
+    },
+    {
+      "epoch": 1.557451078775715,
+      "grad_norm": 3.014297858810689,
+      "learning_rate": 5.536270401136145e-07,
+      "logits/chosen": 0.4835987687110901,
+      "logits/rejected": 0.9562643766403198,
+      "logps/chosen": -4.336380958557129,
+      "logps/rejected": -7.383638858795166,
+      "loss": 0.6491,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.336380958557129,
+      "rewards/margins": 3.0472583770751953,
+      "rewards/rejected": -7.383638858795166,
+      "step": 2910
+    },
+    {
+      "epoch": 1.5601271115571165,
+      "grad_norm": 4.11937962671298,
+      "learning_rate": 5.520783634613667e-07,
+      "logits/chosen": 0.6388608813285828,
+      "logits/rejected": 0.9776711463928223,
+      "logps/chosen": -4.981477737426758,
+      "logps/rejected": -7.892369747161865,
+      "loss": 0.6667,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.981477737426758,
+      "rewards/margins": 2.9108920097351074,
+      "rewards/rejected": -7.892369747161865,
+      "step": 2915
+    },
+    {
+      "epoch": 1.5628031443385182,
+      "grad_norm": 4.137191350864541,
+      "learning_rate": 5.505291815446082e-07,
+      "logits/chosen": 0.5212299823760986,
+      "logits/rejected": 0.7920260429382324,
+      "logps/chosen": -4.761508464813232,
+      "logps/rejected": -7.3172502517700195,
+      "loss": 0.6829,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.761508464813232,
+      "rewards/margins": 2.555741310119629,
+      "rewards/rejected": -7.3172502517700195,
+      "step": 2920
+    },
+    {
+      "epoch": 1.5654791771199197,
+      "grad_norm": 3.227492668634208,
+      "learning_rate": 5.489795093935089e-07,
+      "logits/chosen": 0.5472073554992676,
+      "logits/rejected": 0.9888783693313599,
+      "logps/chosen": -4.479696273803711,
+      "logps/rejected": -7.8708953857421875,
+      "loss": 0.6571,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.479696273803711,
+      "rewards/margins": 3.3911991119384766,
+      "rewards/rejected": -7.8708953857421875,
+      "step": 2925
+    },
+    {
+      "epoch": 1.5681552099013212,
+      "grad_norm": 2.4173201158521884,
+      "learning_rate": 5.474293620429946e-07,
+      "logits/chosen": 0.3226237893104553,
+      "logits/rejected": 0.8754428625106812,
+      "logps/chosen": -4.2056989669799805,
+      "logps/rejected": -8.222943305969238,
+      "loss": 0.653,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.2056989669799805,
+      "rewards/margins": 4.017243385314941,
+      "rewards/rejected": -8.222943305969238,
+      "step": 2930
+    },
+    {
+      "epoch": 1.570831242682723,
+      "grad_norm": 5.2696118083649495,
+      "learning_rate": 5.458787545326018e-07,
+      "logits/chosen": 0.4318312704563141,
+      "logits/rejected": 0.8220310211181641,
+      "logps/chosen": -4.616839408874512,
+      "logps/rejected": -7.033632755279541,
+      "loss": 0.672,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.616839408874512,
+      "rewards/margins": 2.41679310798645,
+      "rewards/rejected": -7.033632755279541,
+      "step": 2935
+    },
+    {
+      "epoch": 1.5735072754641244,
+      "grad_norm": 5.800019911342011,
+      "learning_rate": 5.443277019063311e-07,
+      "logits/chosen": 0.3738608956336975,
+      "logits/rejected": 0.8496935963630676,
+      "logps/chosen": -4.309153079986572,
+      "logps/rejected": -7.437277317047119,
+      "loss": 0.6655,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.309153079986572,
+      "rewards/margins": 3.128124713897705,
+      "rewards/rejected": -7.437277317047119,
+      "step": 2940
+    },
+    {
+      "epoch": 1.5761833082455259,
+      "grad_norm": 7.447918641361906,
+      "learning_rate": 5.427762192125023e-07,
+      "logits/chosen": 0.5561836361885071,
+      "logits/rejected": 0.8606193661689758,
+      "logps/chosen": -5.683553695678711,
+      "logps/rejected": -7.377643585205078,
+      "loss": 0.6659,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -5.683553695678711,
+      "rewards/margins": 1.6940898895263672,
+      "rewards/rejected": -7.377643585205078,
+      "step": 2945
+    },
+    {
+      "epoch": 1.5788593410269276,
+      "grad_norm": 6.2333456480560185,
+      "learning_rate": 5.41224321503607e-07,
+      "logits/chosen": 0.6866841316223145,
+      "logits/rejected": 0.9957801699638367,
+      "logps/chosen": -5.446958541870117,
+      "logps/rejected": -7.7111711502075195,
+      "loss": 0.6729,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -5.446958541870117,
+      "rewards/margins": 2.2642123699188232,
+      "rewards/rejected": -7.7111711502075195,
+      "step": 2950
+    },
+    {
+      "epoch": 1.5815353738083293,
+      "grad_norm": 5.75668845621592,
+      "learning_rate": 5.396720238361637e-07,
+      "logits/chosen": 0.6823299527168274,
+      "logits/rejected": 1.0507533550262451,
+      "logps/chosen": -5.438050746917725,
+      "logps/rejected": -8.14564323425293,
+      "loss": 0.6715,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -5.438050746917725,
+      "rewards/margins": 2.7075929641723633,
+      "rewards/rejected": -8.14564323425293,
+      "step": 2955
+    },
+    {
+      "epoch": 1.5842114065897306,
+      "grad_norm": 3.9348166158049755,
+      "learning_rate": 5.381193412705711e-07,
+      "logits/chosen": 0.5944125056266785,
+      "logits/rejected": 1.1707426309585571,
+      "logps/chosen": -5.608029365539551,
+      "logps/rejected": -9.530790328979492,
+      "loss": 0.6522,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -5.608029365539551,
+      "rewards/margins": 3.922760009765625,
+      "rewards/rejected": -9.530790328979492,
+      "step": 2960
+    },
+    {
+      "epoch": 1.5868874393711323,
+      "grad_norm": 5.982246380843602,
+      "learning_rate": 5.365662888709622e-07,
+      "logits/chosen": 0.6211116313934326,
+      "logits/rejected": 1.3079092502593994,
+      "logps/chosen": -5.796026229858398,
+      "logps/rejected": -10.381234169006348,
+      "loss": 0.6693,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -5.796026229858398,
+      "rewards/margins": 4.585208415985107,
+      "rewards/rejected": -10.381234169006348,
+      "step": 2965
+    },
+    {
+      "epoch": 1.589563472152534,
+      "grad_norm": 5.134749756328359,
+      "learning_rate": 5.350128817050585e-07,
+      "logits/chosen": 0.4572945535182953,
+      "logits/rejected": 1.1549270153045654,
+      "logps/chosen": -5.122225284576416,
+      "logps/rejected": -8.820940017700195,
+      "loss": 0.6649,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -5.122225284576416,
+      "rewards/margins": 3.6987144947052,
+      "rewards/rejected": -8.820940017700195,
+      "step": 2970
+    },
+    {
+      "epoch": 1.5922395049339353,
+      "grad_norm": 4.726169104227263,
+      "learning_rate": 5.334591348440229e-07,
+      "logits/chosen": 0.5736401677131653,
+      "logits/rejected": 1.037611722946167,
+      "logps/chosen": -5.320729732513428,
+      "logps/rejected": -8.281679153442383,
+      "loss": 0.6684,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -5.320729732513428,
+      "rewards/margins": 2.9609498977661133,
+      "rewards/rejected": -8.281679153442383,
+      "step": 2975
+    },
+    {
+      "epoch": 1.594915537715337,
+      "grad_norm": 5.402936515991711,
+      "learning_rate": 5.319050633623141e-07,
+      "logits/chosen": 0.4811383783817291,
+      "logits/rejected": 0.887110710144043,
+      "logps/chosen": -5.136692523956299,
+      "logps/rejected": -7.474466800689697,
+      "loss": 0.6794,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -5.136692523956299,
+      "rewards/margins": 2.3377745151519775,
+      "rewards/rejected": -7.474466800689697,
+      "step": 2980
+    },
+    {
+      "epoch": 1.5975915704967387,
+      "grad_norm": 3.9862367243524055,
+      "learning_rate": 5.303506823375409e-07,
+      "logits/chosen": 0.5122233033180237,
+      "logits/rejected": 0.8712376356124878,
+      "logps/chosen": -5.095685005187988,
+      "logps/rejected": -7.4881086349487305,
+      "loss": 0.6762,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -5.095685005187988,
+      "rewards/margins": 2.392423629760742,
+      "rewards/rejected": -7.4881086349487305,
+      "step": 2985
+    },
+    {
+      "epoch": 1.60026760327814,
+      "grad_norm": 9.753359434583151,
+      "learning_rate": 5.287960068503143e-07,
+      "logits/chosen": 0.37681570649147034,
+      "logits/rejected": 0.70493483543396,
+      "logps/chosen": -4.5189337730407715,
+      "logps/rejected": -6.55059814453125,
+      "loss": 0.6553,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.5189337730407715,
+      "rewards/margins": 2.0316643714904785,
+      "rewards/rejected": -6.55059814453125,
+      "step": 2990
+    },
+    {
+      "epoch": 1.6029436360595417,
+      "grad_norm": 2.9289432270296385,
+      "learning_rate": 5.272410519841032e-07,
+      "logits/chosen": 0.3510708212852478,
+      "logits/rejected": 1.017427682876587,
+      "logps/chosen": -4.248021125793457,
+      "logps/rejected": -8.995039939880371,
+      "loss": 0.65,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.248021125793457,
+      "rewards/margins": 4.747018814086914,
+      "rewards/rejected": -8.995039939880371,
+      "step": 2995
+    },
+    {
+      "epoch": 1.6056196688409434,
+      "grad_norm": 2.85241795838451,
+      "learning_rate": 5.256858328250861e-07,
+      "logits/chosen": 0.5438185334205627,
+      "logits/rejected": 1.0563455820083618,
+      "logps/chosen": -5.394139766693115,
+      "logps/rejected": -8.63255500793457,
+      "loss": 0.6707,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -5.394139766693115,
+      "rewards/margins": 3.238415479660034,
+      "rewards/rejected": -8.63255500793457,
+      "step": 3000
+    },
+    {
+      "epoch": 1.608295701622345,
+      "grad_norm": 3.048733466904856,
+      "learning_rate": 5.241303644620063e-07,
+      "logits/chosen": 0.5721206665039062,
+      "logits/rejected": 0.7040168046951294,
+      "logps/chosen": -5.922365665435791,
+      "logps/rejected": -7.277604103088379,
+      "loss": 0.6777,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -5.922365665435791,
+      "rewards/margins": 1.355238914489746,
+      "rewards/rejected": -7.277604103088379,
+      "step": 3005
+    },
+    {
+      "epoch": 1.6109717344037464,
+      "grad_norm": 4.483576673862505,
+      "learning_rate": 5.225746619860248e-07,
+      "logits/chosen": 0.5236977934837341,
+      "logits/rejected": 0.9320691823959351,
+      "logps/chosen": -5.152453899383545,
+      "logps/rejected": -8.04359245300293,
+      "loss": 0.6745,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -5.152453899383545,
+      "rewards/margins": 2.891138792037964,
+      "rewards/rejected": -8.04359245300293,
+      "step": 3010
+    },
+    {
+      "epoch": 1.6136477671851481,
+      "grad_norm": 5.009674636888851,
+      "learning_rate": 5.210187404905735e-07,
+      "logits/chosen": 0.5780618786811829,
+      "logits/rejected": 1.057121992111206,
+      "logps/chosen": -5.319887161254883,
+      "logps/rejected": -8.295541763305664,
+      "loss": 0.6658,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -5.319887161254883,
+      "rewards/margins": 2.9756534099578857,
+      "rewards/rejected": -8.295541763305664,
+      "step": 3015
+    },
+    {
+      "epoch": 1.6163237999665496,
+      "grad_norm": 3.077968652506768,
+      "learning_rate": 5.194626150712098e-07,
+      "logits/chosen": 0.4693143367767334,
+      "logits/rejected": 0.9520252346992493,
+      "logps/chosen": -5.2622504234313965,
+      "logps/rejected": -7.701306343078613,
+      "loss": 0.6734,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -5.2622504234313965,
+      "rewards/margins": 2.4390549659729004,
+      "rewards/rejected": -7.701306343078613,
+      "step": 3020
+    },
+    {
+      "epoch": 1.6189998327479511,
+      "grad_norm": 4.298792782658865,
+      "learning_rate": 5.179063008254695e-07,
+      "logits/chosen": 0.6581199765205383,
+      "logits/rejected": 0.9134021997451782,
+      "logps/chosen": -5.953556537628174,
+      "logps/rejected": -7.705558776855469,
+      "loss": 0.6713,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -5.953556537628174,
+      "rewards/margins": 1.7520027160644531,
+      "rewards/rejected": -7.705558776855469,
+      "step": 3025
+    },
+    {
+      "epoch": 1.6216758655293528,
+      "grad_norm": 3.6193592087270274,
+      "learning_rate": 5.163498128527199e-07,
+      "logits/chosen": 0.5181972980499268,
+      "logits/rejected": 0.9898965954780579,
+      "logps/chosen": -5.613431453704834,
+      "logps/rejected": -8.23238754272461,
+      "loss": 0.6708,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -5.613431453704834,
+      "rewards/margins": 2.6189568042755127,
+      "rewards/rejected": -8.23238754272461,
+      "step": 3030
+    },
+    {
+      "epoch": 1.6243518983107543,
+      "grad_norm": 4.750906690203447,
+      "learning_rate": 5.147931662540144e-07,
+      "logits/chosen": 0.5471720099449158,
+      "logits/rejected": 0.9917758107185364,
+      "logps/chosen": -4.7455854415893555,
+      "logps/rejected": -7.251121520996094,
+      "loss": 0.6731,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.7455854415893555,
+      "rewards/margins": 2.5055360794067383,
+      "rewards/rejected": -7.251121520996094,
+      "step": 3035
+    },
+    {
+      "epoch": 1.6270279310921558,
+      "grad_norm": 6.790489502759956,
+      "learning_rate": 5.132363761319449e-07,
+      "logits/chosen": 0.4837442934513092,
+      "logits/rejected": 0.8884152173995972,
+      "logps/chosen": -4.820464134216309,
+      "logps/rejected": -7.918983459472656,
+      "loss": 0.656,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.820464134216309,
+      "rewards/margins": 3.0985193252563477,
+      "rewards/rejected": -7.918983459472656,
+      "step": 3040
+    },
+    {
+      "epoch": 1.6297039638735575,
+      "grad_norm": 3.9458736979792413,
+      "learning_rate": 5.116794575904962e-07,
+      "logits/chosen": 0.5177257061004639,
+      "logits/rejected": 0.955532431602478,
+      "logps/chosen": -4.5562286376953125,
+      "logps/rejected": -7.046250343322754,
+      "loss": 0.6611,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.5562286376953125,
+      "rewards/margins": 2.490021228790283,
+      "rewards/rejected": -7.046250343322754,
+      "step": 3045
+    },
+    {
+      "epoch": 1.632379996654959,
+      "grad_norm": 3.898971175351481,
+      "learning_rate": 5.101224257348987e-07,
+      "logits/chosen": 0.4544016420841217,
+      "logits/rejected": 0.887833297252655,
+      "logps/chosen": -4.651072978973389,
+      "logps/rejected": -7.357897758483887,
+      "loss": 0.6665,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.651072978973389,
+      "rewards/margins": 2.706824779510498,
+      "rewards/rejected": -7.357897758483887,
+      "step": 3050
+    },
+    {
+      "epoch": 1.6350560294363605,
+      "grad_norm": 2.776157515813491,
+      "learning_rate": 5.085652956714823e-07,
+      "logits/chosen": 0.699173092842102,
+      "logits/rejected": 1.0971158742904663,
+      "logps/chosen": -6.07081413269043,
+      "logps/rejected": -8.559813499450684,
+      "loss": 0.6607,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -6.07081413269043,
+      "rewards/margins": 2.488999843597412,
+      "rewards/rejected": -8.559813499450684,
+      "step": 3055
+    },
+    {
+      "epoch": 1.6377320622177622,
+      "grad_norm": 5.673592751669144,
+      "learning_rate": 5.070080825075298e-07,
+      "logits/chosen": 0.5794312357902527,
+      "logits/rejected": 0.8734928965568542,
+      "logps/chosen": -5.387199401855469,
+      "logps/rejected": -7.4864182472229,
+      "loss": 0.6733,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -5.387199401855469,
+      "rewards/margins": 2.099219799041748,
+      "rewards/rejected": -7.4864182472229,
+      "step": 3060
+    },
+    {
+      "epoch": 1.6404080949991637,
+      "grad_norm": 7.376127074634331,
+      "learning_rate": 5.0545080135113e-07,
+      "logits/chosen": 0.5828385353088379,
+      "logits/rejected": 1.0233455896377563,
+      "logps/chosen": -5.084925651550293,
+      "logps/rejected": -8.580615997314453,
+      "loss": 0.6623,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -5.084925651550293,
+      "rewards/margins": 3.4956908226013184,
+      "rewards/rejected": -8.580615997314453,
+      "step": 3065
+    },
+    {
+      "epoch": 1.6430841277805652,
+      "grad_norm": 3.1997853199482744,
+      "learning_rate": 5.038934673110316e-07,
+      "logits/chosen": 0.6661735773086548,
+      "logits/rejected": 1.115176796913147,
+      "logps/chosen": -6.2261762619018555,
+      "logps/rejected": -9.238046646118164,
+      "loss": 0.6593,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -6.2261762619018555,
+      "rewards/margins": 3.011871337890625,
+      "rewards/rejected": -9.238046646118164,
+      "step": 3070
+    },
+    {
+      "epoch": 1.645760160561967,
+      "grad_norm": 6.176103184208479,
+      "learning_rate": 5.023360954964963e-07,
+      "logits/chosen": 0.8922260999679565,
+      "logits/rejected": 1.143858551979065,
+      "logps/chosen": -6.561410427093506,
+      "logps/rejected": -8.872845649719238,
+      "loss": 0.6674,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -6.561410427093506,
+      "rewards/margins": 2.3114349842071533,
+      "rewards/rejected": -8.872845649719238,
+      "step": 3075
+    },
+    {
+      "epoch": 1.6484361933433684,
+      "grad_norm": 6.004387026200529,
+      "learning_rate": 5.007787010171524e-07,
+      "logits/chosen": 0.5648599863052368,
+      "logits/rejected": 1.0675740242004395,
+      "logps/chosen": -5.108396530151367,
+      "logps/rejected": -9.04470157623291,
+      "loss": 0.6571,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -5.108396530151367,
+      "rewards/margins": 3.936305284500122,
+      "rewards/rejected": -9.04470157623291,
+      "step": 3080
+    },
+    {
+      "epoch": 1.65111222612477,
+      "grad_norm": 5.006979839148598,
+      "learning_rate": 4.992212989828477e-07,
+      "logits/chosen": 0.7814642786979675,
+      "logits/rejected": 1.1484363079071045,
+      "logps/chosen": -5.69457483291626,
+      "logps/rejected": -8.227791786193848,
+      "loss": 0.6669,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -5.69457483291626,
+      "rewards/margins": 2.533216953277588,
+      "rewards/rejected": -8.227791786193848,
+      "step": 3085
+    },
+    {
+      "epoch": 1.6537882589061716,
+      "grad_norm": 7.861403645292318,
+      "learning_rate": 4.976639045035036e-07,
+      "logits/chosen": 0.6518059968948364,
+      "logits/rejected": 1.0227234363555908,
+      "logps/chosen": -5.655069828033447,
+      "logps/rejected": -8.017797470092773,
+      "loss": 0.6805,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -5.655069828033447,
+      "rewards/margins": 2.3627266883850098,
+      "rewards/rejected": -8.017797470092773,
+      "step": 3090
+    },
+    {
+      "epoch": 1.6564642916875731,
+      "grad_norm": 7.939571091130441,
+      "learning_rate": 4.961065326889683e-07,
+      "logits/chosen": 0.699022650718689,
+      "logits/rejected": 1.0907307863235474,
+      "logps/chosen": -5.782702445983887,
+      "logps/rejected": -8.830089569091797,
+      "loss": 0.6768,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -5.782702445983887,
+      "rewards/margins": 3.047386884689331,
+      "rewards/rejected": -8.830089569091797,
+      "step": 3095
+    },
+    {
+      "epoch": 1.6591403244689746,
+      "grad_norm": 4.339089835457751,
+      "learning_rate": 4.9454919864887e-07,
+      "logits/chosen": 0.5124965906143188,
+      "logits/rejected": 0.9299848675727844,
+      "logps/chosen": -5.872725963592529,
+      "logps/rejected": -8.123193740844727,
+      "loss": 0.6712,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -5.872725963592529,
+      "rewards/margins": 2.2504684925079346,
+      "rewards/rejected": -8.123193740844727,
+      "step": 3100
+    },
+    {
+      "epoch": 1.6618163572503764,
+      "grad_norm": 2.8648133287974544,
+      "learning_rate": 4.929919174924701e-07,
+      "logits/chosen": 0.546550452709198,
+      "logits/rejected": 0.8062145113945007,
+      "logps/chosen": -5.374571800231934,
+      "logps/rejected": -6.9999589920043945,
+      "loss": 0.6776,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -5.374571800231934,
+      "rewards/margins": 1.6253865957260132,
+      "rewards/rejected": -6.9999589920043945,
+      "step": 3105
+    },
+    {
+      "epoch": 1.6644923900317778,
+      "grad_norm": 2.959594988126024,
+      "learning_rate": 4.914347043285177e-07,
+      "logits/chosen": 0.5598922967910767,
+      "logits/rejected": 0.9223871231079102,
+      "logps/chosen": -5.308091163635254,
+      "logps/rejected": -8.301132202148438,
+      "loss": 0.6555,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -5.308091163635254,
+      "rewards/margins": 2.9930405616760254,
+      "rewards/rejected": -8.301132202148438,
+      "step": 3110
+    },
+    {
+      "epoch": 1.6671684228131793,
+      "grad_norm": 2.8531033508141506,
+      "learning_rate": 4.898775742651013e-07,
+      "logits/chosen": 0.6701597571372986,
+      "logits/rejected": 0.9836909174919128,
+      "logps/chosen": -5.183486461639404,
+      "logps/rejected": -7.901432037353516,
+      "loss": 0.6661,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -5.183486461639404,
+      "rewards/margins": 2.7179460525512695,
+      "rewards/rejected": -7.901432037353516,
+      "step": 3115
+    },
+    {
+      "epoch": 1.669844455594581,
+      "grad_norm": 4.386413590293837,
+      "learning_rate": 4.883205424095037e-07,
+      "logits/chosen": 0.48722219467163086,
+      "logits/rejected": 0.9049069285392761,
+      "logps/chosen": -4.602488040924072,
+      "logps/rejected": -7.275084495544434,
+      "loss": 0.6659,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.602488040924072,
+      "rewards/margins": 2.6725974082946777,
+      "rewards/rejected": -7.275084495544434,
+      "step": 3120
+    },
+    {
+      "epoch": 1.6725204883759828,
+      "grad_norm": 5.7283758259683495,
+      "learning_rate": 4.86763623868055e-07,
+      "logits/chosen": 0.5263519883155823,
+      "logits/rejected": 0.8598620295524597,
+      "logps/chosen": -4.6285810470581055,
+      "logps/rejected": -7.445448398590088,
+      "loss": 0.6688,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.6285810470581055,
+      "rewards/margins": 2.8168673515319824,
+      "rewards/rejected": -7.445448398590088,
+      "step": 3125
+    },
+    {
+      "epoch": 1.675196521157384,
+      "grad_norm": 4.623319867811598,
+      "learning_rate": 4.852068337459856e-07,
+      "logits/chosen": 0.5010684728622437,
+      "logits/rejected": 0.8111163973808289,
+      "logps/chosen": -4.829095363616943,
+      "logps/rejected": -6.6501054763793945,
+      "loss": 0.6704,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.829095363616943,
+      "rewards/margins": 1.821009874343872,
+      "rewards/rejected": -6.6501054763793945,
+      "step": 3130
+    },
+    {
+      "epoch": 1.6778725539387858,
+      "grad_norm": 4.186174539749044,
+      "learning_rate": 4.8365018714728e-07,
+      "logits/chosen": 0.6984392404556274,
+      "logits/rejected": 1.0659825801849365,
+      "logps/chosen": -5.266923427581787,
+      "logps/rejected": -7.703289031982422,
+      "loss": 0.669,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -5.266923427581787,
+      "rewards/margins": 2.4363653659820557,
+      "rewards/rejected": -7.703289031982422,
+      "step": 3135
+    },
+    {
+      "epoch": 1.6805485867201875,
+      "grad_norm": 7.1535987849422815,
+      "learning_rate": 4.820936991745304e-07,
+      "logits/chosen": 0.4620518684387207,
+      "logits/rejected": 0.8315756916999817,
+      "logps/chosen": -4.8800811767578125,
+      "logps/rejected": -7.534416198730469,
+      "loss": 0.6609,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.8800811767578125,
+      "rewards/margins": 2.654334306716919,
+      "rewards/rejected": -7.534416198730469,
+      "step": 3140
+    },
+    {
+      "epoch": 1.6832246195015887,
+      "grad_norm": 6.81698198481438,
+      "learning_rate": 4.8053738492879e-07,
+      "logits/chosen": 0.395782470703125,
+      "logits/rejected": 0.9724879264831543,
+      "logps/chosen": -4.2613911628723145,
+      "logps/rejected": -7.715605735778809,
+      "loss": 0.6632,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.2613911628723145,
+      "rewards/margins": 3.454214572906494,
+      "rewards/rejected": -7.715605735778809,
+      "step": 3145
+    },
+    {
+      "epoch": 1.6859006522829905,
+      "grad_norm": 4.756880676318941,
+      "learning_rate": 4.789812595094265e-07,
+      "logits/chosen": 0.32011279463768005,
+      "logits/rejected": 0.7607211470603943,
+      "logps/chosen": -4.255454063415527,
+      "logps/rejected": -7.169301509857178,
+      "loss": 0.6605,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.255454063415527,
+      "rewards/margins": 2.9138481616973877,
+      "rewards/rejected": -7.169301509857178,
+      "step": 3150
+    },
+    {
+      "epoch": 1.6885766850643922,
+      "grad_norm": 3.968974821400292,
+      "learning_rate": 4.774253380139752e-07,
+      "logits/chosen": 0.4557875096797943,
+      "logits/rejected": 0.9234863519668579,
+      "logps/chosen": -4.5074262619018555,
+      "logps/rejected": -7.799464225769043,
+      "loss": 0.6688,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.5074262619018555,
+      "rewards/margins": 3.292039155960083,
+      "rewards/rejected": -7.799464225769043,
+      "step": 3155
+    },
+    {
+      "epoch": 1.6912527178457935,
+      "grad_norm": 4.245738125838862,
+      "learning_rate": 4.758696355379936e-07,
+      "logits/chosen": 0.4622158110141754,
+      "logits/rejected": 0.8049589991569519,
+      "logps/chosen": -4.675787925720215,
+      "logps/rejected": -7.658982753753662,
+      "loss": 0.6628,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.675787925720215,
+      "rewards/margins": 2.9831948280334473,
+      "rewards/rejected": -7.658982753753662,
+      "step": 3160
+    },
+    {
+      "epoch": 1.6939287506271952,
+      "grad_norm": 4.071929760233066,
+      "learning_rate": 4.743141671749138e-07,
+      "logits/chosen": 0.4790673851966858,
+      "logits/rejected": 0.7104719877243042,
+      "logps/chosen": -5.718655109405518,
+      "logps/rejected": -7.399728298187256,
+      "loss": 0.6754,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -5.718655109405518,
+      "rewards/margins": 1.6810745000839233,
+      "rewards/rejected": -7.399728298187256,
+      "step": 3165
+    },
+    {
+      "epoch": 1.6966047834085969,
+      "grad_norm": 2.477310734559971,
+      "learning_rate": 4.727589480158968e-07,
+      "logits/chosen": 0.38287216424942017,
+      "logits/rejected": 0.956261932849884,
+      "logps/chosen": -5.030936241149902,
+      "logps/rejected": -8.560375213623047,
+      "loss": 0.6687,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -5.030936241149902,
+      "rewards/margins": 3.5294387340545654,
+      "rewards/rejected": -8.560375213623047,
+      "step": 3170
+    },
+    {
+      "epoch": 1.6992808161899984,
+      "grad_norm": 5.410475822656681,
+      "learning_rate": 4.712039931496855e-07,
+      "logits/chosen": 0.43522852659225464,
+      "logits/rejected": 0.8723834156990051,
+      "logps/chosen": -4.626428604125977,
+      "logps/rejected": -7.244049072265625,
+      "loss": 0.6842,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.626428604125977,
+      "rewards/margins": 2.617621421813965,
+      "rewards/rejected": -7.244049072265625,
+      "step": 3175
+    },
+    {
+      "epoch": 1.7019568489713999,
+      "grad_norm": 2.9629463327662235,
+      "learning_rate": 4.6964931766245905e-07,
+      "logits/chosen": 0.6233412027359009,
+      "logits/rejected": 0.9540999531745911,
+      "logps/chosen": -5.334083557128906,
+      "logps/rejected": -8.109797477722168,
+      "loss": 0.6664,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -5.334083557128906,
+      "rewards/margins": 2.775714874267578,
+      "rewards/rejected": -8.109797477722168,
+      "step": 3180
+    },
+    {
+      "epoch": 1.7046328817528016,
+      "grad_norm": 4.4986974122154875,
+      "learning_rate": 4.6809493663768575e-07,
+      "logits/chosen": 0.5591728091239929,
+      "logits/rejected": 0.7440407872200012,
+      "logps/chosen": -5.12191104888916,
+      "logps/rejected": -6.48370885848999,
+      "loss": 0.6757,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -5.12191104888916,
+      "rewards/margins": 1.3617980480194092,
+      "rewards/rejected": -6.48370885848999,
+      "step": 3185
+    },
+    {
+      "epoch": 1.707308914534203,
+      "grad_norm": 3.358127983547076,
+      "learning_rate": 4.6654086515597716e-07,
+      "logits/chosen": 0.7406435012817383,
+      "logits/rejected": 0.9592331051826477,
+      "logps/chosen": -5.942282676696777,
+      "logps/rejected": -8.265983581542969,
+      "loss": 0.6627,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -5.942282676696777,
+      "rewards/margins": 2.3237013816833496,
+      "rewards/rejected": -8.265983581542969,
+      "step": 3190
+    },
+    {
+      "epoch": 1.7099849473156046,
+      "grad_norm": 4.2108906311997005,
+      "learning_rate": 4.6498711829494154e-07,
+      "logits/chosen": 0.4864862561225891,
+      "logits/rejected": 0.8411710858345032,
+      "logps/chosen": -4.813061237335205,
+      "logps/rejected": -7.783934593200684,
+      "loss": 0.6615,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.813061237335205,
+      "rewards/margins": 2.9708735942840576,
+      "rewards/rejected": -7.783934593200684,
+      "step": 3195
+    },
+    {
+      "epoch": 1.7126609800970063,
+      "grad_norm": 4.047776264871089,
+      "learning_rate": 4.6343371112903777e-07,
+      "logits/chosen": 0.6882549524307251,
+      "logits/rejected": 0.8133997917175293,
+      "logps/chosen": -5.753777027130127,
+      "logps/rejected": -7.26686954498291,
+      "loss": 0.6746,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -5.753777027130127,
+      "rewards/margins": 1.513091802597046,
+      "rewards/rejected": -7.26686954498291,
+      "step": 3200
+    },
+    {
+      "epoch": 1.7126609800970063,
+      "eval_logits/chosen": 0.699344277381897,
+      "eval_logits/rejected": 1.0711151361465454,
+      "eval_logps/chosen": -5.101490020751953,
+      "eval_logps/rejected": -8.220707893371582,
+      "eval_loss": 0.664119303226471,
+      "eval_rewards/accuracies": 0.6802670359611511,
+      "eval_rewards/chosen": -5.101490020751953,
+      "eval_rewards/margins": 3.119217872619629,
+      "eval_rewards/rejected": -8.220707893371582,
+      "eval_runtime": 40.7564,
+      "eval_samples_per_second": 33.001,
+      "eval_steps_per_second": 8.269,
+      "step": 3200
+    },
+    {
+      "epoch": 1.7153370128784078,
+      "grad_norm": 3.5587022384859304,
+      "learning_rate": 4.618806587294291e-07,
+      "logits/chosen": 0.4815581738948822,
+      "logits/rejected": 0.935097873210907,
+      "logps/chosen": -4.884929656982422,
+      "logps/rejected": -8.489778518676758,
+      "loss": 0.6611,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.884929656982422,
+      "rewards/margins": 3.604849338531494,
+      "rewards/rejected": -8.489778518676758,
+      "step": 3205
+    },
+    {
+      "epoch": 1.7180130456598093,
+      "grad_norm": 4.4736159434962515,
+      "learning_rate": 4.603279761638365e-07,
+      "logits/chosen": 0.41907739639282227,
+      "logits/rejected": 0.6237906217575073,
+      "logps/chosen": -5.430784702301025,
+      "logps/rejected": -7.202707767486572,
+      "loss": 0.6809,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -5.430784702301025,
+      "rewards/margins": 1.7719230651855469,
+      "rewards/rejected": -7.202707767486572,
+      "step": 3210
+    },
+    {
+      "epoch": 1.720689078441211,
+      "grad_norm": 3.6947123910512674,
+      "learning_rate": 4.5877567849639315e-07,
+      "logits/chosen": 0.5396860241889954,
+      "logits/rejected": 1.02396559715271,
+      "logps/chosen": -5.245244026184082,
+      "logps/rejected": -8.331345558166504,
+      "loss": 0.665,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -5.245244026184082,
+      "rewards/margins": 3.0861012935638428,
+      "rewards/rejected": -8.331345558166504,
+      "step": 3215
+    },
+    {
+      "epoch": 1.7233651112226125,
+      "grad_norm": 7.639725343626722,
+      "learning_rate": 4.572237807874979e-07,
+      "logits/chosen": 0.7041572332382202,
+      "logits/rejected": 0.9497126340866089,
+      "logps/chosen": -6.765023231506348,
+      "logps/rejected": -8.318975448608398,
+      "loss": 0.6783,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -6.765023231506348,
+      "rewards/margins": 1.5539530515670776,
+      "rewards/rejected": -8.318975448608398,
+      "step": 3220
+    },
+    {
+      "epoch": 1.726041144004014,
+      "grad_norm": 4.933908128851974,
+      "learning_rate": 4.5567229809366895e-07,
+      "logits/chosen": 0.5847693085670471,
+      "logits/rejected": 1.0567636489868164,
+      "logps/chosen": -5.342928409576416,
+      "logps/rejected": -8.867692947387695,
+      "loss": 0.6692,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -5.342928409576416,
+      "rewards/margins": 3.5247650146484375,
+      "rewards/rejected": -8.867692947387695,
+      "step": 3225
+    },
+    {
+      "epoch": 1.7287171767854157,
+      "grad_norm": 2.3797637479934544,
+      "learning_rate": 4.541212454673984e-07,
+      "logits/chosen": 0.7964419722557068,
+      "logits/rejected": 1.0569311380386353,
+      "logps/chosen": -6.4530792236328125,
+      "logps/rejected": -8.868021011352539,
+      "loss": 0.6653,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -6.4530792236328125,
+      "rewards/margins": 2.4149413108825684,
+      "rewards/rejected": -8.868021011352539,
+      "step": 3230
+    },
+    {
+      "epoch": 1.7313932095668172,
+      "grad_norm": 3.8945825108759506,
+      "learning_rate": 4.525706379570055e-07,
+      "logits/chosen": 0.8039935827255249,
+      "logits/rejected": 1.268603801727295,
+      "logps/chosen": -5.975590705871582,
+      "logps/rejected": -8.999273300170898,
+      "loss": 0.6573,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -5.975590705871582,
+      "rewards/margins": 3.023682117462158,
+      "rewards/rejected": -8.999273300170898,
+      "step": 3235
+    },
+    {
+      "epoch": 1.7340692423482187,
+      "grad_norm": 3.269392422102451,
+      "learning_rate": 4.510204906064911e-07,
+      "logits/chosen": 0.6911996603012085,
+      "logits/rejected": 1.2800791263580322,
+      "logps/chosen": -5.978976726531982,
+      "logps/rejected": -10.316455841064453,
+      "loss": 0.6642,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -5.978976726531982,
+      "rewards/margins": 4.337479114532471,
+      "rewards/rejected": -10.316455841064453,
+      "step": 3240
+    },
+    {
+      "epoch": 1.7367452751296204,
+      "grad_norm": 6.904078549516082,
+      "learning_rate": 4.4947081845539177e-07,
+      "logits/chosen": 0.7624322175979614,
+      "logits/rejected": 1.080706000328064,
+      "logps/chosen": -6.239609718322754,
+      "logps/rejected": -9.026888847351074,
+      "loss": 0.6659,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -6.239609718322754,
+      "rewards/margins": 2.7872796058654785,
+      "rewards/rejected": -9.026888847351074,
+      "step": 3245
+    },
+    {
+      "epoch": 1.739421307911022,
+      "grad_norm": 5.62464746770437,
+      "learning_rate": 4.479216365386333e-07,
+      "logits/chosen": 0.5803827047348022,
+      "logits/rejected": 1.280809998512268,
+      "logps/chosen": -5.526041030883789,
+      "logps/rejected": -9.4818754196167,
+      "loss": 0.663,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -5.526041030883789,
+      "rewards/margins": 3.9558346271514893,
+      "rewards/rejected": -9.4818754196167,
+      "step": 3250
+    },
+    {
+      "epoch": 1.7420973406924234,
+      "grad_norm": 4.5823974923556205,
+      "learning_rate": 4.4637295988638555e-07,
+      "logits/chosen": 0.7770492434501648,
+      "logits/rejected": 1.1709457635879517,
+      "logps/chosen": -5.78334903717041,
+      "logps/rejected": -9.09637451171875,
+      "loss": 0.6613,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -5.78334903717041,
+      "rewards/margins": 3.313025951385498,
+      "rewards/rejected": -9.09637451171875,
+      "step": 3255
+    },
+    {
+      "epoch": 1.744773373473825,
+      "grad_norm": 3.3968243524775117,
+      "learning_rate": 4.4482480352391623e-07,
+      "logits/chosen": 0.6010516285896301,
+      "logits/rejected": 1.217570424079895,
+      "logps/chosen": -5.541954040527344,
+      "logps/rejected": -9.428802490234375,
+      "loss": 0.6629,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -5.541954040527344,
+      "rewards/margins": 3.8868484497070312,
+      "rewards/rejected": -9.428802490234375,
+      "step": 3260
+    },
+    {
+      "epoch": 1.7474494062552266,
+      "grad_norm": 4.5871503835404175,
+      "learning_rate": 4.4327718247144507e-07,
+      "logits/chosen": 0.6734186410903931,
+      "logits/rejected": 1.0896427631378174,
+      "logps/chosen": -5.313602924346924,
+      "logps/rejected": -7.844895839691162,
+      "loss": 0.6671,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -5.313602924346924,
+      "rewards/margins": 2.53129243850708,
+      "rewards/rejected": -7.844895839691162,
+      "step": 3265
+    },
+    {
+      "epoch": 1.750125439036628,
+      "grad_norm": 4.486069857877553,
+      "learning_rate": 4.417301117439984e-07,
+      "logits/chosen": 0.7627257704734802,
+      "logits/rejected": 1.1285130977630615,
+      "logps/chosen": -5.594837188720703,
+      "logps/rejected": -8.218804359436035,
+      "loss": 0.6635,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -5.594837188720703,
+      "rewards/margins": 2.623966932296753,
+      "rewards/rejected": -8.218804359436035,
+      "step": 3270
+    },
+    {
+      "epoch": 1.7528014718180298,
+      "grad_norm": 3.8809036271383146,
+      "learning_rate": 4.401836063512631e-07,
+      "logits/chosen": 0.45250430703163147,
+      "logits/rejected": 1.0316245555877686,
+      "logps/chosen": -4.718832969665527,
+      "logps/rejected": -7.615536689758301,
+      "loss": 0.6614,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.718832969665527,
+      "rewards/margins": 2.8967034816741943,
+      "rewards/rejected": -7.615536689758301,
+      "step": 3275
+    },
+    {
+      "epoch": 1.7554775045994313,
+      "grad_norm": 5.24008134985795,
+      "learning_rate": 4.386376812974413e-07,
+      "logits/chosen": 0.4453478753566742,
+      "logits/rejected": 0.9081689715385437,
+      "logps/chosen": -4.267691612243652,
+      "logps/rejected": -7.873740196228027,
+      "loss": 0.6604,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.267691612243652,
+      "rewards/margins": 3.6060492992401123,
+      "rewards/rejected": -7.873740196228027,
+      "step": 3280
+    },
+    {
+      "epoch": 1.7581535373808328,
+      "grad_norm": 3.9275670628014634,
+      "learning_rate": 4.370923515811048e-07,
+      "logits/chosen": 0.5199798941612244,
+      "logits/rejected": 0.9599331617355347,
+      "logps/chosen": -5.136094570159912,
+      "logps/rejected": -7.247191429138184,
+      "loss": 0.6639,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -5.136094570159912,
+      "rewards/margins": 2.1110968589782715,
+      "rewards/rejected": -7.247191429138184,
+      "step": 3285
+    },
+    {
+      "epoch": 1.7608295701622345,
+      "grad_norm": 3.8340533795779552,
+      "learning_rate": 4.35547632195049e-07,
+      "logits/chosen": 0.49817007780075073,
+      "logits/rejected": 1.047565221786499,
+      "logps/chosen": -4.700753211975098,
+      "logps/rejected": -7.907202243804932,
+      "loss": 0.6654,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.700753211975098,
+      "rewards/margins": 3.206449031829834,
+      "rewards/rejected": -7.907202243804932,
+      "step": 3290
+    },
+    {
+      "epoch": 1.763505602943636,
+      "grad_norm": 4.620355743847193,
+      "learning_rate": 4.340035381261484e-07,
+      "logits/chosen": 0.5925118923187256,
+      "logits/rejected": 1.0531352758407593,
+      "logps/chosen": -5.424226760864258,
+      "logps/rejected": -7.77655553817749,
+      "loss": 0.6759,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -5.424226760864258,
+      "rewards/margins": 2.3523290157318115,
+      "rewards/rejected": -7.77655553817749,
+      "step": 3295
+    },
+    {
+      "epoch": 1.7661816357250375,
+      "grad_norm": 6.399811989049656,
+      "learning_rate": 4.324600843552104e-07,
+      "logits/chosen": 0.5855228304862976,
+      "logits/rejected": 1.068809151649475,
+      "logps/chosen": -5.175127029418945,
+      "logps/rejected": -8.335172653198242,
+      "loss": 0.665,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -5.175127029418945,
+      "rewards/margins": 3.160045623779297,
+      "rewards/rejected": -8.335172653198242,
+      "step": 3300
+    },
+    {
+      "epoch": 1.7688576685064392,
+      "grad_norm": 3.3007515030678003,
+      "learning_rate": 4.309172858568302e-07,
+      "logits/chosen": 0.5936049222946167,
+      "logits/rejected": 1.107150912284851,
+      "logps/chosen": -5.0815229415893555,
+      "logps/rejected": -9.045859336853027,
+      "loss": 0.6648,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -5.0815229415893555,
+      "rewards/margins": 3.964336395263672,
+      "rewards/rejected": -9.045859336853027,
+      "step": 3305
+    },
+    {
+      "epoch": 1.771533701287841,
+      "grad_norm": 3.3259384640236456,
+      "learning_rate": 4.293751575992455e-07,
+      "logits/chosen": 0.8595949411392212,
+      "logits/rejected": 1.4078086614608765,
+      "logps/chosen": -5.843766689300537,
+      "logps/rejected": -9.990134239196777,
+      "loss": 0.6718,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -5.843766689300537,
+      "rewards/margins": 4.146368026733398,
+      "rewards/rejected": -9.990134239196777,
+      "step": 3310
+    },
+    {
+      "epoch": 1.7742097340692422,
+      "grad_norm": 3.3824501996121614,
+      "learning_rate": 4.278337145441916e-07,
+      "logits/chosen": 0.7068182229995728,
+      "logits/rejected": 1.2271534204483032,
+      "logps/chosen": -6.040187835693359,
+      "logps/rejected": -8.56459903717041,
+      "loss": 0.6591,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -6.040187835693359,
+      "rewards/margins": 2.5244109630584717,
+      "rewards/rejected": -8.56459903717041,
+      "step": 3315
+    },
+    {
+      "epoch": 1.776885766850644,
+      "grad_norm": 5.50968450366947,
+      "learning_rate": 4.262929716467556e-07,
+      "logits/chosen": 0.7869489192962646,
+      "logits/rejected": 1.1056265830993652,
+      "logps/chosen": -5.990656852722168,
+      "logps/rejected": -8.01071548461914,
+      "loss": 0.6771,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -5.990656852722168,
+      "rewards/margins": 2.0200583934783936,
+      "rewards/rejected": -8.01071548461914,
+      "step": 3320
+    },
+    {
+      "epoch": 1.7795617996320456,
+      "grad_norm": 4.023099996072839,
+      "learning_rate": 4.247529438552321e-07,
+      "logits/chosen": 0.9098905324935913,
+      "logits/rejected": 1.1957557201385498,
+      "logps/chosen": -6.179778575897217,
+      "logps/rejected": -9.155012130737305,
+      "loss": 0.6679,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -6.179778575897217,
+      "rewards/margins": 2.975234031677246,
+      "rewards/rejected": -9.155012130737305,
+      "step": 3325
+    },
+    {
+      "epoch": 1.782237832413447,
+      "grad_norm": 4.075035960403828,
+      "learning_rate": 4.232136461109773e-07,
+      "logits/chosen": 0.9311989545822144,
+      "logits/rejected": 1.2302860021591187,
+      "logps/chosen": -5.581821441650391,
+      "logps/rejected": -9.156855583190918,
+      "loss": 0.656,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -5.581821441650391,
+      "rewards/margins": 3.5750339031219482,
+      "rewards/rejected": -9.156855583190918,
+      "step": 3330
+    },
+    {
+      "epoch": 1.7849138651948486,
+      "grad_norm": 6.574270669452065,
+      "learning_rate": 4.216750933482646e-07,
+      "logits/chosen": 0.7370100617408752,
+      "logits/rejected": 1.1170061826705933,
+      "logps/chosen": -5.826984405517578,
+      "logps/rejected": -8.546550750732422,
+      "loss": 0.6696,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -5.826984405517578,
+      "rewards/margins": 2.7195663452148438,
+      "rewards/rejected": -8.546550750732422,
+      "step": 3335
+    },
+    {
+      "epoch": 1.7875898979762503,
+      "grad_norm": 2.929527872015368,
+      "learning_rate": 4.2013730049413986e-07,
+      "logits/chosen": 0.4875618815422058,
+      "logits/rejected": 0.9929645657539368,
+      "logps/chosen": -4.091117858886719,
+      "logps/rejected": -7.950071811676025,
+      "loss": 0.6517,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.091117858886719,
+      "rewards/margins": 3.8589534759521484,
+      "rewards/rejected": -7.950071811676025,
+      "step": 3340
+    },
+    {
+      "epoch": 1.7902659307576518,
+      "grad_norm": 5.255590954429728,
+      "learning_rate": 4.1860028246827594e-07,
+      "logits/chosen": 0.5470582842826843,
+      "logits/rejected": 0.9300987124443054,
+      "logps/chosen": -4.381545066833496,
+      "logps/rejected": -7.4505743980407715,
+      "loss": 0.67,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.381545066833496,
+      "rewards/margins": 3.0690295696258545,
+      "rewards/rejected": -7.4505743980407715,
+      "step": 3345
+    },
+    {
+      "epoch": 1.7929419635390533,
+      "grad_norm": 2.7164296764815985,
+      "learning_rate": 4.170640541828285e-07,
+      "logits/chosen": 0.48592671751976013,
+      "logits/rejected": 1.028541922569275,
+      "logps/chosen": -4.612813472747803,
+      "logps/rejected": -7.239218235015869,
+      "loss": 0.662,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.612813472747803,
+      "rewards/margins": 2.6264050006866455,
+      "rewards/rejected": -7.239218235015869,
+      "step": 3350
+    },
+    {
+      "epoch": 1.795617996320455,
+      "grad_norm": 5.693969841509044,
+      "learning_rate": 4.1552863054229116e-07,
+      "logits/chosen": 0.4584383964538574,
+      "logits/rejected": 1.0258220434188843,
+      "logps/chosen": -4.255452632904053,
+      "logps/rejected": -7.215727806091309,
+      "loss": 0.6689,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.255452632904053,
+      "rewards/margins": 2.960275650024414,
+      "rewards/rejected": -7.215727806091309,
+      "step": 3355
+    },
+    {
+      "epoch": 1.7982940291018565,
+      "grad_norm": 3.5795897299308153,
+      "learning_rate": 4.139940264433508e-07,
+      "logits/chosen": 0.4342985153198242,
+      "logits/rejected": 0.5621687769889832,
+      "logps/chosen": -4.236763000488281,
+      "logps/rejected": -5.723001003265381,
+      "loss": 0.6774,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -4.236763000488281,
+      "rewards/margins": 1.4862384796142578,
+      "rewards/rejected": -5.723001003265381,
+      "step": 3360
+    },
+    {
+      "epoch": 1.800970061883258,
+      "grad_norm": 3.9306659139411546,
+      "learning_rate": 4.1246025677474303e-07,
+      "logits/chosen": 0.5122398734092712,
+      "logits/rejected": 0.8382307887077332,
+      "logps/chosen": -5.126894950866699,
+      "logps/rejected": -6.632272243499756,
+      "loss": 0.6751,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -5.126894950866699,
+      "rewards/margins": 1.505378007888794,
+      "rewards/rejected": -6.632272243499756,
+      "step": 3365
+    },
+    {
+      "epoch": 1.8036460946646597,
+      "grad_norm": 3.860102964657749,
+      "learning_rate": 4.10927336417108e-07,
+      "logits/chosen": 0.4885980486869812,
+      "logits/rejected": 0.6637006402015686,
+      "logps/chosen": -4.47636079788208,
+      "logps/rejected": -5.636756420135498,
+      "loss": 0.6778,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.47636079788208,
+      "rewards/margins": 1.160395622253418,
+      "rewards/rejected": -5.636756420135498,
+      "step": 3370
+    },
+    {
+      "epoch": 1.8063221274460612,
+      "grad_norm": 3.5504753746966444,
+      "learning_rate": 4.093952802428457e-07,
+      "logits/chosen": 0.538927435874939,
+      "logits/rejected": 0.9301983118057251,
+      "logps/chosen": -4.4421706199646,
+      "logps/rejected": -7.030969142913818,
+      "loss": 0.6692,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.4421706199646,
+      "rewards/margins": 2.588798999786377,
+      "rewards/rejected": -7.030969142913818,
+      "step": 3375
+    },
+    {
+      "epoch": 1.8089981602274627,
+      "grad_norm": 3.3579293985007923,
+      "learning_rate": 4.0786410311597184e-07,
+      "logits/chosen": 0.46699851751327515,
+      "logits/rejected": 0.8269960284233093,
+      "logps/chosen": -4.928410530090332,
+      "logps/rejected": -7.2203049659729,
+      "loss": 0.6793,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -4.928410530090332,
+      "rewards/margins": 2.2918946743011475,
+      "rewards/rejected": -7.2203049659729,
+      "step": 3380
+    },
+    {
+      "epoch": 1.8116741930088645,
+      "grad_norm": 4.343689183815147,
+      "learning_rate": 4.063338198919737e-07,
+      "logits/chosen": 0.5209351181983948,
+      "logits/rejected": 0.7413983941078186,
+      "logps/chosen": -5.026631832122803,
+      "logps/rejected": -7.217681884765625,
+      "loss": 0.6665,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -5.026631832122803,
+      "rewards/margins": 2.191049575805664,
+      "rewards/rejected": -7.217681884765625,
+      "step": 3385
+    },
+    {
+      "epoch": 1.814350225790266,
+      "grad_norm": 4.191374337063945,
+      "learning_rate": 4.0480444541766575e-07,
+      "logits/chosen": 0.5499339699745178,
+      "logits/rejected": 0.8942667841911316,
+      "logps/chosen": -5.267385959625244,
+      "logps/rejected": -7.687216281890869,
+      "loss": 0.6757,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -5.267385959625244,
+      "rewards/margins": 2.419830799102783,
+      "rewards/rejected": -7.687216281890869,
+      "step": 3390
+    },
+    {
+      "epoch": 1.8170262585716674,
+      "grad_norm": 4.845703569934548,
+      "learning_rate": 4.0327599453104606e-07,
+      "logits/chosen": 0.5659745931625366,
+      "logits/rejected": 1.1608030796051025,
+      "logps/chosen": -4.578476905822754,
+      "logps/rejected": -9.056882858276367,
+      "loss": 0.658,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.578476905822754,
+      "rewards/margins": 4.478405952453613,
+      "rewards/rejected": -9.056882858276367,
+      "step": 3395
+    },
+    {
+      "epoch": 1.8197022913530692,
+      "grad_norm": 6.88030737302693,
+      "learning_rate": 4.017484820611514e-07,
+      "logits/chosen": 0.56777024269104,
+      "logits/rejected": 0.8355584144592285,
+      "logps/chosen": -4.964198112487793,
+      "logps/rejected": -7.212578773498535,
+      "loss": 0.6634,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.964198112487793,
+      "rewards/margins": 2.248379945755005,
+      "rewards/rejected": -7.212578773498535,
+      "step": 3400
+    },
+    {
+      "epoch": 1.8223783241344707,
+      "grad_norm": 8.379201161092084,
+      "learning_rate": 4.002219228279148e-07,
+      "logits/chosen": 0.415731817483902,
+      "logits/rejected": 0.8794049024581909,
+      "logps/chosen": -4.7508745193481445,
+      "logps/rejected": -7.837724208831787,
+      "loss": 0.6769,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.7508745193481445,
+      "rewards/margins": 3.0868496894836426,
+      "rewards/rejected": -7.837724208831787,
+      "step": 3405
+    },
+    {
+      "epoch": 1.8250543569158721,
+      "grad_norm": 2.441256128072914,
+      "learning_rate": 3.9869633164202045e-07,
+      "logits/chosen": 0.6133834719657898,
+      "logits/rejected": 0.9773615002632141,
+      "logps/chosen": -5.406148433685303,
+      "logps/rejected": -8.143996238708496,
+      "loss": 0.668,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -5.406148433685303,
+      "rewards/margins": 2.7378487586975098,
+      "rewards/rejected": -8.143996238708496,
+      "step": 3410
+    },
+    {
+      "epoch": 1.8277303896972739,
+      "grad_norm": 3.6823691788501756,
+      "learning_rate": 3.9717172330476077e-07,
+      "logits/chosen": 0.6086641550064087,
+      "logits/rejected": 0.9942671656608582,
+      "logps/chosen": -5.326874732971191,
+      "logps/rejected": -7.5008134841918945,
+      "loss": 0.6555,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -5.326874732971191,
+      "rewards/margins": 2.173938512802124,
+      "rewards/rejected": -7.5008134841918945,
+      "step": 3415
+    },
+    {
+      "epoch": 1.8304064224786754,
+      "grad_norm": 3.9279046149458225,
+      "learning_rate": 3.956481126078927e-07,
+      "logits/chosen": 0.7402667999267578,
+      "logits/rejected": 0.9905534982681274,
+      "logps/chosen": -6.1569929122924805,
+      "logps/rejected": -7.7348127365112305,
+      "loss": 0.678,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -6.1569929122924805,
+      "rewards/margins": 1.5778197050094604,
+      "rewards/rejected": -7.7348127365112305,
+      "step": 3420
+    },
+    {
+      "epoch": 1.8330824552600768,
+      "grad_norm": 4.846905837081264,
+      "learning_rate": 3.941255143334937e-07,
+      "logits/chosen": 0.44442033767700195,
+      "logits/rejected": 0.9233905076980591,
+      "logps/chosen": -4.770227909088135,
+      "logps/rejected": -8.789190292358398,
+      "loss": 0.6591,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.770227909088135,
+      "rewards/margins": 4.018962860107422,
+      "rewards/rejected": -8.789190292358398,
+      "step": 3425
+    },
+    {
+      "epoch": 1.8357584880414786,
+      "grad_norm": 4.5693997308782635,
+      "learning_rate": 3.9260394325381895e-07,
+      "logits/chosen": 0.3542768061161041,
+      "logits/rejected": 0.9803436398506165,
+      "logps/chosen": -4.564907073974609,
+      "logps/rejected": -8.667490005493164,
+      "loss": 0.6607,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.564907073974609,
+      "rewards/margins": 4.102583885192871,
+      "rewards/rejected": -8.667490005493164,
+      "step": 3430
+    },
+    {
+      "epoch": 1.83843452082288,
+      "grad_norm": 5.7714015035430855,
+      "learning_rate": 3.9108341413115784e-07,
+      "logits/chosen": 0.4231301248073578,
+      "logits/rejected": 0.840813934803009,
+      "logps/chosen": -4.577905178070068,
+      "logps/rejected": -7.557101249694824,
+      "loss": 0.6582,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.577905178070068,
+      "rewards/margins": 2.979196071624756,
+      "rewards/rejected": -7.557101249694824,
+      "step": 3435
+    },
+    {
+      "epoch": 1.8411105536042816,
+      "grad_norm": 2.572985010632432,
+      "learning_rate": 3.895639417176905e-07,
+      "logits/chosen": 0.40942350029945374,
+      "logits/rejected": 0.744761049747467,
+      "logps/chosen": -4.44787073135376,
+      "logps/rejected": -7.42244815826416,
+      "loss": 0.6773,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.44787073135376,
+      "rewards/margins": 2.9745774269104004,
+      "rewards/rejected": -7.42244815826416,
+      "step": 3440
+    },
+    {
+      "epoch": 1.8437865863856833,
+      "grad_norm": 5.696937374557431,
+      "learning_rate": 3.8804554075534497e-07,
+      "logits/chosen": 0.40642499923706055,
+      "logits/rejected": 0.8030750155448914,
+      "logps/chosen": -4.6965131759643555,
+      "logps/rejected": -7.086382865905762,
+      "loss": 0.6616,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.6965131759643555,
+      "rewards/margins": 2.38986873626709,
+      "rewards/rejected": -7.086382865905762,
+      "step": 3445
+    },
+    {
+      "epoch": 1.8464626191670848,
+      "grad_norm": 6.329103409944875,
+      "learning_rate": 3.8652822597565403e-07,
+      "logits/chosen": 0.40786781907081604,
+      "logits/rejected": 0.7251715660095215,
+      "logps/chosen": -4.586820602416992,
+      "logps/rejected": -7.394270420074463,
+      "loss": 0.6721,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.586820602416992,
+      "rewards/margins": 2.80745005607605,
+      "rewards/rejected": -7.394270420074463,
+      "step": 3450
+    },
+    {
+      "epoch": 1.8491386519484863,
+      "grad_norm": 3.2475708753369115,
+      "learning_rate": 3.850120120996123e-07,
+      "logits/chosen": 0.2819128930568695,
+      "logits/rejected": 0.6387428641319275,
+      "logps/chosen": -4.174250602722168,
+      "logps/rejected": -6.440793037414551,
+      "loss": 0.6703,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.174250602722168,
+      "rewards/margins": 2.266542673110962,
+      "rewards/rejected": -6.440793037414551,
+      "step": 3455
+    },
+    {
+      "epoch": 1.851814684729888,
+      "grad_norm": 4.1797730222640785,
+      "learning_rate": 3.8349691383753356e-07,
+      "logits/chosen": 0.3826242983341217,
+      "logits/rejected": 0.948569655418396,
+      "logps/chosen": -4.638914585113525,
+      "logps/rejected": -7.874539375305176,
+      "loss": 0.6727,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.638914585113525,
+      "rewards/margins": 3.2356247901916504,
+      "rewards/rejected": -7.874539375305176,
+      "step": 3460
+    },
+    {
+      "epoch": 1.8544907175112895,
+      "grad_norm": 2.3663235748188787,
+      "learning_rate": 3.819829458889078e-07,
+      "logits/chosen": 0.330669105052948,
+      "logits/rejected": 0.7199997901916504,
+      "logps/chosen": -4.2842326164245605,
+      "logps/rejected": -6.967148780822754,
+      "loss": 0.6633,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.2842326164245605,
+      "rewards/margins": 2.6829161643981934,
+      "rewards/rejected": -6.967148780822754,
+      "step": 3465
+    },
+    {
+      "epoch": 1.857166750292691,
+      "grad_norm": 3.149207739073661,
+      "learning_rate": 3.804701229422585e-07,
+      "logits/chosen": 0.4026687741279602,
+      "logits/rejected": 0.8423298001289368,
+      "logps/chosen": -4.666319370269775,
+      "logps/rejected": -7.809033393859863,
+      "loss": 0.664,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.666319370269775,
+      "rewards/margins": 3.142714262008667,
+      "rewards/rejected": -7.809033393859863,
+      "step": 3470
+    },
+    {
+      "epoch": 1.8598427830740927,
+      "grad_norm": 3.258389242395978,
+      "learning_rate": 3.789584596750007e-07,
+      "logits/chosen": 0.49451884627342224,
+      "logits/rejected": 0.9638348817825317,
+      "logps/chosen": -4.890847206115723,
+      "logps/rejected": -7.970078945159912,
+      "loss": 0.6575,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.890847206115723,
+      "rewards/margins": 3.0792319774627686,
+      "rewards/rejected": -7.970078945159912,
+      "step": 3475
+    },
+    {
+      "epoch": 1.8625188158554944,
+      "grad_norm": 8.567508500907035,
+      "learning_rate": 3.77447970753298e-07,
+      "logits/chosen": 0.6142381429672241,
+      "logits/rejected": 1.0127321481704712,
+      "logps/chosen": -5.169547080993652,
+      "logps/rejected": -8.107711791992188,
+      "loss": 0.6609,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -5.169547080993652,
+      "rewards/margins": 2.938164234161377,
+      "rewards/rejected": -8.107711791992188,
+      "step": 3480
+    },
+    {
+      "epoch": 1.8651948486368957,
+      "grad_norm": 6.072359492809332,
+      "learning_rate": 3.7593867083192057e-07,
+      "logits/chosen": 0.5890600681304932,
+      "logits/rejected": 1.0903421640396118,
+      "logps/chosen": -5.219073295593262,
+      "logps/rejected": -8.709348678588867,
+      "loss": 0.675,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -5.219073295593262,
+      "rewards/margins": 3.4902756214141846,
+      "rewards/rejected": -8.709348678588867,
+      "step": 3485
+    },
+    {
+      "epoch": 1.8678708814182974,
+      "grad_norm": 3.9675752905046817,
+      "learning_rate": 3.7443057455410276e-07,
+      "logits/chosen": 0.5476547479629517,
+      "logits/rejected": 1.1254818439483643,
+      "logps/chosen": -4.5131072998046875,
+      "logps/rejected": -8.537757873535156,
+      "loss": 0.6542,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.5131072998046875,
+      "rewards/margins": 4.0246500968933105,
+      "rewards/rejected": -8.537757873535156,
+      "step": 3490
+    },
+    {
+      "epoch": 1.870546914199699,
+      "grad_norm": 3.191658869701444,
+      "learning_rate": 3.7292369655140145e-07,
+      "logits/chosen": 0.7475110292434692,
+      "logits/rejected": 1.1323316097259521,
+      "logps/chosen": -5.589471340179443,
+      "logps/rejected": -8.331764221191406,
+      "loss": 0.6692,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -5.589471340179443,
+      "rewards/margins": 2.742292881011963,
+      "rewards/rejected": -8.331764221191406,
+      "step": 3495
+    },
+    {
+      "epoch": 1.8732229469811004,
+      "grad_norm": 3.030681345140318,
+      "learning_rate": 3.714180514435534e-07,
+      "logits/chosen": 0.47294846177101135,
+      "logits/rejected": 0.8193120956420898,
+      "logps/chosen": -4.742311954498291,
+      "logps/rejected": -7.077642917633057,
+      "loss": 0.6652,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.742311954498291,
+      "rewards/margins": 2.3353309631347656,
+      "rewards/rejected": -7.077642917633057,
+      "step": 3500
+    },
+    {
+      "epoch": 1.875898979762502,
+      "grad_norm": 4.435444919822571,
+      "learning_rate": 3.6991365383833426e-07,
+      "logits/chosen": 0.46030712127685547,
+      "logits/rejected": 0.8646756410598755,
+      "logps/chosen": -4.395931243896484,
+      "logps/rejected": -7.443209648132324,
+      "loss": 0.6719,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.395931243896484,
+      "rewards/margins": 3.047278642654419,
+      "rewards/rejected": -7.443209648132324,
+      "step": 3505
+    },
+    {
+      "epoch": 1.8785750125439038,
+      "grad_norm": 4.39318949239719,
+      "learning_rate": 3.684105183314162e-07,
+      "logits/chosen": 0.39090561866760254,
+      "logits/rejected": 0.9344631433486938,
+      "logps/chosen": -4.440518379211426,
+      "logps/rejected": -7.802041053771973,
+      "loss": 0.6641,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.440518379211426,
+      "rewards/margins": 3.361522674560547,
+      "rewards/rejected": -7.802041053771973,
+      "step": 3510
+    },
+    {
+      "epoch": 1.881251045325305,
+      "grad_norm": 3.6362459679311296,
+      "learning_rate": 3.669086595062263e-07,
+      "logits/chosen": 0.6779913902282715,
+      "logits/rejected": 1.0349311828613281,
+      "logps/chosen": -5.230127811431885,
+      "logps/rejected": -8.138174057006836,
+      "loss": 0.682,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -5.230127811431885,
+      "rewards/margins": 2.9080452919006348,
+      "rewards/rejected": -8.138174057006836,
+      "step": 3515
+    },
+    {
+      "epoch": 1.8839270781067068,
+      "grad_norm": 4.460843163067645,
+      "learning_rate": 3.654080919338056e-07,
+      "logits/chosen": 0.5372574925422668,
+      "logits/rejected": 0.8569413423538208,
+      "logps/chosen": -4.792227745056152,
+      "logps/rejected": -7.337124824523926,
+      "loss": 0.6734,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.792227745056152,
+      "rewards/margins": 2.5448975563049316,
+      "rewards/rejected": -7.337124824523926,
+      "step": 3520
+    },
+    {
+      "epoch": 1.8866031108881085,
+      "grad_norm": 6.584839721402313,
+      "learning_rate": 3.639088301726673e-07,
+      "logits/chosen": 0.6621233820915222,
+      "logits/rejected": 1.087973713874817,
+      "logps/chosen": -5.601874351501465,
+      "logps/rejected": -8.230483055114746,
+      "loss": 0.674,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -5.601874351501465,
+      "rewards/margins": 2.6286094188690186,
+      "rewards/rejected": -8.230483055114746,
+      "step": 3525
+    },
+    {
+      "epoch": 1.88927914366951,
+      "grad_norm": 2.994469056324305,
+      "learning_rate": 3.624108887686556e-07,
+      "logits/chosen": 0.6639381051063538,
+      "logits/rejected": 0.8746098279953003,
+      "logps/chosen": -5.200750827789307,
+      "logps/rejected": -7.160675048828125,
+      "loss": 0.6644,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -5.200750827789307,
+      "rewards/margins": 1.9599227905273438,
+      "rewards/rejected": -7.160675048828125,
+      "step": 3530
+    },
+    {
+      "epoch": 1.8919551764509115,
+      "grad_norm": 4.174579332590993,
+      "learning_rate": 3.6091428225480433e-07,
+      "logits/chosen": 0.4854133129119873,
+      "logits/rejected": 1.0605086088180542,
+      "logps/chosen": -5.057211875915527,
+      "logps/rejected": -7.8533477783203125,
+      "loss": 0.6664,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -5.057211875915527,
+      "rewards/margins": 2.796135902404785,
+      "rewards/rejected": -7.8533477783203125,
+      "step": 3535
+    },
+    {
+      "epoch": 1.8946312092323132,
+      "grad_norm": 4.1663982716308015,
+      "learning_rate": 3.5941902515119674e-07,
+      "logits/chosen": 0.5708856582641602,
+      "logits/rejected": 0.9139571189880371,
+      "logps/chosen": -4.982260227203369,
+      "logps/rejected": -7.1112260818481445,
+      "loss": 0.6636,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.982260227203369,
+      "rewards/margins": 2.1289665699005127,
+      "rewards/rejected": -7.1112260818481445,
+      "step": 3540
+    },
+    {
+      "epoch": 1.8973072420137147,
+      "grad_norm": 3.6183161620405886,
+      "learning_rate": 3.5792513196482373e-07,
+      "logits/chosen": 0.3628273904323578,
+      "logits/rejected": 0.8742368817329407,
+      "logps/chosen": -4.186161994934082,
+      "logps/rejected": -7.238897800445557,
+      "loss": 0.6656,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.186161994934082,
+      "rewards/margins": 3.0527353286743164,
+      "rewards/rejected": -7.238897800445557,
+      "step": 3545
+    },
+    {
+      "epoch": 1.8999832747951162,
+      "grad_norm": 3.9855579764721543,
+      "learning_rate": 3.5643261718944346e-07,
+      "logits/chosen": 0.5879573822021484,
+      "logits/rejected": 0.8058868646621704,
+      "logps/chosen": -4.913398742675781,
+      "logps/rejected": -6.783103942871094,
+      "loss": 0.6772,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -4.913398742675781,
+      "rewards/margins": 1.8697052001953125,
+      "rewards/rejected": -6.783103942871094,
+      "step": 3550
+    },
+    {
+      "epoch": 1.902659307576518,
+      "grad_norm": 6.256915878552453,
+      "learning_rate": 3.5494149530544087e-07,
+      "logits/chosen": 0.5619211792945862,
+      "logits/rejected": 0.9288104176521301,
+      "logps/chosen": -4.609374046325684,
+      "logps/rejected": -6.9960174560546875,
+      "loss": 0.6705,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.609374046325684,
+      "rewards/margins": 2.386643648147583,
+      "rewards/rejected": -6.9960174560546875,
+      "step": 3555
+    },
+    {
+      "epoch": 1.9053353403579194,
+      "grad_norm": 5.131243117683389,
+      "learning_rate": 3.534517807796871e-07,
+      "logits/chosen": 0.5741761922836304,
+      "logits/rejected": 0.7838306427001953,
+      "logps/chosen": -4.652432918548584,
+      "logps/rejected": -6.956583499908447,
+      "loss": 0.6815,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.652432918548584,
+      "rewards/margins": 2.3041505813598633,
+      "rewards/rejected": -6.956583499908447,
+      "step": 3560
+    },
+    {
+      "epoch": 1.908011373139321,
+      "grad_norm": 3.957123165657345,
+      "learning_rate": 3.519634880653988e-07,
+      "logits/chosen": 0.7055962681770325,
+      "logits/rejected": 1.5045572519302368,
+      "logps/chosen": -4.95360803604126,
+      "logps/rejected": -9.937471389770508,
+      "loss": 0.6594,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.95360803604126,
+      "rewards/margins": 4.98386287689209,
+      "rewards/rejected": -9.937471389770508,
+      "step": 3565
+    },
+    {
+      "epoch": 1.9106874059207226,
+      "grad_norm": 3.931250719044019,
+      "learning_rate": 3.504766316019987e-07,
+      "logits/chosen": 0.4875578284263611,
+      "logits/rejected": 1.0084998607635498,
+      "logps/chosen": -4.695186614990234,
+      "logps/rejected": -7.444551944732666,
+      "loss": 0.6538,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.695186614990234,
+      "rewards/margins": 2.7493653297424316,
+      "rewards/rejected": -7.444551944732666,
+      "step": 3570
+    },
+    {
+      "epoch": 1.913363438702124,
+      "grad_norm": 5.221705207797446,
+      "learning_rate": 3.489912258149745e-07,
+      "logits/chosen": 0.7003227472305298,
+      "logits/rejected": 1.0682531595230103,
+      "logps/chosen": -5.211117267608643,
+      "logps/rejected": -7.573805332183838,
+      "loss": 0.6644,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -5.211117267608643,
+      "rewards/margins": 2.3626880645751953,
+      "rewards/rejected": -7.573805332183838,
+      "step": 3575
+    },
+    {
+      "epoch": 1.9160394714835256,
+      "grad_norm": 4.4433702031285005,
+      "learning_rate": 3.475072851157397e-07,
+      "logits/chosen": 0.5896960496902466,
+      "logits/rejected": 1.0145801305770874,
+      "logps/chosen": -4.557041168212891,
+      "logps/rejected": -7.538638114929199,
+      "loss": 0.6664,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.557041168212891,
+      "rewards/margins": 2.9815969467163086,
+      "rewards/rejected": -7.538638114929199,
+      "step": 3580
+    },
+    {
+      "epoch": 1.9187155042649273,
+      "grad_norm": 3.278137604981305,
+      "learning_rate": 3.460248239014936e-07,
+      "logits/chosen": 0.6733719110488892,
+      "logits/rejected": 1.1145843267440796,
+      "logps/chosen": -5.353055477142334,
+      "logps/rejected": -8.858613014221191,
+      "loss": 0.6473,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -5.353055477142334,
+      "rewards/margins": 3.5055572986602783,
+      "rewards/rejected": -8.858613014221191,
+      "step": 3585
+    },
+    {
+      "epoch": 1.9213915370463288,
+      "grad_norm": 4.044527644065297,
+      "learning_rate": 3.4454385655508134e-07,
+      "logits/chosen": 0.7416501641273499,
+      "logits/rejected": 1.0753076076507568,
+      "logps/chosen": -5.474855422973633,
+      "logps/rejected": -7.960033416748047,
+      "loss": 0.6644,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -5.474855422973633,
+      "rewards/margins": 2.485178232192993,
+      "rewards/rejected": -7.960033416748047,
+      "step": 3590
+    },
+    {
+      "epoch": 1.9240675698277303,
+      "grad_norm": 4.395283993670073,
+      "learning_rate": 3.4306439744485447e-07,
+      "logits/chosen": 0.5754159092903137,
+      "logits/rejected": 0.9927362203598022,
+      "logps/chosen": -5.303385257720947,
+      "logps/rejected": -7.740556240081787,
+      "loss": 0.6697,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -5.303385257720947,
+      "rewards/margins": 2.4371705055236816,
+      "rewards/rejected": -7.740556240081787,
+      "step": 3595
+    },
+    {
+      "epoch": 1.926743602609132,
+      "grad_norm": 4.825852668987415,
+      "learning_rate": 3.415864609245322e-07,
+      "logits/chosen": 0.6607574224472046,
+      "logits/rejected": 0.9625824689865112,
+      "logps/chosen": -5.11215353012085,
+      "logps/rejected": -7.137121677398682,
+      "loss": 0.6634,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -5.11215353012085,
+      "rewards/margins": 2.024968385696411,
+      "rewards/rejected": -7.137121677398682,
+      "step": 3600
+    },
+    {
+      "epoch": 1.926743602609132,
+      "eval_logits/chosen": 0.7086397409439087,
+      "eval_logits/rejected": 1.0737833976745605,
+      "eval_logps/chosen": -4.7410993576049805,
+      "eval_logps/rejected": -7.857568264007568,
+      "eval_loss": 0.6629143953323364,
+      "eval_rewards/accuracies": 0.6854599118232727,
+      "eval_rewards/chosen": -4.7410993576049805,
+      "eval_rewards/margins": 3.1164684295654297,
+      "eval_rewards/rejected": -7.857568264007568,
+      "eval_runtime": 40.7965,
+      "eval_samples_per_second": 32.969,
+      "eval_steps_per_second": 8.261,
+      "step": 3600
+    },
+    {
+      "epoch": 1.9294196353905335,
+      "grad_norm": 4.001621180975842,
+      "learning_rate": 3.401100613330605e-07,
+      "logits/chosen": 0.4974862039089203,
+      "logits/rejected": 0.9388201832771301,
+      "logps/chosen": -4.413834571838379,
+      "logps/rejected": -7.337038993835449,
+      "loss": 0.6591,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.413834571838379,
+      "rewards/margins": 2.9232046604156494,
+      "rewards/rejected": -7.337038993835449,
+      "step": 3605
+    },
+    {
+      "epoch": 1.932095668171935,
+      "grad_norm": 4.914833771889251,
+      "learning_rate": 3.3863521299447514e-07,
+      "logits/chosen": 0.4793081283569336,
+      "logits/rejected": 0.8154508471488953,
+      "logps/chosen": -4.569281578063965,
+      "logps/rejected": -6.677482604980469,
+      "loss": 0.6654,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.569281578063965,
+      "rewards/margins": 2.108201265335083,
+      "rewards/rejected": -6.677482604980469,
+      "step": 3610
+    },
+    {
+      "epoch": 1.9347717009533367,
+      "grad_norm": 5.917873373196923,
+      "learning_rate": 3.371619302177609e-07,
+      "logits/chosen": 0.5080233216285706,
+      "logits/rejected": 0.8888565301895142,
+      "logps/chosen": -4.222813129425049,
+      "logps/rejected": -6.783210754394531,
+      "loss": 0.6698,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.222813129425049,
+      "rewards/margins": 2.560398578643799,
+      "rewards/rejected": -6.783210754394531,
+      "step": 3615
+    },
+    {
+      "epoch": 1.9374477337347382,
+      "grad_norm": 3.972848733928864,
+      "learning_rate": 3.3569022729671393e-07,
+      "logits/chosen": 0.5850021839141846,
+      "logits/rejected": 0.9067463874816895,
+      "logps/chosen": -4.980602741241455,
+      "logps/rejected": -7.229658603668213,
+      "loss": 0.6603,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.980602741241455,
+      "rewards/margins": 2.2490553855895996,
+      "rewards/rejected": -7.229658603668213,
+      "step": 3620
+    },
+    {
+      "epoch": 1.9401237665161397,
+      "grad_norm": 5.0360368337271,
+      "learning_rate": 3.342201185098024e-07,
+      "logits/chosen": 0.6728376746177673,
+      "logits/rejected": 0.9961096048355103,
+      "logps/chosen": -4.853075981140137,
+      "logps/rejected": -7.585652828216553,
+      "loss": 0.6498,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.853075981140137,
+      "rewards/margins": 2.732577085494995,
+      "rewards/rejected": -7.585652828216553,
+      "step": 3625
+    },
+    {
+      "epoch": 1.9427997992975414,
+      "grad_norm": 3.3697121343043555,
+      "learning_rate": 3.3275161812002807e-07,
+      "logits/chosen": 0.5583959817886353,
+      "logits/rejected": 1.0206317901611328,
+      "logps/chosen": -5.070122718811035,
+      "logps/rejected": -8.004772186279297,
+      "loss": 0.659,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -5.070122718811035,
+      "rewards/margins": 2.9346494674682617,
+      "rewards/rejected": -8.004772186279297,
+      "step": 3630
+    },
+    {
+      "epoch": 1.945475832078943,
+      "grad_norm": 4.291272178099334,
+      "learning_rate": 3.312847403747883e-07,
+      "logits/chosen": 0.4955342411994934,
+      "logits/rejected": 1.0955837965011597,
+      "logps/chosen": -4.710444450378418,
+      "logps/rejected": -8.480511665344238,
+      "loss": 0.6628,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.710444450378418,
+      "rewards/margins": 3.770066738128662,
+      "rewards/rejected": -8.480511665344238,
+      "step": 3635
+    },
+    {
+      "epoch": 1.9481518648603444,
+      "grad_norm": 3.90534202632478,
+      "learning_rate": 3.2981949950573733e-07,
+      "logits/chosen": 0.7087478637695312,
+      "logits/rejected": 1.1205369234085083,
+      "logps/chosen": -5.479466438293457,
+      "logps/rejected": -8.259045600891113,
+      "loss": 0.668,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -5.479466438293457,
+      "rewards/margins": 2.7795798778533936,
+      "rewards/rejected": -8.259045600891113,
+      "step": 3640
+    },
+    {
+      "epoch": 1.9508278976417461,
+      "grad_norm": 5.642598782730567,
+      "learning_rate": 3.283559097286486e-07,
+      "logits/chosen": 0.6035521030426025,
+      "logits/rejected": 1.028377890586853,
+      "logps/chosen": -5.634242057800293,
+      "logps/rejected": -7.993752479553223,
+      "loss": 0.6662,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -5.634242057800293,
+      "rewards/margins": 2.3595097064971924,
+      "rewards/rejected": -7.993752479553223,
+      "step": 3645
+    },
+    {
+      "epoch": 1.9535039304231478,
+      "grad_norm": 5.967948694822902,
+      "learning_rate": 3.268939852432765e-07,
+      "logits/chosen": 0.5537285208702087,
+      "logits/rejected": 0.9856319427490234,
+      "logps/chosen": -5.078745365142822,
+      "logps/rejected": -8.015796661376953,
+      "loss": 0.665,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -5.078745365142822,
+      "rewards/margins": 2.9370505809783936,
+      "rewards/rejected": -8.015796661376953,
+      "step": 3650
+    },
+    {
+      "epoch": 1.9561799632045491,
+      "grad_norm": 8.298761584632427,
+      "learning_rate": 3.254337402332187e-07,
+      "logits/chosen": 0.5040444135665894,
+      "logits/rejected": 0.9765418171882629,
+      "logps/chosen": -4.4548420906066895,
+      "logps/rejected": -7.409937381744385,
+      "loss": 0.6802,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.4548420906066895,
+      "rewards/margins": 2.9550957679748535,
+      "rewards/rejected": -7.409937381744385,
+      "step": 3655
+    },
+    {
+      "epoch": 1.9588559959859508,
+      "grad_norm": 11.181621850937342,
+      "learning_rate": 3.239751888657788e-07,
+      "logits/chosen": 0.4553329348564148,
+      "logits/rejected": 0.8989969491958618,
+      "logps/chosen": -4.338444709777832,
+      "logps/rejected": -6.994523525238037,
+      "loss": 0.6686,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.338444709777832,
+      "rewards/margins": 2.6560795307159424,
+      "rewards/rejected": -6.994523525238037,
+      "step": 3660
+    },
+    {
+      "epoch": 1.9615320287673526,
+      "grad_norm": 7.537651243030505,
+      "learning_rate": 3.2251834529182856e-07,
+      "logits/chosen": 0.49521002173423767,
+      "logits/rejected": 0.8646965026855469,
+      "logps/chosen": -4.419943332672119,
+      "logps/rejected": -7.2947258949279785,
+      "loss": 0.66,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.419943332672119,
+      "rewards/margins": 2.8747825622558594,
+      "rewards/rejected": -7.2947258949279785,
+      "step": 3665
+    },
+    {
+      "epoch": 1.9642080615487538,
+      "grad_norm": 2.8241822841123967,
+      "learning_rate": 3.2106322364567075e-07,
+      "logits/chosen": 0.541882336139679,
+      "logits/rejected": 0.97935950756073,
+      "logps/chosen": -4.698421478271484,
+      "logps/rejected": -8.372888565063477,
+      "loss": 0.6437,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.698421478271484,
+      "rewards/margins": 3.674467086791992,
+      "rewards/rejected": -8.372888565063477,
+      "step": 3670
+    },
+    {
+      "epoch": 1.9668840943301555,
+      "grad_norm": 3.449040580316544,
+      "learning_rate": 3.1960983804490183e-07,
+      "logits/chosen": 0.6194535493850708,
+      "logits/rejected": 0.8201051950454712,
+      "logps/chosen": -5.020188808441162,
+      "logps/rejected": -7.544220924377441,
+      "loss": 0.662,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -5.020188808441162,
+      "rewards/margins": 2.524031162261963,
+      "rewards/rejected": -7.544220924377441,
+      "step": 3675
+    },
+    {
+      "epoch": 1.9695601271115573,
+      "grad_norm": 9.291222506289872,
+      "learning_rate": 3.1815820259027537e-07,
+      "logits/chosen": 0.35654065012931824,
+      "logits/rejected": 0.9410832524299622,
+      "logps/chosen": -4.0460405349731445,
+      "logps/rejected": -6.9808549880981445,
+      "loss": 0.669,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.0460405349731445,
+      "rewards/margins": 2.9348137378692627,
+      "rewards/rejected": -6.9808549880981445,
+      "step": 3680
+    },
+    {
+      "epoch": 1.9722361598929585,
+      "grad_norm": 4.465496581829361,
+      "learning_rate": 3.16708331365565e-07,
+      "logits/chosen": 0.552197277545929,
+      "logits/rejected": 0.9555072784423828,
+      "logps/chosen": -5.1199631690979,
+      "logps/rejected": -7.627871513366699,
+      "loss": 0.6522,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -5.1199631690979,
+      "rewards/margins": 2.507908582687378,
+      "rewards/rejected": -7.627871513366699,
+      "step": 3685
+    },
+    {
+      "epoch": 1.9749121926743602,
+      "grad_norm": 6.705912428142108,
+      "learning_rate": 3.152602384374275e-07,
+      "logits/chosen": 0.5311455726623535,
+      "logits/rejected": 0.8408058285713196,
+      "logps/chosen": -4.5480804443359375,
+      "logps/rejected": -7.144596099853516,
+      "loss": 0.6758,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -4.5480804443359375,
+      "rewards/margins": 2.596515417098999,
+      "rewards/rejected": -7.144596099853516,
+      "step": 3690
+    },
+    {
+      "epoch": 1.977588225455762,
+      "grad_norm": 3.2419504001029655,
+      "learning_rate": 3.1381393785526697e-07,
+      "logits/chosen": 0.3926595449447632,
+      "logits/rejected": 0.8124720454216003,
+      "logps/chosen": -3.783808946609497,
+      "logps/rejected": -7.1068549156188965,
+      "loss": 0.6559,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -3.783808946609497,
+      "rewards/margins": 3.3230462074279785,
+      "rewards/rejected": -7.1068549156188965,
+      "step": 3695
+    },
+    {
+      "epoch": 1.9802642582371635,
+      "grad_norm": 5.9164326012592765,
+      "learning_rate": 3.123694436510979e-07,
+      "logits/chosen": 0.5512126684188843,
+      "logits/rejected": 0.7995705604553223,
+      "logps/chosen": -4.323787689208984,
+      "logps/rejected": -6.298802375793457,
+      "loss": 0.6684,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.323787689208984,
+      "rewards/margins": 1.9750146865844727,
+      "rewards/rejected": -6.298802375793457,
+      "step": 3700
+    },
+    {
+      "epoch": 1.982940291018565,
+      "grad_norm": 4.681393072493192,
+      "learning_rate": 3.1092676983940946e-07,
+      "logits/chosen": 0.44990020990371704,
+      "logits/rejected": 0.8111467361450195,
+      "logps/chosen": -3.712167263031006,
+      "logps/rejected": -7.03220272064209,
+      "loss": 0.6598,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -3.712167263031006,
+      "rewards/margins": 3.320035457611084,
+      "rewards/rejected": -7.03220272064209,
+      "step": 3705
+    },
+    {
+      "epoch": 1.9856163237999667,
+      "grad_norm": 3.848498447852249,
+      "learning_rate": 3.094859304170293e-07,
+      "logits/chosen": 0.5502398610115051,
+      "logits/rejected": 0.8433178663253784,
+      "logps/chosen": -4.158316612243652,
+      "logps/rejected": -6.1366119384765625,
+      "loss": 0.6732,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.158316612243652,
+      "rewards/margins": 1.9782955646514893,
+      "rewards/rejected": -6.1366119384765625,
+      "step": 3710
+    },
+    {
+      "epoch": 1.9882923565813682,
+      "grad_norm": 6.296445513373104,
+      "learning_rate": 3.0804693936298795e-07,
+      "logits/chosen": 0.5430026054382324,
+      "logits/rejected": 0.8904650807380676,
+      "logps/chosen": -4.4812750816345215,
+      "logps/rejected": -6.942924499511719,
+      "loss": 0.6553,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.4812750816345215,
+      "rewards/margins": 2.4616494178771973,
+      "rewards/rejected": -6.942924499511719,
+      "step": 3715
+    },
+    {
+      "epoch": 1.9909683893627697,
+      "grad_norm": 6.051999859069768,
+      "learning_rate": 3.066098106383826e-07,
+      "logits/chosen": 0.472832053899765,
+      "logits/rejected": 0.9841610193252563,
+      "logps/chosen": -4.161350727081299,
+      "logps/rejected": -6.923689842224121,
+      "loss": 0.6744,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.161350727081299,
+      "rewards/margins": 2.7623391151428223,
+      "rewards/rejected": -6.923689842224121,
+      "step": 3720
+    },
+    {
+      "epoch": 1.9936444221441714,
+      "grad_norm": 3.2550008444953176,
+      "learning_rate": 3.0517455818624263e-07,
+      "logits/chosen": 0.5394463539123535,
+      "logits/rejected": 1.0628547668457031,
+      "logps/chosen": -4.546173572540283,
+      "logps/rejected": -8.234067916870117,
+      "loss": 0.6573,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.546173572540283,
+      "rewards/margins": 3.687894821166992,
+      "rewards/rejected": -8.234067916870117,
+      "step": 3725
+    },
+    {
+      "epoch": 1.9963204549255729,
+      "grad_norm": 5.365185187908445,
+      "learning_rate": 3.037411959313936e-07,
+      "logits/chosen": 0.6335793733596802,
+      "logits/rejected": 0.9163691401481628,
+      "logps/chosen": -4.605353355407715,
+      "logps/rejected": -7.284852504730225,
+      "loss": 0.6578,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.605353355407715,
+      "rewards/margins": 2.679499387741089,
+      "rewards/rejected": -7.284852504730225,
+      "step": 3730
+    },
+    {
+      "epoch": 1.9989964877069744,
+      "grad_norm": 3.9182968374378095,
+      "learning_rate": 3.023097377803224e-07,
+      "logits/chosen": 0.6478672027587891,
+      "logits/rejected": 1.024259328842163,
+      "logps/chosen": -4.996875286102295,
+      "logps/rejected": -6.983926296234131,
+      "loss": 0.6775,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.996875286102295,
+      "rewards/margins": 1.9870504140853882,
+      "rewards/rejected": -6.983926296234131,
+      "step": 3735
+    },
+    {
+      "epoch": 2.001672520488376,
+      "grad_norm": 5.0942647398757686,
+      "learning_rate": 3.008801976210423e-07,
+      "logits/chosen": 0.7724747657775879,
+      "logits/rejected": 1.2603263854980469,
+      "logps/chosen": -4.9839277267456055,
+      "logps/rejected": -9.064435958862305,
+      "loss": 0.6602,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.9839277267456055,
+      "rewards/margins": 4.080507278442383,
+      "rewards/rejected": -9.064435958862305,
+      "step": 3740
+    },
+    {
+      "epoch": 2.0043485532697773,
+      "grad_norm": 3.553914267154261,
+      "learning_rate": 2.994525893229581e-07,
+      "logits/chosen": 0.6246925592422485,
+      "logits/rejected": 1.1714510917663574,
+      "logps/chosen": -5.254703521728516,
+      "logps/rejected": -8.828350067138672,
+      "loss": 0.6499,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -5.254703521728516,
+      "rewards/margins": 3.5736465454101562,
+      "rewards/rejected": -8.828350067138672,
+      "step": 3745
+    },
+    {
+      "epoch": 2.007024586051179,
+      "grad_norm": 4.911655197455444,
+      "learning_rate": 2.98026926736732e-07,
+      "logits/chosen": 0.5825676321983337,
+      "logits/rejected": 1.0285228490829468,
+      "logps/chosen": -4.681671142578125,
+      "logps/rejected": -8.130693435668945,
+      "loss": 0.6491,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.681671142578125,
+      "rewards/margins": 3.449021816253662,
+      "rewards/rejected": -8.130693435668945,
+      "step": 3750
+    },
+    {
+      "epoch": 2.0097006188325808,
+      "grad_norm": 4.55151015230584,
+      "learning_rate": 2.9660322369414846e-07,
+      "logits/chosen": 0.8097246885299683,
+      "logits/rejected": 1.17612886428833,
+      "logps/chosen": -5.188439846038818,
+      "logps/rejected": -8.423254013061523,
+      "loss": 0.6497,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -5.188439846038818,
+      "rewards/margins": 3.234814405441284,
+      "rewards/rejected": -8.423254013061523,
+      "step": 3755
+    },
+    {
+      "epoch": 2.0123766516139825,
+      "grad_norm": 4.847184617708044,
+      "learning_rate": 2.9518149400798063e-07,
+      "logits/chosen": 0.5694905519485474,
+      "logits/rejected": 0.9265146255493164,
+      "logps/chosen": -4.927008152008057,
+      "logps/rejected": -8.375517845153809,
+      "loss": 0.6558,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.927008152008057,
+      "rewards/margins": 3.4485104084014893,
+      "rewards/rejected": -8.375517845153809,
+      "step": 3760
+    },
+    {
+      "epoch": 2.0150526843953838,
+      "grad_norm": 4.082084448294874,
+      "learning_rate": 2.9376175147185633e-07,
+      "logits/chosen": 0.8116804361343384,
+      "logits/rejected": 1.2267146110534668,
+      "logps/chosen": -5.974259376525879,
+      "logps/rejected": -8.46102237701416,
+      "loss": 0.6582,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -5.974259376525879,
+      "rewards/margins": 2.486762285232544,
+      "rewards/rejected": -8.46102237701416,
+      "step": 3765
+    },
+    {
+      "epoch": 2.0177287171767855,
+      "grad_norm": 4.332508128523993,
+      "learning_rate": 2.9234400986012376e-07,
+      "logits/chosen": 0.5680437684059143,
+      "logits/rejected": 0.8916395902633667,
+      "logps/chosen": -4.9265828132629395,
+      "logps/rejected": -8.32175350189209,
+      "loss": 0.643,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.9265828132629395,
+      "rewards/margins": 3.3951709270477295,
+      "rewards/rejected": -8.32175350189209,
+      "step": 3770
+    },
+    {
+      "epoch": 2.020404749958187,
+      "grad_norm": 4.1204729205460655,
+      "learning_rate": 2.9092828292771817e-07,
+      "logits/chosen": 0.7949905395507812,
+      "logits/rejected": 1.2284226417541504,
+      "logps/chosen": -5.523741245269775,
+      "logps/rejected": -8.606515884399414,
+      "loss": 0.6519,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -5.523741245269775,
+      "rewards/margins": 3.082775831222534,
+      "rewards/rejected": -8.606515884399414,
+      "step": 3775
+    },
+    {
+      "epoch": 2.0230807827395885,
+      "grad_norm": 4.569121982659876,
+      "learning_rate": 2.8951458441002875e-07,
+      "logits/chosen": 0.5115416646003723,
+      "logits/rejected": 0.9616246223449707,
+      "logps/chosen": -4.2246832847595215,
+      "logps/rejected": -7.207086086273193,
+      "loss": 0.6525,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.2246832847595215,
+      "rewards/margins": 2.9824025630950928,
+      "rewards/rejected": -7.207086086273193,
+      "step": 3780
+    },
+    {
+      "epoch": 2.02575681552099,
+      "grad_norm": 3.6076043838863887,
+      "learning_rate": 2.881029280227643e-07,
+      "logits/chosen": 0.674122154712677,
+      "logits/rejected": 0.9840105175971985,
+      "logps/chosen": -4.986670017242432,
+      "logps/rejected": -7.535124778747559,
+      "loss": 0.6509,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.986670017242432,
+      "rewards/margins": 2.5484559535980225,
+      "rewards/rejected": -7.535124778747559,
+      "step": 3785
+    },
+    {
+      "epoch": 2.028432848302392,
+      "grad_norm": 3.7623921317599325,
+      "learning_rate": 2.8669332746182177e-07,
+      "logits/chosen": 0.48682260513305664,
+      "logits/rejected": 1.0269490480422974,
+      "logps/chosen": -4.287054538726807,
+      "logps/rejected": -8.12656307220459,
+      "loss": 0.6379,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.287054538726807,
+      "rewards/margins": 3.8395087718963623,
+      "rewards/rejected": -8.12656307220459,
+      "step": 3790
+    },
+    {
+      "epoch": 2.031108881083793,
+      "grad_norm": 4.134033157512106,
+      "learning_rate": 2.8528579640315156e-07,
+      "logits/chosen": 0.6849793195724487,
+      "logits/rejected": 1.0777286291122437,
+      "logps/chosen": -5.24420166015625,
+      "logps/rejected": -8.08556842803955,
+      "loss": 0.6515,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -5.24420166015625,
+      "rewards/margins": 2.8413662910461426,
+      "rewards/rejected": -8.08556842803955,
+      "step": 3795
+    },
+    {
+      "epoch": 2.033784913865195,
+      "grad_norm": 5.002670160479487,
+      "learning_rate": 2.8388034850262646e-07,
+      "logits/chosen": 0.7173837423324585,
+      "logits/rejected": 1.0184799432754517,
+      "logps/chosen": -5.327506065368652,
+      "logps/rejected": -7.429425239562988,
+      "loss": 0.6481,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -5.327506065368652,
+      "rewards/margins": 2.1019186973571777,
+      "rewards/rejected": -7.429425239562988,
+      "step": 3800
+    },
+    {
+      "epoch": 2.0364609466465966,
+      "grad_norm": 7.440070091478831,
+      "learning_rate": 2.824769973959079e-07,
+      "logits/chosen": 0.6762133836746216,
+      "logits/rejected": 1.1280977725982666,
+      "logps/chosen": -4.955260276794434,
+      "logps/rejected": -7.8950090408325195,
+      "loss": 0.658,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.955260276794434,
+      "rewards/margins": 2.9397482872009277,
+      "rewards/rejected": -7.8950090408325195,
+      "step": 3805
+    },
+    {
+      "epoch": 2.039136979427998,
+      "grad_norm": 4.440283715859192,
+      "learning_rate": 2.81075756698315e-07,
+      "logits/chosen": 0.6264491081237793,
+      "logits/rejected": 1.0991246700286865,
+      "logps/chosen": -5.023207187652588,
+      "logps/rejected": -7.944009304046631,
+      "loss": 0.6419,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -5.023207187652588,
+      "rewards/margins": 2.920802593231201,
+      "rewards/rejected": -7.944009304046631,
+      "step": 3810
+    },
+    {
+      "epoch": 2.0418130122093996,
+      "grad_norm": 3.6445458491025033,
+      "learning_rate": 2.7967664000469035e-07,
+      "logits/chosen": 0.6065690517425537,
+      "logits/rejected": 1.2127424478530884,
+      "logps/chosen": -5.554740905761719,
+      "logps/rejected": -8.760030746459961,
+      "loss": 0.6517,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -5.554740905761719,
+      "rewards/margins": 3.2052910327911377,
+      "rewards/rejected": -8.760030746459961,
+      "step": 3815
+    },
+    {
+      "epoch": 2.0444890449908013,
+      "grad_norm": 2.7848255840969283,
+      "learning_rate": 2.7827966088927095e-07,
+      "logits/chosen": 0.6752180457115173,
+      "logits/rejected": 1.1875120401382446,
+      "logps/chosen": -5.403857707977295,
+      "logps/rejected": -8.200937271118164,
+      "loss": 0.6602,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -5.403857707977295,
+      "rewards/margins": 2.7970786094665527,
+      "rewards/rejected": -8.200937271118164,
+      "step": 3820
+    },
+    {
+      "epoch": 2.0471650777722026,
+      "grad_norm": 7.856724910061622,
+      "learning_rate": 2.768848329055538e-07,
+      "logits/chosen": 0.6781426668167114,
+      "logits/rejected": 1.21895170211792,
+      "logps/chosen": -4.603583335876465,
+      "logps/rejected": -8.874626159667969,
+      "loss": 0.6552,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.603583335876465,
+      "rewards/margins": 4.271042823791504,
+      "rewards/rejected": -8.874626159667969,
+      "step": 3825
+    },
+    {
+      "epoch": 2.0498411105536043,
+      "grad_norm": 3.848185062373218,
+      "learning_rate": 2.7549216958616657e-07,
+      "logits/chosen": 0.693032443523407,
+      "logits/rejected": 1.2147791385650635,
+      "logps/chosen": -5.530742645263672,
+      "logps/rejected": -9.122186660766602,
+      "loss": 0.6555,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -5.530742645263672,
+      "rewards/margins": 3.591444492340088,
+      "rewards/rejected": -9.122186660766602,
+      "step": 3830
+    },
+    {
+      "epoch": 2.052517143335006,
+      "grad_norm": 4.0332378324176394,
+      "learning_rate": 2.741016844427344e-07,
+      "logits/chosen": 0.5781368613243103,
+      "logits/rejected": 1.0955333709716797,
+      "logps/chosen": -4.33101749420166,
+      "logps/rejected": -8.092267036437988,
+      "loss": 0.667,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.33101749420166,
+      "rewards/margins": 3.76124906539917,
+      "rewards/rejected": -8.092267036437988,
+      "step": 3835
+    },
+    {
+      "epoch": 2.0551931761164073,
+      "grad_norm": 4.083048070974939,
+      "learning_rate": 2.7271339096575073e-07,
+      "logits/chosen": 0.6468622088432312,
+      "logits/rejected": 1.1245617866516113,
+      "logps/chosen": -4.643099308013916,
+      "logps/rejected": -7.652809143066406,
+      "loss": 0.6459,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.643099308013916,
+      "rewards/margins": 3.0097107887268066,
+      "rewards/rejected": -7.652809143066406,
+      "step": 3840
+    },
+    {
+      "epoch": 2.057869208897809,
+      "grad_norm": 3.045053830100292,
+      "learning_rate": 2.713273026244446e-07,
+      "logits/chosen": 0.6768208742141724,
+      "logits/rejected": 1.0716087818145752,
+      "logps/chosen": -5.627554893493652,
+      "logps/rejected": -7.699488162994385,
+      "loss": 0.6676,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -5.627554893493652,
+      "rewards/margins": 2.0719332695007324,
+      "rewards/rejected": -7.699488162994385,
+      "step": 3845
+    },
+    {
+      "epoch": 2.0605452416792107,
+      "grad_norm": 4.546400494469088,
+      "learning_rate": 2.6994343286665156e-07,
+      "logits/chosen": 0.8181160092353821,
+      "logits/rejected": 1.0489778518676758,
+      "logps/chosen": -5.62258243560791,
+      "logps/rejected": -7.2288689613342285,
+      "loss": 0.6534,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -5.62258243560791,
+      "rewards/margins": 1.606286644935608,
+      "rewards/rejected": -7.2288689613342285,
+      "step": 3850
+    },
+    {
+      "epoch": 2.063221274460612,
+      "grad_norm": 8.29929258879728,
+      "learning_rate": 2.6856179511868156e-07,
+      "logits/chosen": 0.9014382362365723,
+      "logits/rejected": 1.1278889179229736,
+      "logps/chosen": -5.4712700843811035,
+      "logps/rejected": -7.761249542236328,
+      "loss": 0.6616,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -5.4712700843811035,
+      "rewards/margins": 2.2899794578552246,
+      "rewards/rejected": -7.761249542236328,
+      "step": 3855
+    },
+    {
+      "epoch": 2.0658973072420137,
+      "grad_norm": 7.396886201795468,
+      "learning_rate": 2.6718240278519056e-07,
+      "logits/chosen": 0.6725633144378662,
+      "logits/rejected": 1.051861047744751,
+      "logps/chosen": -4.809172630310059,
+      "logps/rejected": -8.067784309387207,
+      "loss": 0.6528,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.809172630310059,
+      "rewards/margins": 3.2586112022399902,
+      "rewards/rejected": -8.067784309387207,
+      "step": 3860
+    },
+    {
+      "epoch": 2.0685733400234154,
+      "grad_norm": 10.760433304515132,
+      "learning_rate": 2.6580526924904866e-07,
+      "logits/chosen": 0.5745353698730469,
+      "logits/rejected": 1.2705421447753906,
+      "logps/chosen": -5.009461879730225,
+      "logps/rejected": -8.865117073059082,
+      "loss": 0.6562,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -5.009461879730225,
+      "rewards/margins": 3.855653762817383,
+      "rewards/rejected": -8.865117073059082,
+      "step": 3865
+    },
+    {
+      "epoch": 2.0712493728048167,
+      "grad_norm": 8.135704429627515,
+      "learning_rate": 2.6443040787121186e-07,
+      "logits/chosen": 0.6934847831726074,
+      "logits/rejected": 1.328850507736206,
+      "logps/chosen": -4.997049808502197,
+      "logps/rejected": -9.373119354248047,
+      "loss": 0.6435,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -4.997049808502197,
+      "rewards/margins": 4.376069068908691,
+      "rewards/rejected": -9.373119354248047,
+      "step": 3870
+    },
+    {
+      "epoch": 2.0739254055862184,
+      "grad_norm": 3.1027323336264985,
+      "learning_rate": 2.6305783199059084e-07,
+      "logits/chosen": 0.8809429407119751,
+      "logits/rejected": 1.2663222551345825,
+      "logps/chosen": -5.6389031410217285,
+      "logps/rejected": -8.28916072845459,
+      "loss": 0.6665,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -5.6389031410217285,
+      "rewards/margins": 2.6502585411071777,
+      "rewards/rejected": -8.28916072845459,
+      "step": 3875
+    },
+    {
+      "epoch": 2.07660143836762,
+      "grad_norm": 3.2977015768050073,
+      "learning_rate": 2.6168755492392324e-07,
+      "logits/chosen": 0.6581215262413025,
+      "logits/rejected": 1.2826836109161377,
+      "logps/chosen": -4.886979103088379,
+      "logps/rejected": -8.701908111572266,
+      "loss": 0.6302,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.886979103088379,
+      "rewards/margins": 3.814929485321045,
+      "rewards/rejected": -8.701908111572266,
+      "step": 3880
+    },
+    {
+      "epoch": 2.0792774711490214,
+      "grad_norm": 5.074077045774385,
+      "learning_rate": 2.6031958996564274e-07,
+      "logits/chosen": 0.6497079133987427,
+      "logits/rejected": 1.2873790264129639,
+      "logps/chosen": -4.808619976043701,
+      "logps/rejected": -9.165671348571777,
+      "loss": 0.6455,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.808619976043701,
+      "rewards/margins": 4.357051849365234,
+      "rewards/rejected": -9.165671348571777,
+      "step": 3885
+    },
+    {
+      "epoch": 2.081953503930423,
+      "grad_norm": 13.59844240883991,
+      "learning_rate": 2.589539503877518e-07,
+      "logits/chosen": 0.6917709708213806,
+      "logits/rejected": 1.009840965270996,
+      "logps/chosen": -5.020747661590576,
+      "logps/rejected": -7.292848110198975,
+      "loss": 0.6635,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -5.020747661590576,
+      "rewards/margins": 2.2721004486083984,
+      "rewards/rejected": -7.292848110198975,
+      "step": 3890
+    },
+    {
+      "epoch": 2.084629536711825,
+      "grad_norm": 12.212998297254355,
+      "learning_rate": 2.5759064943969125e-07,
+      "logits/chosen": 0.5760279297828674,
+      "logits/rejected": 1.0745757818222046,
+      "logps/chosen": -4.440577030181885,
+      "logps/rejected": -7.489468574523926,
+      "loss": 0.6575,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.440577030181885,
+      "rewards/margins": 3.04889178276062,
+      "rewards/rejected": -7.489468574523926,
+      "step": 3895
+    },
+    {
+      "epoch": 2.087305569493226,
+      "grad_norm": 5.299774110512319,
+      "learning_rate": 2.562297003482131e-07,
+      "logits/chosen": 0.6553961038589478,
+      "logits/rejected": 1.193622350692749,
+      "logps/chosen": -4.683090686798096,
+      "logps/rejected": -8.005773544311523,
+      "loss": 0.6434,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.683090686798096,
+      "rewards/margins": 3.3226826190948486,
+      "rewards/rejected": -8.005773544311523,
+      "step": 3900
+    },
+    {
+      "epoch": 2.089981602274628,
+      "grad_norm": 4.865845379060866,
+      "learning_rate": 2.548711163172512e-07,
+      "logits/chosen": 0.6448699235916138,
+      "logits/rejected": 1.0494476556777954,
+      "logps/chosen": -4.847794532775879,
+      "logps/rejected": -7.57025146484375,
+      "loss": 0.6626,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.847794532775879,
+      "rewards/margins": 2.7224576473236084,
+      "rewards/rejected": -7.57025146484375,
+      "step": 3905
+    },
+    {
+      "epoch": 2.0926576350560295,
+      "grad_norm": 4.918009414011563,
+      "learning_rate": 2.53514910527794e-07,
+      "logits/chosen": 0.6293562650680542,
+      "logits/rejected": 1.0439144372940063,
+      "logps/chosen": -4.076495170593262,
+      "logps/rejected": -7.361364841461182,
+      "loss": 0.651,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.076495170593262,
+      "rewards/margins": 3.28486967086792,
+      "rewards/rejected": -7.361364841461182,
+      "step": 3910
+    },
+    {
+      "epoch": 2.095333667837431,
+      "grad_norm": 4.41867756169221,
+      "learning_rate": 2.5216109613775573e-07,
+      "logits/chosen": 0.6109418869018555,
+      "logits/rejected": 0.9633005857467651,
+      "logps/chosen": -5.479954242706299,
+      "logps/rejected": -7.57854700088501,
+      "loss": 0.6655,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -5.479954242706299,
+      "rewards/margins": 2.0985920429229736,
+      "rewards/rejected": -7.57854700088501,
+      "step": 3915
+    },
+    {
+      "epoch": 2.0980097006188325,
+      "grad_norm": 2.910352036253663,
+      "learning_rate": 2.5080968628184993e-07,
+      "logits/chosen": 0.5730571746826172,
+      "logits/rejected": 0.9870290756225586,
+      "logps/chosen": -4.575341701507568,
+      "logps/rejected": -7.196019172668457,
+      "loss": 0.6548,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.575341701507568,
+      "rewards/margins": 2.6206765174865723,
+      "rewards/rejected": -7.196019172668457,
+      "step": 3920
+    },
+    {
+      "epoch": 2.1006857334002342,
+      "grad_norm": 4.012942625485214,
+      "learning_rate": 2.494606940714605e-07,
+      "logits/chosen": 0.5769037008285522,
+      "logits/rejected": 1.0240081548690796,
+      "logps/chosen": -4.330559253692627,
+      "logps/rejected": -7.727107048034668,
+      "loss": 0.6322,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.330559253692627,
+      "rewards/margins": 3.3965487480163574,
+      "rewards/rejected": -7.727107048034668,
+      "step": 3925
+    },
+    {
+      "epoch": 2.103361766181636,
+      "grad_norm": 4.270441004971415,
+      "learning_rate": 2.4811413259451625e-07,
+      "logits/chosen": 0.4952712953090668,
+      "logits/rejected": 0.8577522039413452,
+      "logps/chosen": -4.004376411437988,
+      "logps/rejected": -6.819216728210449,
+      "loss": 0.6427,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.004376411437988,
+      "rewards/margins": 2.8148415088653564,
+      "rewards/rejected": -6.819216728210449,
+      "step": 3930
+    },
+    {
+      "epoch": 2.106037798963037,
+      "grad_norm": 5.225531078002105,
+      "learning_rate": 2.46770014915362e-07,
+      "logits/chosen": 0.6533412933349609,
+      "logits/rejected": 0.9399615526199341,
+      "logps/chosen": -4.84005069732666,
+      "logps/rejected": -7.41485071182251,
+      "loss": 0.6575,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.84005069732666,
+      "rewards/margins": 2.5748000144958496,
+      "rewards/rejected": -7.41485071182251,
+      "step": 3935
+    },
+    {
+      "epoch": 2.108713831744439,
+      "grad_norm": 6.590866405397109,
+      "learning_rate": 2.45428354074634e-07,
+      "logits/chosen": 0.5922633409500122,
+      "logits/rejected": 0.8697918653488159,
+      "logps/chosen": -4.644097328186035,
+      "logps/rejected": -6.799752235412598,
+      "loss": 0.6509,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.644097328186035,
+      "rewards/margins": 2.1556553840637207,
+      "rewards/rejected": -6.799752235412598,
+      "step": 3940
+    },
+    {
+      "epoch": 2.1113898645258407,
+      "grad_norm": 7.510847346086081,
+      "learning_rate": 2.4408916308913105e-07,
+      "logits/chosen": 0.6811133623123169,
+      "logits/rejected": 0.9791873097419739,
+      "logps/chosen": -5.0890889167785645,
+      "logps/rejected": -7.459744453430176,
+      "loss": 0.6758,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -5.0890889167785645,
+      "rewards/margins": 2.3706564903259277,
+      "rewards/rejected": -7.459744453430176,
+      "step": 3945
+    },
+    {
+      "epoch": 2.114065897307242,
+      "grad_norm": 5.457046305385777,
+      "learning_rate": 2.4275245495169025e-07,
+      "logits/chosen": 0.5841856598854065,
+      "logits/rejected": 1.2661731243133545,
+      "logps/chosen": -4.209464073181152,
+      "logps/rejected": -8.867825508117676,
+      "loss": 0.648,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -4.209464073181152,
+      "rewards/margins": 4.65836238861084,
+      "rewards/rejected": -8.867825508117676,
+      "step": 3950
+    },
+    {
+      "epoch": 2.1167419300886436,
+      "grad_norm": 5.133620061840492,
+      "learning_rate": 2.414182426310597e-07,
+      "logits/chosen": 0.6282256245613098,
+      "logits/rejected": 1.2959620952606201,
+      "logps/chosen": -4.85838508605957,
+      "logps/rejected": -9.278965950012207,
+      "loss": 0.6475,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.85838508605957,
+      "rewards/margins": 4.420580863952637,
+      "rewards/rejected": -9.278965950012207,
+      "step": 3955
+    },
+    {
+      "epoch": 2.1194179628700454,
+      "grad_norm": 5.317672400977429,
+      "learning_rate": 2.400865390717734e-07,
+      "logits/chosen": 0.6487271785736084,
+      "logits/rejected": 1.164998173713684,
+      "logps/chosen": -4.51699161529541,
+      "logps/rejected": -8.211235046386719,
+      "loss": 0.6424,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.51699161529541,
+      "rewards/margins": 3.694244861602783,
+      "rewards/rejected": -8.211235046386719,
+      "step": 3960
+    },
+    {
+      "epoch": 2.1220939956514466,
+      "grad_norm": 5.699823310249397,
+      "learning_rate": 2.3875735719402475e-07,
+      "logits/chosen": 0.752027153968811,
+      "logits/rejected": 1.29035484790802,
+      "logps/chosen": -5.5525803565979,
+      "logps/rejected": -8.787683486938477,
+      "loss": 0.6538,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -5.5525803565979,
+      "rewards/margins": 3.235102891921997,
+      "rewards/rejected": -8.787683486938477,
+      "step": 3965
+    },
+    {
+      "epoch": 2.1247700284328483,
+      "grad_norm": 3.5415316345001258,
+      "learning_rate": 2.3743070989354258e-07,
+      "logits/chosen": 0.7930649518966675,
+      "logits/rejected": 1.201524019241333,
+      "logps/chosen": -5.231817245483398,
+      "logps/rejected": -8.273914337158203,
+      "loss": 0.649,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -5.231817245483398,
+      "rewards/margins": 3.0420970916748047,
+      "rewards/rejected": -8.273914337158203,
+      "step": 3970
+    },
+    {
+      "epoch": 2.12744606121425,
+      "grad_norm": 5.530314141892127,
+      "learning_rate": 2.3610661004146454e-07,
+      "logits/chosen": 0.6344776749610901,
+      "logits/rejected": 1.108520746231079,
+      "logps/chosen": -4.176750183105469,
+      "logps/rejected": -7.429540157318115,
+      "loss": 0.6401,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.176750183105469,
+      "rewards/margins": 3.2527904510498047,
+      "rewards/rejected": -7.429540157318115,
+      "step": 3975
+    },
+    {
+      "epoch": 2.1301220939956513,
+      "grad_norm": 6.968728807074978,
+      "learning_rate": 2.3478507048421314e-07,
+      "logits/chosen": 0.6452749967575073,
+      "logits/rejected": 1.0167276859283447,
+      "logps/chosen": -4.509917736053467,
+      "logps/rejected": -8.03312873840332,
+      "loss": 0.6412,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.509917736053467,
+      "rewards/margins": 3.5232110023498535,
+      "rewards/rejected": -8.03312873840332,
+      "step": 3980
+    },
+    {
+      "epoch": 2.132798126777053,
+      "grad_norm": 4.365601568396005,
+      "learning_rate": 2.334661040433713e-07,
+      "logits/chosen": 0.7223197221755981,
+      "logits/rejected": 0.9214617609977722,
+      "logps/chosen": -5.100761890411377,
+      "logps/rejected": -7.472434043884277,
+      "loss": 0.6468,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -5.100761890411377,
+      "rewards/margins": 2.3716723918914795,
+      "rewards/rejected": -7.472434043884277,
+      "step": 3985
+    },
+    {
+      "epoch": 2.1354741595584548,
+      "grad_norm": 4.037124811806548,
+      "learning_rate": 2.321497235155568e-07,
+      "logits/chosen": 0.3878096342086792,
+      "logits/rejected": 0.9574130177497864,
+      "logps/chosen": -3.6397719383239746,
+      "logps/rejected": -7.497265815734863,
+      "loss": 0.6344,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -3.6397719383239746,
+      "rewards/margins": 3.8574938774108887,
+      "rewards/rejected": -7.497265815734863,
+      "step": 3990
+    },
+    {
+      "epoch": 2.138150192339856,
+      "grad_norm": 5.203815628561266,
+      "learning_rate": 2.3083594167229965e-07,
+      "logits/chosen": 0.6156367063522339,
+      "logits/rejected": 0.9322867393493652,
+      "logps/chosen": -5.295657157897949,
+      "logps/rejected": -7.517223358154297,
+      "loss": 0.6661,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -5.295657157897949,
+      "rewards/margins": 2.2215654850006104,
+      "rewards/rejected": -7.517223358154297,
+      "step": 3995
+    },
+    {
+      "epoch": 2.1408262251212578,
+      "grad_norm": 6.903241618415471,
+      "learning_rate": 2.295247712599167e-07,
+      "logits/chosen": 0.45621052384376526,
+      "logits/rejected": 0.8468198776245117,
+      "logps/chosen": -3.753781795501709,
+      "logps/rejected": -7.075902462005615,
+      "loss": 0.6224,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -3.753781795501709,
+      "rewards/margins": 3.3221206665039062,
+      "rewards/rejected": -7.075902462005615,
+      "step": 4000
+    },
+    {
+      "epoch": 2.1408262251212578,
+      "eval_logits/chosen": 0.7335026860237122,
+      "eval_logits/rejected": 1.1107698678970337,
+      "eval_logps/chosen": -4.652297496795654,
+      "eval_logps/rejected": -7.886660575866699,
+      "eval_loss": 0.660657525062561,
+      "eval_rewards/accuracies": 0.6817507147789001,
+      "eval_rewards/chosen": -4.652297496795654,
+      "eval_rewards/margins": 3.234363079071045,
+      "eval_rewards/rejected": -7.886660575866699,
+      "eval_runtime": 40.7966,
+      "eval_samples_per_second": 32.968,
+      "eval_steps_per_second": 8.26,
+      "step": 4000
+    },
+    {
+      "epoch": 2.1435022579026595,
+      "grad_norm": 4.583960820890794,
+      "learning_rate": 2.2821622499938948e-07,
+      "logits/chosen": 0.4935286045074463,
+      "logits/rejected": 1.104052186012268,
+      "logps/chosen": -4.642902851104736,
+      "logps/rejected": -7.847242832183838,
+      "loss": 0.6638,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.642902851104736,
+      "rewards/margins": 3.2043399810791016,
+      "rewards/rejected": -7.847242832183838,
+      "step": 4005
+    },
+    {
+      "epoch": 2.1461782906840607,
+      "grad_norm": 4.27869684687148,
+      "learning_rate": 2.269103155862391e-07,
+      "logits/chosen": 0.5886538624763489,
+      "logits/rejected": 0.9757359623908997,
+      "logps/chosen": -4.8091349601745605,
+      "logps/rejected": -8.04555892944336,
+      "loss": 0.6559,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.8091349601745605,
+      "rewards/margins": 3.2364234924316406,
+      "rewards/rejected": -8.04555892944336,
+      "step": 4010
+    },
+    {
+      "epoch": 2.1488543234654625,
+      "grad_norm": 6.5179009659982095,
+      "learning_rate": 2.2560705569040483e-07,
+      "logits/chosen": 0.6606079936027527,
+      "logits/rejected": 1.064867615699768,
+      "logps/chosen": -5.457742214202881,
+      "logps/rejected": -7.959450721740723,
+      "loss": 0.6666,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -5.457742214202881,
+      "rewards/margins": 2.5017082691192627,
+      "rewards/rejected": -7.959450721740723,
+      "step": 4015
+    },
+    {
+      "epoch": 2.151530356246864,
+      "grad_norm": 3.726855209464774,
+      "learning_rate": 2.2430645795611963e-07,
+      "logits/chosen": 0.5882696509361267,
+      "logits/rejected": 1.23526930809021,
+      "logps/chosen": -4.822445869445801,
+      "logps/rejected": -9.17939567565918,
+      "loss": 0.6484,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.822445869445801,
+      "rewards/margins": 4.356949806213379,
+      "rewards/rejected": -9.17939567565918,
+      "step": 4020
+    },
+    {
+      "epoch": 2.1542063890282654,
+      "grad_norm": 6.494597949090722,
+      "learning_rate": 2.230085350017884e-07,
+      "logits/chosen": 0.8560161590576172,
+      "logits/rejected": 1.1432278156280518,
+      "logps/chosen": -5.597212791442871,
+      "logps/rejected": -8.00193977355957,
+      "loss": 0.6711,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -5.597212791442871,
+      "rewards/margins": 2.40472674369812,
+      "rewards/rejected": -8.00193977355957,
+      "step": 4025
+    },
+    {
+      "epoch": 2.156882421809667,
+      "grad_norm": 5.777312993602814,
+      "learning_rate": 2.2171329941986554e-07,
+      "logits/chosen": 0.7859684228897095,
+      "logits/rejected": 1.3144736289978027,
+      "logps/chosen": -5.023255825042725,
+      "logps/rejected": -9.240660667419434,
+      "loss": 0.6383,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -5.023255825042725,
+      "rewards/margins": 4.217404842376709,
+      "rewards/rejected": -9.240660667419434,
+      "step": 4030
+    },
+    {
+      "epoch": 2.159558454591069,
+      "grad_norm": 10.69076537230497,
+      "learning_rate": 2.2042076377673202e-07,
+      "logits/chosen": 0.6916903853416443,
+      "logits/rejected": 1.1172786951065063,
+      "logps/chosen": -4.914486885070801,
+      "logps/rejected": -7.599175930023193,
+      "loss": 0.6669,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.914486885070801,
+      "rewards/margins": 2.6846890449523926,
+      "rewards/rejected": -7.599175930023193,
+      "step": 4035
+    },
+    {
+      "epoch": 2.16223448737247,
+      "grad_norm": 6.048728543447373,
+      "learning_rate": 2.1913094061257476e-07,
+      "logits/chosen": 0.6838670969009399,
+      "logits/rejected": 1.1847387552261353,
+      "logps/chosen": -4.5417866706848145,
+      "logps/rejected": -8.538542747497559,
+      "loss": 0.6443,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.5417866706848145,
+      "rewards/margins": 3.9967567920684814,
+      "rewards/rejected": -8.538542747497559,
+      "step": 4040
+    },
+    {
+      "epoch": 2.164910520153872,
+      "grad_norm": 7.035487529070744,
+      "learning_rate": 2.178438424412633e-07,
+      "logits/chosen": 0.6040331125259399,
+      "logits/rejected": 1.023630976676941,
+      "logps/chosen": -4.75737190246582,
+      "logps/rejected": -7.841941833496094,
+      "loss": 0.6503,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.75737190246582,
+      "rewards/margins": 3.0845699310302734,
+      "rewards/rejected": -7.841941833496094,
+      "step": 4045
+    },
+    {
+      "epoch": 2.1675865529352736,
+      "grad_norm": 3.555313076764686,
+      "learning_rate": 2.165594817502302e-07,
+      "logits/chosen": 0.8315707445144653,
+      "logits/rejected": 1.0971678495407104,
+      "logps/chosen": -5.644009590148926,
+      "logps/rejected": -7.811171054840088,
+      "loss": 0.6652,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -5.644009590148926,
+      "rewards/margins": 2.1671605110168457,
+      "rewards/rejected": -7.811171054840088,
+      "step": 4050
+    },
+    {
+      "epoch": 2.170262585716675,
+      "grad_norm": 6.038864745043228,
+      "learning_rate": 2.1527787100034806e-07,
+      "logits/chosen": 0.8532511591911316,
+      "logits/rejected": 1.2538522481918335,
+      "logps/chosen": -5.597521781921387,
+      "logps/rejected": -7.93221378326416,
+      "loss": 0.6588,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -5.597521781921387,
+      "rewards/margins": 2.3346915245056152,
+      "rewards/rejected": -7.93221378326416,
+      "step": 4055
+    },
+    {
+      "epoch": 2.1729386184980766,
+      "grad_norm": 5.992075022055659,
+      "learning_rate": 2.1399902262581037e-07,
+      "logits/chosen": 1.0516215562820435,
+      "logits/rejected": 1.3338295221328735,
+      "logps/chosen": -5.653298377990723,
+      "logps/rejected": -8.622682571411133,
+      "loss": 0.6637,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -5.653298377990723,
+      "rewards/margins": 2.969383955001831,
+      "rewards/rejected": -8.622682571411133,
+      "step": 4060
+    },
+    {
+      "epoch": 2.1756146512794783,
+      "grad_norm": 4.633380704727173,
+      "learning_rate": 2.127229490340094e-07,
+      "logits/chosen": 0.7253440618515015,
+      "logits/rejected": 1.3344320058822632,
+      "logps/chosen": -5.337162494659424,
+      "logps/rejected": -9.180313110351562,
+      "loss": 0.6549,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -5.337162494659424,
+      "rewards/margins": 3.843151569366455,
+      "rewards/rejected": -9.180313110351562,
+      "step": 4065
+    },
+    {
+      "epoch": 2.1782906840608796,
+      "grad_norm": 5.186513947437672,
+      "learning_rate": 2.1144966260541698e-07,
+      "logits/chosen": 0.8931434750556946,
+      "logits/rejected": 1.3092409372329712,
+      "logps/chosen": -5.29010534286499,
+      "logps/rejected": -8.529533386230469,
+      "loss": 0.6521,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -5.29010534286499,
+      "rewards/margins": 3.2394275665283203,
+      "rewards/rejected": -8.529533386230469,
+      "step": 4070
+    },
+    {
+      "epoch": 2.1809667168422813,
+      "grad_norm": 3.5175088677067827,
+      "learning_rate": 2.1017917569346332e-07,
+      "logits/chosen": 0.6528345346450806,
+      "logits/rejected": 1.1931852102279663,
+      "logps/chosen": -4.7965617179870605,
+      "logps/rejected": -8.177209854125977,
+      "loss": 0.6488,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.7965617179870605,
+      "rewards/margins": 3.380648374557495,
+      "rewards/rejected": -8.177209854125977,
+      "step": 4075
+    },
+    {
+      "epoch": 2.183642749623683,
+      "grad_norm": 4.683191842845829,
+      "learning_rate": 2.0891150062441837e-07,
+      "logits/chosen": 0.6069825291633606,
+      "logits/rejected": 1.019539475440979,
+      "logps/chosen": -4.933394908905029,
+      "logps/rejected": -8.157915115356445,
+      "loss": 0.658,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.933394908905029,
+      "rewards/margins": 3.2245190143585205,
+      "rewards/rejected": -8.157915115356445,
+      "step": 4080
+    },
+    {
+      "epoch": 2.1863187824050843,
+      "grad_norm": 6.185567441502656,
+      "learning_rate": 2.0764664969727086e-07,
+      "logits/chosen": 0.6634326577186584,
+      "logits/rejected": 1.2841718196868896,
+      "logps/chosen": -4.380050182342529,
+      "logps/rejected": -8.057024955749512,
+      "loss": 0.6532,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.380050182342529,
+      "rewards/margins": 3.6769752502441406,
+      "rewards/rejected": -8.057024955749512,
+      "step": 4085
+    },
+    {
+      "epoch": 2.188994815186486,
+      "grad_norm": 9.842712792131461,
+      "learning_rate": 2.0638463518361033e-07,
+      "logits/chosen": 0.5819960236549377,
+      "logits/rejected": 1.06165611743927,
+      "logps/chosen": -4.513581275939941,
+      "logps/rejected": -7.145085334777832,
+      "loss": 0.6625,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.513581275939941,
+      "rewards/margins": 2.6315033435821533,
+      "rewards/rejected": -7.145085334777832,
+      "step": 4090
+    },
+    {
+      "epoch": 2.1916708479678877,
+      "grad_norm": 5.016633588582747,
+      "learning_rate": 2.0512546932750702e-07,
+      "logits/chosen": 0.6788409352302551,
+      "logits/rejected": 1.0429365634918213,
+      "logps/chosen": -4.658480167388916,
+      "logps/rejected": -7.277688026428223,
+      "loss": 0.6471,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.658480167388916,
+      "rewards/margins": 2.6192076206207275,
+      "rewards/rejected": -7.277688026428223,
+      "step": 4095
+    },
+    {
+      "epoch": 2.194346880749289,
+      "grad_norm": 6.646028620968852,
+      "learning_rate": 2.0386916434539343e-07,
+      "logits/chosen": 0.6716817021369934,
+      "logits/rejected": 1.0559837818145752,
+      "logps/chosen": -4.571835517883301,
+      "logps/rejected": -8.187578201293945,
+      "loss": 0.6334,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.571835517883301,
+      "rewards/margins": 3.6157429218292236,
+      "rewards/rejected": -8.187578201293945,
+      "step": 4100
+    },
+    {
+      "epoch": 2.1970229135306907,
+      "grad_norm": 8.96441434782502,
+      "learning_rate": 2.0261573242594627e-07,
+      "logits/chosen": 0.7228914499282837,
+      "logits/rejected": 1.262155294418335,
+      "logps/chosen": -5.633551120758057,
+      "logps/rejected": -8.37105655670166,
+      "loss": 0.6547,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -5.633551120758057,
+      "rewards/margins": 2.7375059127807617,
+      "rewards/rejected": -8.37105655670166,
+      "step": 4105
+    },
+    {
+      "epoch": 2.1996989463120924,
+      "grad_norm": 5.050647748514331,
+      "learning_rate": 2.0136518572996724e-07,
+      "logits/chosen": 0.8371809124946594,
+      "logits/rejected": 1.2711749076843262,
+      "logps/chosen": -5.600058555603027,
+      "logps/rejected": -8.536770820617676,
+      "loss": 0.6487,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -5.600058555603027,
+      "rewards/margins": 2.9367117881774902,
+      "rewards/rejected": -8.536770820617676,
+      "step": 4110
+    },
+    {
+      "epoch": 2.202374979093494,
+      "grad_norm": 6.499172036654939,
+      "learning_rate": 2.0011753639026617e-07,
+      "logits/chosen": 0.6979454755783081,
+      "logits/rejected": 1.1962063312530518,
+      "logps/chosen": -5.409426689147949,
+      "logps/rejected": -8.35353946685791,
+      "loss": 0.6618,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -5.409426689147949,
+      "rewards/margins": 2.9441120624542236,
+      "rewards/rejected": -8.35353946685791,
+      "step": 4115
+    },
+    {
+      "epoch": 2.2050510118748954,
+      "grad_norm": 5.505637759944027,
+      "learning_rate": 1.988727965115421e-07,
+      "logits/chosen": 0.5769757628440857,
+      "logits/rejected": 1.0596833229064941,
+      "logps/chosen": -4.272248268127441,
+      "logps/rejected": -7.903855323791504,
+      "loss": 0.6276,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.272248268127441,
+      "rewards/margins": 3.6316070556640625,
+      "rewards/rejected": -7.903855323791504,
+      "step": 4120
+    },
+    {
+      "epoch": 2.207727044656297,
+      "grad_norm": 3.847800202119962,
+      "learning_rate": 1.9763097817026713e-07,
+      "logits/chosen": 0.7888336181640625,
+      "logits/rejected": 1.2446476221084595,
+      "logps/chosen": -5.320199489593506,
+      "logps/rejected": -8.648067474365234,
+      "loss": 0.6342,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -5.320199489593506,
+      "rewards/margins": 3.3278682231903076,
+      "rewards/rejected": -8.648067474365234,
+      "step": 4125
+    },
+    {
+      "epoch": 2.210403077437699,
+      "grad_norm": 4.575361651359957,
+      "learning_rate": 1.9639209341456796e-07,
+      "logits/chosen": 0.5934714674949646,
+      "logits/rejected": 1.061612844467163,
+      "logps/chosen": -4.741045951843262,
+      "logps/rejected": -8.023757934570312,
+      "loss": 0.6405,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.741045951843262,
+      "rewards/margins": 3.282710552215576,
+      "rewards/rejected": -8.023757934570312,
+      "step": 4130
+    },
+    {
+      "epoch": 2.2130791102191,
+      "grad_norm": 5.47548851868597,
+      "learning_rate": 1.951561542641102e-07,
+      "logits/chosen": 0.6767334938049316,
+      "logits/rejected": 1.0727511644363403,
+      "logps/chosen": -4.970210075378418,
+      "logps/rejected": -7.7707839012146,
+      "loss": 0.6629,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.970210075378418,
+      "rewards/margins": 2.8005733489990234,
+      "rewards/rejected": -7.7707839012146,
+      "step": 4135
+    },
+    {
+      "epoch": 2.215755143000502,
+      "grad_norm": 3.0135013100647816,
+      "learning_rate": 1.939231727099806e-07,
+      "logits/chosen": 0.5016201138496399,
+      "logits/rejected": 0.9960115551948547,
+      "logps/chosen": -4.5830583572387695,
+      "logps/rejected": -7.851959228515625,
+      "loss": 0.6549,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.5830583572387695,
+      "rewards/margins": 3.2689006328582764,
+      "rewards/rejected": -7.851959228515625,
+      "step": 4140
+    },
+    {
+      "epoch": 2.2184311757819035,
+      "grad_norm": 5.418494205942117,
+      "learning_rate": 1.926931607145719e-07,
+      "logits/chosen": 0.8207980394363403,
+      "logits/rejected": 1.0904099941253662,
+      "logps/chosen": -5.554192066192627,
+      "logps/rejected": -7.655829429626465,
+      "loss": 0.669,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -5.554192066192627,
+      "rewards/margins": 2.101637601852417,
+      "rewards/rejected": -7.655829429626465,
+      "step": 4145
+    },
+    {
+      "epoch": 2.221107208563305,
+      "grad_norm": 8.492766519696499,
+      "learning_rate": 1.9146613021146564e-07,
+      "logits/chosen": 0.6360867619514465,
+      "logits/rejected": 1.0704528093338013,
+      "logps/chosen": -4.713377475738525,
+      "logps/rejected": -7.820076942443848,
+      "loss": 0.6513,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.713377475738525,
+      "rewards/margins": 3.1066999435424805,
+      "rewards/rejected": -7.820076942443848,
+      "step": 4150
+    },
+    {
+      "epoch": 2.2237832413447065,
+      "grad_norm": 3.294041258893447,
+      "learning_rate": 1.9024209310531736e-07,
+      "logits/chosen": 0.840248703956604,
+      "logits/rejected": 1.1801815032958984,
+      "logps/chosen": -4.8054890632629395,
+      "logps/rejected": -7.830203056335449,
+      "loss": 0.6409,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.8054890632629395,
+      "rewards/margins": 3.024714946746826,
+      "rewards/rejected": -7.830203056335449,
+      "step": 4155
+    },
+    {
+      "epoch": 2.2264592741261082,
+      "grad_norm": 5.360400507771889,
+      "learning_rate": 1.890210612717401e-07,
+      "logits/chosen": 0.6175212860107422,
+      "logits/rejected": 1.2140053510665894,
+      "logps/chosen": -5.047219276428223,
+      "logps/rejected": -8.567647933959961,
+      "loss": 0.6556,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -5.047219276428223,
+      "rewards/margins": 3.520430088043213,
+      "rewards/rejected": -8.567647933959961,
+      "step": 4160
+    },
+    {
+      "epoch": 2.2291353069075095,
+      "grad_norm": 3.170508684176078,
+      "learning_rate": 1.8780304655719054e-07,
+      "logits/chosen": 0.7066279649734497,
+      "logits/rejected": 1.3121592998504639,
+      "logps/chosen": -4.8012614250183105,
+      "logps/rejected": -9.079513549804688,
+      "loss": 0.6509,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.8012614250183105,
+      "rewards/margins": 4.278252601623535,
+      "rewards/rejected": -9.079513549804688,
+      "step": 4165
+    },
+    {
+      "epoch": 2.231811339688911,
+      "grad_norm": 6.296474076375813,
+      "learning_rate": 1.865880607788523e-07,
+      "logits/chosen": 0.8745888471603394,
+      "logits/rejected": 1.1795707941055298,
+      "logps/chosen": -4.936895370483398,
+      "logps/rejected": -7.591843605041504,
+      "loss": 0.6556,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -4.936895370483398,
+      "rewards/margins": 2.6549484729766846,
+      "rewards/rejected": -7.591843605041504,
+      "step": 4170
+    },
+    {
+      "epoch": 2.234487372470313,
+      "grad_norm": 8.05053361952291,
+      "learning_rate": 1.8537611572452316e-07,
+      "logits/chosen": 0.7784465551376343,
+      "logits/rejected": 1.200213074684143,
+      "logps/chosen": -5.348075866699219,
+      "logps/rejected": -8.372184753417969,
+      "loss": 0.6584,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -5.348075866699219,
+      "rewards/margins": 3.024109125137329,
+      "rewards/rejected": -8.372184753417969,
+      "step": 4175
+    },
+    {
+      "epoch": 2.237163405251714,
+      "grad_norm": 7.062512673775031,
+      "learning_rate": 1.84167223152499e-07,
+      "logits/chosen": 0.7155938148498535,
+      "logits/rejected": 1.1105462312698364,
+      "logps/chosen": -5.126309394836426,
+      "logps/rejected": -8.086161613464355,
+      "loss": 0.6477,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -5.126309394836426,
+      "rewards/margins": 2.9598536491394043,
+      "rewards/rejected": -8.086161613464355,
+      "step": 4180
+    },
+    {
+      "epoch": 2.239839438033116,
+      "grad_norm": 7.468972740740055,
+      "learning_rate": 1.8296139479146112e-07,
+      "logits/chosen": 0.7056856155395508,
+      "logits/rejected": 1.2330639362335205,
+      "logps/chosen": -4.946039199829102,
+      "logps/rejected": -8.816801071166992,
+      "loss": 0.6358,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.946039199829102,
+      "rewards/margins": 3.8707618713378906,
+      "rewards/rejected": -8.816801071166992,
+      "step": 4185
+    },
+    {
+      "epoch": 2.2425154708145176,
+      "grad_norm": 5.859287742790127,
+      "learning_rate": 1.8175864234036132e-07,
+      "logits/chosen": 0.7813374400138855,
+      "logits/rejected": 1.0991300344467163,
+      "logps/chosen": -4.864354610443115,
+      "logps/rejected": -7.530570030212402,
+      "loss": 0.6497,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.864354610443115,
+      "rewards/margins": 2.6662163734436035,
+      "rewards/rejected": -7.530570030212402,
+      "step": 4190
+    },
+    {
+      "epoch": 2.245191503595919,
+      "grad_norm": 4.892303925144629,
+      "learning_rate": 1.805589774683094e-07,
+      "logits/chosen": 0.6302552223205566,
+      "logits/rejected": 1.0579967498779297,
+      "logps/chosen": -4.768343925476074,
+      "logps/rejected": -7.566976070404053,
+      "loss": 0.6646,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.768343925476074,
+      "rewards/margins": 2.7986326217651367,
+      "rewards/rejected": -7.566976070404053,
+      "step": 4195
+    },
+    {
+      "epoch": 2.2478675363773206,
+      "grad_norm": 3.9477635786341825,
+      "learning_rate": 1.79362411814459e-07,
+      "logits/chosen": 0.8069186210632324,
+      "logits/rejected": 1.1487573385238647,
+      "logps/chosen": -4.804275035858154,
+      "logps/rejected": -8.131963729858398,
+      "loss": 0.6472,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.804275035858154,
+      "rewards/margins": 3.3276894092559814,
+      "rewards/rejected": -8.131963729858398,
+      "step": 4200
+    },
+    {
+      "epoch": 2.2505435691587223,
+      "grad_norm": 5.3593314064749675,
+      "learning_rate": 1.7816895698789552e-07,
+      "logits/chosen": 0.5728598833084106,
+      "logits/rejected": 1.0751702785491943,
+      "logps/chosen": -4.695885181427002,
+      "logps/rejected": -7.617256164550781,
+      "loss": 0.6591,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.695885181427002,
+      "rewards/margins": 2.9213714599609375,
+      "rewards/rejected": -7.617256164550781,
+      "step": 4205
+    },
+    {
+      "epoch": 2.2532196019401236,
+      "grad_norm": 4.169915606967582,
+      "learning_rate": 1.7697862456752271e-07,
+      "logits/chosen": 0.7250362634658813,
+      "logits/rejected": 1.1470948457717896,
+      "logps/chosen": -5.405759811401367,
+      "logps/rejected": -8.637968063354492,
+      "loss": 0.6562,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -5.405759811401367,
+      "rewards/margins": 3.2322070598602295,
+      "rewards/rejected": -8.637968063354492,
+      "step": 4210
+    },
+    {
+      "epoch": 2.2558956347215253,
+      "grad_norm": 4.329873662541532,
+      "learning_rate": 1.7579142610195124e-07,
+      "logits/chosen": 0.7127309441566467,
+      "logits/rejected": 1.364197015762329,
+      "logps/chosen": -5.274501323699951,
+      "logps/rejected": -8.8240966796875,
+      "loss": 0.6538,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -5.274501323699951,
+      "rewards/margins": 3.549595355987549,
+      "rewards/rejected": -8.8240966796875,
+      "step": 4215
+    },
+    {
+      "epoch": 2.258571667502927,
+      "grad_norm": 12.18191672396889,
+      "learning_rate": 1.7460737310938568e-07,
+      "logits/chosen": 0.7564809322357178,
+      "logits/rejected": 1.3809926509857178,
+      "logps/chosen": -4.888838768005371,
+      "logps/rejected": -8.84379768371582,
+      "loss": 0.6421,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.888838768005371,
+      "rewards/margins": 3.9549591541290283,
+      "rewards/rejected": -8.84379768371582,
+      "step": 4220
+    },
+    {
+      "epoch": 2.2612477002843283,
+      "grad_norm": 8.005801623503947,
+      "learning_rate": 1.734264770775133e-07,
+      "logits/chosen": 0.6519125699996948,
+      "logits/rejected": 1.1616982221603394,
+      "logps/chosen": -4.484858512878418,
+      "logps/rejected": -8.192185401916504,
+      "loss": 0.661,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.484858512878418,
+      "rewards/margins": 3.7073261737823486,
+      "rewards/rejected": -8.192185401916504,
+      "step": 4225
+    },
+    {
+      "epoch": 2.26392373306573,
+      "grad_norm": 4.4108983991826305,
+      "learning_rate": 1.7224874946339241e-07,
+      "logits/chosen": 0.7080442905426025,
+      "logits/rejected": 1.025758981704712,
+      "logps/chosen": -5.038155555725098,
+      "logps/rejected": -7.844827175140381,
+      "loss": 0.6545,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -5.038155555725098,
+      "rewards/margins": 2.806671142578125,
+      "rewards/rejected": -7.844827175140381,
+      "step": 4230
+    },
+    {
+      "epoch": 2.2665997658471317,
+      "grad_norm": 5.591088487408887,
+      "learning_rate": 1.7107420169334186e-07,
+      "logits/chosen": 0.7961901426315308,
+      "logits/rejected": 1.3041313886642456,
+      "logps/chosen": -5.307501316070557,
+      "logps/rejected": -7.967949867248535,
+      "loss": 0.6555,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -5.307501316070557,
+      "rewards/margins": 2.6604487895965576,
+      "rewards/rejected": -7.967949867248535,
+      "step": 4235
+    },
+    {
+      "epoch": 2.269275798628533,
+      "grad_norm": 4.8085569110781385,
+      "learning_rate": 1.6990284516282893e-07,
+      "logits/chosen": 0.6907963156700134,
+      "logits/rejected": 1.2960927486419678,
+      "logps/chosen": -4.552126884460449,
+      "logps/rejected": -8.79484748840332,
+      "loss": 0.6689,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.552126884460449,
+      "rewards/margins": 4.242720127105713,
+      "rewards/rejected": -8.79484748840332,
+      "step": 4240
+    },
+    {
+      "epoch": 2.2719518314099347,
+      "grad_norm": 3.4489896019083295,
+      "learning_rate": 1.687346912363602e-07,
+      "logits/chosen": 0.7263502478599548,
+      "logits/rejected": 1.1436059474945068,
+      "logps/chosen": -5.112565040588379,
+      "logps/rejected": -8.023330688476562,
+      "loss": 0.6372,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -5.112565040588379,
+      "rewards/margins": 2.9107658863067627,
+      "rewards/rejected": -8.023330688476562,
+      "step": 4245
+    },
+    {
+      "epoch": 2.2746278641913364,
+      "grad_norm": 2.1880452970299684,
+      "learning_rate": 1.675697512473697e-07,
+      "logits/chosen": 0.6986660957336426,
+      "logits/rejected": 1.2515475749969482,
+      "logps/chosen": -4.990495681762695,
+      "logps/rejected": -7.986851692199707,
+      "loss": 0.6589,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.990495681762695,
+      "rewards/margins": 2.9963557720184326,
+      "rewards/rejected": -7.986851692199707,
+      "step": 4250
+    },
+    {
+      "epoch": 2.2773038969727377,
+      "grad_norm": 4.516988542543613,
+      "learning_rate": 1.6640803649811087e-07,
+      "logits/chosen": 0.6611096262931824,
+      "logits/rejected": 1.0282419919967651,
+      "logps/chosen": -5.111964225769043,
+      "logps/rejected": -7.853438377380371,
+      "loss": 0.664,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -5.111964225769043,
+      "rewards/margins": 2.7414743900299072,
+      "rewards/rejected": -7.853438377380371,
+      "step": 4255
+    },
+    {
+      "epoch": 2.2799799297541394,
+      "grad_norm": 6.144449659457795,
+      "learning_rate": 1.6524955825954472e-07,
+      "logits/chosen": 0.7686523795127869,
+      "logits/rejected": 1.2556498050689697,
+      "logps/chosen": -5.048253536224365,
+      "logps/rejected": -8.226381301879883,
+      "loss": 0.6451,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -5.048253536224365,
+      "rewards/margins": 3.1781280040740967,
+      "rewards/rejected": -8.226381301879883,
+      "step": 4260
+    },
+    {
+      "epoch": 2.282655962535541,
+      "grad_norm": 2.974731267604704,
+      "learning_rate": 1.6409432777123277e-07,
+      "logits/chosen": 0.7873324155807495,
+      "logits/rejected": 1.360412359237671,
+      "logps/chosen": -4.741394996643066,
+      "logps/rejected": -9.173967361450195,
+      "loss": 0.6437,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.741394996643066,
+      "rewards/margins": 4.432572364807129,
+      "rewards/rejected": -9.173967361450195,
+      "step": 4265
+    },
+    {
+      "epoch": 2.285331995316943,
+      "grad_norm": 4.878336644967607,
+      "learning_rate": 1.6294235624122577e-07,
+      "logits/chosen": 0.7129805684089661,
+      "logits/rejected": 1.0942606925964355,
+      "logps/chosen": -4.896267890930176,
+      "logps/rejected": -7.1911420822143555,
+      "loss": 0.6608,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.896267890930176,
+      "rewards/margins": 2.294874668121338,
+      "rewards/rejected": -7.1911420822143555,
+      "step": 4270
+    },
+    {
+      "epoch": 2.288008028098344,
+      "grad_norm": 4.009545347100187,
+      "learning_rate": 1.6179365484595697e-07,
+      "logits/chosen": 0.7766197919845581,
+      "logits/rejected": 1.095897912979126,
+      "logps/chosen": -5.103603363037109,
+      "logps/rejected": -7.577892303466797,
+      "loss": 0.6624,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -5.103603363037109,
+      "rewards/margins": 2.4742889404296875,
+      "rewards/rejected": -7.577892303466797,
+      "step": 4275
+    },
+    {
+      "epoch": 2.290684060879746,
+      "grad_norm": 14.28897259843873,
+      "learning_rate": 1.60648234730132e-07,
+      "logits/chosen": 0.6507691144943237,
+      "logits/rejected": 1.2045106887817383,
+      "logps/chosen": -4.483565330505371,
+      "logps/rejected": -8.527327537536621,
+      "loss": 0.6424,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.483565330505371,
+      "rewards/margins": 4.04376220703125,
+      "rewards/rejected": -8.527327537536621,
+      "step": 4280
+    },
+    {
+      "epoch": 2.293360093661147,
+      "grad_norm": 6.470301816680742,
+      "learning_rate": 1.595061070066222e-07,
+      "logits/chosen": 0.767861545085907,
+      "logits/rejected": 1.3582266569137573,
+      "logps/chosen": -4.616291522979736,
+      "logps/rejected": -8.927189826965332,
+      "loss": 0.635,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.616291522979736,
+      "rewards/margins": 4.310898780822754,
+      "rewards/rejected": -8.927189826965332,
+      "step": 4285
+    },
+    {
+      "epoch": 2.296036126442549,
+      "grad_norm": 4.328785232589537,
+      "learning_rate": 1.5836728275635542e-07,
+      "logits/chosen": 0.5635241270065308,
+      "logits/rejected": 1.016815185546875,
+      "logps/chosen": -4.690028190612793,
+      "logps/rejected": -8.05630874633789,
+      "loss": 0.6516,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.690028190612793,
+      "rewards/margins": 3.3662796020507812,
+      "rewards/rejected": -8.05630874633789,
+      "step": 4290
+    },
+    {
+      "epoch": 2.2987121592239506,
+      "grad_norm": 5.245626966559062,
+      "learning_rate": 1.5723177302820984e-07,
+      "logits/chosen": 0.7679174542427063,
+      "logits/rejected": 1.139195203781128,
+      "logps/chosen": -5.114535331726074,
+      "logps/rejected": -7.5207695960998535,
+      "loss": 0.6626,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -5.114535331726074,
+      "rewards/margins": 2.4062342643737793,
+      "rewards/rejected": -7.5207695960998535,
+      "step": 4295
+    },
+    {
+      "epoch": 2.3013881920053523,
+      "grad_norm": 5.140862820130752,
+      "learning_rate": 1.5609958883890544e-07,
+      "logits/chosen": 0.6896184682846069,
+      "logits/rejected": 1.3681724071502686,
+      "logps/chosen": -4.648235321044922,
+      "logps/rejected": -8.494241714477539,
+      "loss": 0.6536,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.648235321044922,
+      "rewards/margins": 3.8460052013397217,
+      "rewards/rejected": -8.494241714477539,
+      "step": 4300
+    },
+    {
+      "epoch": 2.3040642247867535,
+      "grad_norm": 6.330969027611712,
+      "learning_rate": 1.5497074117289865e-07,
+      "logits/chosen": 0.598589301109314,
+      "logits/rejected": 0.9353580474853516,
+      "logps/chosen": -4.0943603515625,
+      "logps/rejected": -7.381089210510254,
+      "loss": 0.6331,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.0943603515625,
+      "rewards/margins": 3.286729097366333,
+      "rewards/rejected": -7.381089210510254,
+      "step": 4305
+    },
+    {
+      "epoch": 2.3067402575681553,
+      "grad_norm": 3.0346098059058666,
+      "learning_rate": 1.5384524098227402e-07,
+      "logits/chosen": 0.6140684485435486,
+      "logits/rejected": 1.159968614578247,
+      "logps/chosen": -4.535906791687012,
+      "logps/rejected": -8.18099594116211,
+      "loss": 0.6451,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.535906791687012,
+      "rewards/margins": 3.6450889110565186,
+      "rewards/rejected": -8.18099594116211,
+      "step": 4310
+    },
+    {
+      "epoch": 2.3094162903495565,
+      "grad_norm": 5.8233272783628625,
+      "learning_rate": 1.5272309918663974e-07,
+      "logits/chosen": 0.7978981137275696,
+      "logits/rejected": 1.041983723640442,
+      "logps/chosen": -5.177875518798828,
+      "logps/rejected": -6.952022552490234,
+      "loss": 0.6712,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -5.177875518798828,
+      "rewards/margins": 1.7741472721099854,
+      "rewards/rejected": -6.952022552490234,
+      "step": 4315
+    },
+    {
+      "epoch": 2.3120923231309582,
+      "grad_norm": 5.687871484363345,
+      "learning_rate": 1.516043266730201e-07,
+      "logits/chosen": 0.6296438574790955,
+      "logits/rejected": 0.9813583493232727,
+      "logps/chosen": -5.144505500793457,
+      "logps/rejected": -7.0942487716674805,
+      "loss": 0.6652,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -5.144505500793457,
+      "rewards/margins": 1.9497438669204712,
+      "rewards/rejected": -7.0942487716674805,
+      "step": 4320
+    },
+    {
+      "epoch": 2.31476835591236,
+      "grad_norm": 6.681316156713009,
+      "learning_rate": 1.504889342957512e-07,
+      "logits/chosen": 0.8050605654716492,
+      "logits/rejected": 1.1804124116897583,
+      "logps/chosen": -5.694845199584961,
+      "logps/rejected": -8.2296142578125,
+      "loss": 0.6583,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -5.694845199584961,
+      "rewards/margins": 2.5347695350646973,
+      "rewards/rejected": -8.2296142578125,
+      "step": 4325
+    },
+    {
+      "epoch": 2.3174443886937617,
+      "grad_norm": 3.8056151649982928,
+      "learning_rate": 1.4937693287637453e-07,
+      "logits/chosen": 0.7173921465873718,
+      "logits/rejected": 1.217578411102295,
+      "logps/chosen": -5.138716220855713,
+      "logps/rejected": -7.676386833190918,
+      "loss": 0.6581,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -5.138716220855713,
+      "rewards/margins": 2.5376713275909424,
+      "rewards/rejected": -7.676386833190918,
+      "step": 4330
+    },
+    {
+      "epoch": 2.320120421475163,
+      "grad_norm": 4.633408781396923,
+      "learning_rate": 1.4826833320353305e-07,
+      "logits/chosen": 0.7019614577293396,
+      "logits/rejected": 1.0986006259918213,
+      "logps/chosen": -5.105649471282959,
+      "logps/rejected": -7.5213494300842285,
+      "loss": 0.6791,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -5.105649471282959,
+      "rewards/margins": 2.4157001972198486,
+      "rewards/rejected": -7.5213494300842285,
+      "step": 4335
+    },
+    {
+      "epoch": 2.3227964542565647,
+      "grad_norm": 4.625668286707405,
+      "learning_rate": 1.4716314603286528e-07,
+      "logits/chosen": 0.8646273612976074,
+      "logits/rejected": 1.0557798147201538,
+      "logps/chosen": -4.975428104400635,
+      "logps/rejected": -7.195790767669678,
+      "loss": 0.6514,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.975428104400635,
+      "rewards/margins": 2.220362901687622,
+      "rewards/rejected": -7.195790767669678,
+      "step": 4340
+    },
+    {
+      "epoch": 2.3254724870379664,
+      "grad_norm": 5.791772668284485,
+      "learning_rate": 1.4606138208690233e-07,
+      "logits/chosen": 0.7779443264007568,
+      "logits/rejected": 1.4017661809921265,
+      "logps/chosen": -5.322023391723633,
+      "logps/rejected": -9.386761665344238,
+      "loss": 0.6673,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -5.322023391723633,
+      "rewards/margins": 4.064737796783447,
+      "rewards/rejected": -9.386761665344238,
+      "step": 4345
+    },
+    {
+      "epoch": 2.3281485198193677,
+      "grad_norm": 2.9203319045966887,
+      "learning_rate": 1.4496305205496251e-07,
+      "logits/chosen": 0.7787855863571167,
+      "logits/rejected": 1.181867241859436,
+      "logps/chosen": -5.071282386779785,
+      "logps/rejected": -8.463969230651855,
+      "loss": 0.6499,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -5.071282386779785,
+      "rewards/margins": 3.3926873207092285,
+      "rewards/rejected": -8.463969230651855,
+      "step": 4350
+    },
+    {
+      "epoch": 2.3308245526007694,
+      "grad_norm": 5.92197392910747,
+      "learning_rate": 1.4386816659304895e-07,
+      "logits/chosen": 0.7114294767379761,
+      "logits/rejected": 1.0922520160675049,
+      "logps/chosen": -4.8944196701049805,
+      "logps/rejected": -7.495493412017822,
+      "loss": 0.6573,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.8944196701049805,
+      "rewards/margins": 2.6010730266571045,
+      "rewards/rejected": -7.495493412017822,
+      "step": 4355
+    },
+    {
+      "epoch": 2.333500585382171,
+      "grad_norm": 5.488109191110414,
+      "learning_rate": 1.4277673632374492e-07,
+      "logits/chosen": 0.7415236830711365,
+      "logits/rejected": 1.1820952892303467,
+      "logps/chosen": -5.055557727813721,
+      "logps/rejected": -8.047971725463867,
+      "loss": 0.6573,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -5.055557727813721,
+      "rewards/margins": 2.992414951324463,
+      "rewards/rejected": -8.047971725463867,
+      "step": 4360
+    },
+    {
+      "epoch": 2.3361766181635724,
+      "grad_norm": 6.923039764264364,
+      "learning_rate": 1.416887718361119e-07,
+      "logits/chosen": 0.8450724482536316,
+      "logits/rejected": 1.171465516090393,
+      "logps/chosen": -4.935488700866699,
+      "logps/rejected": -7.756838798522949,
+      "loss": 0.6513,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.935488700866699,
+      "rewards/margins": 2.821349620819092,
+      "rewards/rejected": -7.756838798522949,
+      "step": 4365
+    },
+    {
+      "epoch": 2.338852650944974,
+      "grad_norm": 5.098280190589263,
+      "learning_rate": 1.406042836855859e-07,
+      "logits/chosen": 0.8013781309127808,
+      "logits/rejected": 1.2317490577697754,
+      "logps/chosen": -4.687870979309082,
+      "logps/rejected": -7.67673397064209,
+      "loss": 0.6484,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.687870979309082,
+      "rewards/margins": 2.988863468170166,
+      "rewards/rejected": -7.67673397064209,
+      "step": 4370
+    },
+    {
+      "epoch": 2.341528683726376,
+      "grad_norm": 5.794189359564772,
+      "learning_rate": 1.3952328239387595e-07,
+      "logits/chosen": 0.6368528008460999,
+      "logits/rejected": 1.1038202047348022,
+      "logps/chosen": -4.759398460388184,
+      "logps/rejected": -7.33819580078125,
+      "loss": 0.6548,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.759398460388184,
+      "rewards/margins": 2.5787971019744873,
+      "rewards/rejected": -7.33819580078125,
+      "step": 4375
+    },
+    {
+      "epoch": 2.344204716507777,
+      "grad_norm": 4.290937422912301,
+      "learning_rate": 1.3844577844886109e-07,
+      "logits/chosen": 0.6940240859985352,
+      "logits/rejected": 1.19984769821167,
+      "logps/chosen": -4.766362190246582,
+      "logps/rejected": -7.738821506500244,
+      "loss": 0.6594,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.766362190246582,
+      "rewards/margins": 2.972459316253662,
+      "rewards/rejected": -7.738821506500244,
+      "step": 4380
+    },
+    {
+      "epoch": 2.346880749289179,
+      "grad_norm": 5.888849048774243,
+      "learning_rate": 1.3737178230448955e-07,
+      "logits/chosen": 0.5540894269943237,
+      "logits/rejected": 1.0812121629714966,
+      "logps/chosen": -4.552201271057129,
+      "logps/rejected": -7.680624961853027,
+      "loss": 0.6575,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.552201271057129,
+      "rewards/margins": 3.1284244060516357,
+      "rewards/rejected": -7.680624961853027,
+      "step": 4385
+    },
+    {
+      "epoch": 2.3495567820705805,
+      "grad_norm": 4.229195457492085,
+      "learning_rate": 1.363013043806764e-07,
+      "logits/chosen": 0.6420809030532837,
+      "logits/rejected": 1.2507293224334717,
+      "logps/chosen": -4.460238456726074,
+      "logps/rejected": -7.65313720703125,
+      "loss": 0.6505,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.460238456726074,
+      "rewards/margins": 3.192899703979492,
+      "rewards/rejected": -7.65313720703125,
+      "step": 4390
+    },
+    {
+      "epoch": 2.3522328148519818,
+      "grad_norm": 4.6888584425565325,
+      "learning_rate": 1.352343550632034e-07,
+      "logits/chosen": 0.7038132548332214,
+      "logits/rejected": 1.0590779781341553,
+      "logps/chosen": -4.868699073791504,
+      "logps/rejected": -7.196985721588135,
+      "loss": 0.6528,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.868699073791504,
+      "rewards/margins": 2.32828688621521,
+      "rewards/rejected": -7.196985721588135,
+      "step": 4395
+    },
+    {
+      "epoch": 2.3549088476333835,
+      "grad_norm": 4.325161118660404,
+      "learning_rate": 1.3417094470361722e-07,
+      "logits/chosen": 0.8569762110710144,
+      "logits/rejected": 0.9415275454521179,
+      "logps/chosen": -5.515634059906006,
+      "logps/rejected": -6.942421913146973,
+      "loss": 0.6604,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -5.515634059906006,
+      "rewards/margins": 1.4267867803573608,
+      "rewards/rejected": -6.942421913146973,
+      "step": 4400
+    },
+    {
+      "epoch": 2.3549088476333835,
+      "eval_logits/chosen": 0.8695406317710876,
+      "eval_logits/rejected": 1.26536226272583,
+      "eval_logps/chosen": -4.774649143218994,
+      "eval_logps/rejected": -8.044668197631836,
+      "eval_loss": 0.6618175506591797,
+      "eval_rewards/accuracies": 0.6780415177345276,
+      "eval_rewards/chosen": -4.774649143218994,
+      "eval_rewards/margins": 3.270019054412842,
+      "eval_rewards/rejected": -8.044668197631836,
+      "eval_runtime": 40.7136,
+      "eval_samples_per_second": 33.036,
+      "eval_steps_per_second": 8.277,
+      "step": 4400
+    },
+    {
+      "epoch": 2.357584880414785,
+      "grad_norm": 4.64417008694542,
+      "learning_rate": 1.3311108361913015e-07,
+      "logits/chosen": 0.6235929727554321,
+      "logits/rejected": 1.0834648609161377,
+      "logps/chosen": -4.757271766662598,
+      "logps/rejected": -8.228525161743164,
+      "loss": 0.6399,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.757271766662598,
+      "rewards/margins": 3.471254825592041,
+      "rewards/rejected": -8.228525161743164,
+      "step": 4405
+    },
+    {
+      "epoch": 2.3602609131961865,
+      "grad_norm": 4.146113088715577,
+      "learning_rate": 1.3205478209251874e-07,
+      "logits/chosen": 0.8090956807136536,
+      "logits/rejected": 1.2492562532424927,
+      "logps/chosen": -5.306336879730225,
+      "logps/rejected": -8.36553955078125,
+      "loss": 0.648,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -5.306336879730225,
+      "rewards/margins": 3.059201717376709,
+      "rewards/rejected": -8.36553955078125,
+      "step": 4410
+    },
+    {
+      "epoch": 2.362936945977588,
+      "grad_norm": 3.7578330620762457,
+      "learning_rate": 1.310020503720254e-07,
+      "logits/chosen": 0.6743098497390747,
+      "logits/rejected": 1.284294843673706,
+      "logps/chosen": -4.903055191040039,
+      "logps/rejected": -8.57181167602539,
+      "loss": 0.6447,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.903055191040039,
+      "rewards/margins": 3.6687569618225098,
+      "rewards/rejected": -8.57181167602539,
+      "step": 4415
+    },
+    {
+      "epoch": 2.36561297875899,
+      "grad_norm": 9.429469167795997,
+      "learning_rate": 1.2995289867125752e-07,
+      "logits/chosen": 0.6294046640396118,
+      "logits/rejected": 1.0454283952713013,
+      "logps/chosen": -4.541731834411621,
+      "logps/rejected": -7.3188886642456055,
+      "loss": 0.6612,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.541731834411621,
+      "rewards/margins": 2.7771565914154053,
+      "rewards/rejected": -7.3188886642456055,
+      "step": 4420
+    },
+    {
+      "epoch": 2.368289011540391,
+      "grad_norm": 6.250629663526953,
+      "learning_rate": 1.2890733716908986e-07,
+      "logits/chosen": 0.49378567934036255,
+      "logits/rejected": 1.0912262201309204,
+      "logps/chosen": -3.8791046142578125,
+      "logps/rejected": -8.059572219848633,
+      "loss": 0.6224,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -3.8791046142578125,
+      "rewards/margins": 4.180467128753662,
+      "rewards/rejected": -8.059572219848633,
+      "step": 4425
+    },
+    {
+      "epoch": 2.370965044321793,
+      "grad_norm": 5.467187570705986,
+      "learning_rate": 1.2786537600956454e-07,
+      "logits/chosen": 0.5463210940361023,
+      "logits/rejected": 1.1899809837341309,
+      "logps/chosen": -4.39423942565918,
+      "logps/rejected": -8.273466110229492,
+      "loss": 0.6435,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.39423942565918,
+      "rewards/margins": 3.8792266845703125,
+      "rewards/rejected": -8.273466110229492,
+      "step": 4430
+    },
+    {
+      "epoch": 2.3736410771031946,
+      "grad_norm": 5.195886476665207,
+      "learning_rate": 1.268270253017933e-07,
+      "logits/chosen": 0.899684727191925,
+      "logits/rejected": 1.232582926750183,
+      "logps/chosen": -5.602095127105713,
+      "logps/rejected": -8.775694847106934,
+      "loss": 0.6552,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -5.602095127105713,
+      "rewards/margins": 3.1735997200012207,
+      "rewards/rejected": -8.775694847106934,
+      "step": 4435
+    },
+    {
+      "epoch": 2.376317109884596,
+      "grad_norm": 7.85842846940594,
+      "learning_rate": 1.257922951198591e-07,
+      "logits/chosen": 0.5811127424240112,
+      "logits/rejected": 1.1005308628082275,
+      "logps/chosen": -4.859737396240234,
+      "logps/rejected": -8.360882759094238,
+      "loss": 0.6471,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.859737396240234,
+      "rewards/margins": 3.501145124435425,
+      "rewards/rejected": -8.360882759094238,
+      "step": 4440
+    },
+    {
+      "epoch": 2.3789931426659976,
+      "grad_norm": 5.135971649151467,
+      "learning_rate": 1.24761195502719e-07,
+      "logits/chosen": 0.7321099042892456,
+      "logits/rejected": 0.974925696849823,
+      "logps/chosen": -5.793312072753906,
+      "logps/rejected": -7.404653072357178,
+      "loss": 0.6845,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -5.793312072753906,
+      "rewards/margins": 1.6113412380218506,
+      "rewards/rejected": -7.404653072357178,
+      "step": 4445
+    },
+    {
+      "epoch": 2.3816691754473993,
+      "grad_norm": 5.483319669508381,
+      "learning_rate": 1.2373373645410573e-07,
+      "logits/chosen": 0.8408491015434265,
+      "logits/rejected": 1.2796143293380737,
+      "logps/chosen": -5.842965602874756,
+      "logps/rejected": -8.220290184020996,
+      "loss": 0.6672,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -5.842965602874756,
+      "rewards/margins": 2.3773245811462402,
+      "rewards/rejected": -8.220290184020996,
+      "step": 4450
+    },
+    {
+      "epoch": 2.384345208228801,
+      "grad_norm": 2.778942788908202,
+      "learning_rate": 1.2270992794243175e-07,
+      "logits/chosen": 0.5991044044494629,
+      "logits/rejected": 1.2203844785690308,
+      "logps/chosen": -5.364320278167725,
+      "logps/rejected": -8.756747245788574,
+      "loss": 0.6628,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -5.364320278167725,
+      "rewards/margins": 3.392427444458008,
+      "rewards/rejected": -8.756747245788574,
+      "step": 4455
+    },
+    {
+      "epoch": 2.3870212410102023,
+      "grad_norm": 5.73774569372886,
+      "learning_rate": 1.2168977990069147e-07,
+      "logits/chosen": 0.7507764101028442,
+      "logits/rejected": 1.2571558952331543,
+      "logps/chosen": -5.068604469299316,
+      "logps/rejected": -8.337202072143555,
+      "loss": 0.6373,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -5.068604469299316,
+      "rewards/margins": 3.2685978412628174,
+      "rewards/rejected": -8.337202072143555,
+      "step": 4460
+    },
+    {
+      "epoch": 2.389697273791604,
+      "grad_norm": 3.3865383173989247,
+      "learning_rate": 1.206733022263659e-07,
+      "logits/chosen": 0.7935504913330078,
+      "logits/rejected": 1.3912582397460938,
+      "logps/chosen": -5.951399326324463,
+      "logps/rejected": -8.96430492401123,
+      "loss": 0.6666,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -5.951399326324463,
+      "rewards/margins": 3.0129058361053467,
+      "rewards/rejected": -8.96430492401123,
+      "step": 4465
+    },
+    {
+      "epoch": 2.3923733065730053,
+      "grad_norm": 6.592865965123559,
+      "learning_rate": 1.1966050478132572e-07,
+      "logits/chosen": 0.7477896809577942,
+      "logits/rejected": 1.1178234815597534,
+      "logps/chosen": -4.867952346801758,
+      "logps/rejected": -8.003345489501953,
+      "loss": 0.652,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.867952346801758,
+      "rewards/margins": 3.135392665863037,
+      "rewards/rejected": -8.003345489501953,
+      "step": 4470
+    },
+    {
+      "epoch": 2.395049339354407,
+      "grad_norm": 6.962759657639781,
+      "learning_rate": 1.1865139739173635e-07,
+      "logits/chosen": 0.8700340390205383,
+      "logits/rejected": 1.2160828113555908,
+      "logps/chosen": -5.943991661071777,
+      "logps/rejected": -8.004270553588867,
+      "loss": 0.6562,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -5.943991661071777,
+      "rewards/margins": 2.060279607772827,
+      "rewards/rejected": -8.004270553588867,
+      "step": 4475
+    },
+    {
+      "epoch": 2.3977253721358087,
+      "grad_norm": 4.911332171637151,
+      "learning_rate": 1.1764598984796187e-07,
+      "logits/chosen": 0.6750605702400208,
+      "logits/rejected": 1.1272200345993042,
+      "logps/chosen": -4.92875337600708,
+      "logps/rejected": -7.238613128662109,
+      "loss": 0.6489,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.92875337600708,
+      "rewards/margins": 2.3098597526550293,
+      "rewards/rejected": -7.238613128662109,
+      "step": 4480
+    },
+    {
+      "epoch": 2.4004014049172104,
+      "grad_norm": 6.388436060699118,
+      "learning_rate": 1.1664429190447095e-07,
+      "logits/chosen": 0.7568793296813965,
+      "logits/rejected": 1.230337381362915,
+      "logps/chosen": -5.279677391052246,
+      "logps/rejected": -8.541528701782227,
+      "loss": 0.6636,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -5.279677391052246,
+      "rewards/margins": 3.2618517875671387,
+      "rewards/rejected": -8.541528701782227,
+      "step": 4485
+    },
+    {
+      "epoch": 2.4030774376986117,
+      "grad_norm": 7.169087640811448,
+      "learning_rate": 1.1564631327974122e-07,
+      "logits/chosen": 0.7785574197769165,
+      "logits/rejected": 1.276386022567749,
+      "logps/chosen": -5.129861354827881,
+      "logps/rejected": -9.105204582214355,
+      "loss": 0.6493,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -5.129861354827881,
+      "rewards/margins": 3.975342273712158,
+      "rewards/rejected": -9.105204582214355,
+      "step": 4490
+    },
+    {
+      "epoch": 2.4057534704800134,
+      "grad_norm": 6.73307755257911,
+      "learning_rate": 1.1465206365616587e-07,
+      "logits/chosen": 0.7546981573104858,
+      "logits/rejected": 1.1353422403335571,
+      "logps/chosen": -5.513069152832031,
+      "logps/rejected": -7.366274833679199,
+      "loss": 0.6604,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -5.513069152832031,
+      "rewards/margins": 1.8532063961029053,
+      "rewards/rejected": -7.366274833679199,
+      "step": 4495
+    },
+    {
+      "epoch": 2.408429503261415,
+      "grad_norm": 4.129603354021001,
+      "learning_rate": 1.1366155267995887e-07,
+      "logits/chosen": 0.8315432667732239,
+      "logits/rejected": 1.2240508794784546,
+      "logps/chosen": -5.080111503601074,
+      "logps/rejected": -7.967687129974365,
+      "loss": 0.6421,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -5.080111503601074,
+      "rewards/margins": 2.887576103210449,
+      "rewards/rejected": -7.967687129974365,
+      "step": 4500
+    },
+    {
+      "epoch": 2.4111055360428164,
+      "grad_norm": 5.800070288080934,
+      "learning_rate": 1.1267478996106228e-07,
+      "logits/chosen": 0.9275251626968384,
+      "logits/rejected": 1.217498540878296,
+      "logps/chosen": -6.047314643859863,
+      "logps/rejected": -7.900041103363037,
+      "loss": 0.6618,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -6.047314643859863,
+      "rewards/margins": 1.852726697921753,
+      "rewards/rejected": -7.900041103363037,
+      "step": 4505
+    },
+    {
+      "epoch": 2.413781568824218,
+      "grad_norm": 9.152640404661213,
+      "learning_rate": 1.116917850730521e-07,
+      "logits/chosen": 0.7317572832107544,
+      "logits/rejected": 1.1840275526046753,
+      "logps/chosen": -5.4841766357421875,
+      "logps/rejected": -8.267538070678711,
+      "loss": 0.6605,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -5.4841766357421875,
+      "rewards/margins": 2.7833614349365234,
+      "rewards/rejected": -8.267538070678711,
+      "step": 4510
+    },
+    {
+      "epoch": 2.41645760160562,
+      "grad_norm": 5.120943579385317,
+      "learning_rate": 1.1071254755304637e-07,
+      "logits/chosen": 0.6067882776260376,
+      "logits/rejected": 1.1805981397628784,
+      "logps/chosen": -4.48081111907959,
+      "logps/rejected": -8.700325012207031,
+      "loss": 0.6442,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.48081111907959,
+      "rewards/margins": 4.219513893127441,
+      "rewards/rejected": -8.700325012207031,
+      "step": 4515
+    },
+    {
+      "epoch": 2.419133634387021,
+      "grad_norm": 4.08357301745016,
+      "learning_rate": 1.0973708690161143e-07,
+      "logits/chosen": 0.9028582572937012,
+      "logits/rejected": 1.2807990312576294,
+      "logps/chosen": -5.521599769592285,
+      "logps/rejected": -8.564383506774902,
+      "loss": 0.643,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -5.521599769592285,
+      "rewards/margins": 3.042783498764038,
+      "rewards/rejected": -8.564383506774902,
+      "step": 4520
+    },
+    {
+      "epoch": 2.421809667168423,
+      "grad_norm": 5.48766220992808,
+      "learning_rate": 1.0876541258267119e-07,
+      "logits/chosen": 0.6939808130264282,
+      "logits/rejected": 1.1409887075424194,
+      "logps/chosen": -5.448697566986084,
+      "logps/rejected": -7.811097621917725,
+      "loss": 0.6531,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -5.448697566986084,
+      "rewards/margins": 2.362400531768799,
+      "rewards/rejected": -7.811097621917725,
+      "step": 4525
+    },
+    {
+      "epoch": 2.4244856999498245,
+      "grad_norm": 3.416962206301364,
+      "learning_rate": 1.0779753402341379e-07,
+      "logits/chosen": 0.7167526483535767,
+      "logits/rejected": 1.138471245765686,
+      "logps/chosen": -5.129691123962402,
+      "logps/rejected": -7.495391845703125,
+      "loss": 0.673,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -5.129691123962402,
+      "rewards/margins": 2.3657007217407227,
+      "rewards/rejected": -7.495391845703125,
+      "step": 4530
+    },
+    {
+      "epoch": 2.427161732731226,
+      "grad_norm": 3.6925137769926244,
+      "learning_rate": 1.0683346061420157e-07,
+      "logits/chosen": 0.7040577530860901,
+      "logits/rejected": 1.3312757015228271,
+      "logps/chosen": -4.218172073364258,
+      "logps/rejected": -8.327013969421387,
+      "loss": 0.6533,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.218172073364258,
+      "rewards/margins": 4.108842372894287,
+      "rewards/rejected": -8.327013969421387,
+      "step": 4535
+    },
+    {
+      "epoch": 2.4298377655126275,
+      "grad_norm": 5.0152956633745704,
+      "learning_rate": 1.0587320170847874e-07,
+      "logits/chosen": 0.8521683812141418,
+      "logits/rejected": 1.2078973054885864,
+      "logps/chosen": -5.691619396209717,
+      "logps/rejected": -8.1076021194458,
+      "loss": 0.6537,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -5.691619396209717,
+      "rewards/margins": 2.4159817695617676,
+      "rewards/rejected": -8.1076021194458,
+      "step": 4540
+    },
+    {
+      "epoch": 2.4325137982940293,
+      "grad_norm": 5.440424711459051,
+      "learning_rate": 1.0491676662268156e-07,
+      "logits/chosen": 0.8367718458175659,
+      "logits/rejected": 1.318353533744812,
+      "logps/chosen": -5.28448486328125,
+      "logps/rejected": -8.917255401611328,
+      "loss": 0.6587,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -5.28448486328125,
+      "rewards/margins": 3.632770538330078,
+      "rewards/rejected": -8.917255401611328,
+      "step": 4545
+    },
+    {
+      "epoch": 2.4351898310754305,
+      "grad_norm": 6.974100553608133,
+      "learning_rate": 1.0396416463614732e-07,
+      "logits/chosen": 0.623924195766449,
+      "logits/rejected": 1.1520938873291016,
+      "logps/chosen": -4.493465900421143,
+      "logps/rejected": -7.918328762054443,
+      "loss": 0.646,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.493465900421143,
+      "rewards/margins": 3.424863338470459,
+      "rewards/rejected": -7.918328762054443,
+      "step": 4550
+    },
+    {
+      "epoch": 2.4378658638568322,
+      "grad_norm": 3.361308407301288,
+      "learning_rate": 1.0301540499102479e-07,
+      "logits/chosen": 0.7184853553771973,
+      "logits/rejected": 1.2356756925582886,
+      "logps/chosen": -5.242965221405029,
+      "logps/rejected": -8.0581636428833,
+      "loss": 0.6542,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -5.242965221405029,
+      "rewards/margins": 2.8151986598968506,
+      "rewards/rejected": -8.0581636428833,
+      "step": 4555
+    },
+    {
+      "epoch": 2.440541896638234,
+      "grad_norm": 3.888464872563384,
+      "learning_rate": 1.0207049689218405e-07,
+      "logits/chosen": 0.7919880151748657,
+      "logits/rejected": 1.030605435371399,
+      "logps/chosen": -5.435633182525635,
+      "logps/rejected": -7.5948333740234375,
+      "loss": 0.6571,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -5.435633182525635,
+      "rewards/margins": 2.1591992378234863,
+      "rewards/rejected": -7.5948333740234375,
+      "step": 4560
+    },
+    {
+      "epoch": 2.4432179294196352,
+      "grad_norm": 3.7708181332866855,
+      "learning_rate": 1.0112944950712782e-07,
+      "logits/chosen": 0.5346997976303101,
+      "logits/rejected": 1.2393728494644165,
+      "logps/chosen": -4.269177436828613,
+      "logps/rejected": -8.470273971557617,
+      "loss": 0.6513,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.269177436828613,
+      "rewards/margins": 4.201096534729004,
+      "rewards/rejected": -8.470273971557617,
+      "step": 4565
+    },
+    {
+      "epoch": 2.445893962201037,
+      "grad_norm": 5.151055190557485,
+      "learning_rate": 1.0019227196590174e-07,
+      "logits/chosen": 0.8576459884643555,
+      "logits/rejected": 1.290848731994629,
+      "logps/chosen": -5.619999885559082,
+      "logps/rejected": -8.348254203796387,
+      "loss": 0.6522,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -5.619999885559082,
+      "rewards/margins": 2.728254795074463,
+      "rewards/rejected": -8.348254203796387,
+      "step": 4570
+    },
+    {
+      "epoch": 2.4485699949824387,
+      "grad_norm": 5.724233711841114,
+      "learning_rate": 9.925897336100664e-08,
+      "logits/chosen": 0.5602216124534607,
+      "logits/rejected": 1.2020868062973022,
+      "logps/chosen": -4.190063953399658,
+      "logps/rejected": -7.966056823730469,
+      "loss": 0.6586,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.190063953399658,
+      "rewards/margins": 3.7759928703308105,
+      "rewards/rejected": -7.966056823730469,
+      "step": 4575
+    },
+    {
+      "epoch": 2.45124602776384,
+      "grad_norm": 5.6630222766475855,
+      "learning_rate": 9.832956274730946e-08,
+      "logits/chosen": 0.5671447515487671,
+      "logits/rejected": 0.9335812330245972,
+      "logps/chosen": -4.812465667724609,
+      "logps/rejected": -7.039891242980957,
+      "loss": 0.6597,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.812465667724609,
+      "rewards/margins": 2.227426052093506,
+      "rewards/rejected": -7.039891242980957,
+      "step": 4580
+    },
+    {
+      "epoch": 2.4539220605452416,
+      "grad_norm": 4.171889721665223,
+      "learning_rate": 9.740404914195633e-08,
+      "logits/chosen": 0.693152904510498,
+      "logits/rejected": 1.191304326057434,
+      "logps/chosen": -4.66318416595459,
+      "logps/rejected": -7.716940402984619,
+      "loss": 0.643,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.66318416595459,
+      "rewards/margins": 3.05375599861145,
+      "rewards/rejected": -7.716940402984619,
+      "step": 4585
+    },
+    {
+      "epoch": 2.4565980933266434,
+      "grad_norm": 5.155228608199212,
+      "learning_rate": 9.648244152428392e-08,
+      "logits/chosen": 0.6991937160491943,
+      "logits/rejected": 1.2253680229187012,
+      "logps/chosen": -5.254127502441406,
+      "logps/rejected": -8.093042373657227,
+      "loss": 0.6557,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -5.254127502441406,
+      "rewards/margins": 2.838914155960083,
+      "rewards/rejected": -8.093042373657227,
+      "step": 4590
+    },
+    {
+      "epoch": 2.4592741261080446,
+      "grad_norm": 5.910461857373014,
+      "learning_rate": 9.556474883573379e-08,
+      "logits/chosen": 0.6341494917869568,
+      "logits/rejected": 1.1397039890289307,
+      "logps/chosen": -4.500366687774658,
+      "logps/rejected": -7.931443214416504,
+      "loss": 0.6453,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.500366687774658,
+      "rewards/margins": 3.431076765060425,
+      "rewards/rejected": -7.931443214416504,
+      "step": 4595
+    },
+    {
+      "epoch": 2.4619501588894463,
+      "grad_norm": 4.08927225304091,
+      "learning_rate": 9.465097997976412e-08,
+      "logits/chosen": 0.7181195020675659,
+      "logits/rejected": 1.1862919330596924,
+      "logps/chosen": -4.921706199645996,
+      "logps/rejected": -8.70015811920166,
+      "loss": 0.6367,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -4.921706199645996,
+      "rewards/margins": 3.7784526348114014,
+      "rewards/rejected": -8.70015811920166,
+      "step": 4600
+    },
+    {
+      "epoch": 2.464626191670848,
+      "grad_norm": 6.20317244337158,
+      "learning_rate": 9.374114382176457e-08,
+      "logits/chosen": 0.8055015802383423,
+      "logits/rejected": 1.1672663688659668,
+      "logps/chosen": -4.992861270904541,
+      "logps/rejected": -7.528120517730713,
+      "loss": 0.649,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.992861270904541,
+      "rewards/margins": 2.5352590084075928,
+      "rewards/rejected": -7.528120517730713,
+      "step": 4605
+    },
+    {
+      "epoch": 2.46730222445225,
+      "grad_norm": 5.387480871069172,
+      "learning_rate": 9.283524918896945e-08,
+      "logits/chosen": 0.8409932851791382,
+      "logits/rejected": 1.16977858543396,
+      "logps/chosen": -5.423148155212402,
+      "logps/rejected": -7.990517616271973,
+      "loss": 0.6581,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -5.423148155212402,
+      "rewards/margins": 2.567368745803833,
+      "rewards/rejected": -7.990517616271973,
+      "step": 4610
+    },
+    {
+      "epoch": 2.469978257233651,
+      "grad_norm": 2.720373830614946,
+      "learning_rate": 9.193330487037232e-08,
+      "logits/chosen": 1.0474634170532227,
+      "logits/rejected": 1.4661476612091064,
+      "logps/chosen": -6.162823677062988,
+      "logps/rejected": -9.273473739624023,
+      "loss": 0.6441,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -6.162823677062988,
+      "rewards/margins": 3.110650062561035,
+      "rewards/rejected": -9.273473739624023,
+      "step": 4615
+    },
+    {
+      "epoch": 2.4726542900150528,
+      "grad_norm": 8.649784078128533,
+      "learning_rate": 9.103531961664118e-08,
+      "logits/chosen": 0.7911742329597473,
+      "logits/rejected": 1.3635225296020508,
+      "logps/chosen": -4.835679054260254,
+      "logps/rejected": -8.196763038635254,
+      "loss": 0.6442,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.835679054260254,
+      "rewards/margins": 3.361084461212158,
+      "rewards/rejected": -8.196763038635254,
+      "step": 4620
+    },
+    {
+      "epoch": 2.475330322796454,
+      "grad_norm": 3.739943813647293,
+      "learning_rate": 9.014130214003269e-08,
+      "logits/chosen": 0.6460190415382385,
+      "logits/rejected": 1.0517737865447998,
+      "logps/chosen": -5.111288070678711,
+      "logps/rejected": -8.27877426147461,
+      "loss": 0.6503,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -5.111288070678711,
+      "rewards/margins": 3.1674861907958984,
+      "rewards/rejected": -8.27877426147461,
+      "step": 4625
+    },
+    {
+      "epoch": 2.4780063555778558,
+      "grad_norm": 7.859546831406479,
+      "learning_rate": 8.925126111430848e-08,
+      "logits/chosen": 0.5666110515594482,
+      "logits/rejected": 1.1186630725860596,
+      "logps/chosen": -3.822451114654541,
+      "logps/rejected": -7.833033561706543,
+      "loss": 0.64,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -3.822451114654541,
+      "rewards/margins": 4.010583400726318,
+      "rewards/rejected": -7.833033561706543,
+      "step": 4630
+    },
+    {
+      "epoch": 2.4806823883592575,
+      "grad_norm": 15.965771208215175,
+      "learning_rate": 8.83652051746504e-08,
+      "logits/chosen": 0.943730354309082,
+      "logits/rejected": 1.2836445569992065,
+      "logps/chosen": -5.809576034545898,
+      "logps/rejected": -8.511076927185059,
+      "loss": 0.6617,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -5.809576034545898,
+      "rewards/margins": 2.701500415802002,
+      "rewards/rejected": -8.511076927185059,
+      "step": 4635
+    },
+    {
+      "epoch": 2.483358421140659,
+      "grad_norm": 3.4784815898703414,
+      "learning_rate": 8.748314291757696e-08,
+      "logits/chosen": 0.7015920281410217,
+      "logits/rejected": 1.0992889404296875,
+      "logps/chosen": -4.7373504638671875,
+      "logps/rejected": -7.632442474365234,
+      "loss": 0.665,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.7373504638671875,
+      "rewards/margins": 2.895092487335205,
+      "rewards/rejected": -7.632442474365234,
+      "step": 4640
+    },
+    {
+      "epoch": 2.4860344539220605,
+      "grad_norm": 4.761924553633939,
+      "learning_rate": 8.660508290086032e-08,
+      "logits/chosen": 0.7006752490997314,
+      "logits/rejected": 1.1958411931991577,
+      "logps/chosen": -4.520000457763672,
+      "logps/rejected": -8.219992637634277,
+      "loss": 0.6433,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.520000457763672,
+      "rewards/margins": 3.6999926567077637,
+      "rewards/rejected": -8.219992637634277,
+      "step": 4645
+    },
+    {
+      "epoch": 2.488710486703462,
+      "grad_norm": 4.600351446706549,
+      "learning_rate": 8.573103364344231e-08,
+      "logits/chosen": 0.644333004951477,
+      "logits/rejected": 1.2011265754699707,
+      "logps/chosen": -4.4600982666015625,
+      "logps/rejected": -7.5825982093811035,
+      "loss": 0.6435,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.4600982666015625,
+      "rewards/margins": 3.122499704360962,
+      "rewards/rejected": -7.5825982093811035,
+      "step": 4650
+    },
+    {
+      "epoch": 2.4913865194848634,
+      "grad_norm": 5.707801337094879,
+      "learning_rate": 8.486100362535292e-08,
+      "logits/chosen": 0.7132748365402222,
+      "logits/rejected": 1.186736822128296,
+      "logps/chosen": -5.034489154815674,
+      "logps/rejected": -8.020044326782227,
+      "loss": 0.6649,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -5.034489154815674,
+      "rewards/margins": 2.98555588722229,
+      "rewards/rejected": -8.020044326782227,
+      "step": 4655
+    },
+    {
+      "epoch": 2.494062552266265,
+      "grad_norm": 2.829496113437484,
+      "learning_rate": 8.399500128762693e-08,
+      "logits/chosen": 0.5554698705673218,
+      "logits/rejected": 1.1394189596176147,
+      "logps/chosen": -4.7488322257995605,
+      "logps/rejected": -8.480119705200195,
+      "loss": 0.6499,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.7488322257995605,
+      "rewards/margins": 3.731288194656372,
+      "rewards/rejected": -8.480119705200195,
+      "step": 4660
+    },
+    {
+      "epoch": 2.496738585047667,
+      "grad_norm": 5.683163790418566,
+      "learning_rate": 8.313303503222313e-08,
+      "logits/chosen": 0.6311101913452148,
+      "logits/rejected": 0.9306130409240723,
+      "logps/chosen": -4.7874603271484375,
+      "logps/rejected": -6.774118900299072,
+      "loss": 0.6561,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.7874603271484375,
+      "rewards/margins": 1.986657738685608,
+      "rewards/rejected": -6.774118900299072,
+      "step": 4665
+    },
+    {
+      "epoch": 2.4994146178290686,
+      "grad_norm": 4.028974692775134,
+      "learning_rate": 8.227511322194164e-08,
+      "logits/chosen": 0.7671923041343689,
+      "logits/rejected": 1.1316936016082764,
+      "logps/chosen": -4.797974109649658,
+      "logps/rejected": -7.6807074546813965,
+      "loss": 0.6607,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.797974109649658,
+      "rewards/margins": 2.8827333450317383,
+      "rewards/rejected": -7.6807074546813965,
+      "step": 4670
+    },
+    {
+      "epoch": 2.50209065061047,
+      "grad_norm": 3.7385015130733787,
+      "learning_rate": 8.142124418034385e-08,
+      "logits/chosen": 0.9632235765457153,
+      "logits/rejected": 1.177288293838501,
+      "logps/chosen": -5.29378604888916,
+      "logps/rejected": -7.645918369293213,
+      "loss": 0.6565,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -5.29378604888916,
+      "rewards/margins": 2.352132797241211,
+      "rewards/rejected": -7.645918369293213,
+      "step": 4675
+    },
+    {
+      "epoch": 2.5047666833918716,
+      "grad_norm": 4.655021140521432,
+      "learning_rate": 8.057143619167073e-08,
+      "logits/chosen": 0.8443503379821777,
+      "logits/rejected": 1.1922781467437744,
+      "logps/chosen": -5.135584831237793,
+      "logps/rejected": -8.259990692138672,
+      "loss": 0.6778,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -5.135584831237793,
+      "rewards/margins": 3.1244049072265625,
+      "rewards/rejected": -8.259990692138672,
+      "step": 4680
+    },
+    {
+      "epoch": 2.507442716173273,
+      "grad_norm": 5.042226957786147,
+      "learning_rate": 7.97256975007633e-08,
+      "logits/chosen": 0.6535072326660156,
+      "logits/rejected": 1.2777506113052368,
+      "logps/chosen": -4.614386558532715,
+      "logps/rejected": -7.933239936828613,
+      "loss": 0.6633,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.614386558532715,
+      "rewards/margins": 3.3188533782958984,
+      "rewards/rejected": -7.933239936828613,
+      "step": 4685
+    },
+    {
+      "epoch": 2.5101187489546746,
+      "grad_norm": 8.739315878963954,
+      "learning_rate": 7.888403631298186e-08,
+      "logits/chosen": 0.6591877341270447,
+      "logits/rejected": 1.216149091720581,
+      "logps/chosen": -4.449265480041504,
+      "logps/rejected": -8.279069900512695,
+      "loss": 0.6497,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.449265480041504,
+      "rewards/margins": 3.8298048973083496,
+      "rewards/rejected": -8.279069900512695,
+      "step": 4690
+    },
+    {
+      "epoch": 2.5127947817360763,
+      "grad_norm": 3.130369516464317,
+      "learning_rate": 7.804646079412719e-08,
+      "logits/chosen": 0.8610504269599915,
+      "logits/rejected": 1.310791015625,
+      "logps/chosen": -5.40376615524292,
+      "logps/rejected": -8.737608909606934,
+      "loss": 0.6472,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -5.40376615524292,
+      "rewards/margins": 3.3338425159454346,
+      "rewards/rejected": -8.737608909606934,
+      "step": 4695
+    },
+    {
+      "epoch": 2.515470814517478,
+      "grad_norm": 5.713722629116274,
+      "learning_rate": 7.72129790703604e-08,
+      "logits/chosen": 0.7484351992607117,
+      "logits/rejected": 1.057403326034546,
+      "logps/chosen": -4.948575019836426,
+      "logps/rejected": -7.44402551651001,
+      "loss": 0.6589,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.948575019836426,
+      "rewards/margins": 2.495450496673584,
+      "rewards/rejected": -7.44402551651001,
+      "step": 4700
+    },
+    {
+      "epoch": 2.5181468472988793,
+      "grad_norm": 3.446215586148225,
+      "learning_rate": 7.638359922812504e-08,
+      "logits/chosen": 0.817892849445343,
+      "logits/rejected": 1.1204694509506226,
+      "logps/chosen": -5.107209205627441,
+      "logps/rejected": -7.472251892089844,
+      "loss": 0.657,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -5.107209205627441,
+      "rewards/margins": 2.365042209625244,
+      "rewards/rejected": -7.472251892089844,
+      "step": 4705
+    },
+    {
+      "epoch": 2.520822880080281,
+      "grad_norm": 4.023618972358758,
+      "learning_rate": 7.555832931406774e-08,
+      "logits/chosen": 0.7196842432022095,
+      "logits/rejected": 0.9853816032409668,
+      "logps/chosen": -5.043528079986572,
+      "logps/rejected": -7.570195198059082,
+      "loss": 0.6576,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -5.043528079986572,
+      "rewards/margins": 2.5266671180725098,
+      "rewards/rejected": -7.570195198059082,
+      "step": 4710
+    },
+    {
+      "epoch": 2.5234989128616827,
+      "grad_norm": 2.9463776513965803,
+      "learning_rate": 7.47371773349611e-08,
+      "logits/chosen": 0.5457627177238464,
+      "logits/rejected": 1.3584779500961304,
+      "logps/chosen": -4.302974700927734,
+      "logps/rejected": -10.006356239318848,
+      "loss": 0.6404,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -4.302974700927734,
+      "rewards/margins": 5.7033820152282715,
+      "rewards/rejected": -10.006356239318848,
+      "step": 4715
+    },
+    {
+      "epoch": 2.526174945643084,
+      "grad_norm": 6.950217713335976,
+      "learning_rate": 7.392015125762496e-08,
+      "logits/chosen": 0.7517132759094238,
+      "logits/rejected": 1.265366554260254,
+      "logps/chosen": -5.316319465637207,
+      "logps/rejected": -8.429070472717285,
+      "loss": 0.655,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -5.316319465637207,
+      "rewards/margins": 3.11275053024292,
+      "rewards/rejected": -8.429070472717285,
+      "step": 4720
+    },
+    {
+      "epoch": 2.5288509784244857,
+      "grad_norm": 5.004130637825614,
+      "learning_rate": 7.310725900885018e-08,
+      "logits/chosen": 0.7253432869911194,
+      "logits/rejected": 1.326133131980896,
+      "logps/chosen": -4.743563175201416,
+      "logps/rejected": -8.32368278503418,
+      "loss": 0.6649,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.743563175201416,
+      "rewards/margins": 3.5801196098327637,
+      "rewards/rejected": -8.32368278503418,
+      "step": 4725
+    },
+    {
+      "epoch": 2.5315270112058874,
+      "grad_norm": 7.6409742364081445,
+      "learning_rate": 7.229850847532076e-08,
+      "logits/chosen": 0.9381262063980103,
+      "logits/rejected": 1.2018539905548096,
+      "logps/chosen": -5.5652360916137695,
+      "logps/rejected": -7.350111961364746,
+      "loss": 0.6594,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -5.5652360916137695,
+      "rewards/margins": 1.7848762273788452,
+      "rewards/rejected": -7.350111961364746,
+      "step": 4730
+    },
+    {
+      "epoch": 2.5342030439872887,
+      "grad_norm": 6.655190644861974,
+      "learning_rate": 7.149390750353779e-08,
+      "logits/chosen": 0.598798394203186,
+      "logits/rejected": 1.398573637008667,
+      "logps/chosen": -3.9796364307403564,
+      "logps/rejected": -8.954061508178711,
+      "loss": 0.6322,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -3.9796364307403564,
+      "rewards/margins": 4.97442626953125,
+      "rewards/rejected": -8.954061508178711,
+      "step": 4735
+    },
+    {
+      "epoch": 2.5368790767686904,
+      "grad_norm": 4.650689474669417,
+      "learning_rate": 7.069346389974374e-08,
+      "logits/chosen": 0.8502882719039917,
+      "logits/rejected": 1.1604156494140625,
+      "logps/chosen": -5.099037170410156,
+      "logps/rejected": -7.653689384460449,
+      "loss": 0.6624,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -5.099037170410156,
+      "rewards/margins": 2.554651975631714,
+      "rewards/rejected": -7.653689384460449,
+      "step": 4740
+    },
+    {
+      "epoch": 2.539555109550092,
+      "grad_norm": 3.5533182165074813,
+      "learning_rate": 6.989718542984563e-08,
+      "logits/chosen": 0.7469674944877625,
+      "logits/rejected": 1.2217400074005127,
+      "logps/chosen": -4.937492370605469,
+      "logps/rejected": -8.551791191101074,
+      "loss": 0.6488,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.937492370605469,
+      "rewards/margins": 3.6142990589141846,
+      "rewards/rejected": -8.551791191101074,
+      "step": 4745
+    },
+    {
+      "epoch": 2.5422311423314934,
+      "grad_norm": 4.365954887284784,
+      "learning_rate": 6.9105079819341e-08,
+      "logits/chosen": 0.9204667210578918,
+      "logits/rejected": 1.3389699459075928,
+      "logps/chosen": -5.320522308349609,
+      "logps/rejected": -8.849835395812988,
+      "loss": 0.65,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -5.320522308349609,
+      "rewards/margins": 3.5293126106262207,
+      "rewards/rejected": -8.849835395812988,
+      "step": 4750
+    },
+    {
+      "epoch": 2.544907175112895,
+      "grad_norm": 5.241612298839937,
+      "learning_rate": 6.831715475324163e-08,
+      "logits/chosen": 0.8501812815666199,
+      "logits/rejected": 1.3282244205474854,
+      "logps/chosen": -5.244772911071777,
+      "logps/rejected": -9.32155704498291,
+      "loss": 0.6459,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -5.244772911071777,
+      "rewards/margins": 4.076783180236816,
+      "rewards/rejected": -9.32155704498291,
+      "step": 4755
+    },
+    {
+      "epoch": 2.547583207894297,
+      "grad_norm": 3.948898221974884,
+      "learning_rate": 6.753341787600026e-08,
+      "logits/chosen": 0.7219231724739075,
+      "logits/rejected": 1.3982890844345093,
+      "logps/chosen": -4.687278747558594,
+      "logps/rejected": -8.527350425720215,
+      "loss": 0.658,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.687278747558594,
+      "rewards/margins": 3.8400721549987793,
+      "rewards/rejected": -8.527350425720215,
+      "step": 4760
+    },
+    {
+      "epoch": 2.5502592406756985,
+      "grad_norm": 4.971898272883295,
+      "learning_rate": 6.67538767914353e-08,
+      "logits/chosen": 0.7897475957870483,
+      "logits/rejected": 1.0537608861923218,
+      "logps/chosen": -5.742798805236816,
+      "logps/rejected": -7.159323215484619,
+      "loss": 0.6887,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -5.742798805236816,
+      "rewards/margins": 1.4165239334106445,
+      "rewards/rejected": -7.159323215484619,
+      "step": 4765
+    },
+    {
+      "epoch": 2.5529352734571,
+      "grad_norm": 5.53643904849308,
+      "learning_rate": 6.597853906265793e-08,
+      "logits/chosen": 0.6269160509109497,
+      "logits/rejected": 1.2914178371429443,
+      "logps/chosen": -4.941379070281982,
+      "logps/rejected": -8.76620101928711,
+      "loss": 0.6466,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.941379070281982,
+      "rewards/margins": 3.8248209953308105,
+      "rewards/rejected": -8.76620101928711,
+      "step": 4770
+    },
+    {
+      "epoch": 2.5556113062385015,
+      "grad_norm": 2.88844179875519,
+      "learning_rate": 6.5207412211998e-08,
+      "logits/chosen": 0.8935686349868774,
+      "logits/rejected": 1.2117592096328735,
+      "logps/chosen": -5.569288730621338,
+      "logps/rejected": -8.03335952758789,
+      "loss": 0.6426,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -5.569288730621338,
+      "rewards/margins": 2.464071273803711,
+      "rewards/rejected": -8.03335952758789,
+      "step": 4775
+    },
+    {
+      "epoch": 2.558287339019903,
+      "grad_norm": 5.2940931111580145,
+      "learning_rate": 6.444050372093186e-08,
+      "logits/chosen": 0.8733730316162109,
+      "logits/rejected": 1.2583779096603394,
+      "logps/chosen": -5.225768566131592,
+      "logps/rejected": -8.476118087768555,
+      "loss": 0.6461,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -5.225768566131592,
+      "rewards/margins": 3.2503504753112793,
+      "rewards/rejected": -8.476118087768555,
+      "step": 4780
+    },
+    {
+      "epoch": 2.5609633718013045,
+      "grad_norm": 5.738069125402751,
+      "learning_rate": 6.367782103000873e-08,
+      "logits/chosen": 0.7506901025772095,
+      "logits/rejected": 1.119620442390442,
+      "logps/chosen": -4.810736179351807,
+      "logps/rejected": -7.645442962646484,
+      "loss": 0.6563,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.810736179351807,
+      "rewards/margins": 2.8347058296203613,
+      "rewards/rejected": -7.645442962646484,
+      "step": 4785
+    },
+    {
+      "epoch": 2.5636394045827062,
+      "grad_norm": 4.713221663517672,
+      "learning_rate": 6.29193715387798e-08,
+      "logits/chosen": 0.7165239453315735,
+      "logits/rejected": 1.1845703125,
+      "logps/chosen": -4.66361141204834,
+      "logps/rejected": -8.259965896606445,
+      "loss": 0.6396,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.66361141204834,
+      "rewards/margins": 3.5963542461395264,
+      "rewards/rejected": -8.259965896606445,
+      "step": 4790
+    },
+    {
+      "epoch": 2.566315437364108,
+      "grad_norm": 8.641726666057119,
+      "learning_rate": 6.216516260572502e-08,
+      "logits/chosen": 0.8041791915893555,
+      "logits/rejected": 1.17696213722229,
+      "logps/chosen": -5.081482887268066,
+      "logps/rejected": -7.604484558105469,
+      "loss": 0.6478,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -5.081482887268066,
+      "rewards/margins": 2.523002862930298,
+      "rewards/rejected": -7.604484558105469,
+      "step": 4795
+    },
+    {
+      "epoch": 2.568991470145509,
+      "grad_norm": 8.678903944072701,
+      "learning_rate": 6.141520154818297e-08,
+      "logits/chosen": 0.9081575274467468,
+      "logits/rejected": 1.2223659753799438,
+      "logps/chosen": -5.786370754241943,
+      "logps/rejected": -7.733120918273926,
+      "loss": 0.6512,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -5.786370754241943,
+      "rewards/margins": 1.9467499256134033,
+      "rewards/rejected": -7.733120918273926,
+      "step": 4800
+    },
+    {
+      "epoch": 2.568991470145509,
+      "eval_logits/chosen": 0.8804936408996582,
+      "eval_logits/rejected": 1.2819464206695557,
+      "eval_logps/chosen": -4.914679050445557,
+      "eval_logps/rejected": -8.277661323547363,
+      "eval_loss": 0.6615036725997925,
+      "eval_rewards/accuracies": 0.6772996783256531,
+      "eval_rewards/chosen": -4.914679050445557,
+      "eval_rewards/margins": 3.362982749938965,
+      "eval_rewards/rejected": -8.277661323547363,
+      "eval_runtime": 40.7683,
+      "eval_samples_per_second": 32.991,
+      "eval_steps_per_second": 8.266,
+      "step": 4800
+    },
+    {
+      "epoch": 2.571667502926911,
+      "grad_norm": 4.279971410568114,
+      "learning_rate": 6.066949564227897e-08,
+      "logits/chosen": 0.6369834542274475,
+      "logits/rejected": 1.1628409624099731,
+      "logps/chosen": -4.9243879318237305,
+      "logps/rejected": -7.921910762786865,
+      "loss": 0.6661,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.9243879318237305,
+      "rewards/margins": 2.997523069381714,
+      "rewards/rejected": -7.921910762786865,
+      "step": 4805
+    },
+    {
+      "epoch": 2.574343535708312,
+      "grad_norm": 4.310577434491807,
+      "learning_rate": 5.992805212285523e-08,
+      "logits/chosen": 0.5950236320495605,
+      "logits/rejected": 1.110089659690857,
+      "logps/chosen": -4.082036018371582,
+      "logps/rejected": -7.518782138824463,
+      "loss": 0.6467,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.082036018371582,
+      "rewards/margins": 3.4367458820343018,
+      "rewards/rejected": -7.518782138824463,
+      "step": 4810
+    },
+    {
+      "epoch": 2.577019568489714,
+      "grad_norm": 6.500963318997128,
+      "learning_rate": 5.9190878183399684e-08,
+      "logits/chosen": 0.7438062429428101,
+      "logits/rejected": 1.1180588006973267,
+      "logps/chosen": -4.664271831512451,
+      "logps/rejected": -8.073579788208008,
+      "loss": 0.6308,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.664271831512451,
+      "rewards/margins": 3.4093070030212402,
+      "rewards/rejected": -8.073579788208008,
+      "step": 4815
+    },
+    {
+      "epoch": 2.5796956012711156,
+      "grad_norm": 3.961283918964112,
+      "learning_rate": 5.845798097597748e-08,
+      "logits/chosen": 0.7926705479621887,
+      "logits/rejected": 1.0163605213165283,
+      "logps/chosen": -5.300661087036133,
+      "logps/rejected": -7.094559669494629,
+      "loss": 0.6594,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -5.300661087036133,
+      "rewards/margins": 1.7938976287841797,
+      "rewards/rejected": -7.094559669494629,
+      "step": 4820
+    },
+    {
+      "epoch": 2.5823716340525174,
+      "grad_norm": 5.486749415866384,
+      "learning_rate": 5.772936761116026e-08,
+      "logits/chosen": 0.7406069040298462,
+      "logits/rejected": 1.2497667074203491,
+      "logps/chosen": -4.851771354675293,
+      "logps/rejected": -7.62369441986084,
+      "loss": 0.6409,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.851771354675293,
+      "rewards/margins": 2.771923542022705,
+      "rewards/rejected": -7.62369441986084,
+      "step": 4825
+    },
+    {
+      "epoch": 2.5850476668339186,
+      "grad_norm": 3.429474108842891,
+      "learning_rate": 5.700504515795829e-08,
+      "logits/chosen": 0.9239054918289185,
+      "logits/rejected": 1.4188076257705688,
+      "logps/chosen": -5.581028938293457,
+      "logps/rejected": -8.997245788574219,
+      "loss": 0.6562,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -5.581028938293457,
+      "rewards/margins": 3.4162163734436035,
+      "rewards/rejected": -8.997245788574219,
+      "step": 4830
+    },
+    {
+      "epoch": 2.5877236996153203,
+      "grad_norm": 12.75124364153754,
+      "learning_rate": 5.628502064375101e-08,
+      "logits/chosen": 0.6405271291732788,
+      "logits/rejected": 1.2279365062713623,
+      "logps/chosen": -4.813295841217041,
+      "logps/rejected": -8.677081108093262,
+      "loss": 0.6724,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.813295841217041,
+      "rewards/margins": 3.8637847900390625,
+      "rewards/rejected": -8.677081108093262,
+      "step": 4835
+    },
+    {
+      "epoch": 2.5903997323967216,
+      "grad_norm": 11.757706590408366,
+      "learning_rate": 5.55693010542197e-08,
+      "logits/chosen": 0.5248925089836121,
+      "logits/rejected": 1.344184160232544,
+      "logps/chosen": -4.3812994956970215,
+      "logps/rejected": -9.260825157165527,
+      "loss": 0.636,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.3812994956970215,
+      "rewards/margins": 4.879525184631348,
+      "rewards/rejected": -9.260825157165527,
+      "step": 4840
+    },
+    {
+      "epoch": 2.5930757651781233,
+      "grad_norm": 4.745097185971385,
+      "learning_rate": 5.485789333327856e-08,
+      "logits/chosen": 0.8376296758651733,
+      "logits/rejected": 1.0887072086334229,
+      "logps/chosen": -5.557162284851074,
+      "logps/rejected": -7.603895664215088,
+      "loss": 0.6519,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -5.557162284851074,
+      "rewards/margins": 2.0467333793640137,
+      "rewards/rejected": -7.603895664215088,
+      "step": 4845
+    },
+    {
+      "epoch": 2.595751797959525,
+      "grad_norm": 7.145236251932616,
+      "learning_rate": 5.4150804383008675e-08,
+      "logits/chosen": 0.7607053518295288,
+      "logits/rejected": 1.1724084615707397,
+      "logps/chosen": -5.397799968719482,
+      "logps/rejected": -8.787528991699219,
+      "loss": 0.6577,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -5.397799968719482,
+      "rewards/margins": 3.3897292613983154,
+      "rewards/rejected": -8.787528991699219,
+      "step": 4850
+    },
+    {
+      "epoch": 2.5984278307409268,
+      "grad_norm": 7.268643810488111,
+      "learning_rate": 5.344804106359002e-08,
+      "logits/chosen": 0.8205873370170593,
+      "logits/rejected": 1.1211081743240356,
+      "logps/chosen": -5.202333927154541,
+      "logps/rejected": -7.698519706726074,
+      "loss": 0.6654,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -5.202333927154541,
+      "rewards/margins": 2.4961860179901123,
+      "rewards/rejected": -7.698519706726074,
+      "step": 4855
+    },
+    {
+      "epoch": 2.601103863522328,
+      "grad_norm": 5.073321343633651,
+      "learning_rate": 5.274961019323559e-08,
+      "logits/chosen": 0.68889319896698,
+      "logits/rejected": 1.036307454109192,
+      "logps/chosen": -4.540287971496582,
+      "logps/rejected": -7.399824619293213,
+      "loss": 0.6466,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.540287971496582,
+      "rewards/margins": 2.859536647796631,
+      "rewards/rejected": -7.399824619293213,
+      "step": 4860
+    },
+    {
+      "epoch": 2.6037798963037297,
+      "grad_norm": 3.0164899705990993,
+      "learning_rate": 5.205551854812451e-08,
+      "logits/chosen": 0.721064031124115,
+      "logits/rejected": 1.2545911073684692,
+      "logps/chosen": -4.972177028656006,
+      "logps/rejected": -8.157731056213379,
+      "loss": 0.6448,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.972177028656006,
+      "rewards/margins": 3.1855525970458984,
+      "rewards/rejected": -8.157731056213379,
+      "step": 4865
+    },
+    {
+      "epoch": 2.606455929085131,
+      "grad_norm": 4.699547855820392,
+      "learning_rate": 5.1365772862337177e-08,
+      "logits/chosen": 0.6260263919830322,
+      "logits/rejected": 1.3591649532318115,
+      "logps/chosen": -3.856776714324951,
+      "logps/rejected": -8.810465812683105,
+      "loss": 0.6417,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -3.856776714324951,
+      "rewards/margins": 4.9536895751953125,
+      "rewards/rejected": -8.810465812683105,
+      "step": 4870
+    },
+    {
+      "epoch": 2.6091319618665327,
+      "grad_norm": 4.675576167370054,
+      "learning_rate": 5.068037982778905e-08,
+      "logits/chosen": 0.6980249285697937,
+      "logits/rejected": 1.1021798849105835,
+      "logps/chosen": -4.335538387298584,
+      "logps/rejected": -7.662753105163574,
+      "loss": 0.6604,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.335538387298584,
+      "rewards/margins": 3.3272156715393066,
+      "rewards/rejected": -7.662753105163574,
+      "step": 4875
+    },
+    {
+      "epoch": 2.6118079946479344,
+      "grad_norm": 4.782822923305197,
+      "learning_rate": 4.999934609416656e-08,
+      "logits/chosen": 0.8684225082397461,
+      "logits/rejected": 1.3131214380264282,
+      "logps/chosen": -4.892592430114746,
+      "logps/rejected": -8.03726863861084,
+      "loss": 0.6361,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.892592430114746,
+      "rewards/margins": 3.1446759700775146,
+      "rewards/rejected": -8.03726863861084,
+      "step": 4880
+    },
+    {
+      "epoch": 2.614484027429336,
+      "grad_norm": 5.138437845944443,
+      "learning_rate": 4.932267826886183e-08,
+      "logits/chosen": 0.7397621870040894,
+      "logits/rejected": 1.329167127609253,
+      "logps/chosen": -4.821078300476074,
+      "logps/rejected": -8.721323013305664,
+      "loss": 0.6508,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.821078300476074,
+      "rewards/margins": 3.9002442359924316,
+      "rewards/rejected": -8.721323013305664,
+      "step": 4885
+    },
+    {
+      "epoch": 2.6171600602107374,
+      "grad_norm": 6.339027833835395,
+      "learning_rate": 4.8650382916909206e-08,
+      "logits/chosen": 0.7806223034858704,
+      "logits/rejected": 1.1464259624481201,
+      "logps/chosen": -5.542832851409912,
+      "logps/rejected": -7.9821672439575195,
+      "loss": 0.6581,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -5.542832851409912,
+      "rewards/margins": 2.439333915710449,
+      "rewards/rejected": -7.9821672439575195,
+      "step": 4890
+    },
+    {
+      "epoch": 2.619836092992139,
+      "grad_norm": 4.54812705046962,
+      "learning_rate": 4.7982466560920976e-08,
+      "logits/chosen": 0.6562093496322632,
+      "logits/rejected": 1.1314470767974854,
+      "logps/chosen": -4.864579200744629,
+      "logps/rejected": -7.264063358306885,
+      "loss": 0.6447,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.864579200744629,
+      "rewards/margins": 2.3994839191436768,
+      "rewards/rejected": -7.264063358306885,
+      "step": 4895
+    },
+    {
+      "epoch": 2.622512125773541,
+      "grad_norm": 4.740445277480145,
+      "learning_rate": 4.7318935681024685e-08,
+      "logits/chosen": 0.7177092432975769,
+      "logits/rejected": 1.2532570362091064,
+      "logps/chosen": -4.715688228607178,
+      "logps/rejected": -7.245957851409912,
+      "loss": 0.6473,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.715688228607178,
+      "rewards/margins": 2.5302700996398926,
+      "rewards/rejected": -7.245957851409912,
+      "step": 4900
+    },
+    {
+      "epoch": 2.625188158554942,
+      "grad_norm": 4.151047012141013,
+      "learning_rate": 4.6659796714799745e-08,
+      "logits/chosen": 0.5941017866134644,
+      "logits/rejected": 1.2570723295211792,
+      "logps/chosen": -4.878970623016357,
+      "logps/rejected": -8.958597183227539,
+      "loss": 0.642,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.878970623016357,
+      "rewards/margins": 4.07962703704834,
+      "rewards/rejected": -8.958597183227539,
+      "step": 4905
+    },
+    {
+      "epoch": 2.627864191336344,
+      "grad_norm": 4.107555839128542,
+      "learning_rate": 4.60050560572155e-08,
+      "logits/chosen": 0.8097479939460754,
+      "logits/rejected": 1.1727063655853271,
+      "logps/chosen": -5.547590732574463,
+      "logps/rejected": -9.446479797363281,
+      "loss": 0.6611,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -5.547590732574463,
+      "rewards/margins": 3.8988888263702393,
+      "rewards/rejected": -9.446479797363281,
+      "step": 4910
+    },
+    {
+      "epoch": 2.6305402241177456,
+      "grad_norm": 5.285782648544581,
+      "learning_rate": 4.535472006056834e-08,
+      "logits/chosen": 0.7398176789283752,
+      "logits/rejected": 0.9699414968490601,
+      "logps/chosen": -4.7341718673706055,
+      "logps/rejected": -6.995150089263916,
+      "loss": 0.6528,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.7341718673706055,
+      "rewards/margins": 2.2609775066375732,
+      "rewards/rejected": -6.995150089263916,
+      "step": 4915
+    },
+    {
+      "epoch": 2.6332162568991473,
+      "grad_norm": 3.0286626955573777,
+      "learning_rate": 4.470879503442132e-08,
+      "logits/chosen": 0.7532659769058228,
+      "logits/rejected": 1.263484239578247,
+      "logps/chosen": -4.842391490936279,
+      "logps/rejected": -8.445043563842773,
+      "loss": 0.6515,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.842391490936279,
+      "rewards/margins": 3.6026523113250732,
+      "rewards/rejected": -8.445043563842773,
+      "step": 4920
+    },
+    {
+      "epoch": 2.6358922896805486,
+      "grad_norm": 3.296139099555667,
+      "learning_rate": 4.406728724554154e-08,
+      "logits/chosen": 0.7089954018592834,
+      "logits/rejected": 1.1598275899887085,
+      "logps/chosen": -4.911757946014404,
+      "logps/rejected": -7.9169602394104,
+      "loss": 0.6441,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.911757946014404,
+      "rewards/margins": 3.005201816558838,
+      "rewards/rejected": -7.9169602394104,
+      "step": 4925
+    },
+    {
+      "epoch": 2.6385683224619503,
+      "grad_norm": 4.0340670980471725,
+      "learning_rate": 4.3430202917840664e-08,
+      "logits/chosen": 0.7764522433280945,
+      "logits/rejected": 1.2171226739883423,
+      "logps/chosen": -5.048651695251465,
+      "logps/rejected": -8.280994415283203,
+      "loss": 0.6523,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -5.048651695251465,
+      "rewards/margins": 3.2323431968688965,
+      "rewards/rejected": -8.280994415283203,
+      "step": 4930
+    },
+    {
+      "epoch": 2.6412443552433515,
+      "grad_norm": 8.089823186325205,
+      "learning_rate": 4.279754823231346e-08,
+      "logits/chosen": 0.6566135287284851,
+      "logits/rejected": 1.1550073623657227,
+      "logps/chosen": -4.816324710845947,
+      "logps/rejected": -7.761317253112793,
+      "loss": 0.6392,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.816324710845947,
+      "rewards/margins": 2.9449920654296875,
+      "rewards/rejected": -7.761317253112793,
+      "step": 4935
+    },
+    {
+      "epoch": 2.6439203880247533,
+      "grad_norm": 3.556643560022813,
+      "learning_rate": 4.216932932697859e-08,
+      "logits/chosen": 0.600483775138855,
+      "logits/rejected": 1.1105464696884155,
+      "logps/chosen": -4.604409694671631,
+      "logps/rejected": -7.887474060058594,
+      "loss": 0.6346,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.604409694671631,
+      "rewards/margins": 3.283064603805542,
+      "rewards/rejected": -7.887474060058594,
+      "step": 4940
+    },
+    {
+      "epoch": 2.646596420806155,
+      "grad_norm": 4.345375061966081,
+      "learning_rate": 4.154555229681844e-08,
+      "logits/chosen": 0.657063901424408,
+      "logits/rejected": 1.065737009048462,
+      "logps/chosen": -4.719033241271973,
+      "logps/rejected": -7.568997383117676,
+      "loss": 0.6561,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.719033241271973,
+      "rewards/margins": 2.849963903427124,
+      "rewards/rejected": -7.568997383117676,
+      "step": 4945
+    },
+    {
+      "epoch": 2.6492724535875567,
+      "grad_norm": 10.321547118572777,
+      "learning_rate": 4.092622319372069e-08,
+      "logits/chosen": 0.7423306703567505,
+      "logits/rejected": 1.1442407369613647,
+      "logps/chosen": -5.146918296813965,
+      "logps/rejected": -7.763352870941162,
+      "loss": 0.6492,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -5.146918296813965,
+      "rewards/margins": 2.6164348125457764,
+      "rewards/rejected": -7.763352870941162,
+      "step": 4950
+    },
+    {
+      "epoch": 2.651948486368958,
+      "grad_norm": 3.9044797710070456,
+      "learning_rate": 4.031134802641889e-08,
+      "logits/chosen": 0.6819875240325928,
+      "logits/rejected": 1.125776767730713,
+      "logps/chosen": -4.858378887176514,
+      "logps/rejected": -7.982049465179443,
+      "loss": 0.6503,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.858378887176514,
+      "rewards/margins": 3.1236705780029297,
+      "rewards/rejected": -7.982049465179443,
+      "step": 4955
+    },
+    {
+      "epoch": 2.6546245191503597,
+      "grad_norm": 4.095640287377882,
+      "learning_rate": 3.970093276043468e-08,
+      "logits/chosen": 0.7103524208068848,
+      "logits/rejected": 1.192828893661499,
+      "logps/chosen": -4.647218227386475,
+      "logps/rejected": -7.942820072174072,
+      "loss": 0.6431,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.647218227386475,
+      "rewards/margins": 3.2956016063690186,
+      "rewards/rejected": -7.942820072174072,
+      "step": 4960
+    },
+    {
+      "epoch": 2.657300551931761,
+      "grad_norm": 3.735733182357967,
+      "learning_rate": 3.9094983318019584e-08,
+      "logits/chosen": 0.8146587610244751,
+      "logits/rejected": 1.1984922885894775,
+      "logps/chosen": -5.257014274597168,
+      "logps/rejected": -8.653731346130371,
+      "loss": 0.6424,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -5.257014274597168,
+      "rewards/margins": 3.396716356277466,
+      "rewards/rejected": -8.653731346130371,
+      "step": 4965
+    },
+    {
+      "epoch": 2.6599765847131627,
+      "grad_norm": 3.7278146575426847,
+      "learning_rate": 3.849350557809789e-08,
+      "logits/chosen": 0.8334561586380005,
+      "logits/rejected": 1.274526834487915,
+      "logps/chosen": -4.764495372772217,
+      "logps/rejected": -8.353806495666504,
+      "loss": 0.6459,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.764495372772217,
+      "rewards/margins": 3.5893101692199707,
+      "rewards/rejected": -8.353806495666504,
+      "step": 4970
+    },
+    {
+      "epoch": 2.6626526174945644,
+      "grad_norm": 5.104965390290842,
+      "learning_rate": 3.789650537620903e-08,
+      "logits/chosen": 0.8470608592033386,
+      "logits/rejected": 1.3156403303146362,
+      "logps/chosen": -5.2490973472595215,
+      "logps/rejected": -8.932024002075195,
+      "loss": 0.6599,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -5.2490973472595215,
+      "rewards/margins": 3.682926654815674,
+      "rewards/rejected": -8.932024002075195,
+      "step": 4975
+    },
+    {
+      "epoch": 2.665328650275966,
+      "grad_norm": 4.061360208375875,
+      "learning_rate": 3.730398850445182e-08,
+      "logits/chosen": 0.9226518869400024,
+      "logits/rejected": 1.2283064126968384,
+      "logps/chosen": -5.5883331298828125,
+      "logps/rejected": -7.936419486999512,
+      "loss": 0.6651,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -5.5883331298828125,
+      "rewards/margins": 2.3480865955352783,
+      "rewards/rejected": -7.936419486999512,
+      "step": 4980
+    },
+    {
+      "epoch": 2.6680046830573674,
+      "grad_norm": 6.70389847420772,
+      "learning_rate": 3.671596071142735e-08,
+      "logits/chosen": 0.8386240005493164,
+      "logits/rejected": 1.2673699855804443,
+      "logps/chosen": -4.821142196655273,
+      "logps/rejected": -8.23372745513916,
+      "loss": 0.6434,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.821142196655273,
+      "rewards/margins": 3.412585496902466,
+      "rewards/rejected": -8.23372745513916,
+      "step": 4985
+    },
+    {
+      "epoch": 2.670680715838769,
+      "grad_norm": 3.1759928637857207,
+      "learning_rate": 3.6132427702183996e-08,
+      "logits/chosen": 0.5948123931884766,
+      "logits/rejected": 1.1813766956329346,
+      "logps/chosen": -4.439202308654785,
+      "logps/rejected": -8.247601509094238,
+      "loss": 0.6378,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.439202308654785,
+      "rewards/margins": 3.808398723602295,
+      "rewards/rejected": -8.247601509094238,
+      "step": 4990
+    },
+    {
+      "epoch": 2.6733567486201704,
+      "grad_norm": 6.6569663494926035,
+      "learning_rate": 3.555339513816147e-08,
+      "logits/chosen": 0.8165515065193176,
+      "logits/rejected": 1.231727957725525,
+      "logps/chosen": -5.326131343841553,
+      "logps/rejected": -8.162468910217285,
+      "loss": 0.6565,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -5.326131343841553,
+      "rewards/margins": 2.8363373279571533,
+      "rewards/rejected": -8.162468910217285,
+      "step": 4995
+    },
+    {
+      "epoch": 2.676032781401572,
+      "grad_norm": 6.354269183639232,
+      "learning_rate": 3.497886863713639e-08,
+      "logits/chosen": 0.8669255375862122,
+      "logits/rejected": 1.1385505199432373,
+      "logps/chosen": -5.6387152671813965,
+      "logps/rejected": -8.072131156921387,
+      "loss": 0.6543,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -5.6387152671813965,
+      "rewards/margins": 2.433415651321411,
+      "rewards/rejected": -8.072131156921387,
+      "step": 5000
+    },
+    {
+      "epoch": 2.678708814182974,
+      "grad_norm": 3.8924989111727455,
+      "learning_rate": 3.440885377316721e-08,
+      "logits/chosen": 0.7822055816650391,
+      "logits/rejected": 1.0404683351516724,
+      "logps/chosen": -4.818753719329834,
+      "logps/rejected": -7.152914524078369,
+      "loss": 0.6618,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.818753719329834,
+      "rewards/margins": 2.334160327911377,
+      "rewards/rejected": -7.152914524078369,
+      "step": 5005
+    },
+    {
+      "epoch": 2.6813848469643755,
+      "grad_norm": 6.3692362226325985,
+      "learning_rate": 3.384335607654082e-08,
+      "logits/chosen": 0.6580092310905457,
+      "logits/rejected": 1.1280157566070557,
+      "logps/chosen": -5.2137250900268555,
+      "logps/rejected": -7.8815155029296875,
+      "loss": 0.6489,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -5.2137250900268555,
+      "rewards/margins": 2.667790412902832,
+      "rewards/rejected": -7.8815155029296875,
+      "step": 5010
+    },
+    {
+      "epoch": 2.684060879745777,
+      "grad_norm": 5.216177855613218,
+      "learning_rate": 3.328238103371811e-08,
+      "logits/chosen": 0.7337587475776672,
+      "logits/rejected": 1.2386358976364136,
+      "logps/chosen": -4.834414958953857,
+      "logps/rejected": -8.52358341217041,
+      "loss": 0.6619,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.834414958953857,
+      "rewards/margins": 3.6891682147979736,
+      "rewards/rejected": -8.52358341217041,
+      "step": 5015
+    },
+    {
+      "epoch": 2.6867369125271785,
+      "grad_norm": 4.85213952029054,
+      "learning_rate": 3.272593408728169e-08,
+      "logits/chosen": 0.7476842999458313,
+      "logits/rejected": 1.2099324464797974,
+      "logps/chosen": -5.0923638343811035,
+      "logps/rejected": -7.906291961669922,
+      "loss": 0.6569,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -5.0923638343811035,
+      "rewards/margins": 2.8139290809631348,
+      "rewards/rejected": -7.906291961669922,
+      "step": 5020
+    },
+    {
+      "epoch": 2.6894129453085798,
+      "grad_norm": 2.272776162210328,
+      "learning_rate": 3.217402063588204e-08,
+      "logits/chosen": 0.7687848806381226,
+      "logits/rejected": 1.2237908840179443,
+      "logps/chosen": -5.517539024353027,
+      "logps/rejected": -8.57043743133545,
+      "loss": 0.6635,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -5.517539024353027,
+      "rewards/margins": 3.0528981685638428,
+      "rewards/rejected": -8.57043743133545,
+      "step": 5025
+    },
+    {
+      "epoch": 2.6920889780899815,
+      "grad_norm": 20.9150886541202,
+      "learning_rate": 3.162664603418608e-08,
+      "logits/chosen": 0.6991812586784363,
+      "logits/rejected": 1.0864553451538086,
+      "logps/chosen": -4.970685005187988,
+      "logps/rejected": -7.723145961761475,
+      "loss": 0.6491,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.970685005187988,
+      "rewards/margins": 2.7524616718292236,
+      "rewards/rejected": -7.723145961761475,
+      "step": 5030
+    },
+    {
+      "epoch": 2.694765010871383,
+      "grad_norm": 4.422861389967058,
+      "learning_rate": 3.1083815592824416e-08,
+      "logits/chosen": 0.7442831993103027,
+      "logits/rejected": 1.3582837581634521,
+      "logps/chosen": -5.225079536437988,
+      "logps/rejected": -8.470558166503906,
+      "loss": 0.6444,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -5.225079536437988,
+      "rewards/margins": 3.245479106903076,
+      "rewards/rejected": -8.470558166503906,
+      "step": 5035
+    },
+    {
+      "epoch": 2.697441043652785,
+      "grad_norm": 6.02796781994894,
+      "learning_rate": 3.054553457834053e-08,
+      "logits/chosen": 0.941321074962616,
+      "logits/rejected": 1.259972333908081,
+      "logps/chosen": -4.82551383972168,
+      "logps/rejected": -7.966601371765137,
+      "loss": 0.6569,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.82551383972168,
+      "rewards/margins": 3.1410865783691406,
+      "rewards/rejected": -7.966601371765137,
+      "step": 5040
+    },
+    {
+      "epoch": 2.700117076434186,
+      "grad_norm": 4.86974208553889,
+      "learning_rate": 3.0011808213139036e-08,
+      "logits/chosen": 0.6564344167709351,
+      "logits/rejected": 1.1294138431549072,
+      "logps/chosen": -4.4033589363098145,
+      "logps/rejected": -7.5861639976501465,
+      "loss": 0.6422,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.4033589363098145,
+      "rewards/margins": 3.182805299758911,
+      "rewards/rejected": -7.5861639976501465,
+      "step": 5045
+    },
+    {
+      "epoch": 2.702793109215588,
+      "grad_norm": 2.825474393454967,
+      "learning_rate": 2.948264167543568e-08,
+      "logits/chosen": 0.7820049524307251,
+      "logits/rejected": 0.9248069524765015,
+      "logps/chosen": -5.152117729187012,
+      "logps/rejected": -6.806935787200928,
+      "loss": 0.6436,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -5.152117729187012,
+      "rewards/margins": 1.6548172235488892,
+      "rewards/rejected": -6.806935787200928,
+      "step": 5050
+    },
+    {
+      "epoch": 2.7054691419969896,
+      "grad_norm": 7.554956325295155,
+      "learning_rate": 2.8958040099206216e-08,
+      "logits/chosen": 0.5635976195335388,
+      "logits/rejected": 0.9739664793014526,
+      "logps/chosen": -4.591999530792236,
+      "logps/rejected": -7.196866512298584,
+      "loss": 0.6593,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.591999530792236,
+      "rewards/margins": 2.604867458343506,
+      "rewards/rejected": -7.196866512298584,
+      "step": 5055
+    },
+    {
+      "epoch": 2.708145174778391,
+      "grad_norm": 7.5611828320652155,
+      "learning_rate": 2.843800857413775e-08,
+      "logits/chosen": 0.6774371862411499,
+      "logits/rejected": 1.1361892223358154,
+      "logps/chosen": -4.818153381347656,
+      "logps/rejected": -8.229283332824707,
+      "loss": 0.6473,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.818153381347656,
+      "rewards/margins": 3.41113018989563,
+      "rewards/rejected": -8.229283332824707,
+      "step": 5060
+    },
+    {
+      "epoch": 2.7108212075597926,
+      "grad_norm": 5.262517411849471,
+      "learning_rate": 2.7922552145578203e-08,
+      "logits/chosen": 0.6611738801002502,
+      "logits/rejected": 1.0948989391326904,
+      "logps/chosen": -4.615340232849121,
+      "logps/rejected": -7.432889461517334,
+      "loss": 0.6405,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.615340232849121,
+      "rewards/margins": 2.81754994392395,
+      "rewards/rejected": -7.432889461517334,
+      "step": 5065
+    },
+    {
+      "epoch": 2.7134972403411943,
+      "grad_norm": 5.6879741489433435,
+      "learning_rate": 2.7411675814488277e-08,
+      "logits/chosen": 0.7506179809570312,
+      "logits/rejected": 1.357236623764038,
+      "logps/chosen": -4.609767436981201,
+      "logps/rejected": -7.511110782623291,
+      "loss": 0.6425,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.609767436981201,
+      "rewards/margins": 2.9013421535491943,
+      "rewards/rejected": -7.511110782623291,
+      "step": 5070
+    },
+    {
+      "epoch": 2.7161732731225956,
+      "grad_norm": 4.332429323241998,
+      "learning_rate": 2.690538453739216e-08,
+      "logits/chosen": 0.8441430926322937,
+      "logits/rejected": 1.1924471855163574,
+      "logps/chosen": -4.761057376861572,
+      "logps/rejected": -7.29510498046875,
+      "loss": 0.6553,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.761057376861572,
+      "rewards/margins": 2.534048318862915,
+      "rewards/rejected": -7.29510498046875,
+      "step": 5075
+    },
+    {
+      "epoch": 2.7188493059039973,
+      "grad_norm": 3.8244859779263978,
+      "learning_rate": 2.6403683226330298e-08,
+      "logits/chosen": 0.7580150365829468,
+      "logits/rejected": 1.0911517143249512,
+      "logps/chosen": -4.872685432434082,
+      "logps/rejected": -7.902780055999756,
+      "loss": 0.647,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.872685432434082,
+      "rewards/margins": 3.0300955772399902,
+      "rewards/rejected": -7.902780055999756,
+      "step": 5080
+    },
+    {
+      "epoch": 2.721525338685399,
+      "grad_norm": 4.559099254766263,
+      "learning_rate": 2.5906576748810804e-08,
+      "logits/chosen": 0.8079412579536438,
+      "logits/rejected": 1.2271184921264648,
+      "logps/chosen": -4.92527437210083,
+      "logps/rejected": -8.161842346191406,
+      "loss": 0.635,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.92527437210083,
+      "rewards/margins": 3.2365689277648926,
+      "rewards/rejected": -8.161842346191406,
+      "step": 5085
+    },
+    {
+      "epoch": 2.7242013714668003,
+      "grad_norm": 6.125772210478291,
+      "learning_rate": 2.5414069927763016e-08,
+      "logits/chosen": 0.7393813729286194,
+      "logits/rejected": 1.1481651067733765,
+      "logps/chosen": -5.079852104187012,
+      "logps/rejected": -7.761559963226318,
+      "loss": 0.6541,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -5.079852104187012,
+      "rewards/margins": 2.6817078590393066,
+      "rewards/rejected": -7.761559963226318,
+      "step": 5090
+    },
+    {
+      "epoch": 2.726877404248202,
+      "grad_norm": 3.9435196883614196,
+      "learning_rate": 2.4926167541490185e-08,
+      "logits/chosen": 0.6550232768058777,
+      "logits/rejected": 0.9510253071784973,
+      "logps/chosen": -5.064509391784668,
+      "logps/rejected": -7.5706787109375,
+      "loss": 0.6523,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -5.064509391784668,
+      "rewards/margins": 2.5061697959899902,
+      "rewards/rejected": -7.5706787109375,
+      "step": 5095
+    },
+    {
+      "epoch": 2.7295534370296037,
+      "grad_norm": 4.029918156024794,
+      "learning_rate": 2.4442874323623574e-08,
+      "logits/chosen": 0.7553450465202332,
+      "logits/rejected": 1.2641026973724365,
+      "logps/chosen": -5.053984642028809,
+      "logps/rejected": -7.932581424713135,
+      "loss": 0.6686,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -5.053984642028809,
+      "rewards/margins": 2.878596305847168,
+      "rewards/rejected": -7.932581424713135,
+      "step": 5100
+    },
+    {
+      "epoch": 2.7322294698110055,
+      "grad_norm": 8.546478357199065,
+      "learning_rate": 2.396419496307589e-08,
+      "logits/chosen": 0.8467805981636047,
+      "logits/rejected": 1.3810200691223145,
+      "logps/chosen": -5.304780006408691,
+      "logps/rejected": -9.014433860778809,
+      "loss": 0.6536,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -5.304780006408691,
+      "rewards/margins": 3.7096545696258545,
+      "rewards/rejected": -9.014433860778809,
+      "step": 5105
+    },
+    {
+      "epoch": 2.7349055025924067,
+      "grad_norm": 9.48945827806927,
+      "learning_rate": 2.349013410399653e-08,
+      "logits/chosen": 0.6786569356918335,
+      "logits/rejected": 1.0582540035247803,
+      "logps/chosen": -4.857218265533447,
+      "logps/rejected": -7.223915100097656,
+      "loss": 0.6405,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.857218265533447,
+      "rewards/margins": 2.36669659614563,
+      "rewards/rejected": -7.223915100097656,
+      "step": 5110
+    },
+    {
+      "epoch": 2.7375815353738084,
+      "grad_norm": 4.496144579485099,
+      "learning_rate": 2.3020696345725954e-08,
+      "logits/chosen": 0.7765499353408813,
+      "logits/rejected": 1.3109071254730225,
+      "logps/chosen": -5.507508754730225,
+      "logps/rejected": -8.216897964477539,
+      "loss": 0.6516,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -5.507508754730225,
+      "rewards/margins": 2.7093899250030518,
+      "rewards/rejected": -8.216897964477539,
+      "step": 5115
+    },
+    {
+      "epoch": 2.7402575681552097,
+      "grad_norm": 5.5115971753390856,
+      "learning_rate": 2.2555886242751398e-08,
+      "logits/chosen": 0.7259373068809509,
+      "logits/rejected": 1.0852547883987427,
+      "logps/chosen": -5.090877532958984,
+      "logps/rejected": -8.290916442871094,
+      "loss": 0.6508,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -5.090877532958984,
+      "rewards/margins": 3.200038194656372,
+      "rewards/rejected": -8.290916442871094,
+      "step": 5120
+    },
+    {
+      "epoch": 2.7429336009366114,
+      "grad_norm": 9.373616298369065,
+      "learning_rate": 2.2095708304662453e-08,
+      "logits/chosen": 0.7150914072990417,
+      "logits/rejected": 1.2580024003982544,
+      "logps/chosen": -4.986987590789795,
+      "logps/rejected": -8.133567810058594,
+      "loss": 0.648,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.986987590789795,
+      "rewards/margins": 3.146580219268799,
+      "rewards/rejected": -8.133567810058594,
+      "step": 5125
+    },
+    {
+      "epoch": 2.745609633718013,
+      "grad_norm": 4.902641319154763,
+      "learning_rate": 2.16401669961076e-08,
+      "logits/chosen": 0.6511356830596924,
+      "logits/rejected": 1.050058126449585,
+      "logps/chosen": -4.838202953338623,
+      "logps/rejected": -8.098820686340332,
+      "loss": 0.6505,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.838202953338623,
+      "rewards/margins": 3.260618209838867,
+      "rewards/rejected": -8.098820686340332,
+      "step": 5130
+    },
+    {
+      "epoch": 2.748285666499415,
+      "grad_norm": 4.486902760283486,
+      "learning_rate": 2.1189266736750532e-08,
+      "logits/chosen": 0.8758920431137085,
+      "logits/rejected": 1.2374738454818726,
+      "logps/chosen": -5.041923999786377,
+      "logps/rejected": -7.332954406738281,
+      "loss": 0.6651,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -5.041923999786377,
+      "rewards/margins": 2.2910313606262207,
+      "rewards/rejected": -7.332954406738281,
+      "step": 5135
+    },
+    {
+      "epoch": 2.750961699280816,
+      "grad_norm": 4.182867378517797,
+      "learning_rate": 2.0743011901227623e-08,
+      "logits/chosen": 0.727471649646759,
+      "logits/rejected": 1.3188221454620361,
+      "logps/chosen": -4.6489152908325195,
+      "logps/rejected": -7.760054588317871,
+      "loss": 0.6638,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.6489152908325195,
+      "rewards/margins": 3.111140012741089,
+      "rewards/rejected": -7.760054588317871,
+      "step": 5140
+    },
+    {
+      "epoch": 2.753637732062218,
+      "grad_norm": 4.655567386222573,
+      "learning_rate": 2.030140681910508e-08,
+      "logits/chosen": 0.8202909231185913,
+      "logits/rejected": 1.0563172101974487,
+      "logps/chosen": -5.1415300369262695,
+      "logps/rejected": -6.918440341949463,
+      "loss": 0.6642,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -5.1415300369262695,
+      "rewards/margins": 1.776910424232483,
+      "rewards/rejected": -6.918440341949463,
+      "step": 5145
+    },
+    {
+      "epoch": 2.756313764843619,
+      "grad_norm": 3.9090710283896812,
+      "learning_rate": 1.986445577483753e-08,
+      "logits/chosen": 0.6967265605926514,
+      "logits/rejected": 1.1769962310791016,
+      "logps/chosen": -4.631322860717773,
+      "logps/rejected": -8.549386024475098,
+      "loss": 0.6503,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.631322860717773,
+      "rewards/margins": 3.918062925338745,
+      "rewards/rejected": -8.549386024475098,
+      "step": 5150
+    },
+    {
+      "epoch": 2.758989797625021,
+      "grad_norm": 3.898228957044156,
+      "learning_rate": 1.9432163007725765e-08,
+      "logits/chosen": 0.7207098007202148,
+      "logits/rejected": 1.1366569995880127,
+      "logps/chosen": -5.253398418426514,
+      "logps/rejected": -8.280084609985352,
+      "loss": 0.6534,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -5.253398418426514,
+      "rewards/margins": 3.026685953140259,
+      "rewards/rejected": -8.280084609985352,
+      "step": 5155
+    },
+    {
+      "epoch": 2.7616658304064226,
+      "grad_norm": 11.17428521929812,
+      "learning_rate": 1.9004532711876297e-08,
+      "logits/chosen": 0.67936772108078,
+      "logits/rejected": 1.1757582426071167,
+      "logps/chosen": -4.6399431228637695,
+      "logps/rejected": -8.0260648727417,
+      "loss": 0.6234,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.6399431228637695,
+      "rewards/margins": 3.3861217498779297,
+      "rewards/rejected": -8.0260648727417,
+      "step": 5160
+    },
+    {
+      "epoch": 2.7643418631878243,
+      "grad_norm": 4.871240519271159,
+      "learning_rate": 1.8581569036159928e-08,
+      "logits/chosen": 0.7223957777023315,
+      "logits/rejected": 1.048445463180542,
+      "logps/chosen": -5.021294593811035,
+      "logps/rejected": -8.160402297973633,
+      "loss": 0.647,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -5.021294593811035,
+      "rewards/margins": 3.1391074657440186,
+      "rewards/rejected": -8.160402297973633,
+      "step": 5165
+    },
+    {
+      "epoch": 2.7670178959692255,
+      "grad_norm": 7.19689060126041,
+      "learning_rate": 1.8163276084172285e-08,
+      "logits/chosen": 0.786199688911438,
+      "logits/rejected": 1.353553056716919,
+      "logps/chosen": -4.8315348625183105,
+      "logps/rejected": -8.008269309997559,
+      "loss": 0.6521,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.8315348625183105,
+      "rewards/margins": 3.176734447479248,
+      "rewards/rejected": -8.008269309997559,
+      "step": 5170
+    },
+    {
+      "epoch": 2.7696939287506273,
+      "grad_norm": 3.9276620235427253,
+      "learning_rate": 1.7749657914193194e-08,
+      "logits/chosen": 0.6994152069091797,
+      "logits/rejected": 1.4060217142105103,
+      "logps/chosen": -4.7247419357299805,
+      "logps/rejected": -9.067811965942383,
+      "loss": 0.6423,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.7247419357299805,
+      "rewards/margins": 4.343069076538086,
+      "rewards/rejected": -9.067811965942383,
+      "step": 5175
+    },
+    {
+      "epoch": 2.7723699615320285,
+      "grad_norm": 2.8000143080059923,
+      "learning_rate": 1.7340718539148203e-08,
+      "logits/chosen": 0.7042562365531921,
+      "logits/rejected": 1.0341382026672363,
+      "logps/chosen": -4.906681537628174,
+      "logps/rejected": -8.11999797821045,
+      "loss": 0.663,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.906681537628174,
+      "rewards/margins": 3.2133171558380127,
+      "rewards/rejected": -8.11999797821045,
+      "step": 5180
+    },
+    {
+      "epoch": 2.7750459943134302,
+      "grad_norm": 10.07316018542988,
+      "learning_rate": 1.6936461926568724e-08,
+      "logits/chosen": 0.8070123791694641,
+      "logits/rejected": 1.2047308683395386,
+      "logps/chosen": -5.294198513031006,
+      "logps/rejected": -8.391801834106445,
+      "loss": 0.6527,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -5.294198513031006,
+      "rewards/margins": 3.0976033210754395,
+      "rewards/rejected": -8.391801834106445,
+      "step": 5185
+    },
+    {
+      "epoch": 2.777722027094832,
+      "grad_norm": 5.612940464941932,
+      "learning_rate": 1.6536891998554346e-08,
+      "logits/chosen": 0.5763866305351257,
+      "logits/rejected": 1.2736634016036987,
+      "logps/chosen": -4.511490345001221,
+      "logps/rejected": -8.78447151184082,
+      "loss": 0.6329,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.511490345001221,
+      "rewards/margins": 4.272981643676758,
+      "rewards/rejected": -8.78447151184082,
+      "step": 5190
+    },
+    {
+      "epoch": 2.7803980598762337,
+      "grad_norm": 4.419327127367603,
+      "learning_rate": 1.6142012631734093e-08,
+      "logits/chosen": 0.8428241014480591,
+      "logits/rejected": 1.3949544429779053,
+      "logps/chosen": -5.459053993225098,
+      "logps/rejected": -8.282559394836426,
+      "loss": 0.6506,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -5.459053993225098,
+      "rewards/margins": 2.823505163192749,
+      "rewards/rejected": -8.282559394836426,
+      "step": 5195
+    },
+    {
+      "epoch": 2.783074092657635,
+      "grad_norm": 3.7161480350594323,
+      "learning_rate": 1.575182765722949e-08,
+      "logits/chosen": 0.7026704549789429,
+      "logits/rejected": 1.252402663230896,
+      "logps/chosen": -5.0085768699646,
+      "logps/rejected": -9.000102043151855,
+      "loss": 0.6594,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -5.0085768699646,
+      "rewards/margins": 3.9915249347686768,
+      "rewards/rejected": -9.000102043151855,
+      "step": 5200
+    },
+    {
+      "epoch": 2.783074092657635,
+      "eval_logits/chosen": 0.8675931692123413,
+      "eval_logits/rejected": 1.2711238861083984,
+      "eval_logps/chosen": -4.980249404907227,
+      "eval_logps/rejected": -8.38592529296875,
+      "eval_loss": 0.6611245274543762,
+      "eval_rewards/accuracies": 0.6795251965522766,
+      "eval_rewards/chosen": -4.980249404907227,
+      "eval_rewards/margins": 3.4056758880615234,
+      "eval_rewards/rejected": -8.38592529296875,
+      "eval_runtime": 40.6956,
+      "eval_samples_per_second": 33.05,
+      "eval_steps_per_second": 8.281,
+      "step": 5200
+    },
+    {
+      "epoch": 2.7857501254390367,
+      "grad_norm": 3.062623012631288,
+      "learning_rate": 1.536634086061672e-08,
+      "logits/chosen": 0.6883572340011597,
+      "logits/rejected": 1.3664032220840454,
+      "logps/chosen": -4.601208209991455,
+      "logps/rejected": -8.885698318481445,
+      "loss": 0.6344,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.601208209991455,
+      "rewards/margins": 4.284489631652832,
+      "rewards/rejected": -8.885698318481445,
+      "step": 5205
+    },
+    {
+      "epoch": 2.788426158220438,
+      "grad_norm": 8.357937920430848,
+      "learning_rate": 1.4985555981890495e-08,
+      "logits/chosen": 0.6404739618301392,
+      "logits/rejected": 1.3054221868515015,
+      "logps/chosen": -4.497799396514893,
+      "logps/rejected": -9.12489128112793,
+      "loss": 0.644,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.497799396514893,
+      "rewards/margins": 4.62709379196167,
+      "rewards/rejected": -9.12489128112793,
+      "step": 5210
+    },
+    {
+      "epoch": 2.7911021910018396,
+      "grad_norm": 5.433802771236063,
+      "learning_rate": 1.4609476715427226e-08,
+      "logits/chosen": 0.7763322591781616,
+      "logits/rejected": 1.2029774188995361,
+      "logps/chosen": -4.571949005126953,
+      "logps/rejected": -8.046785354614258,
+      "loss": 0.6408,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.571949005126953,
+      "rewards/margins": 3.4748356342315674,
+      "rewards/rejected": -8.046785354614258,
+      "step": 5215
+    },
+    {
+      "epoch": 2.7937782237832414,
+      "grad_norm": 12.205179849621402,
+      "learning_rate": 1.4238106709949792e-08,
+      "logits/chosen": 0.6528645753860474,
+      "logits/rejected": 1.1678955554962158,
+      "logps/chosen": -4.669809818267822,
+      "logps/rejected": -8.215921401977539,
+      "loss": 0.641,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.669809818267822,
+      "rewards/margins": 3.5461106300354004,
+      "rewards/rejected": -8.215921401977539,
+      "step": 5220
+    },
+    {
+      "epoch": 2.796454256564643,
+      "grad_norm": 4.337332191309725,
+      "learning_rate": 1.3871449568491511e-08,
+      "logits/chosen": 0.7095218300819397,
+      "logits/rejected": 1.0721855163574219,
+      "logps/chosen": -4.519903659820557,
+      "logps/rejected": -7.826348304748535,
+      "loss": 0.6532,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.519903659820557,
+      "rewards/margins": 3.3064446449279785,
+      "rewards/rejected": -7.826348304748535,
+      "step": 5225
+    },
+    {
+      "epoch": 2.7991302893460444,
+      "grad_norm": 4.145121164733538,
+      "learning_rate": 1.3509508848361606e-08,
+      "logits/chosen": 0.6851939558982849,
+      "logits/rejected": 1.2608139514923096,
+      "logps/chosen": -4.637674808502197,
+      "logps/rejected": -8.47364616394043,
+      "loss": 0.6489,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.637674808502197,
+      "rewards/margins": 3.8359718322753906,
+      "rewards/rejected": -8.47364616394043,
+      "step": 5230
+    },
+    {
+      "epoch": 2.801806322127446,
+      "grad_norm": 3.8596864191107083,
+      "learning_rate": 1.3152288061110517e-08,
+      "logits/chosen": 0.584857702255249,
+      "logits/rejected": 1.0855541229248047,
+      "logps/chosen": -4.063097953796387,
+      "logps/rejected": -7.513545989990234,
+      "loss": 0.6419,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.063097953796387,
+      "rewards/margins": 3.4504477977752686,
+      "rewards/rejected": -7.513545989990234,
+      "step": 5235
+    },
+    {
+      "epoch": 2.804482354908848,
+      "grad_norm": 6.540232524697452,
+      "learning_rate": 1.2799790672495814e-08,
+      "logits/chosen": 0.711187481880188,
+      "logits/rejected": 1.111051321029663,
+      "logps/chosen": -4.64725399017334,
+      "logps/rejected": -7.065045356750488,
+      "loss": 0.6497,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.64725399017334,
+      "rewards/margins": 2.4177908897399902,
+      "rewards/rejected": -7.065045356750488,
+      "step": 5240
+    },
+    {
+      "epoch": 2.807158387690249,
+      "grad_norm": 3.419807216626517,
+      "learning_rate": 1.2452020102448835e-08,
+      "logits/chosen": 0.8273150324821472,
+      "logits/rejected": 1.1117784976959229,
+      "logps/chosen": -5.221426963806152,
+      "logps/rejected": -7.569383144378662,
+      "loss": 0.6598,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -5.221426963806152,
+      "rewards/margins": 2.3479554653167725,
+      "rewards/rejected": -7.569383144378662,
+      "step": 5245
+    },
+    {
+      "epoch": 2.8098344204716508,
+      "grad_norm": 5.470751229030638,
+      "learning_rate": 1.2108979725041103e-08,
+      "logits/chosen": 0.6748159527778625,
+      "logits/rejected": 1.2749698162078857,
+      "logps/chosen": -4.983471870422363,
+      "logps/rejected": -8.979731559753418,
+      "loss": 0.6571,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.983471870422363,
+      "rewards/margins": 3.996260404586792,
+      "rewards/rejected": -8.979731559753418,
+      "step": 5250
+    },
+    {
+      "epoch": 2.8125104532530525,
+      "grad_norm": 4.410017263566023,
+      "learning_rate": 1.1770672868451958e-08,
+      "logits/chosen": 0.7533653974533081,
+      "logits/rejected": 1.2495496273040771,
+      "logps/chosen": -4.9949517250061035,
+      "logps/rejected": -8.551423072814941,
+      "loss": 0.6543,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.9949517250061035,
+      "rewards/margins": 3.556471586227417,
+      "rewards/rejected": -8.551423072814941,
+      "step": 5255
+    },
+    {
+      "epoch": 2.8151864860344538,
+      "grad_norm": 7.867622455202789,
+      "learning_rate": 1.1437102814935872e-08,
+      "logits/chosen": 0.7798458933830261,
+      "logits/rejected": 1.0462673902511597,
+      "logps/chosen": -5.037194728851318,
+      "logps/rejected": -7.549715518951416,
+      "loss": 0.6481,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -5.037194728851318,
+      "rewards/margins": 2.5125207901000977,
+      "rewards/rejected": -7.549715518951416,
+      "step": 5260
+    },
+    {
+      "epoch": 2.8178625188158555,
+      "grad_norm": 4.885128745794862,
+      "learning_rate": 1.1108272800791018e-08,
+      "logits/chosen": 0.5540732741355896,
+      "logits/rejected": 1.120299220085144,
+      "logps/chosen": -4.276062965393066,
+      "logps/rejected": -7.881084442138672,
+      "loss": 0.6502,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.276062965393066,
+      "rewards/margins": 3.6050219535827637,
+      "rewards/rejected": -7.881084442138672,
+      "step": 5265
+    },
+    {
+      "epoch": 2.820538551597257,
+      "grad_norm": 5.247129072154271,
+      "learning_rate": 1.078418601632769e-08,
+      "logits/chosen": 0.7135279178619385,
+      "logits/rejected": 1.2354868650436401,
+      "logps/chosen": -4.823572635650635,
+      "logps/rejected": -8.821221351623535,
+      "loss": 0.6373,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.823572635650635,
+      "rewards/margins": 3.997648239135742,
+      "rewards/rejected": -8.821221351623535,
+      "step": 5270
+    },
+    {
+      "epoch": 2.8232145843786585,
+      "grad_norm": 6.201497680184341,
+      "learning_rate": 1.0464845605837159e-08,
+      "logits/chosen": 0.7134023308753967,
+      "logits/rejected": 1.110018014907837,
+      "logps/chosen": -4.81196928024292,
+      "logps/rejected": -7.620562553405762,
+      "loss": 0.6505,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.81196928024292,
+      "rewards/margins": 2.808593273162842,
+      "rewards/rejected": -7.620562553405762,
+      "step": 5275
+    },
+    {
+      "epoch": 2.82589061716006,
+      "grad_norm": 4.530081624191387,
+      "learning_rate": 1.0150254667561642e-08,
+      "logits/chosen": 0.6728294491767883,
+      "logits/rejected": 1.2038214206695557,
+      "logps/chosen": -4.750038146972656,
+      "logps/rejected": -8.217870712280273,
+      "loss": 0.656,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.750038146972656,
+      "rewards/margins": 3.467832565307617,
+      "rewards/rejected": -8.217870712280273,
+      "step": 5280
+    },
+    {
+      "epoch": 2.828566649941462,
+      "grad_norm": 2.8262126994725207,
+      "learning_rate": 9.840416253663719e-09,
+      "logits/chosen": 0.6198717951774597,
+      "logits/rejected": 1.160111665725708,
+      "logps/chosen": -4.282834529876709,
+      "logps/rejected": -8.838420867919922,
+      "loss": 0.6472,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.282834529876709,
+      "rewards/margins": 4.555586814880371,
+      "rewards/rejected": -8.838420867919922,
+      "step": 5285
+    },
+    {
+      "epoch": 2.8312426827228636,
+      "grad_norm": 6.2611712058426185,
+      "learning_rate": 9.535333370197074e-09,
+      "logits/chosen": 0.7857381105422974,
+      "logits/rejected": 1.2584213018417358,
+      "logps/chosen": -5.17061710357666,
+      "logps/rejected": -7.940934181213379,
+      "loss": 0.6549,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -5.17061710357666,
+      "rewards/margins": 2.7703166007995605,
+      "rewards/rejected": -7.940934181213379,
+      "step": 5290
+    },
+    {
+      "epoch": 2.833918715504265,
+      "grad_norm": 4.902577012699496,
+      "learning_rate": 9.23500897707713e-09,
+      "logits/chosen": 0.6338592767715454,
+      "logits/rejected": 1.2009062767028809,
+      "logps/chosen": -5.010024070739746,
+      "logps/rejected": -8.601913452148438,
+      "loss": 0.6673,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -5.010024070739746,
+      "rewards/margins": 3.5918891429901123,
+      "rewards/rejected": -8.601913452148438,
+      "step": 5295
+    },
+    {
+      "epoch": 2.8365947482856666,
+      "grad_norm": 3.201034998065709,
+      "learning_rate": 8.939445988052574e-09,
+      "logits/chosen": 0.5752130746841431,
+      "logits/rejected": 1.0920628309249878,
+      "logps/chosen": -4.657744407653809,
+      "logps/rejected": -8.575933456420898,
+      "loss": 0.6463,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.657744407653809,
+      "rewards/margins": 3.9181888103485107,
+      "rewards/rejected": -8.575933456420898,
+      "step": 5300
+    },
+    {
+      "epoch": 2.839270781067068,
+      "grad_norm": 4.73485852757271,
+      "learning_rate": 8.648647270676656e-09,
+      "logits/chosen": 0.6592661142349243,
+      "logits/rejected": 1.015424132347107,
+      "logps/chosen": -4.650457382202148,
+      "logps/rejected": -7.496077537536621,
+      "loss": 0.6532,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.650457382202148,
+      "rewards/margins": 2.845620632171631,
+      "rewards/rejected": -7.496077537536621,
+      "step": 5305
+    },
+    {
+      "epoch": 2.8419468138484696,
+      "grad_norm": 5.40049945776553,
+      "learning_rate": 8.362615646279991e-09,
+      "logits/chosen": 0.7484486699104309,
+      "logits/rejected": 1.230944275856018,
+      "logps/chosen": -5.467637062072754,
+      "logps/rejected": -8.668144226074219,
+      "loss": 0.656,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -5.467637062072754,
+      "rewards/margins": 3.2005069255828857,
+      "rewards/rejected": -8.668144226074219,
+      "step": 5310
+    },
+    {
+      "epoch": 2.8446228466298713,
+      "grad_norm": 6.282747383961463,
+      "learning_rate": 8.081353889942466e-09,
+      "logits/chosen": 0.8602693676948547,
+      "logits/rejected": 1.2941738367080688,
+      "logps/chosen": -5.2561798095703125,
+      "logps/rejected": -7.681174278259277,
+      "loss": 0.6589,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -5.2561798095703125,
+      "rewards/margins": 2.424994945526123,
+      "rewards/rejected": -7.681174278259277,
+      "step": 5315
+    },
+    {
+      "epoch": 2.847298879411273,
+      "grad_norm": 13.934375751473082,
+      "learning_rate": 7.804864730467042e-09,
+      "logits/chosen": 0.8031798601150513,
+      "logits/rejected": 1.3267239332199097,
+      "logps/chosen": -5.0023322105407715,
+      "logps/rejected": -8.536434173583984,
+      "loss": 0.6482,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -5.0023322105407715,
+      "rewards/margins": 3.534101963043213,
+      "rewards/rejected": -8.536434173583984,
+      "step": 5320
+    },
+    {
+      "epoch": 2.8499749121926743,
+      "grad_norm": 4.8084644077359515,
+      "learning_rate": 7.533150850352665e-09,
+      "logits/chosen": 0.6395283937454224,
+      "logits/rejected": 1.1656193733215332,
+      "logps/chosen": -4.396655082702637,
+      "logps/rejected": -7.605472564697266,
+      "loss": 0.659,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.396655082702637,
+      "rewards/margins": 3.2088184356689453,
+      "rewards/rejected": -7.605472564697266,
+      "step": 5325
+    },
+    {
+      "epoch": 2.852650944974076,
+      "grad_norm": 7.698053938595231,
+      "learning_rate": 7.2662148857686175e-09,
+      "logits/chosen": 0.784055233001709,
+      "logits/rejected": 1.0784869194030762,
+      "logps/chosen": -4.7419633865356445,
+      "logps/rejected": -8.001951217651367,
+      "loss": 0.6471,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.7419633865356445,
+      "rewards/margins": 3.2599875926971436,
+      "rewards/rejected": -8.001951217651367,
+      "step": 5330
+    },
+    {
+      "epoch": 2.8553269777554773,
+      "grad_norm": 4.049212243943625,
+      "learning_rate": 7.0040594265287635e-09,
+      "logits/chosen": 0.7620818614959717,
+      "logits/rejected": 1.0444839000701904,
+      "logps/chosen": -4.832968235015869,
+      "logps/rejected": -7.354680061340332,
+      "loss": 0.6518,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.832968235015869,
+      "rewards/margins": 2.5217113494873047,
+      "rewards/rejected": -7.354680061340332,
+      "step": 5335
+    },
+    {
+      "epoch": 2.858003010536879,
+      "grad_norm": 4.848523257424049,
+      "learning_rate": 6.746687016066566e-09,
+      "logits/chosen": 0.6461294889450073,
+      "logits/rejected": 1.162700891494751,
+      "logps/chosen": -4.23568058013916,
+      "logps/rejected": -7.820146083831787,
+      "loss": 0.6413,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.23568058013916,
+      "rewards/margins": 3.5844650268554688,
+      "rewards/rejected": -7.820146083831787,
+      "step": 5340
+    },
+    {
+      "epoch": 2.8606790433182807,
+      "grad_norm": 5.000542498288708,
+      "learning_rate": 6.494100151410276e-09,
+      "logits/chosen": 0.5653608441352844,
+      "logits/rejected": 1.1772863864898682,
+      "logps/chosen": -4.551900863647461,
+      "logps/rejected": -8.261563301086426,
+      "loss": 0.6506,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.551900863647461,
+      "rewards/margins": 3.709662675857544,
+      "rewards/rejected": -8.261563301086426,
+      "step": 5345
+    },
+    {
+      "epoch": 2.8633550760996824,
+      "grad_norm": 3.6384301675383024,
+      "learning_rate": 6.246301283158728e-09,
+      "logits/chosen": 0.8431389927864075,
+      "logits/rejected": 1.1242343187332153,
+      "logps/chosen": -5.126715660095215,
+      "logps/rejected": -7.340585231781006,
+      "loss": 0.6561,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -5.126715660095215,
+      "rewards/margins": 2.21386981010437,
+      "rewards/rejected": -7.340585231781006,
+      "step": 5350
+    },
+    {
+      "epoch": 2.8660311088810837,
+      "grad_norm": 4.182671743864516,
+      "learning_rate": 6.0032928154576944e-09,
+      "logits/chosen": 0.8440614938735962,
+      "logits/rejected": 1.2042032480239868,
+      "logps/chosen": -4.9655351638793945,
+      "logps/rejected": -7.578786373138428,
+      "loss": 0.6622,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.9655351638793945,
+      "rewards/margins": 2.613251209259033,
+      "rewards/rejected": -7.578786373138428,
+      "step": 5355
+    },
+    {
+      "epoch": 2.8687071416624854,
+      "grad_norm": 7.263489039863829,
+      "learning_rate": 5.76507710597629e-09,
+      "logits/chosen": 0.7130125761032104,
+      "logits/rejected": 1.1164230108261108,
+      "logps/chosen": -4.803915977478027,
+      "logps/rejected": -7.342498779296875,
+      "loss": 0.6485,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.803915977478027,
+      "rewards/margins": 2.5385825634002686,
+      "rewards/rejected": -7.342498779296875,
+      "step": 5360
+    },
+    {
+      "epoch": 2.8713831744438867,
+      "grad_norm": 6.712935520395192,
+      "learning_rate": 5.531656465884438e-09,
+      "logits/chosen": 0.6347378492355347,
+      "logits/rejected": 1.1770563125610352,
+      "logps/chosen": -4.63676643371582,
+      "logps/rejected": -8.414531707763672,
+      "loss": 0.638,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.63676643371582,
+      "rewards/margins": 3.7777652740478516,
+      "rewards/rejected": -8.414531707763672,
+      "step": 5365
+    },
+    {
+      "epoch": 2.8740592072252884,
+      "grad_norm": 4.551832795540113,
+      "learning_rate": 5.303033159830217e-09,
+      "logits/chosen": 0.8593635559082031,
+      "logits/rejected": 1.4061036109924316,
+      "logps/chosen": -5.534549713134766,
+      "logps/rejected": -8.482142448425293,
+      "loss": 0.6644,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -5.534549713134766,
+      "rewards/margins": 2.947593927383423,
+      "rewards/rejected": -8.482142448425293,
+      "step": 5370
+    },
+    {
+      "epoch": 2.87673524000669,
+      "grad_norm": 5.70391684065922,
+      "learning_rate": 5.079209405917939e-09,
+      "logits/chosen": 0.6803232431411743,
+      "logits/rejected": 1.3061047792434692,
+      "logps/chosen": -4.525296211242676,
+      "logps/rejected": -9.341375350952148,
+      "loss": 0.6256,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.525296211242676,
+      "rewards/margins": 4.816079616546631,
+      "rewards/rejected": -9.341375350952148,
+      "step": 5375
+    },
+    {
+      "epoch": 2.879411272788092,
+      "grad_norm": 4.5889727004213,
+      "learning_rate": 4.860187375686664e-09,
+      "logits/chosen": 0.7376517057418823,
+      "logits/rejected": 1.199014663696289,
+      "logps/chosen": -4.787007808685303,
+      "logps/rejected": -7.74468469619751,
+      "loss": 0.6478,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.787007808685303,
+      "rewards/margins": 2.9576759338378906,
+      "rewards/rejected": -7.74468469619751,
+      "step": 5380
+    },
+    {
+      "epoch": 2.882087305569493,
+      "grad_norm": 4.842939608943848,
+      "learning_rate": 4.64596919408905e-09,
+      "logits/chosen": 0.7767267227172852,
+      "logits/rejected": 1.3040108680725098,
+      "logps/chosen": -4.65799617767334,
+      "logps/rejected": -8.474201202392578,
+      "loss": 0.6464,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.65799617767334,
+      "rewards/margins": 3.81620454788208,
+      "rewards/rejected": -8.474201202392578,
+      "step": 5385
+    },
+    {
+      "epoch": 2.884763338350895,
+      "grad_norm": 5.364476766432849,
+      "learning_rate": 4.436556939470814e-09,
+      "logits/chosen": 0.8850129842758179,
+      "logits/rejected": 1.4347807168960571,
+      "logps/chosen": -5.417617321014404,
+      "logps/rejected": -8.298930168151855,
+      "loss": 0.6363,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -5.417617321014404,
+      "rewards/margins": 2.8813138008117676,
+      "rewards/rejected": -8.298930168151855,
+      "step": 5390
+    },
+    {
+      "epoch": 2.887439371132296,
+      "grad_norm": 6.143354758944635,
+      "learning_rate": 4.23195264355064e-09,
+      "logits/chosen": 0.6164458990097046,
+      "logits/rejected": 0.9420022964477539,
+      "logps/chosen": -4.959438800811768,
+      "logps/rejected": -7.6174116134643555,
+      "loss": 0.6404,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.959438800811768,
+      "rewards/margins": 2.657973527908325,
+      "rewards/rejected": -7.6174116134643555,
+      "step": 5395
+    },
+    {
+      "epoch": 2.890115403913698,
+      "grad_norm": 4.604990067874132,
+      "learning_rate": 4.032158291400245e-09,
+      "logits/chosen": 0.6131589412689209,
+      "logits/rejected": 1.3442531824111938,
+      "logps/chosen": -4.6462225914001465,
+      "logps/rejected": -9.304888725280762,
+      "loss": 0.6431,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.6462225914001465,
+      "rewards/margins": 4.658665657043457,
+      "rewards/rejected": -9.304888725280762,
+      "step": 5400
+    },
+    {
+      "epoch": 2.8927914366950995,
+      "grad_norm": 2.2489040982491333,
+      "learning_rate": 3.837175821425398e-09,
+      "logits/chosen": 0.7480042576789856,
+      "logits/rejected": 1.1878314018249512,
+      "logps/chosen": -5.378551006317139,
+      "logps/rejected": -7.976735591888428,
+      "loss": 0.6556,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -5.378551006317139,
+      "rewards/margins": 2.598184823989868,
+      "rewards/rejected": -7.976735591888428,
+      "step": 5405
+    },
+    {
+      "epoch": 2.8954674694765012,
+      "grad_norm": 5.618197552392695,
+      "learning_rate": 3.6470071253467683e-09,
+      "logits/chosen": 0.848590075969696,
+      "logits/rejected": 1.2208644151687622,
+      "logps/chosen": -4.995560646057129,
+      "logps/rejected": -8.311874389648438,
+      "loss": 0.6405,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.995560646057129,
+      "rewards/margins": 3.316314220428467,
+      "rewards/rejected": -8.311874389648438,
+      "step": 5410
+    },
+    {
+      "epoch": 2.8981435022579025,
+      "grad_norm": 3.942963319624137,
+      "learning_rate": 3.461654048181939e-09,
+      "logits/chosen": 0.9013597369194031,
+      "logits/rejected": 1.2442383766174316,
+      "logps/chosen": -5.621943473815918,
+      "logps/rejected": -8.132493019104004,
+      "loss": 0.6588,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -5.621943473815918,
+      "rewards/margins": 2.5105483531951904,
+      "rewards/rejected": -8.132493019104004,
+      "step": 5415
+    },
+    {
+      "epoch": 2.9008195350393042,
+      "grad_norm": 20.530931856839814,
+      "learning_rate": 3.281118388227255e-09,
+      "logits/chosen": 0.8555103540420532,
+      "logits/rejected": 1.135647177696228,
+      "logps/chosen": -5.475981712341309,
+      "logps/rejected": -7.752043724060059,
+      "loss": 0.6766,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -5.475981712341309,
+      "rewards/margins": 2.27606201171875,
+      "rewards/rejected": -7.752043724060059,
+      "step": 5420
+    },
+    {
+      "epoch": 2.903495567820706,
+      "grad_norm": 7.691011995449598,
+      "learning_rate": 3.1054018970405048e-09,
+      "logits/chosen": 0.5834836363792419,
+      "logits/rejected": 1.2541258335113525,
+      "logps/chosen": -4.0094404220581055,
+      "logps/rejected": -8.730401039123535,
+      "loss": 0.6425,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.0094404220581055,
+      "rewards/margins": 4.720961570739746,
+      "rewards/rejected": -8.730401039123535,
+      "step": 5425
+    },
+    {
+      "epoch": 2.906171600602107,
+      "grad_norm": 11.017066550922252,
+      "learning_rate": 2.9345062794238207e-09,
+      "logits/chosen": 0.741051971912384,
+      "logits/rejected": 1.2039616107940674,
+      "logps/chosen": -4.614964008331299,
+      "logps/rejected": -7.694038391113281,
+      "loss": 0.6479,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.614964008331299,
+      "rewards/margins": 3.0790746212005615,
+      "rewards/rejected": -7.694038391113281,
+      "step": 5430
+    },
+    {
+      "epoch": 2.908847633383509,
+      "grad_norm": 5.499837186088628,
+      "learning_rate": 2.7684331934072492e-09,
+      "logits/chosen": 0.6573973894119263,
+      "logits/rejected": 1.016038417816162,
+      "logps/chosen": -4.930825710296631,
+      "logps/rejected": -8.391733169555664,
+      "loss": 0.6498,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.930825710296631,
+      "rewards/margins": 3.460906505584717,
+      "rewards/rejected": -8.391733169555664,
+      "step": 5435
+    },
+    {
+      "epoch": 2.9115236661649107,
+      "grad_norm": 3.2378807538661425,
+      "learning_rate": 2.6071842502326526e-09,
+      "logits/chosen": 0.7494984865188599,
+      "logits/rejected": 1.0847275257110596,
+      "logps/chosen": -5.468812465667725,
+      "logps/rejected": -7.7908034324646,
+      "loss": 0.6543,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -5.468812465667725,
+      "rewards/margins": 2.321991443634033,
+      "rewards/rejected": -7.7908034324646,
+      "step": 5440
+    },
+    {
+      "epoch": 2.9141996989463124,
+      "grad_norm": 2.6952757469940933,
+      "learning_rate": 2.450761014337888e-09,
+      "logits/chosen": 0.8897698521614075,
+      "logits/rejected": 1.0474538803100586,
+      "logps/chosen": -5.160197734832764,
+      "logps/rejected": -7.62185525894165,
+      "loss": 0.6494,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -5.160197734832764,
+      "rewards/margins": 2.461657762527466,
+      "rewards/rejected": -7.62185525894165,
+      "step": 5445
+    },
+    {
+      "epoch": 2.9168757317277136,
+      "grad_norm": 5.323043588757468,
+      "learning_rate": 2.299165003341985e-09,
+      "logits/chosen": 0.8049535751342773,
+      "logits/rejected": 1.23271906375885,
+      "logps/chosen": -4.754616737365723,
+      "logps/rejected": -7.836543083190918,
+      "loss": 0.657,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.754616737365723,
+      "rewards/margins": 3.081926107406616,
+      "rewards/rejected": -7.836543083190918,
+      "step": 5450
+    },
+    {
+      "epoch": 2.9195517645091154,
+      "grad_norm": 5.153104180953037,
+      "learning_rate": 2.1523976880299945e-09,
+      "logits/chosen": 0.7475239634513855,
+      "logits/rejected": 1.0843391418457031,
+      "logps/chosen": -5.352973461151123,
+      "logps/rejected": -7.309823513031006,
+      "loss": 0.6669,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -5.352973461151123,
+      "rewards/margins": 1.9568496942520142,
+      "rewards/rejected": -7.309823513031006,
+      "step": 5455
+    },
+    {
+      "epoch": 2.9222277972905166,
+      "grad_norm": 5.646529311029514,
+      "learning_rate": 2.010460492339161e-09,
+      "logits/chosen": 0.6191426515579224,
+      "logits/rejected": 1.1166914701461792,
+      "logps/chosen": -4.401597499847412,
+      "logps/rejected": -7.922074317932129,
+      "loss": 0.6366,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.401597499847412,
+      "rewards/margins": 3.520477294921875,
+      "rewards/rejected": -7.922074317932129,
+      "step": 5460
+    },
+    {
+      "epoch": 2.9249038300719183,
+      "grad_norm": 5.174325122004295,
+      "learning_rate": 1.8733547933446614e-09,
+      "logits/chosen": 0.7639790773391724,
+      "logits/rejected": 1.1621696949005127,
+      "logps/chosen": -5.398471355438232,
+      "logps/rejected": -7.774838447570801,
+      "loss": 0.6746,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -5.398471355438232,
+      "rewards/margins": 2.3763668537139893,
+      "rewards/rejected": -7.774838447570801,
+      "step": 5465
+    },
+    {
+      "epoch": 2.92757986285332,
+      "grad_norm": 4.483115323654616,
+      "learning_rate": 1.7410819212467231e-09,
+      "logits/chosen": 0.9727827906608582,
+      "logits/rejected": 1.2395334243774414,
+      "logps/chosen": -5.637606143951416,
+      "logps/rejected": -7.744087219238281,
+      "loss": 0.6586,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -5.637606143951416,
+      "rewards/margins": 2.106480598449707,
+      "rewards/rejected": -7.744087219238281,
+      "step": 5470
+    },
+    {
+      "epoch": 2.9302558956347218,
+      "grad_norm": 4.346309690383488,
+      "learning_rate": 1.613643159357192e-09,
+      "logits/chosen": 0.8396733403205872,
+      "logits/rejected": 1.2059072256088257,
+      "logps/chosen": -4.783254623413086,
+      "logps/rejected": -7.817492485046387,
+      "loss": 0.6579,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.783254623413086,
+      "rewards/margins": 3.034238338470459,
+      "rewards/rejected": -7.817492485046387,
+      "step": 5475
+    },
+    {
+      "epoch": 2.932931928416123,
+      "grad_norm": 4.268911332357838,
+      "learning_rate": 1.4910397440875967e-09,
+      "logits/chosen": 0.6452018618583679,
+      "logits/rejected": 1.161171793937683,
+      "logps/chosen": -4.669070243835449,
+      "logps/rejected": -8.265832901000977,
+      "loss": 0.6493,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.669070243835449,
+      "rewards/margins": 3.5967631340026855,
+      "rewards/rejected": -8.265832901000977,
+      "step": 5480
+    },
+    {
+      "epoch": 2.9356079611975248,
+      "grad_norm": 5.941003138325341,
+      "learning_rate": 1.3732728649368253e-09,
+      "logits/chosen": 0.8127827644348145,
+      "logits/rejected": 1.2987315654754639,
+      "logps/chosen": -4.778433799743652,
+      "logps/rejected": -7.561098575592041,
+      "loss": 0.6487,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.778433799743652,
+      "rewards/margins": 2.7826645374298096,
+      "rewards/rejected": -7.561098575592041,
+      "step": 5485
+    },
+    {
+      "epoch": 2.938283993978926,
+      "grad_norm": 8.134527457416398,
+      "learning_rate": 1.260343664479524e-09,
+      "logits/chosen": 0.6006717681884766,
+      "logits/rejected": 1.1379812955856323,
+      "logps/chosen": -4.241616725921631,
+      "logps/rejected": -7.810811519622803,
+      "loss": 0.6643,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.241616725921631,
+      "rewards/margins": 3.5691940784454346,
+      "rewards/rejected": -7.810811519622803,
+      "step": 5490
+    },
+    {
+      "epoch": 2.9409600267603278,
+      "grad_norm": 6.195229428169329,
+      "learning_rate": 1.1522532383554384e-09,
+      "logits/chosen": 0.8391780853271484,
+      "logits/rejected": 1.1937634944915771,
+      "logps/chosen": -5.3445844650268555,
+      "logps/rejected": -8.344155311584473,
+      "loss": 0.6345,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -5.3445844650268555,
+      "rewards/margins": 2.999569892883301,
+      "rewards/rejected": -8.344155311584473,
+      "step": 5495
+    },
+    {
+      "epoch": 2.9436360595417295,
+      "grad_norm": 3.7574430794775653,
+      "learning_rate": 1.049002635258256e-09,
+      "logits/chosen": 0.831591010093689,
+      "logits/rejected": 1.245152235031128,
+      "logps/chosen": -5.037074089050293,
+      "logps/rejected": -8.381568908691406,
+      "loss": 0.6587,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -5.037074089050293,
+      "rewards/margins": 3.3444950580596924,
+      "rewards/rejected": -8.381568908691406,
+      "step": 5500
+    },
+    {
+      "epoch": 2.946312092323131,
+      "grad_norm": 6.3912661145530905,
+      "learning_rate": 9.505928569258358e-10,
+      "logits/chosen": 0.6976084113121033,
+      "logits/rejected": 1.17734694480896,
+      "logps/chosen": -4.4580488204956055,
+      "logps/rejected": -7.948143005371094,
+      "loss": 0.6311,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.4580488204956055,
+      "rewards/margins": 3.490093231201172,
+      "rewards/rejected": -7.948143005371094,
+      "step": 5505
+    },
+    {
+      "epoch": 2.9489881251045325,
+      "grad_norm": 4.202794971739333,
+      "learning_rate": 8.57024858130273e-10,
+      "logits/chosen": 0.8722679018974304,
+      "logits/rejected": 1.2229573726654053,
+      "logps/chosen": -5.284964561462402,
+      "logps/rejected": -7.936923980712891,
+      "loss": 0.6455,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -5.284964561462402,
+      "rewards/margins": 2.651959180831909,
+      "rewards/rejected": -7.936923980712891,
+      "step": 5510
+    },
+    {
+      "epoch": 2.951664157885934,
+      "grad_norm": 3.883664932958451,
+      "learning_rate": 7.682995466686826e-10,
+      "logits/chosen": 0.8018722534179688,
+      "logits/rejected": 1.1032469272613525,
+      "logps/chosen": -4.956485748291016,
+      "logps/rejected": -8.055354118347168,
+      "loss": 0.6459,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.956485748291016,
+      "rewards/margins": 3.0988681316375732,
+      "rewards/rejected": -8.055354118347168,
+      "step": 5515
+    },
+    {
+      "epoch": 2.9543401906673354,
+      "grad_norm": 6.207069760900057,
+      "learning_rate": 6.844177833543741e-10,
+      "logits/chosen": 0.7384808659553528,
+      "logits/rejected": 1.2157642841339111,
+      "logps/chosen": -4.56369686126709,
+      "logps/rejected": -8.084737777709961,
+      "loss": 0.6554,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.56369686126709,
+      "rewards/margins": 3.5210413932800293,
+      "rewards/rejected": -8.084737777709961,
+      "step": 5520
+    },
+    {
+      "epoch": 2.957016223448737,
+      "grad_norm": 2.3694684197221543,
+      "learning_rate": 6.053803820087467e-10,
+      "logits/chosen": 0.8647111058235168,
+      "logits/rejected": 1.2884724140167236,
+      "logps/chosen": -5.8626556396484375,
+      "logps/rejected": -8.692947387695312,
+      "loss": 0.6598,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -5.8626556396484375,
+      "rewards/margins": 2.830291271209717,
+      "rewards/rejected": -8.692947387695312,
+      "step": 5525
+    },
+    {
+      "epoch": 2.959692256230139,
+      "grad_norm": 12.124646973021934,
+      "learning_rate": 5.311881094528514e-10,
+      "logits/chosen": 0.6612305641174316,
+      "logits/rejected": 1.0340888500213623,
+      "logps/chosen": -5.102568626403809,
+      "logps/rejected": -7.586211204528809,
+      "loss": 0.6764,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -5.102568626403809,
+      "rewards/margins": 2.483643054962158,
+      "rewards/rejected": -7.586211204528809,
+      "step": 5530
+    },
+    {
+      "epoch": 2.9623682890115406,
+      "grad_norm": 6.208219110922504,
+      "learning_rate": 4.6184168550050806e-10,
+      "logits/chosen": 0.790997326374054,
+      "logits/rejected": 1.1555887460708618,
+      "logps/chosen": -5.7118821144104,
+      "logps/rejected": -8.248160362243652,
+      "loss": 0.6465,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -5.7118821144104,
+      "rewards/margins": 2.5362777709960938,
+      "rewards/rejected": -8.248160362243652,
+      "step": 5535
+    },
+    {
+      "epoch": 2.965044321792942,
+      "grad_norm": 6.528585659568342,
+      "learning_rate": 3.973417829510328e-10,
+      "logits/chosen": 0.6299843788146973,
+      "logits/rejected": 1.1741900444030762,
+      "logps/chosen": -4.422912120819092,
+      "logps/rejected": -8.294031143188477,
+      "loss": 0.6436,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.422912120819092,
+      "rewards/margins": 3.8711180686950684,
+      "rewards/rejected": -8.294031143188477,
+      "step": 5540
+    },
+    {
+      "epoch": 2.9677203545743436,
+      "grad_norm": 7.872413261041495,
+      "learning_rate": 3.3768902758274377e-10,
+      "logits/chosen": 0.785802960395813,
+      "logits/rejected": 1.3659334182739258,
+      "logps/chosen": -5.193419933319092,
+      "logps/rejected": -8.989534378051758,
+      "loss": 0.6534,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -5.193419933319092,
+      "rewards/margins": 3.796113967895508,
+      "rewards/rejected": -8.989534378051758,
+      "step": 5545
+    },
+    {
+      "epoch": 2.970396387355745,
+      "grad_norm": 3.8963789997815024,
+      "learning_rate": 2.8288399814691e-10,
+      "logits/chosen": 0.7000380754470825,
+      "logits/rejected": 1.1333086490631104,
+      "logps/chosen": -4.185264587402344,
+      "logps/rejected": -7.0287981033325195,
+      "loss": 0.6445,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.185264587402344,
+      "rewards/margins": 2.8435332775115967,
+      "rewards/rejected": -7.0287981033325195,
+      "step": 5550
+    },
+    {
+      "epoch": 2.9730724201371466,
+      "grad_norm": 6.346487718784554,
+      "learning_rate": 2.3292722636220066e-10,
+      "logits/chosen": 0.7657510042190552,
+      "logits/rejected": 1.211764931678772,
+      "logps/chosen": -5.01204252243042,
+      "logps/rejected": -8.254289627075195,
+      "loss": 0.6601,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -5.01204252243042,
+      "rewards/margins": 3.2422471046447754,
+      "rewards/rejected": -8.254289627075195,
+      "step": 5555
+    },
+    {
+      "epoch": 2.9757484529185483,
+      "grad_norm": 7.9201711965525625,
+      "learning_rate": 1.8781919690946668e-10,
+      "logits/chosen": 0.6117281913757324,
+      "logits/rejected": 1.0833332538604736,
+      "logps/chosen": -4.630134105682373,
+      "logps/rejected": -7.466133117675781,
+      "loss": 0.6562,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.630134105682373,
+      "rewards/margins": 2.835998058319092,
+      "rewards/rejected": -7.466133117675781,
+      "step": 5560
+    },
+    {
+      "epoch": 2.97842448569995,
+      "grad_norm": 4.357836082663546,
+      "learning_rate": 1.4756034742696711e-10,
+      "logits/chosen": 0.8454300761222839,
+      "logits/rejected": 1.3807599544525146,
+      "logps/chosen": -5.376292705535889,
+      "logps/rejected": -8.327980995178223,
+      "loss": 0.6512,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -5.376292705535889,
+      "rewards/margins": 2.951687812805176,
+      "rewards/rejected": -8.327980995178223,
+      "step": 5565
+    },
+    {
+      "epoch": 2.9811005184813513,
+      "grad_norm": 12.239194841449368,
+      "learning_rate": 1.12151068506261e-10,
+      "logits/chosen": 0.6639296412467957,
+      "logits/rejected": 1.3309894800186157,
+      "logps/chosen": -4.308588981628418,
+      "logps/rejected": -8.366358757019043,
+      "loss": 0.6425,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.308588981628418,
+      "rewards/margins": 4.05776834487915,
+      "rewards/rejected": -8.366358757019043,
+      "step": 5570
+    },
+    {
+      "epoch": 2.983776551262753,
+      "grad_norm": 7.891806718350854,
+      "learning_rate": 8.159170368826629e-11,
+      "logits/chosen": 0.656947135925293,
+      "logits/rejected": 1.101847767829895,
+      "logps/chosen": -4.486090660095215,
+      "logps/rejected": -7.260919094085693,
+      "loss": 0.6586,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.486090660095215,
+      "rewards/margins": 2.7748284339904785,
+      "rewards/rejected": -7.260919094085693,
+      "step": 5575
+    },
+    {
+      "epoch": 2.9864525840441547,
+      "grad_norm": 5.7981567628302235,
+      "learning_rate": 5.588254946015114e-11,
+      "logits/chosen": 0.7003321051597595,
+      "logits/rejected": 1.1341341733932495,
+      "logps/chosen": -4.928320407867432,
+      "logps/rejected": -7.317234992980957,
+      "loss": 0.647,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.928320407867432,
+      "rewards/margins": 2.3889148235321045,
+      "rewards/rejected": -7.317234992980957,
+      "step": 5580
+    },
+    {
+      "epoch": 2.989128616825556,
+      "grad_norm": 16.659667673326833,
+      "learning_rate": 3.502385525216978e-11,
+      "logits/chosen": 0.6201304197311401,
+      "logits/rejected": 1.0062357187271118,
+      "logps/chosen": -4.330212593078613,
+      "logps/rejected": -7.518198490142822,
+      "loss": 0.6703,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.330212593078613,
+      "rewards/margins": 3.1879847049713135,
+      "rewards/rejected": -7.518198490142822,
+      "step": 5585
+    },
+    {
+      "epoch": 2.9918046496069577,
+      "grad_norm": 3.7080891057019976,
+      "learning_rate": 1.901582343555308e-11,
+      "logits/chosen": 0.7568680644035339,
+      "logits/rejected": 1.2104458808898926,
+      "logps/chosen": -5.141021728515625,
+      "logps/rejected": -8.142678260803223,
+      "loss": 0.6616,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -5.141021728515625,
+      "rewards/margins": 3.0016565322875977,
+      "rewards/rejected": -8.142678260803223,
+      "step": 5590
+    },
+    {
+      "epoch": 2.9944806823883594,
+      "grad_norm": 5.875620623195355,
+      "learning_rate": 7.858609320232634e-12,
+      "logits/chosen": 0.6780782341957092,
+      "logits/rejected": 1.2257006168365479,
+      "logps/chosen": -4.683114051818848,
+      "logps/rejected": -7.827949523925781,
+      "loss": 0.6462,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.683114051818848,
+      "rewards/margins": 3.1448357105255127,
+      "rewards/rejected": -7.827949523925781,
+      "step": 5595
+    },
+    {
+      "epoch": 2.9971567151697607,
+      "grad_norm": 4.142935074247411,
+      "learning_rate": 1.5523211535639624e-12,
+      "logits/chosen": 0.7273890972137451,
+      "logits/rejected": 1.1526896953582764,
+      "logps/chosen": -4.603427886962891,
+      "logps/rejected": -8.202547073364258,
+      "loss": 0.6402,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.603427886962891,
+      "rewards/margins": 3.599118709564209,
+      "rewards/rejected": -8.202547073364258,
+      "step": 5600
+    },
+    {
+      "epoch": 2.9971567151697607,
+      "eval_logits/chosen": 0.9296243190765381,
+      "eval_logits/rejected": 1.3373146057128906,
+      "eval_logps/chosen": -4.961299896240234,
+      "eval_logps/rejected": -8.357985496520996,
+      "eval_loss": 0.6612382531166077,
+      "eval_rewards/accuracies": 0.6765578389167786,
+      "eval_rewards/chosen": -4.961299896240234,
+      "eval_rewards/margins": 3.396685838699341,
+      "eval_rewards/rejected": -8.357985496520996,
+      "eval_runtime": 40.4958,
+      "eval_samples_per_second": 33.213,
+      "eval_steps_per_second": 8.322,
+      "step": 5600
+    },
+    {
+      "epoch": 2.999297541394882,
+      "step": 5604,
+      "total_flos": 0.0,
+      "train_loss": 0.6706608949092182,
+      "train_runtime": 30654.4044,
+      "train_samples_per_second": 5.851,
+      "train_steps_per_second": 0.183
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 5604,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 1000000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}