diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,18200 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.999297541394882,
+  "eval_steps": 400,
+  "global_step": 5604,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.002676032781401572,
+      "grad_norm": 2.5517993897632163,
+      "learning_rate": 8.9126559714795e-09,
+      "logits/chosen": -0.060342513024806976,
+      "logits/rejected": 0.15198828279972076,
+      "logps/chosen": -1.7157150506973267,
+      "logps/rejected": -1.8899202346801758,
+      "loss": 0.2027,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -1.7157150506973267,
+      "rewards/margins": 0.17420509457588196,
+      "rewards/rejected": -1.8899202346801758,
+      "sft_loss": 1.4683821201324463,
+      "step": 5
+    },
+    {
+      "epoch": 0.005352065562803144,
+      "grad_norm": 1.6881869363267583,
+      "learning_rate": 1.7825311942959e-08,
+      "logits/chosen": 0.01580592431128025,
+      "logits/rejected": 0.14093606173992157,
+      "logps/chosen": -1.8028850555419922,
+      "logps/rejected": -1.8463115692138672,
+      "loss": 0.2079,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -1.8028850555419922,
+      "rewards/margins": 0.04342658072710037,
+      "rewards/rejected": -1.8463115692138672,
+      "sft_loss": 1.5084385871887207,
+      "step": 10
+    },
+    {
+      "epoch": 0.008028098344204716,
+      "grad_norm": 2.08872868547191,
+      "learning_rate": 2.67379679144385e-08,
+      "logits/chosen": -0.037088386714458466,
+      "logits/rejected": 0.06292067468166351,
+      "logps/chosen": -1.6352245807647705,
+      "logps/rejected": -1.7644548416137695,
+      "loss": 0.233,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -1.6352245807647705,
+      "rewards/margins": 0.12923024594783783,
+      "rewards/rejected": -1.7644548416137695,
+      "sft_loss": 1.5005210638046265,
+      "step": 15
+    },
+    {
+      "epoch": 0.010704131125606288,
+      "grad_norm": 2.666536867260419,
+      "learning_rate": 3.5650623885918e-08,
+      "logits/chosen": -0.05209019035100937,
+      "logits/rejected": 0.03605952113866806,
+      "logps/chosen": -1.7257559299468994,
+      "logps/rejected": -1.8052446842193604,
+      "loss": 0.228,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -1.7257559299468994,
+      "rewards/margins": 0.07948889583349228,
+      "rewards/rejected": -1.8052446842193604,
+      "sft_loss": 1.5005673170089722,
+      "step": 20
+    },
+    {
+      "epoch": 0.013380163907007862,
+      "grad_norm": 2.724261942986969,
+      "learning_rate": 4.45632798573975e-08,
+      "logits/chosen": -0.06255056709051132,
+      "logits/rejected": 0.022401634603738785,
+      "logps/chosen": -1.8699411153793335,
+      "logps/rejected": -1.7783015966415405,
+      "loss": 0.2564,
+      "rewards/accuracies": 0.3812499940395355,
+      "rewards/chosen": -1.8699411153793335,
+      "rewards/margins": -0.09163953363895416,
+      "rewards/rejected": -1.7783015966415405,
+      "sft_loss": 1.5462677478790283,
+      "step": 25
+    },
+    {
+      "epoch": 0.016056196688409432,
+      "grad_norm": 1.9807327110980395,
+      "learning_rate": 5.3475935828877e-08,
+      "logits/chosen": -0.09974177926778793,
+      "logits/rejected": -0.006263503339141607,
+      "logps/chosen": -1.9092490673065186,
+      "logps/rejected": -1.8333038091659546,
+      "loss": 0.2079,
+      "rewards/accuracies": 0.4437499940395355,
+      "rewards/chosen": -1.9092490673065186,
+      "rewards/margins": -0.0759454220533371,
+      "rewards/rejected": -1.8333038091659546,
+      "sft_loss": 1.6469275951385498,
+      "step": 30
+    },
+    {
+      "epoch": 0.018732229469811006,
+      "grad_norm": 2.7289585545074577,
+      "learning_rate": 6.23885918003565e-08,
+      "logits/chosen": -0.045177824795246124,
+      "logits/rejected": 0.12003231048583984,
+      "logps/chosen": -1.8494408130645752,
+      "logps/rejected": -1.999485969543457,
+      "loss": 0.2239,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -1.8494408130645752,
+      "rewards/margins": 0.1500452309846878,
+      "rewards/rejected": -1.999485969543457,
+      "sft_loss": 1.5628007650375366,
+      "step": 35
+    },
+    {
+      "epoch": 0.021408262251212576,
+      "grad_norm": 2.05592651881969,
+      "learning_rate": 7.1301247771836e-08,
+      "logits/chosen": 0.033259421586990356,
+      "logits/rejected": 0.20962993800640106,
+      "logps/chosen": -1.8854608535766602,
+      "logps/rejected": -1.7456992864608765,
+      "loss": 0.2288,
+      "rewards/accuracies": 0.45625001192092896,
+      "rewards/chosen": -1.8854608535766602,
+      "rewards/margins": -0.13976149260997772,
+      "rewards/rejected": -1.7456992864608765,
+      "sft_loss": 1.5202932357788086,
+      "step": 40
+    },
+    {
+      "epoch": 0.02408429503261415,
+      "grad_norm": 2.4443688144646596,
+      "learning_rate": 8.021390374331551e-08,
+      "logits/chosen": 0.030009519308805466,
+      "logits/rejected": 0.23175004124641418,
+      "logps/chosen": -1.8416560888290405,
+      "logps/rejected": -1.876493215560913,
+      "loss": 0.2214,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -1.8416560888290405,
+      "rewards/margins": 0.03483711928129196,
+      "rewards/rejected": -1.876493215560913,
+      "sft_loss": 1.5380895137786865,
+      "step": 45
+    },
+    {
+      "epoch": 0.026760327814015723,
+      "grad_norm": 2.630447917916181,
+      "learning_rate": 8.9126559714795e-08,
+      "logits/chosen": -0.04315720126032829,
+      "logits/rejected": 0.11193470656871796,
+      "logps/chosen": -1.9053493738174438,
+      "logps/rejected": -1.783015489578247,
+      "loss": 0.222,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.9053493738174438,
+      "rewards/margins": -0.12233398109674454,
+      "rewards/rejected": -1.783015489578247,
+      "sft_loss": 1.585577368736267,
+      "step": 50
+    },
+    {
+      "epoch": 0.029436360595417294,
+      "grad_norm": 2.2629886950410265,
+      "learning_rate": 9.80392156862745e-08,
+      "logits/chosen": -0.117431640625,
+      "logits/rejected": 0.10506489127874374,
+      "logps/chosen": -1.8451545238494873,
+      "logps/rejected": -1.879084825515747,
+      "loss": 0.2128,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -1.8451545238494873,
+      "rewards/margins": 0.03393043577671051,
+      "rewards/rejected": -1.879084825515747,
+      "sft_loss": 1.5884572267532349,
+      "step": 55
+    },
+    {
+      "epoch": 0.032112393376818864,
+      "grad_norm": 2.338561500458837,
+      "learning_rate": 1.06951871657754e-07,
+      "logits/chosen": -0.0742352232336998,
+      "logits/rejected": 0.12219759076833725,
+      "logps/chosen": -1.805419921875,
+      "logps/rejected": -1.9098402261734009,
+      "loss": 0.2039,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -1.805419921875,
+      "rewards/margins": 0.10442020744085312,
+      "rewards/rejected": -1.9098402261734009,
+      "sft_loss": 1.5493417978286743,
+      "step": 60
+    },
+    {
+      "epoch": 0.03478842615822044,
+      "grad_norm": 1.7981205640913585,
+      "learning_rate": 1.158645276292335e-07,
+      "logits/chosen": -0.034505169838666916,
+      "logits/rejected": 0.11524344980716705,
+      "logps/chosen": -1.6530320644378662,
+      "logps/rejected": -1.7859472036361694,
+      "loss": 0.2193,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.6530320644378662,
+      "rewards/margins": 0.13291515409946442,
+      "rewards/rejected": -1.7859472036361694,
+      "sft_loss": 1.4817430973052979,
+      "step": 65
+    },
+    {
+      "epoch": 0.03746445893962201,
+      "grad_norm": 3.345104582482527,
+      "learning_rate": 1.24777183600713e-07,
+      "logits/chosen": -0.058969296514987946,
+      "logits/rejected": 0.0972469225525856,
+      "logps/chosen": -1.789322853088379,
+      "logps/rejected": -1.8369725942611694,
+      "loss": 0.2273,
+      "rewards/accuracies": 0.41874998807907104,
+      "rewards/chosen": -1.789322853088379,
+      "rewards/margins": 0.04764973372220993,
+      "rewards/rejected": -1.8369725942611694,
+      "sft_loss": 1.6432716846466064,
+      "step": 70
+    },
+    {
+      "epoch": 0.04014049172102358,
+      "grad_norm": 1.9956416646533002,
+      "learning_rate": 1.3368983957219251e-07,
+      "logits/chosen": -0.05575738102197647,
+      "logits/rejected": 0.13211320340633392,
+      "logps/chosen": -1.8173309564590454,
+      "logps/rejected": -2.084980010986328,
+      "loss": 0.2031,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.8173309564590454,
+      "rewards/margins": 0.26764941215515137,
+      "rewards/rejected": -2.084980010986328,
+      "sft_loss": 1.5819857120513916,
+      "step": 75
+    },
+    {
+      "epoch": 0.04281652450242515,
+      "grad_norm": 2.029903037156413,
+      "learning_rate": 1.42602495543672e-07,
+      "logits/chosen": 0.012477383017539978,
+      "logits/rejected": 0.12156625092029572,
+      "logps/chosen": -1.7673463821411133,
+      "logps/rejected": -1.8001142740249634,
+      "loss": 0.2234,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -1.7673463821411133,
+      "rewards/margins": 0.03276783227920532,
+      "rewards/rejected": -1.8001142740249634,
+      "sft_loss": 1.548523187637329,
+      "step": 80
+    },
+    {
+      "epoch": 0.04549255728382673,
+      "grad_norm": 1.9099806770028345,
+      "learning_rate": 1.5151515151515152e-07,
+      "logits/chosen": -0.15445740520954132,
+      "logits/rejected": 0.09701220691204071,
+      "logps/chosen": -1.8502569198608398,
+      "logps/rejected": -2.039699077606201,
+      "loss": 0.2162,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.8502569198608398,
+      "rewards/margins": 0.18944212794303894,
+      "rewards/rejected": -2.039699077606201,
+      "sft_loss": 1.5177948474884033,
+      "step": 85
+    },
+    {
+      "epoch": 0.0481685900652283,
+      "grad_norm": 1.831509195002992,
+      "learning_rate": 1.6042780748663102e-07,
+      "logits/chosen": 0.0975041389465332,
+      "logits/rejected": 0.06015370413661003,
+      "logps/chosen": -1.8252804279327393,
+      "logps/rejected": -1.8346067667007446,
+      "loss": 0.2283,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -1.8252804279327393,
+      "rewards/margins": 0.009326416999101639,
+      "rewards/rejected": -1.8346067667007446,
+      "sft_loss": 1.4809902906417847,
+      "step": 90
+    },
+    {
+      "epoch": 0.05084462284662987,
+      "grad_norm": 1.5432952584343675,
+      "learning_rate": 1.693404634581105e-07,
+      "logits/chosen": -0.06612877547740936,
+      "logits/rejected": 0.08961978554725647,
+      "logps/chosen": -1.9097493886947632,
+      "logps/rejected": -1.9913265705108643,
+      "loss": 0.2116,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -1.9097493886947632,
+      "rewards/margins": 0.08157713711261749,
+      "rewards/rejected": -1.9913265705108643,
+      "sft_loss": 1.562781810760498,
+      "step": 95
+    },
+    {
+      "epoch": 0.05352065562803145,
+      "grad_norm": 1.7047366447625705,
+      "learning_rate": 1.7825311942959e-07,
+      "logits/chosen": -0.027822494506835938,
+      "logits/rejected": 0.039358578622341156,
+      "logps/chosen": -1.7809836864471436,
+      "logps/rejected": -1.8937673568725586,
+      "loss": 0.2092,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.7809836864471436,
+      "rewards/margins": 0.11278347671031952,
+      "rewards/rejected": -1.8937673568725586,
+      "sft_loss": 1.5269221067428589,
+      "step": 100
+    },
+    {
+      "epoch": 0.05619668840943302,
+      "grad_norm": 1.487137343319396,
+      "learning_rate": 1.8716577540106952e-07,
+      "logits/chosen": 0.04071933031082153,
+      "logits/rejected": 0.06978709995746613,
+      "logps/chosen": -1.7587201595306396,
+      "logps/rejected": -1.9216934442520142,
+      "loss": 0.2053,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -1.7587201595306396,
+      "rewards/margins": 0.16297322511672974,
+      "rewards/rejected": -1.9216934442520142,
+      "sft_loss": 1.4855430126190186,
+      "step": 105
+    },
+    {
+      "epoch": 0.05887272119083459,
+      "grad_norm": 1.7093056938051936,
+      "learning_rate": 1.96078431372549e-07,
+      "logits/chosen": 0.01955454982817173,
+      "logits/rejected": 0.11668159067630768,
+      "logps/chosen": -1.8462550640106201,
+      "logps/rejected": -1.9052842855453491,
+      "loss": 0.2213,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -1.8462550640106201,
+      "rewards/margins": 0.059028930962085724,
+      "rewards/rejected": -1.9052842855453491,
+      "sft_loss": 1.5440876483917236,
+      "step": 110
+    },
+    {
+      "epoch": 0.06154875397223616,
+      "grad_norm": 1.8143653214202433,
+      "learning_rate": 2.049910873440285e-07,
+      "logits/chosen": 0.04305744916200638,
+      "logits/rejected": 0.2593908905982971,
+      "logps/chosen": -1.8372834920883179,
+      "logps/rejected": -2.1679937839508057,
+      "loss": 0.1906,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -1.8372834920883179,
+      "rewards/margins": 0.33071017265319824,
+      "rewards/rejected": -2.1679937839508057,
+      "sft_loss": 1.6622670888900757,
+      "step": 115
+    },
+    {
+      "epoch": 0.06422478675363773,
+      "grad_norm": 1.2953390821472486,
+      "learning_rate": 2.13903743315508e-07,
+      "logits/chosen": -0.040084242820739746,
+      "logits/rejected": 0.14749838411808014,
+      "logps/chosen": -1.9821832180023193,
+      "logps/rejected": -2.126786947250366,
+      "loss": 0.1955,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -1.9821832180023193,
+      "rewards/margins": 0.14460386335849762,
+      "rewards/rejected": -2.126786947250366,
+      "sft_loss": 1.6855179071426392,
+      "step": 120
+    },
+    {
+      "epoch": 0.0669008195350393,
+      "grad_norm": 2.0321252565929697,
+      "learning_rate": 2.2281639928698751e-07,
+      "logits/chosen": -0.08258192241191864,
+      "logits/rejected": 0.054262615740299225,
+      "logps/chosen": -1.880631685256958,
+      "logps/rejected": -1.7970972061157227,
+      "loss": 0.228,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -1.880631685256958,
+      "rewards/margins": -0.08353424817323685,
+      "rewards/rejected": -1.7970972061157227,
+      "sft_loss": 1.6010761260986328,
+      "step": 125
+    },
+    {
+      "epoch": 0.06957685231644088,
+      "grad_norm": 1.7026815978099512,
+      "learning_rate": 2.31729055258467e-07,
+      "logits/chosen": 0.06078602001070976,
+      "logits/rejected": 0.20385079085826874,
+      "logps/chosen": -1.9472017288208008,
+      "logps/rejected": -2.069502592086792,
+      "loss": 0.196,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -1.9472017288208008,
+      "rewards/margins": 0.1223008781671524,
+      "rewards/rejected": -2.069502592086792,
+      "sft_loss": 1.6835273504257202,
+      "step": 130
+    },
+    {
+      "epoch": 0.07225288509784245,
+      "grad_norm": 1.6062457326323307,
+      "learning_rate": 2.406417112299465e-07,
+      "logits/chosen": -0.031107574701309204,
+      "logits/rejected": 0.09176994860172272,
+      "logps/chosen": -2.0292229652404785,
+      "logps/rejected": -2.003788948059082,
+      "loss": 0.2079,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -2.0292229652404785,
+      "rewards/margins": -0.025434067472815514,
+      "rewards/rejected": -2.003788948059082,
+      "sft_loss": 1.6492774486541748,
+      "step": 135
+    },
+    {
+      "epoch": 0.07492891787924402,
+      "grad_norm": 2.259464856729891,
+      "learning_rate": 2.49554367201426e-07,
+      "logits/chosen": -0.012007193639874458,
+      "logits/rejected": 0.16535402834415436,
+      "logps/chosen": -1.9944820404052734,
+      "logps/rejected": -2.249605655670166,
+      "loss": 0.1844,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.9944820404052734,
+      "rewards/margins": 0.2551235854625702,
+      "rewards/rejected": -2.249605655670166,
+      "sft_loss": 1.687748908996582,
+      "step": 140
+    },
+    {
+      "epoch": 0.0776049506606456,
+      "grad_norm": 1.4936273484540237,
+      "learning_rate": 2.5846702317290554e-07,
+      "logits/chosen": 0.006977976765483618,
+      "logits/rejected": 0.17071905732154846,
+      "logps/chosen": -1.963129997253418,
+      "logps/rejected": -2.1104207038879395,
+      "loss": 0.1951,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -1.963129997253418,
+      "rewards/margins": 0.14729070663452148,
+      "rewards/rejected": -2.1104207038879395,
+      "sft_loss": 1.6211490631103516,
+      "step": 145
+    },
+    {
+      "epoch": 0.08028098344204716,
+      "grad_norm": 1.7689496460453615,
+      "learning_rate": 2.6737967914438503e-07,
+      "logits/chosen": -0.0678483173251152,
+      "logits/rejected": 0.09999240934848785,
+      "logps/chosen": -1.953213095664978,
+      "logps/rejected": -1.9564449787139893,
+      "loss": 0.211,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -1.953213095664978,
+      "rewards/margins": 0.003231614828109741,
+      "rewards/rejected": -1.9564449787139893,
+      "sft_loss": 1.4884393215179443,
+      "step": 150
+    },
+    {
+      "epoch": 0.08295701622344874,
+      "grad_norm": 1.7151720647649684,
+      "learning_rate": 2.762923351158645e-07,
+      "logits/chosen": -0.03358950838446617,
+      "logits/rejected": 0.019218524917960167,
+      "logps/chosen": -2.072007179260254,
+      "logps/rejected": -2.119513750076294,
+      "loss": 0.2029,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -2.072007179260254,
+      "rewards/margins": 0.04750671610236168,
+      "rewards/rejected": -2.119513750076294,
+      "sft_loss": 1.65902578830719,
+      "step": 155
+    },
+    {
+      "epoch": 0.0856330490048503,
+      "grad_norm": 1.6343524925886426,
+      "learning_rate": 2.85204991087344e-07,
+      "logits/chosen": -0.12475484609603882,
+      "logits/rejected": 0.024783272296190262,
+      "logps/chosen": -2.2778027057647705,
+      "logps/rejected": -2.253194808959961,
+      "loss": 0.2021,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -2.2778027057647705,
+      "rewards/margins": -0.024608034640550613,
+      "rewards/rejected": -2.253194808959961,
+      "sft_loss": 1.7526813745498657,
+      "step": 160
+    },
+    {
+      "epoch": 0.08830908178625188,
+      "grad_norm": 1.5477966487473764,
+      "learning_rate": 2.941176470588235e-07,
+      "logits/chosen": -0.022026468068361282,
+      "logits/rejected": 0.16233108937740326,
+      "logps/chosen": -2.0127928256988525,
+      "logps/rejected": -2.321645498275757,
+      "loss": 0.1968,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -2.0127928256988525,
+      "rewards/margins": 0.3088526725769043,
+      "rewards/rejected": -2.321645498275757,
+      "sft_loss": 1.6046435832977295,
+      "step": 165
+    },
+    {
+      "epoch": 0.09098511456765346,
+      "grad_norm": 1.5880755708383005,
+      "learning_rate": 3.0303030303030305e-07,
+      "logits/chosen": -0.06025642156600952,
+      "logits/rejected": -0.0005133814993314445,
+      "logps/chosen": -2.291651964187622,
+      "logps/rejected": -2.2674951553344727,
+      "loss": 0.1919,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": -2.291651964187622,
+      "rewards/margins": -0.02415713295340538,
+      "rewards/rejected": -2.2674951553344727,
+      "sft_loss": 1.7107826471328735,
+      "step": 170
+    },
+    {
+      "epoch": 0.09366114734905502,
+      "grad_norm": 1.5517515613243185,
+      "learning_rate": 3.1194295900178254e-07,
+      "logits/chosen": 0.0844862088561058,
+      "logits/rejected": 0.08593183755874634,
+      "logps/chosen": -2.1726529598236084,
+      "logps/rejected": -2.2376325130462646,
+      "loss": 0.2159,
+      "rewards/accuracies": 0.4625000059604645,
+      "rewards/chosen": -2.1726529598236084,
+      "rewards/margins": 0.06497963517904282,
+      "rewards/rejected": -2.2376325130462646,
+      "sft_loss": 1.7399260997772217,
+      "step": 175
+    },
+    {
+      "epoch": 0.0963371801304566,
+      "grad_norm": 1.1933283310629328,
+      "learning_rate": 3.2085561497326203e-07,
+      "logits/chosen": 0.05188767984509468,
+      "logits/rejected": 0.05493194982409477,
+      "logps/chosen": -2.267704486846924,
+      "logps/rejected": -2.23276948928833,
+      "loss": 0.1912,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -2.267704486846924,
+      "rewards/margins": -0.03493490070104599,
+      "rewards/rejected": -2.23276948928833,
+      "sft_loss": 1.7221721410751343,
+      "step": 180
+    },
+    {
+      "epoch": 0.09901321291185818,
+      "grad_norm": 1.6240826562624862,
+      "learning_rate": 3.297682709447415e-07,
+      "logits/chosen": -0.10158465057611465,
+      "logits/rejected": -0.0017013490432873368,
+      "logps/chosen": -2.2014870643615723,
+      "logps/rejected": -2.292428493499756,
+      "loss": 0.2065,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -2.2014870643615723,
+      "rewards/margins": 0.09094144403934479,
+      "rewards/rejected": -2.292428493499756,
+      "sft_loss": 1.690645456314087,
+      "step": 185
+    },
+    {
+      "epoch": 0.10168924569325974,
+      "grad_norm": 1.8581379824371342,
+      "learning_rate": 3.38680926916221e-07,
+      "logits/chosen": -0.013663304038345814,
+      "logits/rejected": 0.1274757981300354,
+      "logps/chosen": -2.723717212677002,
+      "logps/rejected": -2.5758001804351807,
+      "loss": 0.1793,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -2.723717212677002,
+      "rewards/margins": -0.14791707694530487,
+      "rewards/rejected": -2.5758001804351807,
+      "sft_loss": 1.9519933462142944,
+      "step": 190
+    },
+    {
+      "epoch": 0.10436527847466132,
+      "grad_norm": 1.279414711742926,
+      "learning_rate": 3.475935828877005e-07,
+      "logits/chosen": 0.06727501004934311,
+      "logits/rejected": 0.24111323058605194,
+      "logps/chosen": -2.0977349281311035,
+      "logps/rejected": -2.1614160537719727,
+      "loss": 0.1942,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -2.0977349281311035,
+      "rewards/margins": 0.0636814683675766,
+      "rewards/rejected": -2.1614160537719727,
+      "sft_loss": 1.5931757688522339,
+      "step": 195
+    },
+    {
+      "epoch": 0.1070413112560629,
+      "grad_norm": 1.2596713424200148,
+      "learning_rate": 3.5650623885918e-07,
+      "logits/chosen": -0.039129838347435,
+      "logits/rejected": 0.11606033891439438,
+      "logps/chosen": -2.5230002403259277,
+      "logps/rejected": -2.2674851417541504,
+      "loss": 0.1945,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -2.5230002403259277,
+      "rewards/margins": -0.2555150091648102,
+      "rewards/rejected": -2.2674851417541504,
+      "sft_loss": 1.848797082901001,
+      "step": 200
+    },
+    {
+      "epoch": 0.10971734403746446,
+      "grad_norm": 1.8905095199988196,
+      "learning_rate": 3.654188948306595e-07,
+      "logits/chosen": -0.00120444071944803,
+      "logits/rejected": 0.16811782121658325,
+      "logps/chosen": -2.7844417095184326,
+      "logps/rejected": -2.4819183349609375,
+      "loss": 0.181,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -2.7844417095184326,
+      "rewards/margins": -0.3025234341621399,
+      "rewards/rejected": -2.4819183349609375,
+      "sft_loss": 1.793636679649353,
+      "step": 205
+    },
+    {
+      "epoch": 0.11239337681886603,
+      "grad_norm": 2.08986879761342,
+      "learning_rate": 3.7433155080213904e-07,
+      "logits/chosen": -0.10828781127929688,
+      "logits/rejected": 0.11393757164478302,
+      "logps/chosen": -2.6332404613494873,
+      "logps/rejected": -3.001659393310547,
+      "loss": 0.1531,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -2.6332404613494873,
+      "rewards/margins": 0.36841899156570435,
+      "rewards/rejected": -3.001659393310547,
+      "sft_loss": 1.8840652704238892,
+      "step": 210
+    },
+    {
+      "epoch": 0.1150694096002676,
+      "grad_norm": 1.556477272945386,
+      "learning_rate": 3.8324420677361853e-07,
+      "logits/chosen": -0.15626616775989532,
+      "logits/rejected": 0.11343065649271011,
+      "logps/chosen": -2.4919705390930176,
+      "logps/rejected": -2.6139535903930664,
+      "loss": 0.1646,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -2.4919705390930176,
+      "rewards/margins": 0.1219831258058548,
+      "rewards/rejected": -2.6139535903930664,
+      "sft_loss": 1.9114834070205688,
+      "step": 215
+    },
+    {
+      "epoch": 0.11774544238166917,
+      "grad_norm": 1.5614653858453136,
+      "learning_rate": 3.92156862745098e-07,
+      "logits/chosen": 0.09368538856506348,
+      "logits/rejected": 0.21061570942401886,
+      "logps/chosen": -2.9574270248413086,
+      "logps/rejected": -3.3937385082244873,
+      "loss": 0.1471,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -2.9574270248413086,
+      "rewards/margins": 0.4363115429878235,
+      "rewards/rejected": -3.3937385082244873,
+      "sft_loss": 2.102970600128174,
+      "step": 220
+    },
+    {
+      "epoch": 0.12042147516307075,
+      "grad_norm": 1.7111893932334987,
+      "learning_rate": 4.010695187165775e-07,
+      "logits/chosen": -0.0913754254579544,
+      "logits/rejected": 0.09513075649738312,
+      "logps/chosen": -2.7998054027557373,
+      "logps/rejected": -2.8793394565582275,
+      "loss": 0.1501,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -2.7998054027557373,
+      "rewards/margins": 0.0795338898897171,
+      "rewards/rejected": -2.8793394565582275,
+      "sft_loss": 1.8550370931625366,
+      "step": 225
+    },
+    {
+      "epoch": 0.12309750794447231,
+      "grad_norm": 1.7000316768173236,
+      "learning_rate": 4.09982174688057e-07,
+      "logits/chosen": 0.02866949513554573,
+      "logits/rejected": 0.12128983438014984,
+      "logps/chosen": -3.437687397003174,
+      "logps/rejected": -3.4442496299743652,
+      "loss": 0.15,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -3.437687397003174,
+      "rewards/margins": 0.006562310270965099,
+      "rewards/rejected": -3.4442496299743652,
+      "sft_loss": 2.032752513885498,
+      "step": 230
+    },
+    {
+      "epoch": 0.1257735407258739,
+      "grad_norm": 1.281692620921789,
+      "learning_rate": 4.188948306595365e-07,
+      "logits/chosen": 0.06882531940937042,
+      "logits/rejected": 0.2392643392086029,
+      "logps/chosen": -3.251737594604492,
+      "logps/rejected": -3.5582058429718018,
+      "loss": 0.1298,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -3.251737594604492,
+      "rewards/margins": 0.30646806955337524,
+      "rewards/rejected": -3.5582058429718018,
+      "sft_loss": 2.0187947750091553,
+      "step": 235
+    },
+    {
+      "epoch": 0.12844957350727546,
+      "grad_norm": 1.1499528240929935,
+      "learning_rate": 4.27807486631016e-07,
+      "logits/chosen": 0.028048083186149597,
+      "logits/rejected": 0.17282453179359436,
+      "logps/chosen": -3.5118629932403564,
+      "logps/rejected": -3.5622477531433105,
+      "loss": 0.1436,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -3.5118629932403564,
+      "rewards/margins": 0.05038491636514664,
+      "rewards/rejected": -3.5622477531433105,
+      "sft_loss": 2.316441059112549,
+      "step": 240
+    },
+    {
+      "epoch": 0.13112560628867703,
+      "grad_norm": 1.5446112266160616,
+      "learning_rate": 4.3672014260249554e-07,
+      "logits/chosen": 0.05068688839673996,
+      "logits/rejected": 0.18657876551151276,
+      "logps/chosen": -3.1922247409820557,
+      "logps/rejected": -3.881798267364502,
+      "loss": 0.1367,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -3.1922247409820557,
+      "rewards/margins": 0.6895732879638672,
+      "rewards/rejected": -3.881798267364502,
+      "sft_loss": 2.2884485721588135,
+      "step": 245
+    },
+    {
+      "epoch": 0.1338016390700786,
+      "grad_norm": 1.9912640093824259,
+      "learning_rate": 4.4563279857397503e-07,
+      "logits/chosen": 0.0045858025550842285,
+      "logits/rejected": 0.19180163741111755,
+      "logps/chosen": -4.6741437911987305,
+      "logps/rejected": -4.537530422210693,
+      "loss": 0.1404,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -4.6741437911987305,
+      "rewards/margins": -0.13661305606365204,
+      "rewards/rejected": -4.537530422210693,
+      "sft_loss": 2.375713586807251,
+      "step": 250
+    },
+    {
+      "epoch": 0.1364776718514802,
+      "grad_norm": 1.2089128075632967,
+      "learning_rate": 4.545454545454545e-07,
+      "logits/chosen": 0.06856288760900497,
+      "logits/rejected": 0.2534184455871582,
+      "logps/chosen": -3.3681182861328125,
+      "logps/rejected": -4.01994514465332,
+      "loss": 0.1305,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -3.3681182861328125,
+      "rewards/margins": 0.6518270373344421,
+      "rewards/rejected": -4.01994514465332,
+      "sft_loss": 2.0931496620178223,
+      "step": 255
+    },
+    {
+      "epoch": 0.13915370463288176,
+      "grad_norm": 1.1096542333679758,
+      "learning_rate": 4.63458110516934e-07,
+      "logits/chosen": -0.1298966407775879,
+      "logits/rejected": -0.0005374863976612687,
+      "logps/chosen": -4.578324317932129,
+      "logps/rejected": -3.95367693901062,
+      "loss": 0.1116,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -4.578324317932129,
+      "rewards/margins": -0.6246479749679565,
+      "rewards/rejected": -3.95367693901062,
+      "sft_loss": 2.523958683013916,
+      "step": 260
+    },
+    {
+      "epoch": 0.1418297374142833,
+      "grad_norm": 0.8605577330641169,
+      "learning_rate": 4.723707664884135e-07,
+      "logits/chosen": 0.050699442625045776,
+      "logits/rejected": 0.14836013317108154,
+      "logps/chosen": -5.576754570007324,
+      "logps/rejected": -4.454700469970703,
+      "loss": 0.1309,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -5.576754570007324,
+      "rewards/margins": -1.1220548152923584,
+      "rewards/rejected": -4.454700469970703,
+      "sft_loss": 3.4844889640808105,
+      "step": 265
+    },
+    {
+      "epoch": 0.1445057701956849,
+      "grad_norm": 0.7199701899731111,
+      "learning_rate": 4.81283422459893e-07,
+      "logits/chosen": 0.06923731416463852,
+      "logits/rejected": 0.24406075477600098,
+      "logps/chosen": -4.500152587890625,
+      "logps/rejected": -4.8329010009765625,
+      "loss": 0.1177,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -4.500152587890625,
+      "rewards/margins": 0.3327482342720032,
+      "rewards/rejected": -4.8329010009765625,
+      "sft_loss": 2.607105016708374,
+      "step": 270
+    },
+    {
+      "epoch": 0.14718180297708647,
+      "grad_norm": 1.0179457688057774,
+      "learning_rate": 4.901960784313725e-07,
+      "logits/chosen": 0.17000171542167664,
+      "logits/rejected": 0.28452375531196594,
+      "logps/chosen": -5.071500778198242,
+      "logps/rejected": -5.720163822174072,
+      "loss": 0.1316,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -5.071500778198242,
+      "rewards/margins": 0.6486631631851196,
+      "rewards/rejected": -5.720163822174072,
+      "sft_loss": 3.0609421730041504,
+      "step": 275
+    },
+    {
+      "epoch": 0.14985783575848804,
+      "grad_norm": 0.6412985802048274,
+      "learning_rate": 4.99108734402852e-07,
+      "logits/chosen": 0.04470293968915939,
+      "logits/rejected": 0.25358039140701294,
+      "logps/chosen": -5.8443922996521,
+      "logps/rejected": -5.567430019378662,
+      "loss": 0.1129,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -5.8443922996521,
+      "rewards/margins": -0.27696239948272705,
+      "rewards/rejected": -5.567430019378662,
+      "sft_loss": 3.784735918045044,
+      "step": 280
+    },
+    {
+      "epoch": 0.15253386853988962,
+      "grad_norm": 0.8813090112976013,
+      "learning_rate": 5.080213903743315e-07,
+      "logits/chosen": 0.07595234364271164,
+      "logits/rejected": 0.25720566511154175,
+      "logps/chosen": -5.359448432922363,
+      "logps/rejected": -5.277853012084961,
+      "loss": 0.1197,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -5.359448432922363,
+      "rewards/margins": -0.08159530162811279,
+      "rewards/rejected": -5.277853012084961,
+      "sft_loss": 2.7771780490875244,
+      "step": 285
+    },
+    {
+      "epoch": 0.1552099013212912,
+      "grad_norm": 0.4794779219011389,
+      "learning_rate": 5.169340463458111e-07,
+      "logits/chosen": 0.04704619571566582,
+      "logits/rejected": 0.41382455825805664,
+      "logps/chosen": -4.340124130249023,
+      "logps/rejected": -5.529976844787598,
+      "loss": 0.0862,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.340124130249023,
+      "rewards/margins": 1.1898523569107056,
+      "rewards/rejected": -5.529976844787598,
+      "sft_loss": 2.859290838241577,
+      "step": 290
+    },
+    {
+      "epoch": 0.15788593410269275,
+      "grad_norm": 0.5494597370346566,
+      "learning_rate": 5.258467023172905e-07,
+      "logits/chosen": 0.11807294189929962,
+      "logits/rejected": 0.19528234004974365,
+      "logps/chosen": -5.867804050445557,
+      "logps/rejected": -5.229619979858398,
+      "loss": 0.1151,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -5.867804050445557,
+      "rewards/margins": -0.6381844878196716,
+      "rewards/rejected": -5.229619979858398,
+      "sft_loss": 3.5657966136932373,
+      "step": 295
+    },
+    {
+      "epoch": 0.16056196688409433,
+      "grad_norm": 0.8770023678854048,
+      "learning_rate": 5.347593582887701e-07,
+      "logits/chosen": 0.07271115481853485,
+      "logits/rejected": 0.29299789667129517,
+      "logps/chosen": -6.19558048248291,
+      "logps/rejected": -6.314692497253418,
+      "loss": 0.1101,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -6.19558048248291,
+      "rewards/margins": 0.11911182105541229,
+      "rewards/rejected": -6.314692497253418,
+      "sft_loss": 3.1685125827789307,
+      "step": 300
+    },
+    {
+      "epoch": 0.1632379996654959,
+      "grad_norm": 0.8025435563243098,
+      "learning_rate": 5.436720142602496e-07,
+      "logits/chosen": 0.13048218190670013,
+      "logits/rejected": 0.22332973778247833,
+      "logps/chosen": -5.2905073165893555,
+      "logps/rejected": -5.286152362823486,
+      "loss": 0.1116,
+      "rewards/accuracies": 0.48124998807907104,
+      "rewards/chosen": -5.2905073165893555,
+      "rewards/margins": -0.004354977514594793,
+      "rewards/rejected": -5.286152362823486,
+      "sft_loss": 3.451859712600708,
+      "step": 305
+    },
+    {
+      "epoch": 0.16591403244689748,
+      "grad_norm": 0.5576591445870365,
+      "learning_rate": 5.52584670231729e-07,
+      "logits/chosen": -0.046368349343538284,
+      "logits/rejected": 0.07756895571947098,
+      "logps/chosen": -6.2411417961120605,
+      "logps/rejected": -6.300403118133545,
+      "loss": 0.1,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -6.2411417961120605,
+      "rewards/margins": 0.05926106125116348,
+      "rewards/rejected": -6.300403118133545,
+      "sft_loss": 4.0066680908203125,
+      "step": 310
+    },
+    {
+      "epoch": 0.16859006522829906,
+      "grad_norm": 0.5933139332180281,
+      "learning_rate": 5.614973262032086e-07,
+      "logits/chosen": 0.1319553405046463,
+      "logits/rejected": 0.33041349053382874,
+      "logps/chosen": -5.88442850112915,
+      "logps/rejected": -5.793405055999756,
+      "loss": 0.1007,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -5.88442850112915,
+      "rewards/margins": -0.09102272987365723,
+      "rewards/rejected": -5.793405055999756,
+      "sft_loss": 3.6491355895996094,
+      "step": 315
+    },
+    {
+      "epoch": 0.1712660980097006,
+      "grad_norm": 1.2532265917187888,
+      "learning_rate": 5.70409982174688e-07,
+      "logits/chosen": 0.09381995350122452,
+      "logits/rejected": 0.25648611783981323,
+      "logps/chosen": -5.447043418884277,
+      "logps/rejected": -5.1138739585876465,
+      "loss": 0.1122,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -5.447043418884277,
+      "rewards/margins": -0.3331696391105652,
+      "rewards/rejected": -5.1138739585876465,
+      "sft_loss": 3.5053176879882812,
+      "step": 320
+    },
+    {
+      "epoch": 0.17394213079110218,
+      "grad_norm": 2.11143261163554,
+      "learning_rate": 5.793226381461676e-07,
+      "logits/chosen": -0.032501980662345886,
+      "logits/rejected": 0.14243759214878082,
+      "logps/chosen": -6.821471214294434,
+      "logps/rejected": -7.138516426086426,
+      "loss": 0.0944,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -6.821471214294434,
+      "rewards/margins": 0.3170450031757355,
+      "rewards/rejected": -7.138516426086426,
+      "sft_loss": 3.6288344860076904,
+      "step": 325
+    },
+    {
+      "epoch": 0.17661816357250376,
+      "grad_norm": 1.4994788163352675,
+      "learning_rate": 5.88235294117647e-07,
+      "logits/chosen": 0.023045741021633148,
+      "logits/rejected": 0.22768893837928772,
+      "logps/chosen": -5.550793170928955,
+      "logps/rejected": -7.549263000488281,
+      "loss": 0.0808,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -5.550793170928955,
+      "rewards/margins": 1.9984703063964844,
+      "rewards/rejected": -7.549263000488281,
+      "sft_loss": 4.235267162322998,
+      "step": 330
+    },
+    {
+      "epoch": 0.17929419635390534,
+      "grad_norm": 2.6929457580136336,
+      "learning_rate": 5.971479500891266e-07,
+      "logits/chosen": 0.055640123784542084,
+      "logits/rejected": 0.23773732781410217,
+      "logps/chosen": -8.425851821899414,
+      "logps/rejected": -8.300704956054688,
+      "loss": 0.0703,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -8.425851821899414,
+      "rewards/margins": -0.12514746189117432,
+      "rewards/rejected": -8.300704956054688,
+      "sft_loss": 5.204246997833252,
+      "step": 335
+    },
+    {
+      "epoch": 0.18197022913530692,
+      "grad_norm": 2.975215002215455,
+      "learning_rate": 6.060606060606061e-07,
+      "logits/chosen": 0.018163323402404785,
+      "logits/rejected": 0.23212413489818573,
+      "logps/chosen": -7.2970075607299805,
+      "logps/rejected": -7.756361961364746,
+      "loss": 0.0668,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -7.2970075607299805,
+      "rewards/margins": 0.4593542218208313,
+      "rewards/rejected": -7.756361961364746,
+      "sft_loss": 5.14938497543335,
+      "step": 340
+    },
+    {
+      "epoch": 0.1846462619167085,
+      "grad_norm": 2.8114828827778773,
+      "learning_rate": 6.149732620320855e-07,
+      "logits/chosen": 0.06455346941947937,
+      "logits/rejected": 0.13964493572711945,
+      "logps/chosen": -7.02465295791626,
+      "logps/rejected": -7.177855491638184,
+      "loss": 0.0678,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -7.02465295791626,
+      "rewards/margins": 0.15320374071598053,
+      "rewards/rejected": -7.177855491638184,
+      "sft_loss": 4.932073593139648,
+      "step": 345
+    },
+    {
+      "epoch": 0.18732229469811004,
+      "grad_norm": 2.9644531334099495,
+      "learning_rate": 6.238859180035651e-07,
+      "logits/chosen": 0.033121854066848755,
+      "logits/rejected": 0.1822817325592041,
+      "logps/chosen": -6.74831485748291,
+      "logps/rejected": -6.182015895843506,
+      "loss": 0.0692,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -6.74831485748291,
+      "rewards/margins": -0.5662989616394043,
+      "rewards/rejected": -6.182015895843506,
+      "sft_loss": 4.699161529541016,
+      "step": 350
+    },
+    {
+      "epoch": 0.18999832747951162,
+      "grad_norm": 2.2205782453631104,
+      "learning_rate": 6.327985739750445e-07,
+      "logits/chosen": -0.08365367352962494,
+      "logits/rejected": 0.1948840171098709,
+      "logps/chosen": -6.755923271179199,
+      "logps/rejected": -6.674230098724365,
+      "loss": 0.0638,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -6.755923271179199,
+      "rewards/margins": -0.08169381320476532,
+      "rewards/rejected": -6.674230098724365,
+      "sft_loss": 5.259469032287598,
+      "step": 355
+    },
+    {
+      "epoch": 0.1926743602609132,
+      "grad_norm": 5.005317134698349,
+      "learning_rate": 6.417112299465241e-07,
+      "logits/chosen": -0.03285945951938629,
+      "logits/rejected": 0.06617248058319092,
+      "logps/chosen": -5.783998489379883,
+      "logps/rejected": -5.964906215667725,
+      "loss": 0.0631,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -5.783998489379883,
+      "rewards/margins": 0.18090778589248657,
+      "rewards/rejected": -5.964906215667725,
+      "sft_loss": 4.496619701385498,
+      "step": 360
+    },
+    {
+      "epoch": 0.19535039304231477,
+      "grad_norm": 6.547065361331459,
+      "learning_rate": 6.506238859180035e-07,
+      "logits/chosen": -0.03726925700902939,
+      "logits/rejected": 0.07639211416244507,
+      "logps/chosen": -5.752418518066406,
+      "logps/rejected": -5.402900218963623,
+      "loss": 0.0633,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -5.752418518066406,
+      "rewards/margins": -0.3495188355445862,
+      "rewards/rejected": -5.402900218963623,
+      "sft_loss": 4.719771385192871,
+      "step": 365
+    },
+    {
+      "epoch": 0.19802642582371635,
+      "grad_norm": 1.5837236667583166,
+      "learning_rate": 6.59536541889483e-07,
+      "logits/chosen": 0.01798037253320217,
+      "logits/rejected": 0.17442074418067932,
+      "logps/chosen": -5.352682113647461,
+      "logps/rejected": -5.345147132873535,
+      "loss": 0.0676,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -5.352682113647461,
+      "rewards/margins": -0.007535439915955067,
+      "rewards/rejected": -5.345147132873535,
+      "sft_loss": 4.657355785369873,
+      "step": 370
+    },
+    {
+      "epoch": 0.2007024586051179,
+      "grad_norm": 2.264336894541911,
+      "learning_rate": 6.684491978609626e-07,
+      "logits/chosen": -0.014740332961082458,
+      "logits/rejected": 0.19818945229053497,
+      "logps/chosen": -5.521324634552002,
+      "logps/rejected": -6.3202056884765625,
+      "loss": 0.0604,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -5.521324634552002,
+      "rewards/margins": 0.7988806366920471,
+      "rewards/rejected": -6.3202056884765625,
+      "sft_loss": 4.7174763679504395,
+      "step": 375
+    },
+    {
+      "epoch": 0.20337849138651948,
+      "grad_norm": 8.231545263083387,
+      "learning_rate": 6.77361853832442e-07,
+      "logits/chosen": 0.05013446882367134,
+      "logits/rejected": 0.16159534454345703,
+      "logps/chosen": -5.799169540405273,
+      "logps/rejected": -6.049595832824707,
+      "loss": 0.0594,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -5.799169540405273,
+      "rewards/margins": 0.250426322221756,
+      "rewards/rejected": -6.049595832824707,
+      "sft_loss": 4.898706912994385,
+      "step": 380
+    },
+    {
+      "epoch": 0.20605452416792105,
+      "grad_norm": 7.4747221469572045,
+      "learning_rate": 6.862745098039216e-07,
+      "logits/chosen": 0.11043745279312134,
+      "logits/rejected": 0.2178896963596344,
+      "logps/chosen": -5.294905662536621,
+      "logps/rejected": -5.705230236053467,
+      "loss": 0.0569,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -5.294905662536621,
+      "rewards/margins": 0.41032448410987854,
+      "rewards/rejected": -5.705230236053467,
+      "sft_loss": 4.584171772003174,
+      "step": 385
+    },
+    {
+      "epoch": 0.20873055694932263,
+      "grad_norm": 1.5104841571006231,
+      "learning_rate": 6.95187165775401e-07,
+      "logits/chosen": 0.3990798890590668,
+      "logits/rejected": 0.6201587915420532,
+      "logps/chosen": -5.837296962738037,
+      "logps/rejected": -6.025527000427246,
+      "loss": 0.0619,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -5.837296962738037,
+      "rewards/margins": 0.18823003768920898,
+      "rewards/rejected": -6.025527000427246,
+      "sft_loss": 5.096214771270752,
+      "step": 390
+    },
+    {
+      "epoch": 0.2114065897307242,
+      "grad_norm": 1.563130468991321,
+      "learning_rate": 7.040998217468806e-07,
+      "logits/chosen": 0.2377520501613617,
+      "logits/rejected": 0.4436890184879303,
+      "logps/chosen": -5.419901371002197,
+      "logps/rejected": -5.661240577697754,
+      "loss": 0.0602,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -5.419901371002197,
+      "rewards/margins": 0.2413390874862671,
+      "rewards/rejected": -5.661240577697754,
+      "sft_loss": 4.7854814529418945,
+      "step": 395
+    },
+    {
+      "epoch": 0.2140826225121258,
+      "grad_norm": 1.9335448052453914,
+      "learning_rate": 7.1301247771836e-07,
+      "logits/chosen": 0.05647904798388481,
+      "logits/rejected": 0.21013203263282776,
+      "logps/chosen": -5.136744976043701,
+      "logps/rejected": -5.403138160705566,
+      "loss": 0.0548,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -5.136744976043701,
+      "rewards/margins": 0.26639291644096375,
+      "rewards/rejected": -5.403138160705566,
+      "sft_loss": 4.459831714630127,
+      "step": 400
+    },
+    {
+      "epoch": 0.2140826225121258,
+      "eval_logits/chosen": 0.3255070745944977,
+      "eval_logits/rejected": 0.44398289918899536,
+      "eval_logps/chosen": -4.828823566436768,
+      "eval_logps/rejected": -5.042104244232178,
+      "eval_loss": 0.05636167898774147,
+      "eval_rewards/accuracies": 0.5378338098526001,
+      "eval_rewards/chosen": -4.828823566436768,
+      "eval_rewards/margins": 0.21327993273735046,
+      "eval_rewards/rejected": -5.042104244232178,
+      "eval_runtime": 44.2626,
+      "eval_samples_per_second": 30.387,
+      "eval_sft_loss": 4.225595474243164,
+      "eval_steps_per_second": 7.614,
+      "step": 400
+    },
+    {
+      "epoch": 0.21675865529352734,
+      "grad_norm": 3.2425814489856877,
+      "learning_rate": 7.219251336898395e-07,
+      "logits/chosen": -0.10482490062713623,
+      "logits/rejected": 0.020466437563300133,
+      "logps/chosen": -5.138674736022949,
+      "logps/rejected": -5.437190532684326,
+      "loss": 0.0593,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -5.138674736022949,
+      "rewards/margins": 0.29851633310317993,
+      "rewards/rejected": -5.437190532684326,
+      "sft_loss": 4.819356441497803,
+      "step": 405
+    },
+    {
+      "epoch": 0.2194346880749289,
+      "grad_norm": 1.246234153314806,
+      "learning_rate": 7.30837789661319e-07,
+      "logits/chosen": -0.23379549384117126,
+      "logits/rejected": -0.03975790739059448,
+      "logps/chosen": -4.86204195022583,
+      "logps/rejected": -5.128736972808838,
+      "loss": 0.0558,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -4.86204195022583,
+      "rewards/margins": 0.2666948735713959,
+      "rewards/rejected": -5.128736972808838,
+      "sft_loss": 4.390920639038086,
+      "step": 410
+    },
+    {
+      "epoch": 0.2221107208563305,
+      "grad_norm": 1.2675726442317068,
+      "learning_rate": 7.397504456327985e-07,
+      "logits/chosen": -0.3350445628166199,
+      "logits/rejected": -0.23910042643547058,
+      "logps/chosen": -5.285843849182129,
+      "logps/rejected": -5.223616123199463,
+      "loss": 0.0574,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -5.285843849182129,
+      "rewards/margins": -0.062228064984083176,
+      "rewards/rejected": -5.223616123199463,
+      "sft_loss": 4.837762355804443,
+      "step": 415
+    },
+    {
+      "epoch": 0.22478675363773207,
+      "grad_norm": 4.053916998217198,
+      "learning_rate": 7.486631016042781e-07,
+      "logits/chosen": -0.4638137221336365,
+      "logits/rejected": -0.07929669320583344,
+      "logps/chosen": -4.68533992767334,
+      "logps/rejected": -4.774787902832031,
+      "loss": 0.0572,
+      "rewards/accuracies": 0.518750011920929,
+      "rewards/chosen": -4.68533992767334,
+      "rewards/margins": 0.08944790065288544,
+      "rewards/rejected": -4.774787902832031,
+      "sft_loss": 4.436319828033447,
+      "step": 420
+    },
+    {
+      "epoch": 0.22746278641913364,
+      "grad_norm": 0.9091036273428365,
+      "learning_rate": 7.575757575757575e-07,
+      "logits/chosen": -0.4252164363861084,
+      "logits/rejected": -0.1790289580821991,
+      "logps/chosen": -5.022044658660889,
+      "logps/rejected": -5.134105682373047,
+      "loss": 0.0559,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -5.022044658660889,
+      "rewards/margins": 0.11206040531396866,
+      "rewards/rejected": -5.134105682373047,
+      "sft_loss": 4.640767574310303,
+      "step": 425
+    },
+    {
+      "epoch": 0.2301388192005352,
+      "grad_norm": 3.526198495194285,
+      "learning_rate": 7.664884135472371e-07,
+      "logits/chosen": -0.5839825868606567,
+      "logits/rejected": -0.2868669033050537,
+      "logps/chosen": -4.790310382843018,
+      "logps/rejected": -5.191717624664307,
+      "loss": 0.0553,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.790310382843018,
+      "rewards/margins": 0.40140730142593384,
+      "rewards/rejected": -5.191717624664307,
+      "sft_loss": 4.370650768280029,
+      "step": 430
+    },
+    {
+      "epoch": 0.23281485198193677,
+      "grad_norm": 1.6131020839641845,
+      "learning_rate": 7.754010695187165e-07,
+      "logits/chosen": -0.3908002972602844,
+      "logits/rejected": -0.2709645628929138,
+      "logps/chosen": -5.2322797775268555,
+      "logps/rejected": -5.146048069000244,
+      "loss": 0.0566,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -5.2322797775268555,
+      "rewards/margins": -0.08623187988996506,
+      "rewards/rejected": -5.146048069000244,
+      "sft_loss": 4.601866245269775,
+      "step": 435
+    },
+    {
+      "epoch": 0.23549088476333835,
+      "grad_norm": 1.816790371136798,
+      "learning_rate": 7.84313725490196e-07,
+      "logits/chosen": -0.38437846302986145,
+      "logits/rejected": -0.2163679152727127,
+      "logps/chosen": -4.9463982582092285,
+      "logps/rejected": -5.161070346832275,
+      "loss": 0.0574,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -4.9463982582092285,
+      "rewards/margins": 0.21467189490795135,
+      "rewards/rejected": -5.161070346832275,
+      "sft_loss": 4.722719669342041,
+      "step": 440
+    },
+    {
+      "epoch": 0.23816691754473993,
+      "grad_norm": 0.8271388620405082,
+      "learning_rate": 7.932263814616755e-07,
+      "logits/chosen": -0.3277231752872467,
+      "logits/rejected": -0.17920419573783875,
+      "logps/chosen": -4.781705379486084,
+      "logps/rejected": -5.05832052230835,
+      "loss": 0.0553,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -4.781705379486084,
+      "rewards/margins": 0.27661508321762085,
+      "rewards/rejected": -5.05832052230835,
+      "sft_loss": 4.421566009521484,
+      "step": 445
+    },
+    {
+      "epoch": 0.2408429503261415,
+      "grad_norm": 1.9615683734384015,
+      "learning_rate": 8.02139037433155e-07,
+      "logits/chosen": -0.22028562426567078,
+      "logits/rejected": -0.056511811912059784,
+      "logps/chosen": -4.917468547821045,
+      "logps/rejected": -5.084526062011719,
+      "loss": 0.0558,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -4.917468547821045,
+      "rewards/margins": 0.1670571267604828,
+      "rewards/rejected": -5.084526062011719,
+      "sft_loss": 4.554088115692139,
+      "step": 450
+    },
+    {
+      "epoch": 0.24351898310754308,
+      "grad_norm": 2.080793689004357,
+      "learning_rate": 8.110516934046346e-07,
+      "logits/chosen": -0.31141823530197144,
+      "logits/rejected": -0.17871572077274323,
+      "logps/chosen": -4.898928642272949,
+      "logps/rejected": -5.2130937576293945,
+      "loss": 0.0551,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.898928642272949,
+      "rewards/margins": 0.3141648471355438,
+      "rewards/rejected": -5.2130937576293945,
+      "sft_loss": 4.52283239364624,
+      "step": 455
+    },
+    {
+      "epoch": 0.24619501588894463,
+      "grad_norm": 1.6017894339058516,
+      "learning_rate": 8.19964349376114e-07,
+      "logits/chosen": -0.4909995496273041,
+      "logits/rejected": -0.2803053855895996,
+      "logps/chosen": -4.9490742683410645,
+      "logps/rejected": -5.1322760581970215,
+      "loss": 0.058,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -4.9490742683410645,
+      "rewards/margins": 0.18320170044898987,
+      "rewards/rejected": -5.1322760581970215,
+      "sft_loss": 4.714522838592529,
+      "step": 460
+    },
+    {
+      "epoch": 0.2488710486703462,
+      "grad_norm": 2.5245631885951356,
+      "learning_rate": 8.288770053475936e-07,
+      "logits/chosen": -0.20865888893604279,
+      "logits/rejected": -0.11569474637508392,
+      "logps/chosen": -4.929292678833008,
+      "logps/rejected": -5.23386287689209,
+      "loss": 0.0567,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -4.929292678833008,
+      "rewards/margins": 0.3045700192451477,
+      "rewards/rejected": -5.23386287689209,
+      "sft_loss": 4.6611008644104,
+      "step": 465
+    },
+    {
+      "epoch": 0.2515470814517478,
+      "grad_norm": 0.8415235355536271,
+      "learning_rate": 8.37789661319073e-07,
+      "logits/chosen": -0.15159404277801514,
+      "logits/rejected": -0.2845079302787781,
+      "logps/chosen": -4.953354835510254,
+      "logps/rejected": -5.008639335632324,
+      "loss": 0.0583,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": -4.953354835510254,
+      "rewards/margins": 0.055284250527620316,
+      "rewards/rejected": -5.008639335632324,
+      "sft_loss": 4.679515838623047,
+      "step": 470
+    },
+    {
+      "epoch": 0.25422311423314936,
+      "grad_norm": 1.032866302037683,
+      "learning_rate": 8.467023172905525e-07,
+      "logits/chosen": -0.4970584511756897,
+      "logits/rejected": -0.2463860958814621,
+      "logps/chosen": -4.74661922454834,
+      "logps/rejected": -5.329679489135742,
+      "loss": 0.0541,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.74661922454834,
+      "rewards/margins": 0.5830600261688232,
+      "rewards/rejected": -5.329679489135742,
+      "sft_loss": 4.545025825500488,
+      "step": 475
+    },
+    {
+      "epoch": 0.2568991470145509,
+      "grad_norm": 1.4170166320453947,
+      "learning_rate": 8.55614973262032e-07,
+      "logits/chosen": -0.37650805711746216,
+      "logits/rejected": -0.1301797777414322,
+      "logps/chosen": -4.653770923614502,
+      "logps/rejected": -4.959628582000732,
+      "loss": 0.0554,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -4.653770923614502,
+      "rewards/margins": 0.3058578372001648,
+      "rewards/rejected": -4.959628582000732,
+      "sft_loss": 4.32053279876709,
+      "step": 480
+    },
+    {
+      "epoch": 0.2595751797959525,
+      "grad_norm": 2.0286769162845224,
+      "learning_rate": 8.645276292335115e-07,
+      "logits/chosen": -0.2958958148956299,
+      "logits/rejected": -0.20120573043823242,
+      "logps/chosen": -5.155457496643066,
+      "logps/rejected": -5.184266567230225,
+      "loss": 0.0568,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -5.155457496643066,
+      "rewards/margins": 0.028808558359742165,
+      "rewards/rejected": -5.184266567230225,
+      "sft_loss": 4.663354396820068,
+      "step": 485
+    },
+    {
+      "epoch": 0.26225121257735406,
+      "grad_norm": 0.6716969708735916,
+      "learning_rate": 8.734402852049911e-07,
+      "logits/chosen": -0.23128321766853333,
+      "logits/rejected": -0.16980710625648499,
+      "logps/chosen": -4.904732704162598,
+      "logps/rejected": -5.12713623046875,
+      "loss": 0.0561,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -4.904732704162598,
+      "rewards/margins": 0.22240379452705383,
+      "rewards/rejected": -5.12713623046875,
+      "sft_loss": 4.582591533660889,
+      "step": 490
+    },
+    {
+      "epoch": 0.26492724535875567,
+      "grad_norm": 0.6367113828248504,
+      "learning_rate": 8.823529411764705e-07,
+      "logits/chosen": -0.3560214936733246,
+      "logits/rejected": -0.3447813093662262,
+      "logps/chosen": -4.751753807067871,
+      "logps/rejected": -4.852728843688965,
+      "loss": 0.0565,
+      "rewards/accuracies": 0.4749999940395355,
+      "rewards/chosen": -4.751753807067871,
+      "rewards/margins": 0.10097485780715942,
+      "rewards/rejected": -4.852728843688965,
+      "sft_loss": 4.386483669281006,
+      "step": 495
+    },
+    {
+      "epoch": 0.2676032781401572,
+      "grad_norm": 0.948109673277269,
+      "learning_rate": 8.912655971479501e-07,
+      "logits/chosen": -0.42650288343429565,
+      "logits/rejected": -0.29735860228538513,
+      "logps/chosen": -5.083521366119385,
+      "logps/rejected": -5.359811305999756,
+      "loss": 0.0548,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -5.083521366119385,
+      "rewards/margins": 0.2762903869152069,
+      "rewards/rejected": -5.359811305999756,
+      "sft_loss": 4.7258758544921875,
+      "step": 500
+    },
+    {
+      "epoch": 0.27027931092155877,
+      "grad_norm": 0.5712184638942543,
+      "learning_rate": 9.001782531194295e-07,
+      "logits/chosen": -0.41429275274276733,
+      "logits/rejected": -0.2482190579175949,
+      "logps/chosen": -4.613528251647949,
+      "logps/rejected": -4.611102104187012,
+      "loss": 0.0571,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -4.613528251647949,
+      "rewards/margins": -0.0024250433780252934,
+      "rewards/rejected": -4.611102104187012,
+      "sft_loss": 4.284755706787109,
+      "step": 505
+    },
+    {
+      "epoch": 0.2729553437029604,
+      "grad_norm": 1.6672577216332825,
+      "learning_rate": 9.09090909090909e-07,
+      "logits/chosen": -0.23879782855510712,
+      "logits/rejected": -0.15796609222888947,
+      "logps/chosen": -5.014100074768066,
+      "logps/rejected": -5.1457719802856445,
+      "loss": 0.0555,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -5.014100074768066,
+      "rewards/margins": 0.13167230784893036,
+      "rewards/rejected": -5.1457719802856445,
+      "sft_loss": 4.67844820022583,
+      "step": 510
+    },
+    {
+      "epoch": 0.2756313764843619,
+      "grad_norm": 0.6126001083433033,
+      "learning_rate": 9.180035650623885e-07,
+      "logits/chosen": -0.2774892747402191,
+      "logits/rejected": -0.12253077328205109,
+      "logps/chosen": -4.683966636657715,
+      "logps/rejected": -4.783071041107178,
+      "loss": 0.0556,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -4.683966636657715,
+      "rewards/margins": 0.09910430014133453,
+      "rewards/rejected": -4.783071041107178,
+      "sft_loss": 4.369112968444824,
+      "step": 515
+    },
+    {
+      "epoch": 0.27830740926576353,
+      "grad_norm": 0.5485133251322323,
+      "learning_rate": 9.26916221033868e-07,
+      "logits/chosen": -0.3536015748977661,
+      "logits/rejected": -0.1521953046321869,
+      "logps/chosen": -4.809085845947266,
+      "logps/rejected": -5.069105625152588,
+      "loss": 0.055,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.809085845947266,
+      "rewards/margins": 0.2600199580192566,
+      "rewards/rejected": -5.069105625152588,
+      "sft_loss": 4.547854900360107,
+      "step": 520
+    },
+    {
+      "epoch": 0.2809834420471651,
+      "grad_norm": 0.7628792891718944,
+      "learning_rate": 9.358288770053476e-07,
+      "logits/chosen": -0.20002791285514832,
+      "logits/rejected": -0.06340288370847702,
+      "logps/chosen": -4.899701118469238,
+      "logps/rejected": -5.160953044891357,
+      "loss": 0.0551,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -4.899701118469238,
+      "rewards/margins": 0.2612523138523102,
+      "rewards/rejected": -5.160953044891357,
+      "sft_loss": 4.506587028503418,
+      "step": 525
+    },
+    {
+      "epoch": 0.2836594748285666,
+      "grad_norm": 0.6603876049230433,
+      "learning_rate": 9.44741532976827e-07,
+      "logits/chosen": -0.2737922966480255,
+      "logits/rejected": -0.20090286433696747,
+      "logps/chosen": -4.671916961669922,
+      "logps/rejected": -4.830391883850098,
+      "loss": 0.0558,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -4.671916961669922,
+      "rewards/margins": 0.1584751307964325,
+      "rewards/rejected": -4.830391883850098,
+      "sft_loss": 4.443393230438232,
+      "step": 530
+    },
+    {
+      "epoch": 0.28633550760996823,
+      "grad_norm": 2.146121422661777,
+      "learning_rate": 9.536541889483066e-07,
+      "logits/chosen": -0.5321468114852905,
+      "logits/rejected": -0.1265200674533844,
+      "logps/chosen": -4.758464813232422,
+      "logps/rejected": -5.03375768661499,
+      "loss": 0.054,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -4.758464813232422,
+      "rewards/margins": 0.27529221773147583,
+      "rewards/rejected": -5.03375768661499,
+      "sft_loss": 4.482895851135254,
+      "step": 535
+    },
+    {
+      "epoch": 0.2890115403913698,
+      "grad_norm": 0.6991622921289192,
+      "learning_rate": 9.62566844919786e-07,
+      "logits/chosen": -0.36746540665626526,
+      "logits/rejected": -0.23171451687812805,
+      "logps/chosen": -4.749773979187012,
+      "logps/rejected": -4.893668174743652,
+      "loss": 0.0564,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -4.749773979187012,
+      "rewards/margins": 0.14389494061470032,
+      "rewards/rejected": -4.893668174743652,
+      "sft_loss": 4.500009059906006,
+      "step": 540
+    },
+    {
+      "epoch": 0.2916875731727714,
+      "grad_norm": 0.4631796613476105,
+      "learning_rate": 9.714795008912655e-07,
+      "logits/chosen": -0.45513319969177246,
+      "logits/rejected": -0.17640811204910278,
+      "logps/chosen": -4.959817409515381,
+      "logps/rejected": -5.111637592315674,
+      "loss": 0.0547,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.959817409515381,
+      "rewards/margins": 0.15182055532932281,
+      "rewards/rejected": -5.111637592315674,
+      "sft_loss": 4.645677089691162,
+      "step": 545
+    },
+    {
+      "epoch": 0.29436360595417294,
+      "grad_norm": 1.6120582506856742,
+      "learning_rate": 9.80392156862745e-07,
+      "logits/chosen": -0.29082345962524414,
+      "logits/rejected": -0.20808687806129456,
+      "logps/chosen": -4.473280429840088,
+      "logps/rejected": -4.676797866821289,
+      "loss": 0.0557,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -4.473280429840088,
+      "rewards/margins": 0.20351727306842804,
+      "rewards/rejected": -4.676797866821289,
+      "sft_loss": 4.248588562011719,
+      "step": 550
+    },
+    {
+      "epoch": 0.2970396387355745,
+      "grad_norm": 0.7334937725505589,
+      "learning_rate": 9.893048128342244e-07,
+      "logits/chosen": -0.37902534008026123,
+      "logits/rejected": -0.19630546867847443,
+      "logps/chosen": -4.999424934387207,
+      "logps/rejected": -5.183640480041504,
+      "loss": 0.056,
+      "rewards/accuracies": 0.5249999761581421,
+      "rewards/chosen": -4.999424934387207,
+      "rewards/margins": 0.18421539664268494,
+      "rewards/rejected": -5.183640480041504,
+      "sft_loss": 4.761509418487549,
+      "step": 555
+    },
+    {
+      "epoch": 0.2997156715169761,
+      "grad_norm": 0.6277633250946504,
+      "learning_rate": 9.98217468805704e-07,
+      "logits/chosen": -0.3287035822868347,
+      "logits/rejected": -0.26939138770103455,
+      "logps/chosen": -4.630910873413086,
+      "logps/rejected": -4.798611640930176,
+      "loss": 0.0552,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -4.630910873413086,
+      "rewards/margins": 0.16770049929618835,
+      "rewards/rejected": -4.798611640930176,
+      "sft_loss": 4.358144283294678,
+      "step": 560
+    },
+    {
+      "epoch": 0.30239170429837764,
+      "grad_norm": 0.844846498569962,
+      "learning_rate": 9.999984476788462e-07,
+      "logits/chosen": -0.40788155794143677,
+      "logits/rejected": -0.2840214967727661,
+      "logps/chosen": -4.668403625488281,
+      "logps/rejected": -4.911566257476807,
+      "loss": 0.055,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -4.668403625488281,
+      "rewards/margins": 0.24316303431987762,
+      "rewards/rejected": -4.911566257476807,
+      "sft_loss": 4.477941036224365,
+      "step": 565
+    },
+    {
+      "epoch": 0.30506773707977924,
+      "grad_norm": 0.7660023635754369,
+      "learning_rate": 9.999921413906797e-07,
+      "logits/chosen": -0.3872382640838623,
+      "logits/rejected": -0.0676632821559906,
+      "logps/chosen": -4.775974273681641,
+      "logps/rejected": -4.973284721374512,
+      "loss": 0.0546,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.775974273681641,
+      "rewards/margins": 0.19731023907661438,
+      "rewards/rejected": -4.973284721374512,
+      "sft_loss": 4.47655725479126,
+      "step": 570
+    },
+    {
+      "epoch": 0.3077437698611808,
+      "grad_norm": 1.0086652767673534,
+      "learning_rate": 9.999809841765644e-07,
+      "logits/chosen": -0.36274272203445435,
+      "logits/rejected": -0.3295183479785919,
+      "logps/chosen": -4.695250511169434,
+      "logps/rejected": -4.898679256439209,
+      "loss": 0.0561,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -4.695250511169434,
+      "rewards/margins": 0.20342937111854553,
+      "rewards/rejected": -4.898679256439209,
+      "sft_loss": 4.503565788269043,
+      "step": 575
+    },
+    {
+      "epoch": 0.3104198026425824,
+      "grad_norm": 0.48046001270132227,
+      "learning_rate": 9.999649761447477e-07,
+      "logits/chosen": -0.36971116065979004,
+      "logits/rejected": -0.11569008976221085,
+      "logps/chosen": -4.7068705558776855,
+      "logps/rejected": -5.051798343658447,
+      "loss": 0.0545,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.7068705558776855,
+      "rewards/margins": 0.34492722153663635,
+      "rewards/rejected": -5.051798343658447,
+      "sft_loss": 4.506040096282959,
+      "step": 580
+    },
+    {
+      "epoch": 0.31309583542398395,
+      "grad_norm": 0.5578956622360604,
+      "learning_rate": 9.999441174505398e-07,
+      "logits/chosen": -0.3603058159351349,
+      "logits/rejected": -0.2358492612838745,
+      "logps/chosen": -4.69789981842041,
+      "logps/rejected": -4.912814140319824,
+      "loss": 0.0543,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.69789981842041,
+      "rewards/margins": 0.21491391956806183,
+      "rewards/rejected": -4.912814140319824,
+      "sft_loss": 4.283689022064209,
+      "step": 585
+    },
+    {
+      "epoch": 0.3157718682053855,
+      "grad_norm": 1.510950356206102,
+      "learning_rate": 9.999184082963116e-07,
+      "logits/chosen": -0.39853015542030334,
+      "logits/rejected": -0.24396976828575134,
+      "logps/chosen": -4.875593662261963,
+      "logps/rejected": -5.044680118560791,
+      "loss": 0.0546,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -4.875593662261963,
+      "rewards/margins": 0.16908620297908783,
+      "rewards/rejected": -5.044680118560791,
+      "sft_loss": 4.505948066711426,
+      "step": 590
+    },
+    {
+      "epoch": 0.3184479009867871,
+      "grad_norm": 1.2404951790708159,
+      "learning_rate": 9.998878489314937e-07,
+      "logits/chosen": -0.3368639051914215,
+      "logits/rejected": -0.11050537973642349,
+      "logps/chosen": -4.7269487380981445,
+      "logps/rejected": -5.084804058074951,
+      "loss": 0.0553,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.7269487380981445,
+      "rewards/margins": 0.35785627365112305,
+      "rewards/rejected": -5.084804058074951,
+      "sft_loss": 4.460112571716309,
+      "step": 595
+    },
+    {
+      "epoch": 0.32112393376818865,
+      "grad_norm": 0.8285839319543857,
+      "learning_rate": 9.99852439652573e-07,
+      "logits/chosen": -0.40388503670692444,
+      "logits/rejected": -0.1999831646680832,
+      "logps/chosen": -4.579422950744629,
+      "logps/rejected": -4.835511207580566,
+      "loss": 0.0541,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.579422950744629,
+      "rewards/margins": 0.25608840584754944,
+      "rewards/rejected": -4.835511207580566,
+      "sft_loss": 4.344851016998291,
+      "step": 600
+    },
+    {
+      "epoch": 0.32379996654959026,
+      "grad_norm": 0.8321826130171818,
+      "learning_rate": 9.998121808030904e-07,
+      "logits/chosen": -0.357062965631485,
+      "logits/rejected": -0.2568349540233612,
+      "logps/chosen": -4.870299339294434,
+      "logps/rejected": -5.030634880065918,
+      "loss": 0.0555,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.870299339294434,
+      "rewards/margins": 0.1603357344865799,
+      "rewards/rejected": -5.030634880065918,
+      "sft_loss": 4.602914333343506,
+      "step": 605
+    },
+    {
+      "epoch": 0.3264759993309918,
+      "grad_norm": 1.903871359660858,
+      "learning_rate": 9.997670727736379e-07,
+      "logits/chosen": -0.3140949606895447,
+      "logits/rejected": -0.054246217012405396,
+      "logps/chosen": -4.632641792297363,
+      "logps/rejected": -4.857050895690918,
+      "loss": 0.0545,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -4.632641792297363,
+      "rewards/margins": 0.22440913319587708,
+      "rewards/rejected": -4.857050895690918,
+      "sft_loss": 4.307807922363281,
+      "step": 610
+    },
+    {
+      "epoch": 0.32915203211239336,
+      "grad_norm": 0.9977812625827974,
+      "learning_rate": 9.99717116001853e-07,
+      "logits/chosen": -0.32023006677627563,
+      "logits/rejected": -0.19051554799079895,
+      "logps/chosen": -4.718859672546387,
+      "logps/rejected": -5.049766540527344,
+      "loss": 0.0539,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.718859672546387,
+      "rewards/margins": 0.33090734481811523,
+      "rewards/rejected": -5.049766540527344,
+      "sft_loss": 4.482518196105957,
+      "step": 615
+    },
+    {
+      "epoch": 0.33182806489379496,
+      "grad_norm": 1.076283983986177,
+      "learning_rate": 9.996623109724173e-07,
+      "logits/chosen": -0.15702661871910095,
+      "logits/rejected": -0.042694397270679474,
+      "logps/chosen": -4.6651930809021,
+      "logps/rejected": -4.863068580627441,
+      "loss": 0.0546,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.6651930809021,
+      "rewards/margins": 0.19787581264972687,
+      "rewards/rejected": -4.863068580627441,
+      "sft_loss": 4.419007301330566,
+      "step": 620
+    },
+    {
+      "epoch": 0.3345040976751965,
+      "grad_norm": 0.8598104129229739,
+      "learning_rate": 9.996026582170488e-07,
+      "logits/chosen": -0.22388014197349548,
+      "logits/rejected": 0.012990092858672142,
+      "logps/chosen": -4.589800834655762,
+      "logps/rejected": -4.9947404861450195,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.589800834655762,
+      "rewards/margins": 0.40493983030319214,
+      "rewards/rejected": -4.9947404861450195,
+      "sft_loss": 4.264779090881348,
+      "step": 625
+    },
+    {
+      "epoch": 0.3371801304565981,
+      "grad_norm": 0.9323971405554444,
+      "learning_rate": 9.995381583144996e-07,
+      "logits/chosen": -0.2541266977787018,
+      "logits/rejected": -0.09219606220722198,
+      "logps/chosen": -4.74979305267334,
+      "logps/rejected": -5.008017539978027,
+      "loss": 0.0545,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -4.74979305267334,
+      "rewards/margins": 0.2582244277000427,
+      "rewards/rejected": -5.008017539978027,
+      "sft_loss": 4.4836955070495605,
+      "step": 630
+    },
+    {
+      "epoch": 0.33985616323799966,
+      "grad_norm": 1.544263012484942,
+      "learning_rate": 9.994688118905471e-07,
+      "logits/chosen": -0.29842329025268555,
+      "logits/rejected": 0.029618164524435997,
+      "logps/chosen": -4.634865760803223,
+      "logps/rejected": -5.008535861968994,
+      "loss": 0.0541,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.634865760803223,
+      "rewards/margins": 0.37367022037506104,
+      "rewards/rejected": -5.008535861968994,
+      "sft_loss": 4.411338806152344,
+      "step": 635
+    },
+    {
+      "epoch": 0.3425321960194012,
+      "grad_norm": 0.5957628921148597,
+      "learning_rate": 9.993946196179912e-07,
+      "logits/chosen": -0.3504573404788971,
+      "logits/rejected": -0.06777174770832062,
+      "logps/chosen": -4.82008171081543,
+      "logps/rejected": -5.081010341644287,
+      "loss": 0.055,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -4.82008171081543,
+      "rewards/margins": 0.2609284222126007,
+      "rewards/rejected": -5.081010341644287,
+      "sft_loss": 4.516536235809326,
+      "step": 640
+    },
+    {
+      "epoch": 0.3452082288008028,
+      "grad_norm": 0.8031038766853787,
+      "learning_rate": 9.993155822166455e-07,
+      "logits/chosen": -0.36658331751823425,
+      "logits/rejected": -0.2720224857330322,
+      "logps/chosen": -4.595765113830566,
+      "logps/rejected": -4.878204345703125,
+      "loss": 0.0543,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -4.595765113830566,
+      "rewards/margins": 0.2824389934539795,
+      "rewards/rejected": -4.878204345703125,
+      "sft_loss": 4.362072467803955,
+      "step": 645
+    },
+    {
+      "epoch": 0.34788426158220437,
+      "grad_norm": 0.6128000957148086,
+      "learning_rate": 9.992317004533313e-07,
+      "logits/chosen": -0.34441816806793213,
+      "logits/rejected": -0.23594839870929718,
+      "logps/chosen": -4.604920864105225,
+      "logps/rejected": -4.89302396774292,
+      "loss": 0.0551,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -4.604920864105225,
+      "rewards/margins": 0.28810301423072815,
+      "rewards/rejected": -4.89302396774292,
+      "sft_loss": 4.383151531219482,
+      "step": 650
+    },
+    {
+      "epoch": 0.350560294363606,
+      "grad_norm": 0.9221890544863284,
+      "learning_rate": 9.991429751418696e-07,
+      "logits/chosen": -0.34770020842552185,
+      "logits/rejected": -0.32439878582954407,
+      "logps/chosen": -4.752337455749512,
+      "logps/rejected": -5.003704071044922,
+      "loss": 0.0547,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -4.752337455749512,
+      "rewards/margins": 0.2513658106327057,
+      "rewards/rejected": -5.003704071044922,
+      "sft_loss": 4.506827354431152,
+      "step": 655
+    },
+    {
+      "epoch": 0.3532363271450075,
+      "grad_norm": 0.7097128063174312,
+      "learning_rate": 9.99049407143074e-07,
+      "logits/chosen": -0.4512445330619812,
+      "logits/rejected": -0.2543826997280121,
+      "logps/chosen": -4.896830081939697,
+      "logps/rejected": -4.962733745574951,
+      "loss": 0.056,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -4.896830081939697,
+      "rewards/margins": 0.06590431928634644,
+      "rewards/rejected": -4.962733745574951,
+      "sft_loss": 4.547211647033691,
+      "step": 660
+    },
+    {
+      "epoch": 0.35591235992640907,
+      "grad_norm": 0.866563578264519,
+      "learning_rate": 9.989509973647416e-07,
+      "logits/chosen": -0.4053463339805603,
+      "logits/rejected": -0.20467785000801086,
+      "logps/chosen": -4.752414226531982,
+      "logps/rejected": -5.033076763153076,
+      "loss": 0.054,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.752414226531982,
+      "rewards/margins": 0.2806623578071594,
+      "rewards/rejected": -5.033076763153076,
+      "sft_loss": 4.508490562438965,
+      "step": 665
+    },
+    {
+      "epoch": 0.3585883927078107,
+      "grad_norm": 0.810281083281331,
+      "learning_rate": 9.988477467616445e-07,
+      "logits/chosen": -0.3855820298194885,
+      "logits/rejected": -0.1145135909318924,
+      "logps/chosen": -4.608223915100098,
+      "logps/rejected": -4.887753009796143,
+      "loss": 0.0544,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.608223915100098,
+      "rewards/margins": 0.27952879667282104,
+      "rewards/rejected": -4.887753009796143,
+      "sft_loss": 4.343533992767334,
+      "step": 670
+    },
+    {
+      "epoch": 0.3612644254892122,
+      "grad_norm": 0.765948472723309,
+      "learning_rate": 9.987396563355205e-07,
+      "logits/chosen": -0.37758225202560425,
+      "logits/rejected": -0.25603145360946655,
+      "logps/chosen": -4.832340240478516,
+      "logps/rejected": -5.092775344848633,
+      "loss": 0.0554,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.832340240478516,
+      "rewards/margins": 0.2604350447654724,
+      "rewards/rejected": -5.092775344848633,
+      "sft_loss": 4.5779595375061035,
+      "step": 675
+    },
+    {
+      "epoch": 0.36394045827061383,
+      "grad_norm": 1.3162704456915373,
+      "learning_rate": 9.986267271350631e-07,
+      "logits/chosen": -0.30360686779022217,
+      "logits/rejected": -0.10046778619289398,
+      "logps/chosen": -4.6535325050354,
+      "logps/rejected": -4.831793785095215,
+      "loss": 0.0558,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -4.6535325050354,
+      "rewards/margins": 0.1782616376876831,
+      "rewards/rejected": -4.831793785095215,
+      "sft_loss": 4.3908586502075195,
+      "step": 680
+    },
+    {
+      "epoch": 0.3666164910520154,
+      "grad_norm": 0.6702119836091758,
+      "learning_rate": 9.985089602559123e-07,
+      "logits/chosen": -0.3431549072265625,
+      "logits/rejected": -0.09186934679746628,
+      "logps/chosen": -4.754274845123291,
+      "logps/rejected": -5.079950332641602,
+      "loss": 0.0536,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.754274845123291,
+      "rewards/margins": 0.32567575573921204,
+      "rewards/rejected": -5.079950332641602,
+      "sft_loss": 4.485215663909912,
+      "step": 685
+    },
+    {
+      "epoch": 0.369292523833417,
+      "grad_norm": 1.7364082121305755,
+      "learning_rate": 9.983863568406428e-07,
+      "logits/chosen": -0.23817995190620422,
+      "logits/rejected": -0.19120314717292786,
+      "logps/chosen": -4.565924167633057,
+      "logps/rejected": -4.769357681274414,
+      "loss": 0.055,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -4.565924167633057,
+      "rewards/margins": 0.20343346893787384,
+      "rewards/rejected": -4.769357681274414,
+      "sft_loss": 4.296526908874512,
+      "step": 690
+    },
+    {
+      "epoch": 0.37196855661481854,
+      "grad_norm": 0.6355258111678728,
+      "learning_rate": 9.982589180787532e-07,
+      "logits/chosen": -0.3066914975643158,
+      "logits/rejected": -0.16904620826244354,
+      "logps/chosen": -4.9275007247924805,
+      "logps/rejected": -5.171297073364258,
+      "loss": 0.0549,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -4.9275007247924805,
+      "rewards/margins": 0.24379606544971466,
+      "rewards/rejected": -5.171297073364258,
+      "sft_loss": 4.645675182342529,
+      "step": 695
+    },
+    {
+      "epoch": 0.3746445893962201,
+      "grad_norm": 1.0087240174156498,
+      "learning_rate": 9.981266452066553e-07,
+      "logits/chosen": -0.3922460973262787,
+      "logits/rejected": -0.16909214854240417,
+      "logps/chosen": -4.540804862976074,
+      "logps/rejected": -4.852003574371338,
+      "loss": 0.054,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -4.540804862976074,
+      "rewards/margins": 0.3111991286277771,
+      "rewards/rejected": -4.852003574371338,
+      "sft_loss": 4.311594009399414,
+      "step": 700
+    },
+    {
+      "epoch": 0.3773206221776217,
+      "grad_norm": 0.4712790116938098,
+      "learning_rate": 9.979895395076608e-07,
+      "logits/chosen": -0.3389566242694855,
+      "logits/rejected": -0.05373547226190567,
+      "logps/chosen": -4.65207576751709,
+      "logps/rejected": -4.978102684020996,
+      "loss": 0.054,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.65207576751709,
+      "rewards/margins": 0.3260273039340973,
+      "rewards/rejected": -4.978102684020996,
+      "sft_loss": 4.382485389709473,
+      "step": 705
+    },
+    {
+      "epoch": 0.37999665495902324,
+      "grad_norm": 0.588171827288888,
+      "learning_rate": 9.9784760231197e-07,
+      "logits/chosen": -0.2382487952709198,
+      "logits/rejected": -0.08627081662416458,
+      "logps/chosen": -4.65833044052124,
+      "logps/rejected": -4.9416937828063965,
+      "loss": 0.054,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.65833044052124,
+      "rewards/margins": 0.2833631634712219,
+      "rewards/rejected": -4.9416937828063965,
+      "sft_loss": 4.383979797363281,
+      "step": 710
+    },
+    {
+      "epoch": 0.38267268774042484,
+      "grad_norm": 0.5290613154317763,
+      "learning_rate": 9.97700834996658e-07,
+      "logits/chosen": -0.2827507555484772,
+      "logits/rejected": -0.046073682606220245,
+      "logps/chosen": -4.779943943023682,
+      "logps/rejected": -4.971959114074707,
+      "loss": 0.0542,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -4.779943943023682,
+      "rewards/margins": 0.19201482832431793,
+      "rewards/rejected": -4.971959114074707,
+      "sft_loss": 4.34407901763916,
+      "step": 715
+    },
+    {
+      "epoch": 0.3853487205218264,
+      "grad_norm": 0.6632322309729074,
+      "learning_rate": 9.97549238985662e-07,
+      "logits/chosen": -0.2679640054702759,
+      "logits/rejected": -0.01135767437517643,
+      "logps/chosen": -4.855249404907227,
+      "logps/rejected": -5.212562561035156,
+      "loss": 0.054,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.855249404907227,
+      "rewards/margins": 0.3573140501976013,
+      "rewards/rejected": -5.212562561035156,
+      "sft_loss": 4.536675453186035,
+      "step": 720
+    },
+    {
+      "epoch": 0.38802475330322794,
+      "grad_norm": 0.8952742598506807,
+      "learning_rate": 9.973928157497674e-07,
+      "logits/chosen": -0.328164666891098,
+      "logits/rejected": -0.1100778579711914,
+      "logps/chosen": -4.614842891693115,
+      "logps/rejected": -4.933849334716797,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.614842891693115,
+      "rewards/margins": 0.3190067410469055,
+      "rewards/rejected": -4.933849334716797,
+      "sft_loss": 4.245141983032227,
+      "step": 725
+    },
+    {
+      "epoch": 0.39070078608462955,
+      "grad_norm": 1.2109420104021242,
+      "learning_rate": 9.972315668065927e-07,
+      "logits/chosen": -0.36284494400024414,
+      "logits/rejected": -0.19368380308151245,
+      "logps/chosen": -4.754213809967041,
+      "logps/rejected": -5.005512237548828,
+      "loss": 0.0539,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -4.754213809967041,
+      "rewards/margins": 0.251298725605011,
+      "rewards/rejected": -5.005512237548828,
+      "sft_loss": 4.37929105758667,
+      "step": 730
+    },
+    {
+      "epoch": 0.3933768188660311,
+      "grad_norm": 0.5615055732458076,
+      "learning_rate": 9.97065493720576e-07,
+      "logits/chosen": -0.37686052918434143,
+      "logits/rejected": -0.21963970363140106,
+      "logps/chosen": -4.625882148742676,
+      "logps/rejected": -4.9359564781188965,
+      "loss": 0.0541,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.625882148742676,
+      "rewards/margins": 0.31007346510887146,
+      "rewards/rejected": -4.9359564781188965,
+      "sft_loss": 4.357583045959473,
+      "step": 735
+    },
+    {
+      "epoch": 0.3960528516474327,
+      "grad_norm": 0.6515251035088281,
+      "learning_rate": 9.968945981029594e-07,
+      "logits/chosen": -0.3726346790790558,
+      "logits/rejected": -0.1442079395055771,
+      "logps/chosen": -4.788451194763184,
+      "logps/rejected": -5.127315044403076,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.788451194763184,
+      "rewards/margins": 0.3388644754886627,
+      "rewards/rejected": -5.127315044403076,
+      "sft_loss": 4.329194068908691,
+      "step": 740
+    },
+    {
+      "epoch": 0.39872888442883425,
+      "grad_norm": 1.3818832254616853,
+      "learning_rate": 9.967188816117726e-07,
+      "logits/chosen": -0.26630935072898865,
+      "logits/rejected": -0.09576726704835892,
+      "logps/chosen": -4.753724098205566,
+      "logps/rejected": -5.164248466491699,
+      "loss": 0.0552,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.753724098205566,
+      "rewards/margins": 0.41052454710006714,
+      "rewards/rejected": -5.164248466491699,
+      "sft_loss": 4.3944525718688965,
+      "step": 745
+    },
+    {
+      "epoch": 0.4014049172102358,
+      "grad_norm": 0.96053540482909,
+      "learning_rate": 9.965383459518179e-07,
+      "logits/chosen": -0.3140408396720886,
+      "logits/rejected": -0.02686697617173195,
+      "logps/chosen": -4.793798923492432,
+      "logps/rejected": -5.13473653793335,
+      "loss": 0.0541,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.793798923492432,
+      "rewards/margins": 0.34093838930130005,
+      "rewards/rejected": -5.13473653793335,
+      "sft_loss": 4.362773895263672,
+      "step": 750
+    },
+    {
+      "epoch": 0.4040809499916374,
+      "grad_norm": 0.6390136909223184,
+      "learning_rate": 9.963529928746533e-07,
+      "logits/chosen": -0.22388967871665955,
+      "logits/rejected": 0.006080237217247486,
+      "logps/chosen": -4.874973297119141,
+      "logps/rejected": -5.298001289367676,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.874973297119141,
+      "rewards/margins": 0.4230283796787262,
+      "rewards/rejected": -5.298001289367676,
+      "sft_loss": 4.536044120788574,
+      "step": 755
+    },
+    {
+      "epoch": 0.40675698277303896,
+      "grad_norm": 0.7605884515321404,
+      "learning_rate": 9.961628241785746e-07,
+      "logits/chosen": -0.26968467235565186,
+      "logits/rejected": -0.13087113201618195,
+      "logps/chosen": -4.522748947143555,
+      "logps/rejected": -4.854702949523926,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -4.522748947143555,
+      "rewards/margins": 0.3319534361362457,
+      "rewards/rejected": -4.854702949523926,
+      "sft_loss": 4.225506782531738,
+      "step": 760
+    },
+    {
+      "epoch": 0.40943301555444056,
+      "grad_norm": 0.8277696412229419,
+      "learning_rate": 9.959678417085998e-07,
+      "logits/chosen": -0.23488867282867432,
+      "logits/rejected": -0.1160377487540245,
+      "logps/chosen": -4.7814202308654785,
+      "logps/rejected": -4.9829325675964355,
+      "loss": 0.0552,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -4.7814202308654785,
+      "rewards/margins": 0.20151281356811523,
+      "rewards/rejected": -4.9829325675964355,
+      "sft_loss": 4.5762739181518555,
+      "step": 765
+    },
+    {
+      "epoch": 0.4121090483358421,
+      "grad_norm": 0.62157271815199,
+      "learning_rate": 9.957680473564493e-07,
+      "logits/chosen": -0.15221163630485535,
+      "logits/rejected": 0.05266900733113289,
+      "logps/chosen": -4.784552574157715,
+      "logps/rejected": -5.051705360412598,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.784552574157715,
+      "rewards/margins": 0.26715198159217834,
+      "rewards/rejected": -5.051705360412598,
+      "sft_loss": 4.40114688873291,
+      "step": 770
+    },
+    {
+      "epoch": 0.41478508111724366,
+      "grad_norm": 0.468516521364259,
+      "learning_rate": 9.95563443060529e-07,
+      "logits/chosen": -0.32061243057250977,
+      "logits/rejected": -0.0629098191857338,
+      "logps/chosen": -4.645053863525391,
+      "logps/rejected": -4.881279945373535,
+      "loss": 0.054,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -4.645053863525391,
+      "rewards/margins": 0.23622579872608185,
+      "rewards/rejected": -4.881279945373535,
+      "sft_loss": 4.328852653503418,
+      "step": 775
+    },
+    {
+      "epoch": 0.41746111389864526,
+      "grad_norm": 0.6662442429371508,
+      "learning_rate": 9.95354030805911e-07,
+      "logits/chosen": -0.4331437647342682,
+      "logits/rejected": -0.2046525925397873,
+      "logps/chosen": -4.6892008781433105,
+      "logps/rejected": -4.949892997741699,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -4.6892008781433105,
+      "rewards/margins": 0.26069265604019165,
+      "rewards/rejected": -4.949892997741699,
+      "sft_loss": 4.416562080383301,
+      "step": 780
+    },
+    {
+      "epoch": 0.4201371466800468,
+      "grad_norm": 1.4270248860333101,
+      "learning_rate": 9.951398126243133e-07,
+      "logits/chosen": -0.2729490399360657,
+      "logits/rejected": -0.12573480606079102,
+      "logps/chosen": -4.7296462059021,
+      "logps/rejected": -5.007996082305908,
+      "loss": 0.0544,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.7296462059021,
+      "rewards/margins": 0.27834954857826233,
+      "rewards/rejected": -5.007996082305908,
+      "sft_loss": 4.425854682922363,
+      "step": 785
+    },
+    {
+      "epoch": 0.4228131794614484,
+      "grad_norm": 0.6149559348382854,
+      "learning_rate": 9.94920790594082e-07,
+      "logits/chosen": -0.3410663902759552,
+      "logits/rejected": -0.18231642246246338,
+      "logps/chosen": -4.601952075958252,
+      "logps/rejected": -4.903101921081543,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.601952075958252,
+      "rewards/margins": 0.30115026235580444,
+      "rewards/rejected": -4.903101921081543,
+      "sft_loss": 4.3076982498168945,
+      "step": 790
+    },
+    {
+      "epoch": 0.42548921224284997,
+      "grad_norm": 0.45452309863432017,
+      "learning_rate": 9.946969668401696e-07,
+      "logits/chosen": -0.4137187898159027,
+      "logits/rejected": -0.14778129756450653,
+      "logps/chosen": -4.504861354827881,
+      "logps/rejected": -4.95142936706543,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.504861354827881,
+      "rewards/margins": 0.44656771421432495,
+      "rewards/rejected": -4.95142936706543,
+      "sft_loss": 4.271914958953857,
+      "step": 795
+    },
+    {
+      "epoch": 0.4281652450242516,
+      "grad_norm": 0.6182844933475908,
+      "learning_rate": 9.944683435341155e-07,
+      "logits/chosen": -0.26473289728164673,
+      "logits/rejected": -0.16446137428283691,
+      "logps/chosen": -4.790329933166504,
+      "logps/rejected": -5.15310525894165,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.790329933166504,
+      "rewards/margins": 0.3627750277519226,
+      "rewards/rejected": -5.15310525894165,
+      "sft_loss": 4.367070198059082,
+      "step": 800
+    },
+    {
+      "epoch": 0.4281652450242516,
+      "eval_logits/chosen": 0.129440575838089,
+      "eval_logits/rejected": 0.25413477420806885,
+      "eval_logps/chosen": -4.5659942626953125,
+      "eval_logps/rejected": -4.939233779907227,
+      "eval_loss": 0.05261680856347084,
+      "eval_rewards/accuracies": 0.6157270073890686,
+      "eval_rewards/chosen": -4.5659942626953125,
+      "eval_rewards/margins": 0.373240202665329,
+      "eval_rewards/rejected": -4.939233779907227,
+      "eval_runtime": 43.5084,
+      "eval_samples_per_second": 30.914,
+      "eval_sft_loss": 4.0511393547058105,
+      "eval_steps_per_second": 7.746,
+      "step": 800
+    },
+    {
+      "epoch": 0.4308412778056531,
+      "grad_norm": 0.7182798514374586,
+      "learning_rate": 9.942349228940236e-07,
+      "logits/chosen": -0.3890208303928375,
+      "logits/rejected": -0.13592301309108734,
+      "logps/chosen": -4.470497131347656,
+      "logps/rejected": -5.118318557739258,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.470497131347656,
+      "rewards/margins": 0.6478214859962463,
+      "rewards/rejected": -5.118318557739258,
+      "sft_loss": 4.132109642028809,
+      "step": 805
+    },
+    {
+      "epoch": 0.43351731058705467,
+      "grad_norm": 1.2682233477491278,
+      "learning_rate": 9.939967071845424e-07,
+      "logits/chosen": -0.33702266216278076,
+      "logits/rejected": -0.2550191879272461,
+      "logps/chosen": -5.065579891204834,
+      "logps/rejected": -5.333920478820801,
+      "loss": 0.0565,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -5.065579891204834,
+      "rewards/margins": 0.26833999156951904,
+      "rewards/rejected": -5.333920478820801,
+      "sft_loss": 4.749798774719238,
+      "step": 810
+    },
+    {
+      "epoch": 0.4361933433684563,
+      "grad_norm": 2.044570165392625,
+      "learning_rate": 9.937536987168413e-07,
+      "logits/chosen": -0.26246196031570435,
+      "logits/rejected": -0.07664032280445099,
+      "logps/chosen": -4.736085891723633,
+      "logps/rejected": -5.093490123748779,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -4.736085891723633,
+      "rewards/margins": 0.35740429162979126,
+      "rewards/rejected": -5.093490123748779,
+      "sft_loss": 4.295095920562744,
+      "step": 815
+    },
+    {
+      "epoch": 0.4388693761498578,
+      "grad_norm": 0.5535126842165531,
+      "learning_rate": 9.935058998485896e-07,
+      "logits/chosen": -0.24898965656757355,
+      "logits/rejected": -0.23044578731060028,
+      "logps/chosen": -4.624701976776123,
+      "logps/rejected": -4.948840141296387,
+      "loss": 0.0542,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.624701976776123,
+      "rewards/margins": 0.32413753867149353,
+      "rewards/rejected": -4.948840141296387,
+      "sft_loss": 4.249537467956543,
+      "step": 820
+    },
+    {
+      "epoch": 0.44154540893125943,
+      "grad_norm": 1.2057278393769568,
+      "learning_rate": 9.932533129839333e-07,
+      "logits/chosen": -0.4744848608970642,
+      "logits/rejected": -0.30530792474746704,
+      "logps/chosen": -4.787722587585449,
+      "logps/rejected": -4.9824538230896,
+      "loss": 0.0548,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.787722587585449,
+      "rewards/margins": 0.19473133981227875,
+      "rewards/rejected": -4.9824538230896,
+      "sft_loss": 4.53950309753418,
+      "step": 825
+    },
+    {
+      "epoch": 0.444221441712661,
+      "grad_norm": 0.8911826448855662,
+      "learning_rate": 9.929959405734711e-07,
+      "logits/chosen": -0.3885073959827423,
+      "logits/rejected": -0.19365429878234863,
+      "logps/chosen": -4.577334403991699,
+      "logps/rejected": -4.867704391479492,
+      "loss": 0.054,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.577334403991699,
+      "rewards/margins": 0.2903694808483124,
+      "rewards/rejected": -4.867704391479492,
+      "sft_loss": 4.387232780456543,
+      "step": 830
+    },
+    {
+      "epoch": 0.44689747449406253,
+      "grad_norm": 0.46989283083166355,
+      "learning_rate": 9.927337851142314e-07,
+      "logits/chosen": -0.45201578736305237,
+      "logits/rejected": -0.2976873219013214,
+      "logps/chosen": -4.781838417053223,
+      "logps/rejected": -4.962854862213135,
+      "loss": 0.0547,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -4.781838417053223,
+      "rewards/margins": 0.18101605772972107,
+      "rewards/rejected": -4.962854862213135,
+      "sft_loss": 4.531754493713379,
+      "step": 835
+    },
+    {
+      "epoch": 0.44957350727546413,
+      "grad_norm": 0.5984158522488067,
+      "learning_rate": 9.924668491496474e-07,
+      "logits/chosen": -0.5003188848495483,
+      "logits/rejected": -0.19921065866947174,
+      "logps/chosen": -4.628960132598877,
+      "logps/rejected": -4.916049957275391,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.628960132598877,
+      "rewards/margins": 0.2870899438858032,
+      "rewards/rejected": -4.916049957275391,
+      "sft_loss": 4.346935272216797,
+      "step": 840
+    },
+    {
+      "epoch": 0.4522495400568657,
+      "grad_norm": 0.41997151195828325,
+      "learning_rate": 9.92195135269533e-07,
+      "logits/chosen": -0.38395342230796814,
+      "logits/rejected": -0.3282047212123871,
+      "logps/chosen": -4.585183620452881,
+      "logps/rejected": -4.829873561859131,
+      "loss": 0.0539,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.585183620452881,
+      "rewards/margins": 0.24468998610973358,
+      "rewards/rejected": -4.829873561859131,
+      "sft_loss": 4.319632530212402,
+      "step": 845
+    },
+    {
+      "epoch": 0.4549255728382673,
+      "grad_norm": 0.6000201095981429,
+      "learning_rate": 9.919186461100574e-07,
+      "logits/chosen": -0.5364987254142761,
+      "logits/rejected": -0.38975319266319275,
+      "logps/chosen": -4.638622283935547,
+      "logps/rejected": -4.927207946777344,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.638622283935547,
+      "rewards/margins": 0.28858625888824463,
+      "rewards/rejected": -4.927207946777344,
+      "sft_loss": 4.3973798751831055,
+      "step": 850
+    },
+    {
+      "epoch": 0.45760160561966884,
+      "grad_norm": 0.6069321114919756,
+      "learning_rate": 9.9163738435372e-07,
+      "logits/chosen": -0.44249457120895386,
+      "logits/rejected": -0.24139097332954407,
+      "logps/chosen": -4.556834697723389,
+      "logps/rejected": -4.969690799713135,
+      "loss": 0.0543,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.556834697723389,
+      "rewards/margins": 0.41285595297813416,
+      "rewards/rejected": -4.969690799713135,
+      "sft_loss": 4.3291335105896,
+      "step": 855
+    },
+    {
+      "epoch": 0.4602776384010704,
+      "grad_norm": 1.2083997679194676,
+      "learning_rate": 9.913513527293234e-07,
+      "logits/chosen": -0.5292937159538269,
+      "logits/rejected": -0.29144081473350525,
+      "logps/chosen": -4.674202919006348,
+      "logps/rejected": -5.032801628112793,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.674202919006348,
+      "rewards/margins": 0.3585989773273468,
+      "rewards/rejected": -5.032801628112793,
+      "sft_loss": 4.352255344390869,
+      "step": 860
+    },
+    {
+      "epoch": 0.462953671182472,
+      "grad_norm": 1.0394633636666102,
+      "learning_rate": 9.910605540119474e-07,
+      "logits/chosen": -0.4250844419002533,
+      "logits/rejected": -0.2460746318101883,
+      "logps/chosen": -4.961132049560547,
+      "logps/rejected": -5.249226093292236,
+      "loss": 0.0548,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -4.961132049560547,
+      "rewards/margins": 0.28809377551078796,
+      "rewards/rejected": -5.249226093292236,
+      "sft_loss": 4.555105209350586,
+      "step": 865
+    },
+    {
+      "epoch": 0.46562970396387354,
+      "grad_norm": 0.7279167565907648,
+      "learning_rate": 9.907649910229227e-07,
+      "logits/chosen": -0.49782466888427734,
+      "logits/rejected": -0.13824693858623505,
+      "logps/chosen": -4.548239707946777,
+      "logps/rejected": -4.925876617431641,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.548239707946777,
+      "rewards/margins": 0.37763601541519165,
+      "rewards/rejected": -4.925876617431641,
+      "sft_loss": 4.260447025299072,
+      "step": 870
+    },
+    {
+      "epoch": 0.46830573674527515,
+      "grad_norm": 0.5335372480339191,
+      "learning_rate": 9.90464666629803e-07,
+      "logits/chosen": -0.3451125919818878,
+      "logits/rejected": -0.2164868861436844,
+      "logps/chosen": -4.661952018737793,
+      "logps/rejected": -4.983109951019287,
+      "loss": 0.0549,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -4.661952018737793,
+      "rewards/margins": 0.3211577236652374,
+      "rewards/rejected": -4.983109951019287,
+      "sft_loss": 4.368287563323975,
+      "step": 875
+    },
+    {
+      "epoch": 0.4709817695266767,
+      "grad_norm": 0.9231096450635541,
+      "learning_rate": 9.901595837463363e-07,
+      "logits/chosen": -0.3921271860599518,
+      "logits/rejected": -0.14882740378379822,
+      "logps/chosen": -4.827698707580566,
+      "logps/rejected": -5.257676124572754,
+      "loss": 0.0542,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.827698707580566,
+      "rewards/margins": 0.42997655272483826,
+      "rewards/rejected": -5.257676124572754,
+      "sft_loss": 4.5653581619262695,
+      "step": 880
+    },
+    {
+      "epoch": 0.47365780230807825,
+      "grad_norm": 0.4538265940639913,
+      "learning_rate": 9.898497453324384e-07,
+      "logits/chosen": -0.3491121828556061,
+      "logits/rejected": -0.24148587882518768,
+      "logps/chosen": -4.694242477416992,
+      "logps/rejected": -4.968047142028809,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.694242477416992,
+      "rewards/margins": 0.27380457520484924,
+      "rewards/rejected": -4.968047142028809,
+      "sft_loss": 4.328099727630615,
+      "step": 885
+    },
+    {
+      "epoch": 0.47633383508947985,
+      "grad_norm": 0.5980027880690659,
+      "learning_rate": 9.895351543941628e-07,
+      "logits/chosen": -0.4275690019130707,
+      "logits/rejected": -0.25901514291763306,
+      "logps/chosen": -4.467679500579834,
+      "logps/rejected": -4.724061012268066,
+      "loss": 0.0548,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -4.467679500579834,
+      "rewards/margins": 0.2563806176185608,
+      "rewards/rejected": -4.724061012268066,
+      "sft_loss": 4.149270057678223,
+      "step": 890
+    },
+    {
+      "epoch": 0.4790098678708814,
+      "grad_norm": 0.8479129823816223,
+      "learning_rate": 9.892158139836724e-07,
+      "logits/chosen": -0.35013651847839355,
+      "logits/rejected": -0.21295437216758728,
+      "logps/chosen": -5.047763824462891,
+      "logps/rejected": -5.160143852233887,
+      "loss": 0.0551,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -5.047763824462891,
+      "rewards/margins": 0.11238014698028564,
+      "rewards/rejected": -5.160143852233887,
+      "sft_loss": 4.720696926116943,
+      "step": 895
+    },
+    {
+      "epoch": 0.481685900652283,
+      "grad_norm": 1.7723523796984688,
+      "learning_rate": 9.88891727199209e-07,
+      "logits/chosen": -0.42566758394241333,
+      "logits/rejected": -0.3050028383731842,
+      "logps/chosen": -4.764883518218994,
+      "logps/rejected": -4.984330654144287,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.764883518218994,
+      "rewards/margins": 0.21944740414619446,
+      "rewards/rejected": -4.984330654144287,
+      "sft_loss": 4.396208763122559,
+      "step": 900
+    },
+    {
+      "epoch": 0.48436193343368455,
+      "grad_norm": 0.6330269647399032,
+      "learning_rate": 9.885628971850641e-07,
+      "logits/chosen": -0.3774091601371765,
+      "logits/rejected": -0.11741908639669418,
+      "logps/chosen": -4.430881977081299,
+      "logps/rejected": -4.842067718505859,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.430881977081299,
+      "rewards/margins": 0.41118550300598145,
+      "rewards/rejected": -4.842067718505859,
+      "sft_loss": 4.126009941101074,
+      "step": 905
+    },
+    {
+      "epoch": 0.48703796621508616,
+      "grad_norm": 0.4014471425285204,
+      "learning_rate": 9.882293271315481e-07,
+      "logits/chosen": -0.40995973348617554,
+      "logits/rejected": -0.2649644911289215,
+      "logps/chosen": -4.809659004211426,
+      "logps/rejected": -5.131319999694824,
+      "loss": 0.0539,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.809659004211426,
+      "rewards/margins": 0.3216610848903656,
+      "rewards/rejected": -5.131319999694824,
+      "sft_loss": 4.472898006439209,
+      "step": 910
+    },
+    {
+      "epoch": 0.4897139989964877,
+      "grad_norm": 0.6118368817774394,
+      "learning_rate": 9.878910202749589e-07,
+      "logits/chosen": -0.3659573197364807,
+      "logits/rejected": -0.10987311601638794,
+      "logps/chosen": -4.581470489501953,
+      "logps/rejected": -4.995975494384766,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.581470489501953,
+      "rewards/margins": 0.41450485587120056,
+      "rewards/rejected": -4.995975494384766,
+      "sft_loss": 4.3380351066589355,
+      "step": 915
+    },
+    {
+      "epoch": 0.49239003177788926,
+      "grad_norm": 0.6945048585807432,
+      "learning_rate": 9.875479798975512e-07,
+      "logits/chosen": -0.18761909008026123,
+      "logits/rejected": 0.05204455927014351,
+      "logps/chosen": -4.485073089599609,
+      "logps/rejected": -5.013813018798828,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.485073089599609,
+      "rewards/margins": 0.528739869594574,
+      "rewards/rejected": -5.013813018798828,
+      "sft_loss": 4.219447135925293,
+      "step": 920
+    },
+    {
+      "epoch": 0.49506606455929086,
+      "grad_norm": 0.7348924833299391,
+      "learning_rate": 9.87200209327504e-07,
+      "logits/chosen": -0.2375284880399704,
+      "logits/rejected": 0.009685017168521881,
+      "logps/chosen": -4.710652828216553,
+      "logps/rejected": -4.918156147003174,
+      "loss": 0.0541,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -4.710652828216553,
+      "rewards/margins": 0.20750336349010468,
+      "rewards/rejected": -4.918156147003174,
+      "sft_loss": 4.323351860046387,
+      "step": 925
+    },
+    {
+      "epoch": 0.4977420973406924,
+      "grad_norm": 0.8687040936956296,
+      "learning_rate": 9.868477119388894e-07,
+      "logits/chosen": -0.20819273591041565,
+      "logits/rejected": -0.1114293560385704,
+      "logps/chosen": -4.75814962387085,
+      "logps/rejected": -5.279811382293701,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.75814962387085,
+      "rewards/margins": 0.521661639213562,
+      "rewards/rejected": -5.279811382293701,
+      "sft_loss": 4.488502025604248,
+      "step": 930
+    },
+    {
+      "epoch": 0.500418130122094,
+      "grad_norm": 0.6418592693754023,
+      "learning_rate": 9.864904911516383e-07,
+      "logits/chosen": -0.18937340378761292,
+      "logits/rejected": -0.09434659034013748,
+      "logps/chosen": -4.581448554992676,
+      "logps/rejected": -4.985015392303467,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.581448554992676,
+      "rewards/margins": 0.4035671651363373,
+      "rewards/rejected": -4.985015392303467,
+      "sft_loss": 4.2696309089660645,
+      "step": 935
+    },
+    {
+      "epoch": 0.5030941629034956,
+      "grad_norm": 0.7699757525815809,
+      "learning_rate": 9.861285504315084e-07,
+      "logits/chosen": -0.18925723433494568,
+      "logits/rejected": -0.07616479694843292,
+      "logps/chosen": -4.764031410217285,
+      "logps/rejected": -5.095522880554199,
+      "loss": 0.0545,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.764031410217285,
+      "rewards/margins": 0.3314912021160126,
+      "rewards/rejected": -5.095522880554199,
+      "sft_loss": 4.490479469299316,
+      "step": 940
+    },
+    {
+      "epoch": 0.5057701956848971,
+      "grad_norm": 0.37757312786798214,
+      "learning_rate": 9.857618932900502e-07,
+      "logits/chosen": -0.32902026176452637,
+      "logits/rejected": -0.10386113077402115,
+      "logps/chosen": -4.6742939949035645,
+      "logps/rejected": -5.106159210205078,
+      "loss": 0.0539,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.6742939949035645,
+      "rewards/margins": 0.43186482787132263,
+      "rewards/rejected": -5.106159210205078,
+      "sft_loss": 4.532894134521484,
+      "step": 945
+    },
+    {
+      "epoch": 0.5084462284662987,
+      "grad_norm": 1.3493597510734785,
+      "learning_rate": 9.853905232845727e-07,
+      "logits/chosen": -0.2948172092437744,
+      "logits/rejected": -0.09298200905323029,
+      "logps/chosen": -4.439558029174805,
+      "logps/rejected": -4.737777233123779,
+      "loss": 0.0542,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.439558029174805,
+      "rewards/margins": 0.2982189357280731,
+      "rewards/rejected": -4.737777233123779,
+      "sft_loss": 4.231078624725342,
+      "step": 950
+    },
+    {
+      "epoch": 0.5111222612477003,
+      "grad_norm": 0.6918340143624974,
+      "learning_rate": 9.850144440181095e-07,
+      "logits/chosen": -0.27869707345962524,
+      "logits/rejected": -0.016635660082101822,
+      "logps/chosen": -4.659014701843262,
+      "logps/rejected": -4.985834121704102,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.659014701843262,
+      "rewards/margins": 0.32681965827941895,
+      "rewards/rejected": -4.985834121704102,
+      "sft_loss": 4.3856706619262695,
+      "step": 955
+    },
+    {
+      "epoch": 0.5137982940291018,
+      "grad_norm": 0.6120477442496692,
+      "learning_rate": 9.846336591393832e-07,
+      "logits/chosen": -0.29666954278945923,
+      "logits/rejected": -0.11472413688898087,
+      "logps/chosen": -4.67545223236084,
+      "logps/rejected": -5.038684844970703,
+      "loss": 0.0539,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.67545223236084,
+      "rewards/margins": 0.3632330298423767,
+      "rewards/rejected": -5.038684844970703,
+      "sft_loss": 4.411433696746826,
+      "step": 960
+    },
+    {
+      "epoch": 0.5164743268105034,
+      "grad_norm": 0.6360927411394314,
+      "learning_rate": 9.842481723427704e-07,
+      "logits/chosen": -0.2690829336643219,
+      "logits/rejected": -0.21832041442394257,
+      "logps/chosen": -4.791625499725342,
+      "logps/rejected": -5.1202778816223145,
+      "loss": 0.0547,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -4.791625499725342,
+      "rewards/margins": 0.32865259051322937,
+      "rewards/rejected": -5.1202778816223145,
+      "sft_loss": 4.518260955810547,
+      "step": 965
+    },
+    {
+      "epoch": 0.519150359591905,
+      "grad_norm": 0.4585645225121435,
+      "learning_rate": 9.838579873682658e-07,
+      "logits/chosen": -0.23320713639259338,
+      "logits/rejected": -0.23231375217437744,
+      "logps/chosen": -4.707311153411865,
+      "logps/rejected": -4.97707986831665,
+      "loss": 0.0544,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.707311153411865,
+      "rewards/margins": 0.26976901292800903,
+      "rewards/rejected": -4.97707986831665,
+      "sft_loss": 4.333600044250488,
+      "step": 970
+    },
+    {
+      "epoch": 0.5218263923733065,
+      "grad_norm": 0.8276372357867279,
+      "learning_rate": 9.834631080014457e-07,
+      "logits/chosen": -0.4102146625518799,
+      "logits/rejected": -0.1439218968153,
+      "logps/chosen": -4.648586750030518,
+      "logps/rejected": -5.030804634094238,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.648586750030518,
+      "rewards/margins": 0.38221827149391174,
+      "rewards/rejected": -5.030804634094238,
+      "sft_loss": 4.38181209564209,
+      "step": 975
+    },
+    {
+      "epoch": 0.5245024251547081,
+      "grad_norm": 0.3760930016074475,
+      "learning_rate": 9.830635380734312e-07,
+      "logits/chosen": -0.3494859039783478,
+      "logits/rejected": -0.1270177811384201,
+      "logps/chosen": -4.570178985595703,
+      "logps/rejected": -4.84365177154541,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.570178985595703,
+      "rewards/margins": 0.2734723687171936,
+      "rewards/rejected": -4.84365177154541,
+      "sft_loss": 4.262513637542725,
+      "step": 980
+    },
+    {
+      "epoch": 0.5271784579361097,
+      "grad_norm": 0.5004938819787677,
+      "learning_rate": 9.826592814608517e-07,
+      "logits/chosen": -0.31395021080970764,
+      "logits/rejected": -0.05263252183794975,
+      "logps/chosen": -4.681024551391602,
+      "logps/rejected": -4.969412803649902,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -4.681024551391602,
+      "rewards/margins": 0.2883884310722351,
+      "rewards/rejected": -4.969412803649902,
+      "sft_loss": 4.372974872589111,
+      "step": 985
+    },
+    {
+      "epoch": 0.5298544907175113,
+      "grad_norm": 0.49430068887442913,
+      "learning_rate": 9.822503420858067e-07,
+      "logits/chosen": -0.2046670913696289,
+      "logits/rejected": -0.21365389227867126,
+      "logps/chosen": -4.76826810836792,
+      "logps/rejected": -4.973118782043457,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.76826810836792,
+      "rewards/margins": 0.204850435256958,
+      "rewards/rejected": -4.973118782043457,
+      "sft_loss": 4.4387311935424805,
+      "step": 990
+    },
+    {
+      "epoch": 0.5325305234989128,
+      "grad_norm": 1.0140502437702155,
+      "learning_rate": 9.818367239158277e-07,
+      "logits/chosen": -0.17163081467151642,
+      "logits/rejected": -0.09935864061117172,
+      "logps/chosen": -4.683056831359863,
+      "logps/rejected": -4.941170692443848,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -4.683056831359863,
+      "rewards/margins": 0.25811368227005005,
+      "rewards/rejected": -4.941170692443848,
+      "sft_loss": 4.2819976806640625,
+      "step": 995
+    },
+    {
+      "epoch": 0.5352065562803144,
+      "grad_norm": 1.440440192039452,
+      "learning_rate": 9.8141843096384e-07,
+      "logits/chosen": -0.3152937889099121,
+      "logits/rejected": -0.10964254289865494,
+      "logps/chosen": -4.703091621398926,
+      "logps/rejected": -5.163810729980469,
+      "loss": 0.0536,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.703091621398926,
+      "rewards/margins": 0.4607190489768982,
+      "rewards/rejected": -5.163810729980469,
+      "sft_loss": 4.443800449371338,
+      "step": 1000
+    },
+    {
+      "epoch": 0.537882589061716,
+      "grad_norm": 1.0886331697539264,
+      "learning_rate": 9.809954672881237e-07,
+      "logits/chosen": -0.23827552795410156,
+      "logits/rejected": -0.03300425410270691,
+      "logps/chosen": -4.738983154296875,
+      "logps/rejected": -5.138331413269043,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.738983154296875,
+      "rewards/margins": 0.3993481993675232,
+      "rewards/rejected": -5.138331413269043,
+      "sft_loss": 4.400278568267822,
+      "step": 1005
+    },
+    {
+      "epoch": 0.5405586218431175,
+      "grad_norm": 0.6781935583905428,
+      "learning_rate": 9.80567836992274e-07,
+      "logits/chosen": -0.3267586827278137,
+      "logits/rejected": -0.09511874616146088,
+      "logps/chosen": -4.486596584320068,
+      "logps/rejected": -4.954394817352295,
+      "loss": 0.0546,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.486596584320068,
+      "rewards/margins": 0.46779879927635193,
+      "rewards/rejected": -4.954394817352295,
+      "sft_loss": 4.237526893615723,
+      "step": 1010
+    },
+    {
+      "epoch": 0.5432346546245191,
+      "grad_norm": 0.7937538622908429,
+      "learning_rate": 9.801355442251625e-07,
+      "logits/chosen": -0.41196519136428833,
+      "logits/rejected": -0.193257674574852,
+      "logps/chosen": -4.8689775466918945,
+      "logps/rejected": -5.184229850769043,
+      "loss": 0.0548,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.8689775466918945,
+      "rewards/margins": 0.31525272130966187,
+      "rewards/rejected": -5.184229850769043,
+      "sft_loss": 4.687333583831787,
+      "step": 1015
+    },
+    {
+      "epoch": 0.5459106874059207,
+      "grad_norm": 0.48285731675044546,
+      "learning_rate": 9.796985931808949e-07,
+      "logits/chosen": -0.2822311818599701,
+      "logits/rejected": -0.05952910706400871,
+      "logps/chosen": -4.48281192779541,
+      "logps/rejected": -4.878383159637451,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.48281192779541,
+      "rewards/margins": 0.3955712616443634,
+      "rewards/rejected": -4.878383159637451,
+      "sft_loss": 4.184176921844482,
+      "step": 1020
+    },
+    {
+      "epoch": 0.5485867201873222,
+      "grad_norm": 0.5871656423761598,
+      "learning_rate": 9.792569880987724e-07,
+      "logits/chosen": -0.34577301144599915,
+      "logits/rejected": -0.21086709201335907,
+      "logps/chosen": -4.423378944396973,
+      "logps/rejected": -4.85945463180542,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.423378944396973,
+      "rewards/margins": 0.4360765814781189,
+      "rewards/rejected": -4.85945463180542,
+      "sft_loss": 4.179055690765381,
+      "step": 1025
+    },
+    {
+      "epoch": 0.5512627529687238,
+      "grad_norm": 1.0261344581081517,
+      "learning_rate": 9.788107332632493e-07,
+      "logits/chosen": -0.29197707772254944,
+      "logits/rejected": -0.21821355819702148,
+      "logps/chosen": -4.797837257385254,
+      "logps/rejected": -4.941165447235107,
+      "loss": 0.0565,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -4.797837257385254,
+      "rewards/margins": 0.14332810044288635,
+      "rewards/rejected": -4.941165447235107,
+      "sft_loss": 4.4789838790893555,
+      "step": 1030
+    },
+    {
+      "epoch": 0.5539387857501255,
+      "grad_norm": 0.5987597520451127,
+      "learning_rate": 9.783598330038924e-07,
+      "logits/chosen": -0.44390901923179626,
+      "logits/rejected": -0.2985840141773224,
+      "logps/chosen": -4.808468818664551,
+      "logps/rejected": -5.11653995513916,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.808468818664551,
+      "rewards/margins": 0.30807098746299744,
+      "rewards/rejected": -5.11653995513916,
+      "sft_loss": 4.517810344696045,
+      "step": 1035
+    },
+    {
+      "epoch": 0.5566148185315271,
+      "grad_norm": 1.3410928250414738,
+      "learning_rate": 9.779042916953376e-07,
+      "logits/chosen": -0.3711450397968292,
+      "logits/rejected": -0.1343696415424347,
+      "logps/chosen": -4.317525386810303,
+      "logps/rejected": -4.88651704788208,
+      "loss": 0.053,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.317525386810303,
+      "rewards/margins": 0.5689913630485535,
+      "rewards/rejected": -4.88651704788208,
+      "sft_loss": 4.090031147003174,
+      "step": 1040
+    },
+    {
+      "epoch": 0.5592908513129285,
+      "grad_norm": 0.6046804881921751,
+      "learning_rate": 9.774441137572487e-07,
+      "logits/chosen": -0.5188158750534058,
+      "logits/rejected": -0.32569143176078796,
+      "logps/chosen": -4.639094352722168,
+      "logps/rejected": -5.114212989807129,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.639094352722168,
+      "rewards/margins": 0.47511839866638184,
+      "rewards/rejected": -5.114212989807129,
+      "sft_loss": 4.275861740112305,
+      "step": 1045
+    },
+    {
+      "epoch": 0.5619668840943302,
+      "grad_norm": 0.727761819568303,
+      "learning_rate": 9.76979303654274e-07,
+      "logits/chosen": -0.48094066977500916,
+      "logits/rejected": -0.3623979687690735,
+      "logps/chosen": -4.618509769439697,
+      "logps/rejected": -5.188958644866943,
+      "loss": 0.053,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.618509769439697,
+      "rewards/margins": 0.5704492330551147,
+      "rewards/rejected": -5.188958644866943,
+      "sft_loss": 4.3454790115356445,
+      "step": 1050
+    },
+    {
+      "epoch": 0.5646429168757318,
+      "grad_norm": 0.8760146437675645,
+      "learning_rate": 9.765098658960035e-07,
+      "logits/chosen": -0.3562444746494293,
+      "logits/rejected": -0.325128972530365,
+      "logps/chosen": -4.573696613311768,
+      "logps/rejected": -4.9885149002075195,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.573696613311768,
+      "rewards/margins": 0.41481804847717285,
+      "rewards/rejected": -4.9885149002075195,
+      "sft_loss": 4.260937213897705,
+      "step": 1055
+    },
+    {
+      "epoch": 0.5673189496571333,
+      "grad_norm": 0.802977546333446,
+      "learning_rate": 9.76035805036924e-07,
+      "logits/chosen": -0.3120496869087219,
+      "logits/rejected": -0.09086918830871582,
+      "logps/chosen": -4.732306480407715,
+      "logps/rejected": -5.12726354598999,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.732306480407715,
+      "rewards/margins": 0.3949572741985321,
+      "rewards/rejected": -5.12726354598999,
+      "sft_loss": 4.444137096405029,
+      "step": 1060
+    },
+    {
+      "epoch": 0.5699949824385349,
+      "grad_norm": 0.9673416308705352,
+      "learning_rate": 9.755571256763764e-07,
+      "logits/chosen": -0.29424819350242615,
+      "logits/rejected": -0.14790078997612,
+      "logps/chosen": -4.532229423522949,
+      "logps/rejected": -4.993153095245361,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.532229423522949,
+      "rewards/margins": 0.46092328429222107,
+      "rewards/rejected": -4.993153095245361,
+      "sft_loss": 4.201886177062988,
+      "step": 1065
+    },
+    {
+      "epoch": 0.5726710152199365,
+      "grad_norm": 0.5666923403824389,
+      "learning_rate": 9.750738324585097e-07,
+      "logits/chosen": -0.3320736289024353,
+      "logits/rejected": -0.025298094376921654,
+      "logps/chosen": -4.440112113952637,
+      "logps/rejected": -4.799773693084717,
+      "loss": 0.0542,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.440112113952637,
+      "rewards/margins": 0.35966160893440247,
+      "rewards/rejected": -4.799773693084717,
+      "sft_loss": 4.199597358703613,
+      "step": 1070
+    },
+    {
+      "epoch": 0.5753470480013381,
+      "grad_norm": 0.44035129891612645,
+      "learning_rate": 9.74585930072237e-07,
+      "logits/chosen": -0.31049054861068726,
+      "logits/rejected": -0.14209868013858795,
+      "logps/chosen": -4.721621513366699,
+      "logps/rejected": -5.203892230987549,
+      "loss": 0.053,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.721621513366699,
+      "rewards/margins": 0.48226994276046753,
+      "rewards/rejected": -5.203892230987549,
+      "sft_loss": 4.439929008483887,
+      "step": 1075
+    },
+    {
+      "epoch": 0.5780230807827396,
+      "grad_norm": 0.6528718226252765,
+      "learning_rate": 9.740934232511892e-07,
+      "logits/chosen": -0.40927377343177795,
+      "logits/rejected": -0.30179300904273987,
+      "logps/chosen": -4.7493462562561035,
+      "logps/rejected": -4.999502182006836,
+      "loss": 0.054,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.7493462562561035,
+      "rewards/margins": 0.25015532970428467,
+      "rewards/rejected": -4.999502182006836,
+      "sft_loss": 4.381531238555908,
+      "step": 1080
+    },
+    {
+      "epoch": 0.5806991135641412,
+      "grad_norm": 0.7930608365339971,
+      "learning_rate": 9.735963167736698e-07,
+      "logits/chosen": -0.33609557151794434,
+      "logits/rejected": -0.16135595738887787,
+      "logps/chosen": -4.694324016571045,
+      "logps/rejected": -5.053614616394043,
+      "loss": 0.0533,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.694324016571045,
+      "rewards/margins": 0.35929054021835327,
+      "rewards/rejected": -5.053614616394043,
+      "sft_loss": 4.386763572692871,
+      "step": 1085
+    },
+    {
+      "epoch": 0.5833751463455428,
+      "grad_norm": 0.47738026027240005,
+      "learning_rate": 9.730946154626078e-07,
+      "logits/chosen": -0.35580259561538696,
+      "logits/rejected": -0.2341255247592926,
+      "logps/chosen": -4.645298957824707,
+      "logps/rejected": -4.98961877822876,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.645298957824707,
+      "rewards/margins": 0.34431982040405273,
+      "rewards/rejected": -4.98961877822876,
+      "sft_loss": 4.297326564788818,
+      "step": 1090
+    },
+    {
+      "epoch": 0.5860511791269443,
+      "grad_norm": 0.7722621819178563,
+      "learning_rate": 9.725883241855117e-07,
+      "logits/chosen": -0.5120664834976196,
+      "logits/rejected": -0.31316810846328735,
+      "logps/chosen": -4.726617336273193,
+      "logps/rejected": -5.2124552726745605,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.726617336273193,
+      "rewards/margins": 0.4858378469944,
+      "rewards/rejected": -5.2124552726745605,
+      "sft_loss": 4.445858478546143,
+      "step": 1095
+    },
+    {
+      "epoch": 0.5887272119083459,
+      "grad_norm": 0.468710955964014,
+      "learning_rate": 9.720774478544218e-07,
+      "logits/chosen": -0.4066757261753082,
+      "logits/rejected": -0.21096296608448029,
+      "logps/chosen": -4.593195915222168,
+      "logps/rejected": -4.990599632263184,
+      "loss": 0.0533,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.593195915222168,
+      "rewards/margins": 0.3974037170410156,
+      "rewards/rejected": -4.990599632263184,
+      "sft_loss": 4.210533142089844,
+      "step": 1100
+    },
+    {
+      "epoch": 0.5914032446897475,
+      "grad_norm": 0.4850745935833459,
+      "learning_rate": 9.715619914258624e-07,
+      "logits/chosen": -0.5011038184165955,
+      "logits/rejected": -0.40338534116744995,
+      "logps/chosen": -4.631552696228027,
+      "logps/rejected": -4.955190658569336,
+      "loss": 0.0528,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.631552696228027,
+      "rewards/margins": 0.3236382007598877,
+      "rewards/rejected": -4.955190658569336,
+      "sft_loss": 4.248339653015137,
+      "step": 1105
+    },
+    {
+      "epoch": 0.594079277471149,
+      "grad_norm": 0.6066841066596538,
+      "learning_rate": 9.710419599007937e-07,
+      "logits/chosen": -0.42508071660995483,
+      "logits/rejected": -0.22229401767253876,
+      "logps/chosen": -4.685879230499268,
+      "logps/rejected": -4.9598894119262695,
+      "loss": 0.0541,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -4.685879230499268,
+      "rewards/margins": 0.27401041984558105,
+      "rewards/rejected": -4.9598894119262695,
+      "sft_loss": 4.450253486633301,
+      "step": 1110
+    },
+    {
+      "epoch": 0.5967553102525506,
+      "grad_norm": 0.5307786150955166,
+      "learning_rate": 9.705173583245643e-07,
+      "logits/chosen": -0.398245632648468,
+      "logits/rejected": -0.16045819222927094,
+      "logps/chosen": -4.601614475250244,
+      "logps/rejected": -4.990357875823975,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.601614475250244,
+      "rewards/margins": 0.38874366879463196,
+      "rewards/rejected": -4.990357875823975,
+      "sft_loss": 4.338308334350586,
+      "step": 1115
+    },
+    {
+      "epoch": 0.5994313430339522,
+      "grad_norm": 0.6123739703017412,
+      "learning_rate": 9.699881917868609e-07,
+      "logits/chosen": -0.4760781228542328,
+      "logits/rejected": -0.3271307349205017,
+      "logps/chosen": -4.407635688781738,
+      "logps/rejected": -4.813516616821289,
+      "loss": 0.0528,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.407635688781738,
+      "rewards/margins": 0.40588098764419556,
+      "rewards/rejected": -4.813516616821289,
+      "sft_loss": 4.171958923339844,
+      "step": 1120
+    },
+    {
+      "epoch": 0.6021073758153538,
+      "grad_norm": 0.7914767753782926,
+      "learning_rate": 9.694544654216594e-07,
+      "logits/chosen": -0.47358283400535583,
+      "logits/rejected": -0.22326794266700745,
+      "logps/chosen": -4.694055557250977,
+      "logps/rejected": -5.164431571960449,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.694055557250977,
+      "rewards/margins": 0.4703756272792816,
+      "rewards/rejected": -5.164431571960449,
+      "sft_loss": 4.425846576690674,
+      "step": 1125
+    },
+    {
+      "epoch": 0.6047834085967553,
+      "grad_norm": 0.9510732497274381,
+      "learning_rate": 9.689161844071755e-07,
+      "logits/chosen": -0.26579126715660095,
+      "logits/rejected": -0.17085936665534973,
+      "logps/chosen": -4.490574836730957,
+      "logps/rejected": -4.834324836730957,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.490574836730957,
+      "rewards/margins": 0.3437500596046448,
+      "rewards/rejected": -4.834324836730957,
+      "sft_loss": 4.2078447341918945,
+      "step": 1130
+    },
+    {
+      "epoch": 0.6074594413781569,
+      "grad_norm": 0.7961761760788758,
+      "learning_rate": 9.683733539658138e-07,
+      "logits/chosen": -0.42702335119247437,
+      "logits/rejected": -0.15605299174785614,
+      "logps/chosen": -4.632594108581543,
+      "logps/rejected": -5.0079216957092285,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.632594108581543,
+      "rewards/margins": 0.37532731890678406,
+      "rewards/rejected": -5.0079216957092285,
+      "sft_loss": 4.2640180587768555,
+      "step": 1135
+    },
+    {
+      "epoch": 0.6101354741595585,
+      "grad_norm": 0.5850151425803274,
+      "learning_rate": 9.678259793641178e-07,
+      "logits/chosen": -0.4158720374107361,
+      "logits/rejected": -0.3845767080783844,
+      "logps/chosen": -4.806016445159912,
+      "logps/rejected": -5.06912899017334,
+      "loss": 0.0541,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -4.806016445159912,
+      "rewards/margins": 0.2631130516529083,
+      "rewards/rejected": -5.06912899017334,
+      "sft_loss": 4.499835014343262,
+      "step": 1140
+    },
+    {
+      "epoch": 0.61281150694096,
+      "grad_norm": 0.5229820274227105,
+      "learning_rate": 9.672740659127183e-07,
+      "logits/chosen": -0.6019136905670166,
+      "logits/rejected": -0.4428686499595642,
+      "logps/chosen": -4.538661003112793,
+      "logps/rejected": -4.9497575759887695,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.538661003112793,
+      "rewards/margins": 0.4110962450504303,
+      "rewards/rejected": -4.9497575759887695,
+      "sft_loss": 4.195460796356201,
+      "step": 1145
+    },
+    {
+      "epoch": 0.6154875397223616,
+      "grad_norm": 0.5160354493749963,
+      "learning_rate": 9.667176189662818e-07,
+      "logits/chosen": -0.4998350739479065,
+      "logits/rejected": -0.3440905213356018,
+      "logps/chosen": -4.655890464782715,
+      "logps/rejected": -5.005198955535889,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.655890464782715,
+      "rewards/margins": 0.3493087589740753,
+      "rewards/rejected": -5.005198955535889,
+      "sft_loss": 4.31349515914917,
+      "step": 1150
+    },
+    {
+      "epoch": 0.6181635725037632,
+      "grad_norm": 0.6761895399930434,
+      "learning_rate": 9.661566439234592e-07,
+      "logits/chosen": -0.4355531632900238,
+      "logits/rejected": -0.34337887167930603,
+      "logps/chosen": -4.707452297210693,
+      "logps/rejected": -5.022680759429932,
+      "loss": 0.0539,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.707452297210693,
+      "rewards/margins": 0.31522828340530396,
+      "rewards/rejected": -5.022680759429932,
+      "sft_loss": 4.4109697341918945,
+      "step": 1155
+    },
+    {
+      "epoch": 0.6208396052851648,
+      "grad_norm": 0.6315681379626232,
+      "learning_rate": 9.655911462268327e-07,
+      "logits/chosen": -0.441682904958725,
+      "logits/rejected": -0.32130274176597595,
+      "logps/chosen": -4.531501770019531,
+      "logps/rejected": -5.022608757019043,
+      "loss": 0.052,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.531501770019531,
+      "rewards/margins": 0.49110645055770874,
+      "rewards/rejected": -5.022608757019043,
+      "sft_loss": 4.199594974517822,
+      "step": 1160
+    },
+    {
+      "epoch": 0.6235156380665663,
+      "grad_norm": 0.44366417469202174,
+      "learning_rate": 9.650211313628636e-07,
+      "logits/chosen": -0.5553628206253052,
+      "logits/rejected": -0.42054757475852966,
+      "logps/chosen": -4.666576385498047,
+      "logps/rejected": -4.8848371505737305,
+      "loss": 0.0555,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -4.666576385498047,
+      "rewards/margins": 0.21826043725013733,
+      "rewards/rejected": -4.8848371505737305,
+      "sft_loss": 4.408822536468506,
+      "step": 1165
+    },
+    {
+      "epoch": 0.6261916708479679,
+      "grad_norm": 0.6056416701665308,
+      "learning_rate": 9.644466048618386e-07,
+      "logits/chosen": -0.603181779384613,
+      "logits/rejected": -0.40562668442726135,
+      "logps/chosen": -5.020796298980713,
+      "logps/rejected": -5.293118000030518,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -5.020796298980713,
+      "rewards/margins": 0.27232202887535095,
+      "rewards/rejected": -5.293118000030518,
+      "sft_loss": 4.58329963684082,
+      "step": 1170
+    },
+    {
+      "epoch": 0.6288677036293695,
+      "grad_norm": 0.9446275846980485,
+      "learning_rate": 9.63867572297816e-07,
+      "logits/chosen": -0.5902566909790039,
+      "logits/rejected": -0.3584633767604828,
+      "logps/chosen": -4.559604644775391,
+      "logps/rejected": -4.992661476135254,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.559604644775391,
+      "rewards/margins": 0.43305715918540955,
+      "rewards/rejected": -4.992661476135254,
+      "sft_loss": 4.308629512786865,
+      "step": 1175
+    },
+    {
+      "epoch": 0.631543736410771,
+      "grad_norm": 0.4500636348047423,
+      "learning_rate": 9.632840392885727e-07,
+      "logits/chosen": -0.5505040884017944,
+      "logits/rejected": -0.3412054479122162,
+      "logps/chosen": -4.407847881317139,
+      "logps/rejected": -4.881197929382324,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.407847881317139,
+      "rewards/margins": 0.4733501374721527,
+      "rewards/rejected": -4.881197929382324,
+      "sft_loss": 4.203124046325684,
+      "step": 1180
+    },
+    {
+      "epoch": 0.6342197691921726,
+      "grad_norm": 0.5749447318840571,
+      "learning_rate": 9.626960114955483e-07,
+      "logits/chosen": -0.5030391812324524,
+      "logits/rejected": -0.30937737226486206,
+      "logps/chosen": -4.541576385498047,
+      "logps/rejected": -5.093584060668945,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.541576385498047,
+      "rewards/margins": 0.5520075559616089,
+      "rewards/rejected": -5.093584060668945,
+      "sft_loss": 4.283493995666504,
+      "step": 1185
+    },
+    {
+      "epoch": 0.6368958019735742,
+      "grad_norm": 0.7945504810635882,
+      "learning_rate": 9.621034946237909e-07,
+      "logits/chosen": -0.5639665722846985,
+      "logits/rejected": -0.3761981129646301,
+      "logps/chosen": -4.790976524353027,
+      "logps/rejected": -5.246044635772705,
+      "loss": 0.0536,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.790976524353027,
+      "rewards/margins": 0.4550676941871643,
+      "rewards/rejected": -5.246044635772705,
+      "sft_loss": 4.545467376708984,
+      "step": 1190
+    },
+    {
+      "epoch": 0.6395718347549757,
+      "grad_norm": 0.918118566263642,
+      "learning_rate": 9.615064944219021e-07,
+      "logits/chosen": -0.3779866695404053,
+      "logits/rejected": -0.2322605401277542,
+      "logps/chosen": -4.440630912780762,
+      "logps/rejected": -4.815564155578613,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.440630912780762,
+      "rewards/margins": 0.37493371963500977,
+      "rewards/rejected": -4.815564155578613,
+      "sft_loss": 4.151522159576416,
+      "step": 1195
+    },
+    {
+      "epoch": 0.6422478675363773,
+      "grad_norm": 1.124345862015933,
+      "learning_rate": 9.609050166819803e-07,
+      "logits/chosen": -0.3529793620109558,
+      "logits/rejected": -0.3029939532279968,
+      "logps/chosen": -4.427321910858154,
+      "logps/rejected": -4.819708347320557,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.427321910858154,
+      "rewards/margins": 0.3923860490322113,
+      "rewards/rejected": -4.819708347320557,
+      "sft_loss": 4.087639331817627,
+      "step": 1200
+    },
+    {
+      "epoch": 0.6422478675363773,
+      "eval_logits/chosen": 0.13734911382198334,
+      "eval_logits/rejected": 0.25018125772476196,
+      "eval_logps/chosen": -4.564990043640137,
+      "eval_logps/rejected": -5.039029121398926,
+      "eval_loss": 0.051919735968112946,
+      "eval_rewards/accuracies": 0.6387240290641785,
+      "eval_rewards/chosen": -4.564990043640137,
+      "eval_rewards/margins": 0.474039226770401,
+      "eval_rewards/rejected": -5.039029121398926,
+      "eval_runtime": 43.8519,
+      "eval_samples_per_second": 30.671,
+      "eval_sft_loss": 4.1662516593933105,
+      "eval_steps_per_second": 7.685,
+      "step": 1200
+    },
+    {
+      "epoch": 0.6449239003177789,
+      "grad_norm": 0.8910664029545721,
+      "learning_rate": 9.602990672395653e-07,
+      "logits/chosen": -0.44046729803085327,
+      "logits/rejected": -0.23592495918273926,
+      "logps/chosen": -4.662482261657715,
+      "logps/rejected": -5.179293632507324,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.662482261657715,
+      "rewards/margins": 0.5168113708496094,
+      "rewards/rejected": -5.179293632507324,
+      "sft_loss": 4.375925540924072,
+      "step": 1205
+    },
+    {
+      "epoch": 0.6475999330991805,
+      "grad_norm": 0.5819370591266501,
+      "learning_rate": 9.59688651973581e-07,
+      "logits/chosen": -0.37753215432167053,
+      "logits/rejected": -0.10312284529209137,
+      "logps/chosen": -4.637939929962158,
+      "logps/rejected": -5.1369781494140625,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.637939929962158,
+      "rewards/margins": 0.4990386366844177,
+      "rewards/rejected": -5.1369781494140625,
+      "sft_loss": 4.380990982055664,
+      "step": 1210
+    },
+    {
+      "epoch": 0.650275965880582,
+      "grad_norm": 1.7442043523885287,
+      "learning_rate": 9.590737768062792e-07,
+      "logits/chosen": -0.44216713309288025,
+      "logits/rejected": -0.26396456360816956,
+      "logps/chosen": -4.606910228729248,
+      "logps/rejected": -4.996204853057861,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.606910228729248,
+      "rewards/margins": 0.3892943859100342,
+      "rewards/rejected": -4.996204853057861,
+      "sft_loss": 4.274650573730469,
+      "step": 1215
+    },
+    {
+      "epoch": 0.6529519986619836,
+      "grad_norm": 0.5603360085959345,
+      "learning_rate": 9.584544477031816e-07,
+      "logits/chosen": -0.2555375397205353,
+      "logits/rejected": -0.11275085061788559,
+      "logps/chosen": -4.6473894119262695,
+      "logps/rejected": -4.993066787719727,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.6473894119262695,
+      "rewards/margins": 0.3456771969795227,
+      "rewards/rejected": -4.993066787719727,
+      "sft_loss": 4.389549255371094,
+      "step": 1220
+    },
+    {
+      "epoch": 0.6556280314433852,
+      "grad_norm": 0.6020112529125976,
+      "learning_rate": 9.578306706730215e-07,
+      "logits/chosen": -0.5618133544921875,
+      "logits/rejected": -0.2998881936073303,
+      "logps/chosen": -4.613956928253174,
+      "logps/rejected": -4.942728519439697,
+      "loss": 0.0533,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.613956928253174,
+      "rewards/margins": 0.3287719190120697,
+      "rewards/rejected": -4.942728519439697,
+      "sft_loss": 4.343696117401123,
+      "step": 1225
+    },
+    {
+      "epoch": 0.6583040642247867,
+      "grad_norm": 0.5351752048514509,
+      "learning_rate": 9.572024517676865e-07,
+      "logits/chosen": -0.5340681076049805,
+      "logits/rejected": -0.4061642289161682,
+      "logps/chosen": -4.5606770515441895,
+      "logps/rejected": -4.903503894805908,
+      "loss": 0.0548,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.5606770515441895,
+      "rewards/margins": 0.342826783657074,
+      "rewards/rejected": -4.903503894805908,
+      "sft_loss": 4.410942077636719,
+      "step": 1230
+    },
+    {
+      "epoch": 0.6609800970061883,
+      "grad_norm": 0.4114910997444331,
+      "learning_rate": 9.565697970821593e-07,
+      "logits/chosen": -0.6076026558876038,
+      "logits/rejected": -0.3636246621608734,
+      "logps/chosen": -4.661266326904297,
+      "logps/rejected": -5.00773811340332,
+      "loss": 0.053,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.661266326904297,
+      "rewards/margins": 0.34647196531295776,
+      "rewards/rejected": -5.00773811340332,
+      "sft_loss": 4.367223262786865,
+      "step": 1235
+    },
+    {
+      "epoch": 0.6636561297875899,
+      "grad_norm": 0.3821500021648199,
+      "learning_rate": 9.559327127544585e-07,
+      "logits/chosen": -0.6492784023284912,
+      "logits/rejected": -0.5114136934280396,
+      "logps/chosen": -4.658950328826904,
+      "logps/rejected": -4.979151248931885,
+      "loss": 0.053,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -4.658950328826904,
+      "rewards/margins": 0.3202006220817566,
+      "rewards/rejected": -4.979151248931885,
+      "sft_loss": 4.328720569610596,
+      "step": 1240
+    },
+    {
+      "epoch": 0.6663321625689914,
+      "grad_norm": 0.7051365917980006,
+      "learning_rate": 9.552912049655789e-07,
+      "logits/chosen": -0.5670667886734009,
+      "logits/rejected": -0.35566020011901855,
+      "logps/chosen": -4.4767937660217285,
+      "logps/rejected": -4.880216121673584,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.4767937660217285,
+      "rewards/margins": 0.40342283248901367,
+      "rewards/rejected": -4.880216121673584,
+      "sft_loss": 4.220599174499512,
+      "step": 1245
+    },
+    {
+      "epoch": 0.669008195350393,
+      "grad_norm": 0.4327969316914716,
+      "learning_rate": 9.546452799394315e-07,
+      "logits/chosen": -0.632597804069519,
+      "logits/rejected": -0.3552798926830292,
+      "logps/chosen": -4.6764044761657715,
+      "logps/rejected": -5.064030170440674,
+      "loss": 0.0533,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.6764044761657715,
+      "rewards/margins": 0.387625515460968,
+      "rewards/rejected": -5.064030170440674,
+      "sft_loss": 4.441939353942871,
+      "step": 1250
+    },
+    {
+      "epoch": 0.6716842281317946,
+      "grad_norm": 0.4897895362851165,
+      "learning_rate": 9.539949439427846e-07,
+      "logits/chosen": -0.5445303320884705,
+      "logits/rejected": -0.4101172983646393,
+      "logps/chosen": -4.643481254577637,
+      "logps/rejected": -5.050748348236084,
+      "loss": 0.0541,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.643481254577637,
+      "rewards/margins": 0.4072669446468353,
+      "rewards/rejected": -5.050748348236084,
+      "sft_loss": 4.35919189453125,
+      "step": 1255
+    },
+    {
+      "epoch": 0.6743602609131962,
+      "grad_norm": 0.9125178245322394,
+      "learning_rate": 9.533402032852002e-07,
+      "logits/chosen": -0.621421217918396,
+      "logits/rejected": -0.42496657371520996,
+      "logps/chosen": -4.675678730010986,
+      "logps/rejected": -5.152283668518066,
+      "loss": 0.053,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.675678730010986,
+      "rewards/margins": 0.47660455107688904,
+      "rewards/rejected": -5.152283668518066,
+      "sft_loss": 4.386681079864502,
+      "step": 1260
+    },
+    {
+      "epoch": 0.6770362936945977,
+      "grad_norm": 0.6925728581597821,
+      "learning_rate": 9.526810643189754e-07,
+      "logits/chosen": -0.5201088190078735,
+      "logits/rejected": -0.2703739106655121,
+      "logps/chosen": -4.4367218017578125,
+      "logps/rejected": -4.994024753570557,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.4367218017578125,
+      "rewards/margins": 0.5573024749755859,
+      "rewards/rejected": -4.994024753570557,
+      "sft_loss": 4.24059534072876,
+      "step": 1265
+    },
+    {
+      "epoch": 0.6797123264759993,
+      "grad_norm": 0.4113242227701877,
+      "learning_rate": 9.52017533439079e-07,
+      "logits/chosen": -0.556021511554718,
+      "logits/rejected": -0.4820549488067627,
+      "logps/chosen": -4.537997722625732,
+      "logps/rejected": -4.888774871826172,
+      "loss": 0.0546,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.537997722625732,
+      "rewards/margins": 0.35077688097953796,
+      "rewards/rejected": -4.888774871826172,
+      "sft_loss": 4.333981513977051,
+      "step": 1270
+    },
+    {
+      "epoch": 0.6823883592574009,
+      "grad_norm": 0.7023369925776866,
+      "learning_rate": 9.513496170830909e-07,
+      "logits/chosen": -0.6368690729141235,
+      "logits/rejected": -0.5348054766654968,
+      "logps/chosen": -4.6768693923950195,
+      "logps/rejected": -5.018296241760254,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.6768693923950195,
+      "rewards/margins": 0.34142667055130005,
+      "rewards/rejected": -5.018296241760254,
+      "sft_loss": 4.391101360321045,
+      "step": 1275
+    },
+    {
+      "epoch": 0.6850643920388024,
+      "grad_norm": 0.4512299234757288,
+      "learning_rate": 9.506773217311382e-07,
+      "logits/chosen": -0.5606462359428406,
+      "logits/rejected": -0.36210232973098755,
+      "logps/chosen": -4.614706993103027,
+      "logps/rejected": -4.951892375946045,
+      "loss": 0.0528,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.614706993103027,
+      "rewards/margins": 0.3371860086917877,
+      "rewards/rejected": -4.951892375946045,
+      "sft_loss": 4.3583831787109375,
+      "step": 1280
+    },
+    {
+      "epoch": 0.687740424820204,
+      "grad_norm": 0.55139517235157,
+      "learning_rate": 9.500006539058334e-07,
+      "logits/chosen": -0.4866880476474762,
+      "logits/rejected": -0.2489084005355835,
+      "logps/chosen": -4.362078666687012,
+      "logps/rejected": -4.660861015319824,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.362078666687012,
+      "rewards/margins": 0.2987823486328125,
+      "rewards/rejected": -4.660861015319824,
+      "sft_loss": 4.076572895050049,
+      "step": 1285
+    },
+    {
+      "epoch": 0.6904164576016056,
+      "grad_norm": 0.512709938509456,
+      "learning_rate": 9.493196201722109e-07,
+      "logits/chosen": -0.6232250928878784,
+      "logits/rejected": -0.4323170781135559,
+      "logps/chosen": -4.718006610870361,
+      "logps/rejected": -5.043827056884766,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.718006610870361,
+      "rewards/margins": 0.3258204162120819,
+      "rewards/rejected": -5.043827056884766,
+      "sft_loss": 4.426468372344971,
+      "step": 1290
+    },
+    {
+      "epoch": 0.6930924903830072,
+      "grad_norm": 0.6757673766592351,
+      "learning_rate": 9.486342271376628e-07,
+      "logits/chosen": -0.561558723449707,
+      "logits/rejected": -0.6280849575996399,
+      "logps/chosen": -4.768039226531982,
+      "logps/rejected": -5.107489585876465,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -4.768039226531982,
+      "rewards/margins": 0.3394501805305481,
+      "rewards/rejected": -5.107489585876465,
+      "sft_loss": 4.384146213531494,
+      "step": 1295
+    },
+    {
+      "epoch": 0.6957685231644087,
+      "grad_norm": 0.5244603902807886,
+      "learning_rate": 9.479444814518755e-07,
+      "logits/chosen": -0.4877176880836487,
+      "logits/rejected": -0.19087229669094086,
+      "logps/chosen": -4.417885780334473,
+      "logps/rejected": -5.015279769897461,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.417885780334473,
+      "rewards/margins": 0.5973935127258301,
+      "rewards/rejected": -5.015279769897461,
+      "sft_loss": 4.055281639099121,
+      "step": 1300
+    },
+    {
+      "epoch": 0.6984445559458103,
+      "grad_norm": 0.45556924214498573,
+      "learning_rate": 9.472503898067645e-07,
+      "logits/chosen": -0.3125198483467102,
+      "logits/rejected": -0.2833731174468994,
+      "logps/chosen": -4.696566581726074,
+      "logps/rejected": -4.89750862121582,
+      "loss": 0.0551,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -4.696566581726074,
+      "rewards/margins": 0.2009415626525879,
+      "rewards/rejected": -4.89750862121582,
+      "sft_loss": 4.270197868347168,
+      "step": 1305
+    },
+    {
+      "epoch": 0.701120588727212,
+      "grad_norm": 0.7878307361209527,
+      "learning_rate": 9.465519589364099e-07,
+      "logits/chosen": -0.3498966693878174,
+      "logits/rejected": -0.2786726653575897,
+      "logps/chosen": -5.017477035522461,
+      "logps/rejected": -5.288748741149902,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -5.017477035522461,
+      "rewards/margins": 0.27127185463905334,
+      "rewards/rejected": -5.288748741149902,
+      "sft_loss": 4.640139102935791,
+      "step": 1310
+    },
+    {
+      "epoch": 0.7037966215086134,
+      "grad_norm": 0.8920286120051915,
+      "learning_rate": 9.458491956169914e-07,
+      "logits/chosen": -0.37086111307144165,
+      "logits/rejected": -0.16999824345111847,
+      "logps/chosen": -4.605779647827148,
+      "logps/rejected": -4.944549083709717,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.605779647827148,
+      "rewards/margins": 0.33876991271972656,
+      "rewards/rejected": -4.944549083709717,
+      "sft_loss": 4.18860387802124,
+      "step": 1315
+    },
+    {
+      "epoch": 0.706472654290015,
+      "grad_norm": 0.6429781778928714,
+      "learning_rate": 9.451421066667215e-07,
+      "logits/chosen": -0.4545694887638092,
+      "logits/rejected": -0.23324093222618103,
+      "logps/chosen": -4.4322896003723145,
+      "logps/rejected": -4.9902520179748535,
+      "loss": 0.052,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.4322896003723145,
+      "rewards/margins": 0.5579624176025391,
+      "rewards/rejected": -4.9902520179748535,
+      "sft_loss": 4.132850646972656,
+      "step": 1320
+    },
+    {
+      "epoch": 0.7091486870714167,
+      "grad_norm": 0.6153340134014024,
+      "learning_rate": 9.444306989457805e-07,
+      "logits/chosen": -0.1503096967935562,
+      "logits/rejected": -0.038341958075761795,
+      "logps/chosen": -4.885147571563721,
+      "logps/rejected": -5.25883674621582,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -4.885147571563721,
+      "rewards/margins": 0.3736896216869354,
+      "rewards/rejected": -5.25883674621582,
+      "sft_loss": 4.402420997619629,
+      "step": 1325
+    },
+    {
+      "epoch": 0.7118247198528181,
+      "grad_norm": 0.8037003037295921,
+      "learning_rate": 9.437149793562489e-07,
+      "logits/chosen": -0.2632639408111572,
+      "logits/rejected": -0.12199203670024872,
+      "logps/chosen": -4.701368808746338,
+      "logps/rejected": -5.090550899505615,
+      "loss": 0.0539,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.701368808746338,
+      "rewards/margins": 0.38918226957321167,
+      "rewards/rejected": -5.090550899505615,
+      "sft_loss": 4.425740718841553,
+      "step": 1330
+    },
+    {
+      "epoch": 0.7145007526342197,
+      "grad_norm": 0.4360448850251985,
+      "learning_rate": 9.429949548420417e-07,
+      "logits/chosen": -0.3493057191371918,
+      "logits/rejected": -0.23026664555072784,
+      "logps/chosen": -4.643458843231201,
+      "logps/rejected": -5.04981803894043,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.643458843231201,
+      "rewards/margins": 0.4063592553138733,
+      "rewards/rejected": -5.04981803894043,
+      "sft_loss": 4.333911418914795,
+      "step": 1335
+    },
+    {
+      "epoch": 0.7171767854156214,
+      "grad_norm": 0.8043804348967304,
+      "learning_rate": 9.422706323888396e-07,
+      "logits/chosen": -0.34672215580940247,
+      "logits/rejected": -0.3123110234737396,
+      "logps/chosen": -4.475564956665039,
+      "logps/rejected": -4.835718631744385,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.475564956665039,
+      "rewards/margins": 0.3601534962654114,
+      "rewards/rejected": -4.835718631744385,
+      "sft_loss": 4.1928300857543945,
+      "step": 1340
+    },
+    {
+      "epoch": 0.719852818197023,
+      "grad_norm": 0.4814926832557905,
+      "learning_rate": 9.415420190240225e-07,
+      "logits/chosen": -0.43331629037857056,
+      "logits/rejected": -0.17016439139842987,
+      "logps/chosen": -4.414015293121338,
+      "logps/rejected": -5.072504997253418,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.414015293121338,
+      "rewards/margins": 0.6584896445274353,
+      "rewards/rejected": -5.072504997253418,
+      "sft_loss": 4.189124584197998,
+      "step": 1345
+    },
+    {
+      "epoch": 0.7225288509784245,
+      "grad_norm": 0.9425522188384677,
+      "learning_rate": 9.408091218166002e-07,
+      "logits/chosen": -0.40453529357910156,
+      "logits/rejected": -0.34088414907455444,
+      "logps/chosen": -4.602755069732666,
+      "logps/rejected": -4.866302967071533,
+      "loss": 0.054,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.602755069732666,
+      "rewards/margins": 0.26354801654815674,
+      "rewards/rejected": -4.866302967071533,
+      "sft_loss": 4.257199764251709,
+      "step": 1350
+    },
+    {
+      "epoch": 0.7252048837598261,
+      "grad_norm": 0.6496226271390918,
+      "learning_rate": 9.400719478771449e-07,
+      "logits/chosen": -0.49114522337913513,
+      "logits/rejected": -0.13422870635986328,
+      "logps/chosen": -4.723020076751709,
+      "logps/rejected": -5.311274528503418,
+      "loss": 0.052,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.723020076751709,
+      "rewards/margins": 0.5882546901702881,
+      "rewards/rejected": -5.311274528503418,
+      "sft_loss": 4.348706245422363,
+      "step": 1355
+    },
+    {
+      "epoch": 0.7278809165412277,
+      "grad_norm": 0.695843448626192,
+      "learning_rate": 9.393305043577209e-07,
+      "logits/chosen": -0.4368442893028259,
+      "logits/rejected": -0.31888890266418457,
+      "logps/chosen": -4.5367536544799805,
+      "logps/rejected": -5.105678558349609,
+      "loss": 0.053,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.5367536544799805,
+      "rewards/margins": 0.5689245462417603,
+      "rewards/rejected": -5.105678558349609,
+      "sft_loss": 4.157050609588623,
+      "step": 1360
+    },
+    {
+      "epoch": 0.7305569493226292,
+      "grad_norm": 0.5517771060033767,
+      "learning_rate": 9.38584798451817e-07,
+      "logits/chosen": -0.524268388748169,
+      "logits/rejected": -0.3451070189476013,
+      "logps/chosen": -4.593259811401367,
+      "logps/rejected": -5.034455299377441,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.593259811401367,
+      "rewards/margins": 0.4411951005458832,
+      "rewards/rejected": -5.034455299377441,
+      "sft_loss": 4.305645942687988,
+      "step": 1365
+    },
+    {
+      "epoch": 0.7332329821040308,
+      "grad_norm": 0.5616207099026281,
+      "learning_rate": 9.37834837394275e-07,
+      "logits/chosen": -0.4677620530128479,
+      "logits/rejected": -0.3105335831642151,
+      "logps/chosen": -4.563145160675049,
+      "logps/rejected": -5.219221591949463,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.563145160675049,
+      "rewards/margins": 0.6560760736465454,
+      "rewards/rejected": -5.219221591949463,
+      "sft_loss": 4.315075397491455,
+      "step": 1370
+    },
+    {
+      "epoch": 0.7359090148854324,
+      "grad_norm": 0.6524802965194981,
+      "learning_rate": 9.370806284612203e-07,
+      "logits/chosen": -0.5127910375595093,
+      "logits/rejected": -0.3530833125114441,
+      "logps/chosen": -4.410710334777832,
+      "logps/rejected": -4.95272159576416,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.410710334777832,
+      "rewards/margins": 0.542011559009552,
+      "rewards/rejected": -4.95272159576416,
+      "sft_loss": 4.096310615539551,
+      "step": 1375
+    },
+    {
+      "epoch": 0.738585047666834,
+      "grad_norm": 0.49778298805300486,
+      "learning_rate": 9.363221789699912e-07,
+      "logits/chosen": -0.6387651562690735,
+      "logits/rejected": -0.4769114851951599,
+      "logps/chosen": -4.669447898864746,
+      "logps/rejected": -5.161695957183838,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.669447898864746,
+      "rewards/margins": 0.4922473430633545,
+      "rewards/rejected": -5.161695957183838,
+      "sft_loss": 4.3861799240112305,
+      "step": 1380
+    },
+    {
+      "epoch": 0.7412610804482355,
+      "grad_norm": 0.4973033903636775,
+      "learning_rate": 9.355594962790682e-07,
+      "logits/chosen": -0.6316344141960144,
+      "logits/rejected": -0.4682461619377136,
+      "logps/chosen": -4.618069171905518,
+      "logps/rejected": -5.081037998199463,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.618069171905518,
+      "rewards/margins": 0.4629689157009125,
+      "rewards/rejected": -5.081037998199463,
+      "sft_loss": 4.341340065002441,
+      "step": 1385
+    },
+    {
+      "epoch": 0.7439371132296371,
+      "grad_norm": 0.8456456840740274,
+      "learning_rate": 9.34792587788002e-07,
+      "logits/chosen": -0.4881487488746643,
+      "logits/rejected": -0.3412144184112549,
+      "logps/chosen": -4.493343830108643,
+      "logps/rejected": -4.900251865386963,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.493343830108643,
+      "rewards/margins": 0.4069080948829651,
+      "rewards/rejected": -4.900251865386963,
+      "sft_loss": 4.154688358306885,
+      "step": 1390
+    },
+    {
+      "epoch": 0.7466131460110387,
+      "grad_norm": 0.7559539589520664,
+      "learning_rate": 9.34021460937342e-07,
+      "logits/chosen": -0.5141401886940002,
+      "logits/rejected": -0.4605598449707031,
+      "logps/chosen": -4.662027359008789,
+      "logps/rejected": -4.994131088256836,
+      "loss": 0.0541,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.662027359008789,
+      "rewards/margins": 0.332103431224823,
+      "rewards/rejected": -4.994131088256836,
+      "sft_loss": 4.393918037414551,
+      "step": 1395
+    },
+    {
+      "epoch": 0.7492891787924402,
+      "grad_norm": 0.8170734818445623,
+      "learning_rate": 9.332461232085646e-07,
+      "logits/chosen": -0.7511906623840332,
+      "logits/rejected": -0.5474541783332825,
+      "logps/chosen": -4.761792182922363,
+      "logps/rejected": -5.100632667541504,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.761792182922363,
+      "rewards/margins": 0.33883994817733765,
+      "rewards/rejected": -5.100632667541504,
+      "sft_loss": 4.543351650238037,
+      "step": 1400
+    },
+    {
+      "epoch": 0.7519652115738418,
+      "grad_norm": 0.97472817753204,
+      "learning_rate": 9.324665821239998e-07,
+      "logits/chosen": -0.6370987296104431,
+      "logits/rejected": -0.3644103407859802,
+      "logps/chosen": -4.321720123291016,
+      "logps/rejected": -4.904634952545166,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.321720123291016,
+      "rewards/margins": 0.5829154253005981,
+      "rewards/rejected": -4.904634952545166,
+      "sft_loss": 4.122037887573242,
+      "step": 1405
+    },
+    {
+      "epoch": 0.7546412443552434,
+      "grad_norm": 0.8592950845747594,
+      "learning_rate": 9.316828452467583e-07,
+      "logits/chosen": -0.5598332285881042,
+      "logits/rejected": -0.37060433626174927,
+      "logps/chosen": -4.467704772949219,
+      "logps/rejected": -4.977252960205078,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.467704772949219,
+      "rewards/margins": 0.509548008441925,
+      "rewards/rejected": -4.977252960205078,
+      "sft_loss": 4.292231559753418,
+      "step": 1410
+    },
+    {
+      "epoch": 0.7573172771366449,
+      "grad_norm": 0.5166200986995931,
+      "learning_rate": 9.30894920180659e-07,
+      "logits/chosen": -0.4799424111843109,
+      "logits/rejected": -0.37270790338516235,
+      "logps/chosen": -4.589459419250488,
+      "logps/rejected": -4.9591264724731445,
+      "loss": 0.052,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.589459419250488,
+      "rewards/margins": 0.36966708302497864,
+      "rewards/rejected": -4.9591264724731445,
+      "sft_loss": 4.216407775878906,
+      "step": 1415
+    },
+    {
+      "epoch": 0.7599933099180465,
+      "grad_norm": 0.6004760886252983,
+      "learning_rate": 9.301028145701543e-07,
+      "logits/chosen": -0.46649855375289917,
+      "logits/rejected": -0.2898474335670471,
+      "logps/chosen": -4.495982646942139,
+      "logps/rejected": -5.16299295425415,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.495982646942139,
+      "rewards/margins": 0.6670098304748535,
+      "rewards/rejected": -5.16299295425415,
+      "sft_loss": 4.211665153503418,
+      "step": 1420
+    },
+    {
+      "epoch": 0.7626693426994481,
+      "grad_norm": 0.4895814230369738,
+      "learning_rate": 9.293065361002563e-07,
+      "logits/chosen": -0.4595528542995453,
+      "logits/rejected": -0.27548545598983765,
+      "logps/chosen": -4.439912796020508,
+      "logps/rejected": -4.987391948699951,
+      "loss": 0.0544,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.439912796020508,
+      "rewards/margins": 0.5474788546562195,
+      "rewards/rejected": -4.987391948699951,
+      "sft_loss": 4.122614860534668,
+      "step": 1425
+    },
+    {
+      "epoch": 0.7653453754808497,
+      "grad_norm": 0.5974027929532499,
+      "learning_rate": 9.285060924964622e-07,
+      "logits/chosen": -0.6105833053588867,
+      "logits/rejected": -0.47843295335769653,
+      "logps/chosen": -4.8134660720825195,
+      "logps/rejected": -5.239222049713135,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.8134660720825195,
+      "rewards/margins": 0.42575669288635254,
+      "rewards/rejected": -5.239222049713135,
+      "sft_loss": 4.412846088409424,
+      "step": 1430
+    },
+    {
+      "epoch": 0.7680214082622512,
+      "grad_norm": 1.3370991265640622,
+      "learning_rate": 9.277014915246792e-07,
+      "logits/chosen": -0.48557156324386597,
+      "logits/rejected": -0.4342361390590668,
+      "logps/chosen": -4.64917516708374,
+      "logps/rejected": -5.111618518829346,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.64917516708374,
+      "rewards/margins": 0.4624432921409607,
+      "rewards/rejected": -5.111618518829346,
+      "sft_loss": 4.41920804977417,
+      "step": 1435
+    },
+    {
+      "epoch": 0.7706974410436528,
+      "grad_norm": 0.4623632733003547,
+      "learning_rate": 9.268927409911498e-07,
+      "logits/chosen": -0.5703449249267578,
+      "logits/rejected": -0.4686836302280426,
+      "logps/chosen": -4.393882751464844,
+      "logps/rejected": -4.764910697937012,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.393882751464844,
+      "rewards/margins": 0.37102851271629333,
+      "rewards/rejected": -4.764910697937012,
+      "sft_loss": 4.038668155670166,
+      "step": 1440
+    },
+    {
+      "epoch": 0.7733734738250544,
+      "grad_norm": 0.5960811044321848,
+      "learning_rate": 9.260798487423749e-07,
+      "logits/chosen": -0.8741840124130249,
+      "logits/rejected": -0.581978976726532,
+      "logps/chosen": -4.775782108306885,
+      "logps/rejected": -5.2636637687683105,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.775782108306885,
+      "rewards/margins": 0.48788174986839294,
+      "rewards/rejected": -5.2636637687683105,
+      "sft_loss": 4.456978797912598,
+      "step": 1445
+    },
+    {
+      "epoch": 0.7760495066064559,
+      "grad_norm": 0.39021583689376327,
+      "learning_rate": 9.252628226650389e-07,
+      "logits/chosen": -0.6951247453689575,
+      "logits/rejected": -0.6016863584518433,
+      "logps/chosen": -4.730222702026367,
+      "logps/rejected": -4.985324859619141,
+      "loss": 0.053,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -4.730222702026367,
+      "rewards/margins": 0.25510215759277344,
+      "rewards/rejected": -4.985324859619141,
+      "sft_loss": 4.339371681213379,
+      "step": 1450
+    },
+    {
+      "epoch": 0.7787255393878575,
+      "grad_norm": 0.7638939236955999,
+      "learning_rate": 9.244416706859321e-07,
+      "logits/chosen": -0.6787390112876892,
+      "logits/rejected": -0.47037848830223083,
+      "logps/chosen": -4.361235618591309,
+      "logps/rejected": -4.954770565032959,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.361235618591309,
+      "rewards/margins": 0.5935350656509399,
+      "rewards/rejected": -4.954770565032959,
+      "sft_loss": 4.1188554763793945,
+      "step": 1455
+    },
+    {
+      "epoch": 0.7814015721692591,
+      "grad_norm": 0.5921846292678737,
+      "learning_rate": 9.23616400771875e-07,
+      "logits/chosen": -0.6849501729011536,
+      "logits/rejected": -0.4271467328071594,
+      "logps/chosen": -4.496269226074219,
+      "logps/rejected": -4.904913902282715,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.496269226074219,
+      "rewards/margins": 0.40864452719688416,
+      "rewards/rejected": -4.904913902282715,
+      "sft_loss": 4.193645000457764,
+      "step": 1460
+    },
+    {
+      "epoch": 0.7840776049506607,
+      "grad_norm": 0.5140729230742062,
+      "learning_rate": 9.227870209296395e-07,
+      "logits/chosen": -0.5369542837142944,
+      "logits/rejected": -0.361372709274292,
+      "logps/chosen": -4.5893235206604,
+      "logps/rejected": -4.978590965270996,
+      "loss": 0.053,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.5893235206604,
+      "rewards/margins": 0.3892673850059509,
+      "rewards/rejected": -4.978590965270996,
+      "sft_loss": 4.30275821685791,
+      "step": 1465
+    },
+    {
+      "epoch": 0.7867536377320622,
+      "grad_norm": 0.7138005925509934,
+      "learning_rate": 9.219535392058728e-07,
+      "logits/chosen": -0.5888296365737915,
+      "logits/rejected": -0.6135191321372986,
+      "logps/chosen": -4.756964206695557,
+      "logps/rejected": -5.122450351715088,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.756964206695557,
+      "rewards/margins": 0.3654859662055969,
+      "rewards/rejected": -5.122450351715088,
+      "sft_loss": 4.438687801361084,
+      "step": 1470
+    },
+    {
+      "epoch": 0.7894296705134638,
+      "grad_norm": 0.6792551404068019,
+      "learning_rate": 9.211159636870181e-07,
+      "logits/chosen": -0.6354023218154907,
+      "logits/rejected": -0.4295397698879242,
+      "logps/chosen": -4.545450210571289,
+      "logps/rejected": -5.067888259887695,
+      "loss": 0.052,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.545450210571289,
+      "rewards/margins": 0.5224383473396301,
+      "rewards/rejected": -5.067888259887695,
+      "sft_loss": 4.248089790344238,
+      "step": 1475
+    },
+    {
+      "epoch": 0.7921057032948654,
+      "grad_norm": 0.5258294806269868,
+      "learning_rate": 9.202743024992367e-07,
+      "logits/chosen": -0.32514700293540955,
+      "logits/rejected": -0.24210067093372345,
+      "logps/chosen": -4.494786739349365,
+      "logps/rejected": -4.912075996398926,
+      "loss": 0.0536,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.494786739349365,
+      "rewards/margins": 0.41728848218917847,
+      "rewards/rejected": -4.912075996398926,
+      "sft_loss": 4.201507091522217,
+      "step": 1480
+    },
+    {
+      "epoch": 0.7947817360762669,
+      "grad_norm": 0.5507357931137226,
+      "learning_rate": 9.194285638083293e-07,
+      "logits/chosen": -0.467916876077652,
+      "logits/rejected": -0.29624345898628235,
+      "logps/chosen": -4.507334232330322,
+      "logps/rejected": -4.973681449890137,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.507334232330322,
+      "rewards/margins": 0.4663466811180115,
+      "rewards/rejected": -4.973681449890137,
+      "sft_loss": 4.084946632385254,
+      "step": 1485
+    },
+    {
+      "epoch": 0.7974577688576685,
+      "grad_norm": 0.7427535420687216,
+      "learning_rate": 9.185787558196562e-07,
+      "logits/chosen": -0.526081383228302,
+      "logits/rejected": -0.4255906939506531,
+      "logps/chosen": -4.713466644287109,
+      "logps/rejected": -5.259469985961914,
+      "loss": 0.052,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.713466644287109,
+      "rewards/margins": 0.5460041165351868,
+      "rewards/rejected": -5.259469985961914,
+      "sft_loss": 4.401684284210205,
+      "step": 1490
+    },
+    {
+      "epoch": 0.8001338016390701,
+      "grad_norm": 0.5552710520792367,
+      "learning_rate": 9.177248867780583e-07,
+      "logits/chosen": -0.3650432229042053,
+      "logits/rejected": -0.2875578701496124,
+      "logps/chosen": -4.637053966522217,
+      "logps/rejected": -4.971064567565918,
+      "loss": 0.0545,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.637053966522217,
+      "rewards/margins": 0.33401015400886536,
+      "rewards/rejected": -4.971064567565918,
+      "sft_loss": 4.287173271179199,
+      "step": 1495
+    },
+    {
+      "epoch": 0.8028098344204716,
+      "grad_norm": 0.6545200457585186,
+      "learning_rate": 9.168669649677769e-07,
+      "logits/chosen": -0.5093628168106079,
+      "logits/rejected": -0.363433301448822,
+      "logps/chosen": -4.705319881439209,
+      "logps/rejected": -5.057621479034424,
+      "loss": 0.0544,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -4.705319881439209,
+      "rewards/margins": 0.3523014187812805,
+      "rewards/rejected": -5.057621479034424,
+      "sft_loss": 4.391488075256348,
+      "step": 1500
+    },
+    {
+      "epoch": 0.8054858672018732,
+      "grad_norm": 0.693732510533624,
+      "learning_rate": 9.16004998712373e-07,
+      "logits/chosen": -0.48523417115211487,
+      "logits/rejected": -0.3940420150756836,
+      "logps/chosen": -4.654600620269775,
+      "logps/rejected": -5.024332046508789,
+      "loss": 0.0528,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.654600620269775,
+      "rewards/margins": 0.36973097920417786,
+      "rewards/rejected": -5.024332046508789,
+      "sft_loss": 4.359921455383301,
+      "step": 1505
+    },
+    {
+      "epoch": 0.8081618999832748,
+      "grad_norm": 0.8529497338023002,
+      "learning_rate": 9.151389963746472e-07,
+      "logits/chosen": -0.5112209320068359,
+      "logits/rejected": -0.11440527439117432,
+      "logps/chosen": -4.373734951019287,
+      "logps/rejected": -5.003460884094238,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.373734951019287,
+      "rewards/margins": 0.6297259330749512,
+      "rewards/rejected": -5.003460884094238,
+      "sft_loss": 4.172287940979004,
+      "step": 1510
+    },
+    {
+      "epoch": 0.8108379327646764,
+      "grad_norm": 0.5936951496458619,
+      "learning_rate": 9.142689663565577e-07,
+      "logits/chosen": -0.43221956491470337,
+      "logits/rejected": -0.3605879843235016,
+      "logps/chosen": -4.363806247711182,
+      "logps/rejected": -4.777824401855469,
+      "loss": 0.0536,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.363806247711182,
+      "rewards/margins": 0.41401776671409607,
+      "rewards/rejected": -4.777824401855469,
+      "sft_loss": 4.124611854553223,
+      "step": 1515
+    },
+    {
+      "epoch": 0.8135139655460779,
+      "grad_norm": 0.6096514301908822,
+      "learning_rate": 9.133949170991397e-07,
+      "logits/chosen": -0.519121527671814,
+      "logits/rejected": -0.4197636544704437,
+      "logps/chosen": -4.7988386154174805,
+      "logps/rejected": -5.091773986816406,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.7988386154174805,
+      "rewards/margins": 0.2929350435733795,
+      "rewards/rejected": -5.091773986816406,
+      "sft_loss": 4.553792953491211,
+      "step": 1520
+    },
+    {
+      "epoch": 0.8161899983274795,
+      "grad_norm": 0.5746404976910024,
+      "learning_rate": 9.125168570824231e-07,
+      "logits/chosen": -0.6563237309455872,
+      "logits/rejected": -0.4389188885688782,
+      "logps/chosen": -4.557305335998535,
+      "logps/rejected": -4.995202541351318,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.557305335998535,
+      "rewards/margins": 0.43789729475975037,
+      "rewards/rejected": -4.995202541351318,
+      "sft_loss": 4.281145095825195,
+      "step": 1525
+    },
+    {
+      "epoch": 0.8188660311088811,
+      "grad_norm": 0.6697475255234172,
+      "learning_rate": 9.116347948253496e-07,
+      "logits/chosen": -0.6038469672203064,
+      "logits/rejected": -0.4111461639404297,
+      "logps/chosen": -4.381692409515381,
+      "logps/rejected": -4.818631649017334,
+      "loss": 0.053,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.381692409515381,
+      "rewards/margins": 0.43693867325782776,
+      "rewards/rejected": -4.818631649017334,
+      "sft_loss": 4.0797834396362305,
+      "step": 1530
+    },
+    {
+      "epoch": 0.8215420638902826,
+      "grad_norm": 0.801191616568803,
+      "learning_rate": 9.107487388856916e-07,
+      "logits/chosen": -0.7121357917785645,
+      "logits/rejected": -0.4746119976043701,
+      "logps/chosen": -4.668355464935303,
+      "logps/rejected": -5.093571662902832,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.668355464935303,
+      "rewards/margins": 0.42521581053733826,
+      "rewards/rejected": -5.093571662902832,
+      "sft_loss": 4.296419143676758,
+      "step": 1535
+    },
+    {
+      "epoch": 0.8242180966716842,
+      "grad_norm": 1.2012408006490007,
+      "learning_rate": 9.098586978599673e-07,
+      "logits/chosen": -0.584466814994812,
+      "logits/rejected": -0.39628341794013977,
+      "logps/chosen": -4.556466102600098,
+      "logps/rejected": -5.13347864151001,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.556466102600098,
+      "rewards/margins": 0.5770123600959778,
+      "rewards/rejected": -5.13347864151001,
+      "sft_loss": 4.171639442443848,
+      "step": 1540
+    },
+    {
+      "epoch": 0.8268941294530858,
+      "grad_norm": 0.8295755986531027,
+      "learning_rate": 9.089646803833588e-07,
+      "logits/chosen": -0.6427966356277466,
+      "logits/rejected": -0.4719253480434418,
+      "logps/chosen": -4.530259609222412,
+      "logps/rejected": -5.066699981689453,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.530259609222412,
+      "rewards/margins": 0.5364402532577515,
+      "rewards/rejected": -5.066699981689453,
+      "sft_loss": 4.1909990310668945,
+      "step": 1545
+    },
+    {
+      "epoch": 0.8295701622344873,
+      "grad_norm": 0.45575866706602736,
+      "learning_rate": 9.080666951296276e-07,
+      "logits/chosen": -0.7682622671127319,
+      "logits/rejected": -0.4284500479698181,
+      "logps/chosen": -4.4875359535217285,
+      "logps/rejected": -5.184557914733887,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.4875359535217285,
+      "rewards/margins": 0.6970220804214478,
+      "rewards/rejected": -5.184557914733887,
+      "sft_loss": 4.240321159362793,
+      "step": 1550
+    },
+    {
+      "epoch": 0.8322461950158889,
+      "grad_norm": 0.8616340059854087,
+      "learning_rate": 9.071647508110305e-07,
+      "logits/chosen": -0.6000322103500366,
+      "logits/rejected": -0.2714582085609436,
+      "logps/chosen": -4.265105247497559,
+      "logps/rejected": -5.043358325958252,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.265105247497559,
+      "rewards/margins": 0.7782524228096008,
+      "rewards/rejected": -5.043358325958252,
+      "sft_loss": 3.936190128326416,
+      "step": 1555
+    },
+    {
+      "epoch": 0.8349222277972905,
+      "grad_norm": 0.4884634696994051,
+      "learning_rate": 9.062588561782354e-07,
+      "logits/chosen": -0.5129867792129517,
+      "logits/rejected": -0.44472289085388184,
+      "logps/chosen": -4.736525058746338,
+      "logps/rejected": -5.098237037658691,
+      "loss": 0.054,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.736525058746338,
+      "rewards/margins": 0.3617119789123535,
+      "rewards/rejected": -5.098237037658691,
+      "sft_loss": 4.419918060302734,
+      "step": 1560
+    },
+    {
+      "epoch": 0.8375982605786921,
+      "grad_norm": 0.7754204271070733,
+      "learning_rate": 9.053490200202358e-07,
+      "logits/chosen": -0.5212528705596924,
+      "logits/rejected": -0.46367019414901733,
+      "logps/chosen": -4.657750129699707,
+      "logps/rejected": -5.107029914855957,
+      "loss": 0.0541,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.657750129699707,
+      "rewards/margins": 0.44928041100502014,
+      "rewards/rejected": -5.107029914855957,
+      "sft_loss": 4.432155609130859,
+      "step": 1565
+    },
+    {
+      "epoch": 0.8402742933600936,
+      "grad_norm": 0.8302273442870804,
+      "learning_rate": 9.044352511642661e-07,
+      "logits/chosen": -0.6087560057640076,
+      "logits/rejected": -0.5251113176345825,
+      "logps/chosen": -4.926453590393066,
+      "logps/rejected": -5.241636753082275,
+      "loss": 0.0544,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.926453590393066,
+      "rewards/margins": 0.315182626247406,
+      "rewards/rejected": -5.241636753082275,
+      "sft_loss": 4.637141227722168,
+      "step": 1570
+    },
+    {
+      "epoch": 0.8429503261414952,
+      "grad_norm": 0.6513636122476614,
+      "learning_rate": 9.03517558475716e-07,
+      "logits/chosen": -0.6366342306137085,
+      "logits/rejected": -0.5109571218490601,
+      "logps/chosen": -4.386017799377441,
+      "logps/rejected": -4.666255950927734,
+      "loss": 0.0554,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -4.386017799377441,
+      "rewards/margins": 0.2802382707595825,
+      "rewards/rejected": -4.666255950927734,
+      "sft_loss": 4.119292259216309,
+      "step": 1575
+    },
+    {
+      "epoch": 0.8456263589228968,
+      "grad_norm": 0.48291708229520947,
+      "learning_rate": 9.025959508580436e-07,
+      "logits/chosen": -0.7007150650024414,
+      "logits/rejected": -0.39889538288116455,
+      "logps/chosen": -4.4623637199401855,
+      "logps/rejected": -4.950345039367676,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.4623637199401855,
+      "rewards/margins": 0.48798173666000366,
+      "rewards/rejected": -4.950345039367676,
+      "sft_loss": 4.182068824768066,
+      "step": 1580
+    },
+    {
+      "epoch": 0.8483023917042983,
+      "grad_norm": 0.39776745797415436,
+      "learning_rate": 9.016704372526905e-07,
+      "logits/chosen": -0.7292011976242065,
+      "logits/rejected": -0.4912241995334625,
+      "logps/chosen": -4.48965311050415,
+      "logps/rejected": -5.049952983856201,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.48965311050415,
+      "rewards/margins": 0.5602999925613403,
+      "rewards/rejected": -5.049952983856201,
+      "sft_loss": 4.21937370300293,
+      "step": 1585
+    },
+    {
+      "epoch": 0.8509784244856999,
+      "grad_norm": 0.4963728592025816,
+      "learning_rate": 9.007410266389934e-07,
+      "logits/chosen": -0.8241753578186035,
+      "logits/rejected": -0.8256832361221313,
+      "logps/chosen": -4.682924270629883,
+      "logps/rejected": -4.960546016693115,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.682924270629883,
+      "rewards/margins": 0.27762192487716675,
+      "rewards/rejected": -4.960546016693115,
+      "sft_loss": 4.291807174682617,
+      "step": 1590
+    },
+    {
+      "epoch": 0.8536544572671015,
+      "grad_norm": 0.8785993229541145,
+      "learning_rate": 8.998077280340981e-07,
+      "logits/chosen": -0.7560137510299683,
+      "logits/rejected": -0.7197335958480835,
+      "logps/chosen": -4.887700080871582,
+      "logps/rejected": -5.31774377822876,
+      "loss": 0.053,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.887700080871582,
+      "rewards/margins": 0.4300435483455658,
+      "rewards/rejected": -5.31774377822876,
+      "sft_loss": 4.426878929138184,
+      "step": 1595
+    },
+    {
+      "epoch": 0.8563304900485031,
+      "grad_norm": 0.4973210847144275,
+      "learning_rate": 8.988705504928722e-07,
+      "logits/chosen": -0.7660343647003174,
+      "logits/rejected": -0.5587971210479736,
+      "logps/chosen": -4.545231819152832,
+      "logps/rejected": -5.228125095367432,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.545231819152832,
+      "rewards/margins": 0.6828934550285339,
+      "rewards/rejected": -5.228125095367432,
+      "sft_loss": 4.198379993438721,
+      "step": 1600
+    },
+    {
+      "epoch": 0.8563304900485031,
+      "eval_logits/chosen": -0.15241743624210358,
+      "eval_logits/rejected": -0.06400942802429199,
+      "eval_logps/chosen": -4.4388813972473145,
+      "eval_logps/rejected": -4.92216157913208,
+      "eval_loss": 0.05129525065422058,
+      "eval_rewards/accuracies": 0.6357566714286804,
+      "eval_rewards/chosen": -4.4388813972473145,
+      "eval_rewards/margins": 0.4832801818847656,
+      "eval_rewards/rejected": -4.92216157913208,
+      "eval_runtime": 43.7335,
+      "eval_samples_per_second": 30.754,
+      "eval_sft_loss": 3.9593169689178467,
+      "eval_steps_per_second": 7.706,
+      "step": 1600
+    },
+    {
+      "epoch": 0.8590065228299046,
+      "grad_norm": 0.6767207852208145,
+      "learning_rate": 8.979295031078157e-07,
+      "logits/chosen": -0.6382713913917542,
+      "logits/rejected": -0.3558812141418457,
+      "logps/chosen": -4.260805606842041,
+      "logps/rejected": -4.816983699798584,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.260805606842041,
+      "rewards/margins": 0.5561773180961609,
+      "rewards/rejected": -4.816983699798584,
+      "sft_loss": 3.9126687049865723,
+      "step": 1605
+    },
+    {
+      "epoch": 0.8616825556113062,
+      "grad_norm": 0.3932300699915947,
+      "learning_rate": 8.969845950089751e-07,
+      "logits/chosen": -0.7118608355522156,
+      "logits/rejected": -0.4848446249961853,
+      "logps/chosen": -4.688570022583008,
+      "logps/rejected": -5.197754859924316,
+      "loss": 0.0533,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.688570022583008,
+      "rewards/margins": 0.5091850161552429,
+      "rewards/rejected": -5.197754859924316,
+      "sft_loss": 4.411562919616699,
+      "step": 1610
+    },
+    {
+      "epoch": 0.8643585883927078,
+      "grad_norm": 0.5646465640543609,
+      "learning_rate": 8.960358353638526e-07,
+      "logits/chosen": -0.5377587080001831,
+      "logits/rejected": -0.3963084816932678,
+      "logps/chosen": -4.799404144287109,
+      "logps/rejected": -5.366022109985352,
+      "loss": 0.0528,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.799404144287109,
+      "rewards/margins": 0.5666177272796631,
+      "rewards/rejected": -5.366022109985352,
+      "sft_loss": 4.420165061950684,
+      "step": 1615
+    },
+    {
+      "epoch": 0.8670346211741093,
+      "grad_norm": 0.7546229368883803,
+      "learning_rate": 8.950832333773184e-07,
+      "logits/chosen": -0.5079709887504578,
+      "logits/rejected": -0.2918342649936676,
+      "logps/chosen": -4.613333702087402,
+      "logps/rejected": -4.944863319396973,
+      "loss": 0.0543,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.613333702087402,
+      "rewards/margins": 0.33153027296066284,
+      "rewards/rejected": -4.944863319396973,
+      "sft_loss": 4.2668962478637695,
+      "step": 1620
+    },
+    {
+      "epoch": 0.869710653955511,
+      "grad_norm": 1.0035032298444153,
+      "learning_rate": 8.941267982915213e-07,
+      "logits/chosen": -0.4892359673976898,
+      "logits/rejected": -0.41767483949661255,
+      "logps/chosen": -4.5956902503967285,
+      "logps/rejected": -4.8582048416137695,
+      "loss": 0.0555,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.5956902503967285,
+      "rewards/margins": 0.26251503825187683,
+      "rewards/rejected": -4.8582048416137695,
+      "sft_loss": 4.35836124420166,
+      "step": 1625
+    },
+    {
+      "epoch": 0.8723866867369126,
+      "grad_norm": 0.6730402571466032,
+      "learning_rate": 8.931665393857983e-07,
+      "logits/chosen": -0.5518749952316284,
+      "logits/rejected": -0.38204655051231384,
+      "logps/chosen": -4.6279096603393555,
+      "logps/rejected": -5.244035243988037,
+      "loss": 0.053,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.6279096603393555,
+      "rewards/margins": 0.6161251664161682,
+      "rewards/rejected": -5.244035243988037,
+      "sft_loss": 4.439146995544434,
+      "step": 1630
+    },
+    {
+      "epoch": 0.875062719518314,
+      "grad_norm": 0.4254278641604274,
+      "learning_rate": 8.922024659765861e-07,
+      "logits/chosen": -0.7097309827804565,
+      "logits/rejected": -0.5215593576431274,
+      "logps/chosen": -4.5009846687316895,
+      "logps/rejected": -5.05783748626709,
+      "loss": 0.053,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.5009846687316895,
+      "rewards/margins": 0.5568532347679138,
+      "rewards/rejected": -5.05783748626709,
+      "sft_loss": 4.314332962036133,
+      "step": 1635
+    },
+    {
+      "epoch": 0.8777387522997157,
+      "grad_norm": 0.7546877143418629,
+      "learning_rate": 8.912345874173288e-07,
+      "logits/chosen": -0.7383396625518799,
+      "logits/rejected": -0.551169753074646,
+      "logps/chosen": -4.638089656829834,
+      "logps/rejected": -5.158759117126465,
+      "loss": 0.0528,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.638089656829834,
+      "rewards/margins": 0.5206692814826965,
+      "rewards/rejected": -5.158759117126465,
+      "sft_loss": 4.348078727722168,
+      "step": 1640
+    },
+    {
+      "epoch": 0.8804147850811173,
+      "grad_norm": 0.6031239823905878,
+      "learning_rate": 8.902629130983885e-07,
+      "logits/chosen": -0.7252805829048157,
+      "logits/rejected": -0.6789249181747437,
+      "logps/chosen": -4.596409797668457,
+      "logps/rejected": -4.8939619064331055,
+      "loss": 0.0545,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -4.596409797668457,
+      "rewards/margins": 0.2975528836250305,
+      "rewards/rejected": -4.8939619064331055,
+      "sft_loss": 4.3586626052856445,
+      "step": 1645
+    },
+    {
+      "epoch": 0.8830908178625189,
+      "grad_norm": 0.8993478357101632,
+      "learning_rate": 8.892874524469537e-07,
+      "logits/chosen": -0.5916559100151062,
+      "logits/rejected": -0.5924795269966125,
+      "logps/chosen": -4.407802104949951,
+      "logps/rejected": -4.78812313079834,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.407802104949951,
+      "rewards/margins": 0.3803215026855469,
+      "rewards/rejected": -4.78812313079834,
+      "sft_loss": 4.095881462097168,
+      "step": 1650
+    },
+    {
+      "epoch": 0.8857668506439204,
+      "grad_norm": 0.3917673900254094,
+      "learning_rate": 8.883082149269478e-07,
+      "logits/chosen": -0.7587140798568726,
+      "logits/rejected": -0.6571011543273926,
+      "logps/chosen": -4.616636753082275,
+      "logps/rejected": -5.03781795501709,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.616636753082275,
+      "rewards/margins": 0.4211810231208801,
+      "rewards/rejected": -5.03781795501709,
+      "sft_loss": 4.352169990539551,
+      "step": 1655
+    },
+    {
+      "epoch": 0.888442883425322,
+      "grad_norm": 0.7381492187402137,
+      "learning_rate": 8.873252100389377e-07,
+      "logits/chosen": -0.7679041028022766,
+      "logits/rejected": -0.7441409230232239,
+      "logps/chosen": -4.568779945373535,
+      "logps/rejected": -4.98103141784668,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.568779945373535,
+      "rewards/margins": 0.41225147247314453,
+      "rewards/rejected": -4.98103141784668,
+      "sft_loss": 4.325723648071289,
+      "step": 1660
+    },
+    {
+      "epoch": 0.8911189162067236,
+      "grad_norm": 0.7190038370520179,
+      "learning_rate": 8.863384473200411e-07,
+      "logits/chosen": -0.5483480095863342,
+      "logits/rejected": -0.516470193862915,
+      "logps/chosen": -4.5403594970703125,
+      "logps/rejected": -4.815312385559082,
+      "loss": 0.0541,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -4.5403594970703125,
+      "rewards/margins": 0.27495282888412476,
+      "rewards/rejected": -4.815312385559082,
+      "sft_loss": 4.229251861572266,
+      "step": 1665
+    },
+    {
+      "epoch": 0.8937949489881251,
+      "grad_norm": 0.42330802181269084,
+      "learning_rate": 8.853479363438342e-07,
+      "logits/chosen": -0.5562044382095337,
+      "logits/rejected": -0.35739031434059143,
+      "logps/chosen": -4.675718784332275,
+      "logps/rejected": -5.102102279663086,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.675718784332275,
+      "rewards/margins": 0.4263841509819031,
+      "rewards/rejected": -5.102102279663086,
+      "sft_loss": 4.358043670654297,
+      "step": 1670
+    },
+    {
+      "epoch": 0.8964709817695267,
+      "grad_norm": 0.6266125822436,
+      "learning_rate": 8.843536867202588e-07,
+      "logits/chosen": -0.6752755045890808,
+      "logits/rejected": -0.4606241285800934,
+      "logps/chosen": -4.621033668518066,
+      "logps/rejected": -5.194036483764648,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.621033668518066,
+      "rewards/margins": 0.5730026960372925,
+      "rewards/rejected": -5.194036483764648,
+      "sft_loss": 4.461909294128418,
+      "step": 1675
+    },
+    {
+      "epoch": 0.8991470145509283,
+      "grad_norm": 0.529054946655314,
+      "learning_rate": 8.833557080955292e-07,
+      "logits/chosen": -0.7238949537277222,
+      "logits/rejected": -0.5741681456565857,
+      "logps/chosen": -4.534304618835449,
+      "logps/rejected": -4.89604377746582,
+      "loss": 0.0549,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.534304618835449,
+      "rewards/margins": 0.3617388606071472,
+      "rewards/rejected": -4.89604377746582,
+      "sft_loss": 4.328090190887451,
+      "step": 1680
+    },
+    {
+      "epoch": 0.9018230473323299,
+      "grad_norm": 0.4533672050451873,
+      "learning_rate": 8.823540101520381e-07,
+      "logits/chosen": -0.6906416416168213,
+      "logits/rejected": -0.44059187173843384,
+      "logps/chosen": -4.62170934677124,
+      "logps/rejected": -5.1143317222595215,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.62170934677124,
+      "rewards/margins": 0.49262285232543945,
+      "rewards/rejected": -5.1143317222595215,
+      "sft_loss": 4.356869220733643,
+      "step": 1685
+    },
+    {
+      "epoch": 0.9044990801137314,
+      "grad_norm": 0.47576777011310917,
+      "learning_rate": 8.813486026082637e-07,
+      "logits/chosen": -0.6736524701118469,
+      "logits/rejected": -0.4422256350517273,
+      "logps/chosen": -4.475247383117676,
+      "logps/rejected": -5.043217658996582,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.475247383117676,
+      "rewards/margins": 0.5679700970649719,
+      "rewards/rejected": -5.043217658996582,
+      "sft_loss": 4.141319274902344,
+      "step": 1690
+    },
+    {
+      "epoch": 0.907175112895133,
+      "grad_norm": 0.7702839097262766,
+      "learning_rate": 8.803394952186742e-07,
+      "logits/chosen": -0.6925588846206665,
+      "logits/rejected": -0.5139236450195312,
+      "logps/chosen": -4.497862815856934,
+      "logps/rejected": -4.9205546379089355,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.497862815856934,
+      "rewards/margins": 0.4226916432380676,
+      "rewards/rejected": -4.9205546379089355,
+      "sft_loss": 4.133333683013916,
+      "step": 1695
+    },
+    {
+      "epoch": 0.9098511456765346,
+      "grad_norm": 0.6048631380799233,
+      "learning_rate": 8.793266977736342e-07,
+      "logits/chosen": -0.5745486617088318,
+      "logits/rejected": -0.7098916172981262,
+      "logps/chosen": -4.716115474700928,
+      "logps/rejected": -4.911401748657227,
+      "loss": 0.0554,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": -4.716115474700928,
+      "rewards/margins": 0.19528572261333466,
+      "rewards/rejected": -4.911401748657227,
+      "sft_loss": 4.486597537994385,
+      "step": 1700
+    },
+    {
+      "epoch": 0.9125271784579361,
+      "grad_norm": 0.5319797114763031,
+      "learning_rate": 8.783102200993085e-07,
+      "logits/chosen": -0.5865694284439087,
+      "logits/rejected": -0.4831927418708801,
+      "logps/chosen": -4.619460105895996,
+      "logps/rejected": -5.135631084442139,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.619460105895996,
+      "rewards/margins": 0.5161705017089844,
+      "rewards/rejected": -5.135631084442139,
+      "sft_loss": 4.320430278778076,
+      "step": 1705
+    },
+    {
+      "epoch": 0.9152032112393377,
+      "grad_norm": 0.49342138650309186,
+      "learning_rate": 8.772900720575683e-07,
+      "logits/chosen": -0.5916703939437866,
+      "logits/rejected": -0.44493407011032104,
+      "logps/chosen": -4.688383102416992,
+      "logps/rejected": -5.003859519958496,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.688383102416992,
+      "rewards/margins": 0.3154762387275696,
+      "rewards/rejected": -5.003859519958496,
+      "sft_loss": 4.402092933654785,
+      "step": 1710
+    },
+    {
+      "epoch": 0.9178792440207393,
+      "grad_norm": 0.6350320047855338,
+      "learning_rate": 8.762662635458944e-07,
+      "logits/chosen": -0.6132705211639404,
+      "logits/rejected": -0.4131240248680115,
+      "logps/chosen": -4.4683003425598145,
+      "logps/rejected": -4.932827949523926,
+      "loss": 0.0528,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.4683003425598145,
+      "rewards/margins": 0.46452799439430237,
+      "rewards/rejected": -4.932827949523926,
+      "sft_loss": 4.192587375640869,
+      "step": 1715
+    },
+    {
+      "epoch": 0.9205552768021408,
+      "grad_norm": 0.677549094236319,
+      "learning_rate": 8.752388044972811e-07,
+      "logits/chosen": -0.4829626679420471,
+      "logits/rejected": -0.40973132848739624,
+      "logps/chosen": -4.489928722381592,
+      "logps/rejected": -4.995810508728027,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.489928722381592,
+      "rewards/margins": 0.5058815479278564,
+      "rewards/rejected": -4.995810508728027,
+      "sft_loss": 4.183096885681152,
+      "step": 1720
+    },
+    {
+      "epoch": 0.9232313095835424,
+      "grad_norm": 1.1901222995185778,
+      "learning_rate": 8.74207704880141e-07,
+      "logits/chosen": -0.46341466903686523,
+      "logits/rejected": -0.3583849370479584,
+      "logps/chosen": -4.485856533050537,
+      "logps/rejected": -5.075121879577637,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.485856533050537,
+      "rewards/margins": 0.5892654657363892,
+      "rewards/rejected": -5.075121879577637,
+      "sft_loss": 4.133608341217041,
+      "step": 1725
+    },
+    {
+      "epoch": 0.925907342364944,
+      "grad_norm": 0.7088852396963751,
+      "learning_rate": 8.731729746982068e-07,
+      "logits/chosen": -0.5381177663803101,
+      "logits/rejected": -0.5030028820037842,
+      "logps/chosen": -4.645955562591553,
+      "logps/rejected": -5.002853870391846,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.645955562591553,
+      "rewards/margins": 0.35689815878868103,
+      "rewards/rejected": -5.002853870391846,
+      "sft_loss": 4.37051248550415,
+      "step": 1730
+    },
+    {
+      "epoch": 0.9285833751463456,
+      "grad_norm": 0.5727877301628497,
+      "learning_rate": 8.721346239904355e-07,
+      "logits/chosen": -0.7422673106193542,
+      "logits/rejected": -0.4506010413169861,
+      "logps/chosen": -4.574130535125732,
+      "logps/rejected": -5.225555896759033,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.574130535125732,
+      "rewards/margins": 0.6514250636100769,
+      "rewards/rejected": -5.225555896759033,
+      "sft_loss": 4.303416728973389,
+      "step": 1735
+    },
+    {
+      "epoch": 0.9312594079277471,
+      "grad_norm": 0.7587320919835171,
+      "learning_rate": 8.710926628309101e-07,
+      "logits/chosen": -0.7008196115493774,
+      "logits/rejected": -0.4706336557865143,
+      "logps/chosen": -4.421854019165039,
+      "logps/rejected": -4.918593406677246,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.421854019165039,
+      "rewards/margins": 0.49673953652381897,
+      "rewards/rejected": -4.918593406677246,
+      "sft_loss": 4.130229473114014,
+      "step": 1740
+    },
+    {
+      "epoch": 0.9339354407091487,
+      "grad_norm": 0.4268332165756312,
+      "learning_rate": 8.700471013287424e-07,
+      "logits/chosen": -0.4801374077796936,
+      "logits/rejected": -0.528005063533783,
+      "logps/chosen": -4.628663063049316,
+      "logps/rejected": -4.9589996337890625,
+      "loss": 0.0539,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.628663063049316,
+      "rewards/margins": 0.3303371071815491,
+      "rewards/rejected": -4.9589996337890625,
+      "sft_loss": 4.288285255432129,
+      "step": 1745
+    },
+    {
+      "epoch": 0.9366114734905503,
+      "grad_norm": 0.5580153572580532,
+      "learning_rate": 8.689979496279746e-07,
+      "logits/chosen": -0.6604480743408203,
+      "logits/rejected": -0.6128295660018921,
+      "logps/chosen": -4.892617225646973,
+      "logps/rejected": -5.17832088470459,
+      "loss": 0.0556,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -4.892617225646973,
+      "rewards/margins": 0.2857045531272888,
+      "rewards/rejected": -5.17832088470459,
+      "sft_loss": 4.655879974365234,
+      "step": 1750
+    },
+    {
+      "epoch": 0.9392875062719518,
+      "grad_norm": 0.5349767701486052,
+      "learning_rate": 8.679452179074811e-07,
+      "logits/chosen": -0.6491625308990479,
+      "logits/rejected": -0.5268028974533081,
+      "logps/chosen": -4.43688440322876,
+      "logps/rejected": -4.887360095977783,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.43688440322876,
+      "rewards/margins": 0.4504762291908264,
+      "rewards/rejected": -4.887360095977783,
+      "sft_loss": 4.223998069763184,
+      "step": 1755
+    },
+    {
+      "epoch": 0.9419635390533534,
+      "grad_norm": 0.7804616441099496,
+      "learning_rate": 8.668889163808698e-07,
+      "logits/chosen": -0.5952499508857727,
+      "logits/rejected": -0.4252048432826996,
+      "logps/chosen": -4.258835792541504,
+      "logps/rejected": -4.69142484664917,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.258835792541504,
+      "rewards/margins": 0.4325889050960541,
+      "rewards/rejected": -4.69142484664917,
+      "sft_loss": 4.0244221687316895,
+      "step": 1760
+    },
+    {
+      "epoch": 0.944639571834755,
+      "grad_norm": 0.6776172207965506,
+      "learning_rate": 8.658290552963827e-07,
+      "logits/chosen": -0.6151505708694458,
+      "logits/rejected": -0.5384581685066223,
+      "logps/chosen": -4.684379577636719,
+      "logps/rejected": -5.083588123321533,
+      "loss": 0.0543,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.684379577636719,
+      "rewards/margins": 0.39920786023139954,
+      "rewards/rejected": -5.083588123321533,
+      "sft_loss": 4.437235355377197,
+      "step": 1765
+    },
+    {
+      "epoch": 0.9473156046161565,
+      "grad_norm": 0.5500351388760678,
+      "learning_rate": 8.647656449367966e-07,
+      "logits/chosen": -0.6593262553215027,
+      "logits/rejected": -0.46565479040145874,
+      "logps/chosen": -4.797619342803955,
+      "logps/rejected": -5.164062023162842,
+      "loss": 0.054,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -4.797619342803955,
+      "rewards/margins": 0.36644285917282104,
+      "rewards/rejected": -5.164062023162842,
+      "sft_loss": 4.57346248626709,
+      "step": 1770
+    },
+    {
+      "epoch": 0.9499916373975581,
+      "grad_norm": 0.45624990753318817,
+      "learning_rate": 8.636986956193235e-07,
+      "logits/chosen": -0.654018759727478,
+      "logits/rejected": -0.48784175515174866,
+      "logps/chosen": -4.523144721984863,
+      "logps/rejected": -5.065670490264893,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.523144721984863,
+      "rewards/margins": 0.5425251126289368,
+      "rewards/rejected": -5.065670490264893,
+      "sft_loss": 4.223518371582031,
+      "step": 1775
+    },
+    {
+      "epoch": 0.9526676701789597,
+      "grad_norm": 0.7841405500905427,
+      "learning_rate": 8.626282176955104e-07,
+      "logits/chosen": -0.5684636831283569,
+      "logits/rejected": -0.4489496648311615,
+      "logps/chosen": -4.506833076477051,
+      "logps/rejected": -5.011999607086182,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.506833076477051,
+      "rewards/margins": 0.5051662921905518,
+      "rewards/rejected": -5.011999607086182,
+      "sft_loss": 4.277122497558594,
+      "step": 1780
+    },
+    {
+      "epoch": 0.9553437029603613,
+      "grad_norm": 0.5493539692038344,
+      "learning_rate": 8.615542215511389e-07,
+      "logits/chosen": -0.49908360838890076,
+      "logits/rejected": -0.44331297278404236,
+      "logps/chosen": -4.55692195892334,
+      "logps/rejected": -4.80039644241333,
+      "loss": 0.0546,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -4.55692195892334,
+      "rewards/margins": 0.24347443878650665,
+      "rewards/rejected": -4.80039644241333,
+      "sft_loss": 4.2864179611206055,
+      "step": 1785
+    },
+    {
+      "epoch": 0.9580197357417628,
+      "grad_norm": 0.4241587327498337,
+      "learning_rate": 8.604767176061241e-07,
+      "logits/chosen": -0.5102593302726746,
+      "logits/rejected": -0.3818404972553253,
+      "logps/chosen": -4.6066484451293945,
+      "logps/rejected": -5.100188732147217,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.6066484451293945,
+      "rewards/margins": 0.49354037642478943,
+      "rewards/rejected": -5.100188732147217,
+      "sft_loss": 4.308642864227295,
+      "step": 1790
+    },
+    {
+      "epoch": 0.9606957685231644,
+      "grad_norm": 0.4506243224072321,
+      "learning_rate": 8.593957163144141e-07,
+      "logits/chosen": -0.6658445596694946,
+      "logits/rejected": -0.4980512261390686,
+      "logps/chosen": -4.553849220275879,
+      "logps/rejected": -5.111265659332275,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.553849220275879,
+      "rewards/margins": 0.5574164986610413,
+      "rewards/rejected": -5.111265659332275,
+      "sft_loss": 4.330379486083984,
+      "step": 1795
+    },
+    {
+      "epoch": 0.963371801304566,
+      "grad_norm": 0.6366553615073793,
+      "learning_rate": 8.58311228163888e-07,
+      "logits/chosen": -0.5685151815414429,
+      "logits/rejected": -0.5260004997253418,
+      "logps/chosen": -4.504071235656738,
+      "logps/rejected": -4.883088111877441,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.504071235656738,
+      "rewards/margins": 0.379016637802124,
+      "rewards/rejected": -4.883088111877441,
+      "sft_loss": 4.236390590667725,
+      "step": 1800
+    },
+    {
+      "epoch": 0.9660478340859675,
+      "grad_norm": 0.5830645566330016,
+      "learning_rate": 8.57223263676255e-07,
+      "logits/chosen": -0.7408885359764099,
+      "logits/rejected": -0.6090444326400757,
+      "logps/chosen": -4.397838592529297,
+      "logps/rejected": -5.064319133758545,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.397838592529297,
+      "rewards/margins": 0.666480541229248,
+      "rewards/rejected": -5.064319133758545,
+      "sft_loss": 4.16245174407959,
+      "step": 1805
+    },
+    {
+      "epoch": 0.9687238668673691,
+      "grad_norm": 0.6669481125026742,
+      "learning_rate": 8.561318334069511e-07,
+      "logits/chosen": -0.6110736727714539,
+      "logits/rejected": -0.47679129242897034,
+      "logps/chosen": -4.537700653076172,
+      "logps/rejected": -5.0156378746032715,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.537700653076172,
+      "rewards/margins": 0.4779374599456787,
+      "rewards/rejected": -5.0156378746032715,
+      "sft_loss": 4.280492305755615,
+      "step": 1810
+    },
+    {
+      "epoch": 0.9713998996487707,
+      "grad_norm": 0.49609737085138966,
+      "learning_rate": 8.550369479450375e-07,
+      "logits/chosen": -0.5063800811767578,
+      "logits/rejected": -0.38122016191482544,
+      "logps/chosen": -4.3638410568237305,
+      "logps/rejected": -4.9832963943481445,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.3638410568237305,
+      "rewards/margins": 0.6194550395011902,
+      "rewards/rejected": -4.9832963943481445,
+      "sft_loss": 4.111771583557129,
+      "step": 1815
+    },
+    {
+      "epoch": 0.9740759324301723,
+      "grad_norm": 0.541835888004766,
+      "learning_rate": 8.539386179130977e-07,
+      "logits/chosen": -0.4187532067298889,
+      "logits/rejected": -0.39255669713020325,
+      "logps/chosen": -4.4981207847595215,
+      "logps/rejected": -4.965208530426025,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.4981207847595215,
+      "rewards/margins": 0.46708765625953674,
+      "rewards/rejected": -4.965208530426025,
+      "sft_loss": 4.165301322937012,
+      "step": 1820
+    },
+    {
+      "epoch": 0.9767519652115738,
+      "grad_norm": 0.447270816688456,
+      "learning_rate": 8.528368539671347e-07,
+      "logits/chosen": -0.48422351479530334,
+      "logits/rejected": -0.27786120772361755,
+      "logps/chosen": -4.647488594055176,
+      "logps/rejected": -5.433849334716797,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.647488594055176,
+      "rewards/margins": 0.7863603234291077,
+      "rewards/rejected": -5.433849334716797,
+      "sft_loss": 4.390246868133545,
+      "step": 1825
+    },
+    {
+      "epoch": 0.9794279979929754,
+      "grad_norm": 0.6637116050670254,
+      "learning_rate": 8.51731666796467e-07,
+      "logits/chosen": -0.30430883169174194,
+      "logits/rejected": -0.2598590850830078,
+      "logps/chosen": -4.423822402954102,
+      "logps/rejected": -4.9025678634643555,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.423822402954102,
+      "rewards/margins": 0.47874531149864197,
+      "rewards/rejected": -4.9025678634643555,
+      "sft_loss": 4.121176719665527,
+      "step": 1830
+    },
+    {
+      "epoch": 0.982104030774377,
+      "grad_norm": 0.5088849120728498,
+      "learning_rate": 8.506230671236254e-07,
+      "logits/chosen": -0.44984373450279236,
+      "logits/rejected": -0.3657299876213074,
+      "logps/chosen": -4.600604057312012,
+      "logps/rejected": -5.050408840179443,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.600604057312012,
+      "rewards/margins": 0.4498043954372406,
+      "rewards/rejected": -5.050408840179443,
+      "sft_loss": 4.382790565490723,
+      "step": 1835
+    },
+    {
+      "epoch": 0.9847800635557785,
+      "grad_norm": 0.4254169373696342,
+      "learning_rate": 8.495110657042488e-07,
+      "logits/chosen": -0.48050135374069214,
+      "logits/rejected": -0.2589623034000397,
+      "logps/chosen": -4.520174980163574,
+      "logps/rejected": -5.064302921295166,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.520174980163574,
+      "rewards/margins": 0.5441280007362366,
+      "rewards/rejected": -5.064302921295166,
+      "sft_loss": 4.346188545227051,
+      "step": 1840
+    },
+    {
+      "epoch": 0.9874560963371801,
+      "grad_norm": 0.5339016448514589,
+      "learning_rate": 8.483956733269799e-07,
+      "logits/chosen": -0.4132794439792633,
+      "logits/rejected": -0.33570170402526855,
+      "logps/chosen": -4.530377388000488,
+      "logps/rejected": -4.925265312194824,
+      "loss": 0.0528,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.530377388000488,
+      "rewards/margins": 0.39488789439201355,
+      "rewards/rejected": -4.925265312194824,
+      "sft_loss": 4.196063041687012,
+      "step": 1845
+    },
+    {
+      "epoch": 0.9901321291185817,
+      "grad_norm": 0.39421322656444285,
+      "learning_rate": 8.472769008133602e-07,
+      "logits/chosen": -0.7014116048812866,
+      "logits/rejected": -0.5663411021232605,
+      "logps/chosen": -4.50463342666626,
+      "logps/rejected": -4.918387413024902,
+      "loss": 0.0533,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.50463342666626,
+      "rewards/margins": 0.41375404596328735,
+      "rewards/rejected": -4.918387413024902,
+      "sft_loss": 4.1834001541137695,
+      "step": 1850
+    },
+    {
+      "epoch": 0.9928081618999832,
+      "grad_norm": 0.3506549867803869,
+      "learning_rate": 8.461547590177259e-07,
+      "logits/chosen": -0.614751935005188,
+      "logits/rejected": -0.4481315016746521,
+      "logps/chosen": -4.661157131195068,
+      "logps/rejected": -5.182382583618164,
+      "loss": 0.0506,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.661157131195068,
+      "rewards/margins": 0.5212251543998718,
+      "rewards/rejected": -5.182382583618164,
+      "sft_loss": 4.169110298156738,
+      "step": 1855
+    },
+    {
+      "epoch": 0.9954841946813848,
+      "grad_norm": 0.5869023743074764,
+      "learning_rate": 8.450292588271014e-07,
+      "logits/chosen": -0.6788508892059326,
+      "logits/rejected": -0.5645800232887268,
+      "logps/chosen": -4.581757545471191,
+      "logps/rejected": -5.108978271484375,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.581757545471191,
+      "rewards/margins": 0.527221143245697,
+      "rewards/rejected": -5.108978271484375,
+      "sft_loss": 4.166073799133301,
+      "step": 1860
+    },
+    {
+      "epoch": 0.9981602274627864,
+      "grad_norm": 0.515411170688192,
+      "learning_rate": 8.439004111610945e-07,
+      "logits/chosen": -0.6239068508148193,
+      "logits/rejected": -0.5635244846343994,
+      "logps/chosen": -4.709362983703613,
+      "logps/rejected": -5.0980119705200195,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.709362983703613,
+      "rewards/margins": 0.3886488378047943,
+      "rewards/rejected": -5.0980119705200195,
+      "sft_loss": 4.291287422180176,
+      "step": 1865
+    },
+    {
+      "epoch": 1.000836260244188,
+      "grad_norm": 0.33794993218177366,
+      "learning_rate": 8.427682269717901e-07,
+      "logits/chosen": -0.6590108275413513,
+      "logits/rejected": -0.5278070569038391,
+      "logps/chosen": -4.459053039550781,
+      "logps/rejected": -5.024530410766602,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.459053039550781,
+      "rewards/margins": 0.5654768943786621,
+      "rewards/rejected": -5.024530410766602,
+      "sft_loss": 4.135066032409668,
+      "step": 1870
+    },
+    {
+      "epoch": 1.0035122930255895,
+      "grad_norm": 0.5783993800466185,
+      "learning_rate": 8.416327172436446e-07,
+      "logits/chosen": -0.8413249254226685,
+      "logits/rejected": -0.662590742111206,
+      "logps/chosen": -4.73525333404541,
+      "logps/rejected": -5.038640975952148,
+      "loss": 0.0539,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.73525333404541,
+      "rewards/margins": 0.30338791012763977,
+      "rewards/rejected": -5.038640975952148,
+      "sft_loss": 4.415173053741455,
+      "step": 1875
+    },
+    {
+      "epoch": 1.0061883258069912,
+      "grad_norm": 0.41696547083413266,
+      "learning_rate": 8.404938929933778e-07,
+      "logits/chosen": -0.6675068736076355,
+      "logits/rejected": -0.5564897060394287,
+      "logps/chosen": -4.614431858062744,
+      "logps/rejected": -5.160069465637207,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.614431858062744,
+      "rewards/margins": 0.5456372499465942,
+      "rewards/rejected": -5.160069465637207,
+      "sft_loss": 4.223735332489014,
+      "step": 1880
+    },
+    {
+      "epoch": 1.0088643585883927,
+      "grad_norm": 0.36499730825474014,
+      "learning_rate": 8.39351765269868e-07,
+      "logits/chosen": -0.6835610866546631,
+      "logits/rejected": -0.6262503862380981,
+      "logps/chosen": -4.598526477813721,
+      "logps/rejected": -4.998991012573242,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.598526477813721,
+      "rewards/margins": 0.4004649221897125,
+      "rewards/rejected": -4.998991012573242,
+      "sft_loss": 4.258669376373291,
+      "step": 1885
+    },
+    {
+      "epoch": 1.0115403913697942,
+      "grad_norm": 0.8910511813148269,
+      "learning_rate": 8.382063451540431e-07,
+      "logits/chosen": -0.6892908811569214,
+      "logits/rejected": -0.43145257234573364,
+      "logps/chosen": -4.353154182434082,
+      "logps/rejected": -4.833468914031982,
+      "loss": 0.0528,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.353154182434082,
+      "rewards/margins": 0.4803146421909332,
+      "rewards/rejected": -4.833468914031982,
+      "sft_loss": 4.148338317871094,
+      "step": 1890
+    },
+    {
+      "epoch": 1.014216424151196,
+      "grad_norm": 0.4638056210783794,
+      "learning_rate": 8.370576437587742e-07,
+      "logits/chosen": -0.6934480667114258,
+      "logits/rejected": -0.6940955519676208,
+      "logps/chosen": -4.60874080657959,
+      "logps/rejected": -4.962925434112549,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.60874080657959,
+      "rewards/margins": 0.3541843593120575,
+      "rewards/rejected": -4.962925434112549,
+      "sft_loss": 4.259464263916016,
+      "step": 1895
+    },
+    {
+      "epoch": 1.0168924569325974,
+      "grad_norm": 0.5627287270307829,
+      "learning_rate": 8.359056722287674e-07,
+      "logits/chosen": -0.9412443041801453,
+      "logits/rejected": -0.5999225378036499,
+      "logps/chosen": -4.6828742027282715,
+      "logps/rejected": -5.225579261779785,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.6828742027282715,
+      "rewards/margins": 0.5427041053771973,
+      "rewards/rejected": -5.225579261779785,
+      "sft_loss": 4.4391069412231445,
+      "step": 1900
+    },
+    {
+      "epoch": 1.019568489713999,
+      "grad_norm": 0.9109919294767956,
+      "learning_rate": 8.347504417404553e-07,
+      "logits/chosen": -0.6489143967628479,
+      "logits/rejected": -0.5207082629203796,
+      "logps/chosen": -4.5325493812561035,
+      "logps/rejected": -4.916535377502441,
+      "loss": 0.0541,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.5325493812561035,
+      "rewards/margins": 0.3839864730834961,
+      "rewards/rejected": -4.916535377502441,
+      "sft_loss": 4.197742938995361,
+      "step": 1905
+    },
+    {
+      "epoch": 1.0222445224954007,
+      "grad_norm": 0.6385895149089236,
+      "learning_rate": 8.335919635018893e-07,
+      "logits/chosen": -0.8187878727912903,
+      "logits/rejected": -0.692855954170227,
+      "logps/chosen": -4.600462436676025,
+      "logps/rejected": -4.971292495727539,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.600462436676025,
+      "rewards/margins": 0.37083056569099426,
+      "rewards/rejected": -4.971292495727539,
+      "sft_loss": 4.280551910400391,
+      "step": 1910
+    },
+    {
+      "epoch": 1.0249205552768021,
+      "grad_norm": 0.3806885389438999,
+      "learning_rate": 8.324302487526303e-07,
+      "logits/chosen": -0.8339389562606812,
+      "logits/rejected": -0.7144932150840759,
+      "logps/chosen": -4.63545036315918,
+      "logps/rejected": -5.065112113952637,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.63545036315918,
+      "rewards/margins": 0.42966145277023315,
+      "rewards/rejected": -5.065112113952637,
+      "sft_loss": 4.315610885620117,
+      "step": 1915
+    },
+    {
+      "epoch": 1.0275965880582036,
+      "grad_norm": 0.341027383832509,
+      "learning_rate": 8.312653087636398e-07,
+      "logits/chosen": -0.7597075700759888,
+      "logits/rejected": -0.6759294271469116,
+      "logps/chosen": -4.549553871154785,
+      "logps/rejected": -5.001124858856201,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.549553871154785,
+      "rewards/margins": 0.4515710771083832,
+      "rewards/rejected": -5.001124858856201,
+      "sft_loss": 4.2656426429748535,
+      "step": 1920
+    },
+    {
+      "epoch": 1.0302726208396054,
+      "grad_norm": 0.43588735059126427,
+      "learning_rate": 8.300971548371711e-07,
+      "logits/chosen": -0.8216022253036499,
+      "logits/rejected": -0.5611480474472046,
+      "logps/chosen": -4.342541217803955,
+      "logps/rejected": -4.886130332946777,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.342541217803955,
+      "rewards/margins": 0.5435891151428223,
+      "rewards/rejected": -4.886130332946777,
+      "sft_loss": 4.013619899749756,
+      "step": 1925
+    },
+    {
+      "epoch": 1.0329486536210069,
+      "grad_norm": 0.5927348233728311,
+      "learning_rate": 8.289257983066582e-07,
+      "logits/chosen": -0.7210015058517456,
+      "logits/rejected": -0.5344971418380737,
+      "logps/chosen": -4.541009426116943,
+      "logps/rejected": -5.115367412567139,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.541009426116943,
+      "rewards/margins": 0.5743576288223267,
+      "rewards/rejected": -5.115367412567139,
+      "sft_loss": 4.244875431060791,
+      "step": 1930
+    },
+    {
+      "epoch": 1.0356246864024083,
+      "grad_norm": 0.5215888392879336,
+      "learning_rate": 8.277512505366077e-07,
+      "logits/chosen": -0.7188865542411804,
+      "logits/rejected": -0.45655688643455505,
+      "logps/chosen": -4.600197792053223,
+      "logps/rejected": -5.111001014709473,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.600197792053223,
+      "rewards/margins": 0.5108031034469604,
+      "rewards/rejected": -5.111001014709473,
+      "sft_loss": 4.36060905456543,
+      "step": 1935
+    },
+    {
+      "epoch": 1.03830071918381,
+      "grad_norm": 0.7020676703184485,
+      "learning_rate": 8.265735229224868e-07,
+      "logits/chosen": -0.5967748165130615,
+      "logits/rejected": -0.5061367154121399,
+      "logps/chosen": -4.4757161140441895,
+      "logps/rejected": -5.118129253387451,
+      "loss": 0.051,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.4757161140441895,
+      "rewards/margins": 0.6424133777618408,
+      "rewards/rejected": -5.118129253387451,
+      "sft_loss": 4.093837261199951,
+      "step": 1940
+    },
+    {
+      "epoch": 1.0409767519652116,
+      "grad_norm": 0.4171584848961562,
+      "learning_rate": 8.253926268906144e-07,
+      "logits/chosen": -0.6828538179397583,
+      "logits/rejected": -0.5208674073219299,
+      "logps/chosen": -4.653429985046387,
+      "logps/rejected": -5.346189498901367,
+      "loss": 0.052,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.653429985046387,
+      "rewards/margins": 0.6927592158317566,
+      "rewards/rejected": -5.346189498901367,
+      "sft_loss": 4.30237340927124,
+      "step": 1945
+    },
+    {
+      "epoch": 1.043652784746613,
+      "grad_norm": 0.3546205575640321,
+      "learning_rate": 8.242085738980487e-07,
+      "logits/chosen": -0.48705941438674927,
+      "logits/rejected": -0.22814413905143738,
+      "logps/chosen": -4.460116863250732,
+      "logps/rejected": -5.026163101196289,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.460116863250732,
+      "rewards/margins": 0.5660470724105835,
+      "rewards/rejected": -5.026163101196289,
+      "sft_loss": 4.1621503829956055,
+      "step": 1950
+    },
+    {
+      "epoch": 1.0463288175280148,
+      "grad_norm": 0.48678173573302397,
+      "learning_rate": 8.230213754324772e-07,
+      "logits/chosen": -0.6345669031143188,
+      "logits/rejected": -0.5644513964653015,
+      "logps/chosen": -4.458856582641602,
+      "logps/rejected": -4.922835826873779,
+      "loss": 0.0533,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.458856582641602,
+      "rewards/margins": 0.4639790952205658,
+      "rewards/rejected": -4.922835826873779,
+      "sft_loss": 4.2498321533203125,
+      "step": 1955
+    },
+    {
+      "epoch": 1.0490048503094163,
+      "grad_norm": 0.3506005641509898,
+      "learning_rate": 8.218310430121045e-07,
+      "logits/chosen": -0.5607764720916748,
+      "logits/rejected": -0.5480314493179321,
+      "logps/chosen": -4.631636619567871,
+      "logps/rejected": -4.987282752990723,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.631636619567871,
+      "rewards/margins": 0.3556460738182068,
+      "rewards/rejected": -4.987282752990723,
+      "sft_loss": 4.387399673461914,
+      "step": 1960
+    },
+    {
+      "epoch": 1.051680883090818,
+      "grad_norm": 0.596663355978495,
+      "learning_rate": 8.20637588185541e-07,
+      "logits/chosen": -0.5426192879676819,
+      "logits/rejected": -0.4666765332221985,
+      "logps/chosen": -4.4706830978393555,
+      "logps/rejected": -5.0566205978393555,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.4706830978393555,
+      "rewards/margins": 0.5859369039535522,
+      "rewards/rejected": -5.0566205978393555,
+      "sft_loss": 4.242679595947266,
+      "step": 1965
+    },
+    {
+      "epoch": 1.0543569158722195,
+      "grad_norm": 0.4693059687739037,
+      "learning_rate": 8.194410225316906e-07,
+      "logits/chosen": -0.5588054656982422,
+      "logits/rejected": -0.37931281328201294,
+      "logps/chosen": -4.505526065826416,
+      "logps/rejected": -5.0229082107543945,
+      "loss": 0.052,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.505526065826416,
+      "rewards/margins": 0.5173822641372681,
+      "rewards/rejected": -5.0229082107543945,
+      "sft_loss": 4.1600446701049805,
+      "step": 1970
+    },
+    {
+      "epoch": 1.057032948653621,
+      "grad_norm": 0.41002932138245163,
+      "learning_rate": 8.182413576596385e-07,
+      "logits/chosen": -0.3452656865119934,
+      "logits/rejected": -0.3023213744163513,
+      "logps/chosen": -4.485015869140625,
+      "logps/rejected": -5.055952548980713,
+      "loss": 0.0528,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.485015869140625,
+      "rewards/margins": 0.5709371566772461,
+      "rewards/rejected": -5.055952548980713,
+      "sft_loss": 4.246888637542725,
+      "step": 1975
+    },
+    {
+      "epoch": 1.0597089814350227,
+      "grad_norm": 0.378319037298122,
+      "learning_rate": 8.170386052085389e-07,
+      "logits/chosen": -0.4670163691043854,
+      "logits/rejected": -0.34650593996047974,
+      "logps/chosen": -4.661363124847412,
+      "logps/rejected": -5.084620952606201,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.661363124847412,
+      "rewards/margins": 0.4232582449913025,
+      "rewards/rejected": -5.084620952606201,
+      "sft_loss": 4.353013038635254,
+      "step": 1980
+    },
+    {
+      "epoch": 1.0623850142164242,
+      "grad_norm": 0.45885368018202954,
+      "learning_rate": 8.158327768475008e-07,
+      "logits/chosen": -0.582227349281311,
+      "logits/rejected": -0.4262891709804535,
+      "logps/chosen": -4.639840126037598,
+      "logps/rejected": -5.0006208419799805,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.639840126037598,
+      "rewards/margins": 0.36078134179115295,
+      "rewards/rejected": -5.0006208419799805,
+      "sft_loss": 4.249001979827881,
+      "step": 1985
+    },
+    {
+      "epoch": 1.0650610469978257,
+      "grad_norm": 0.44103614728408597,
+      "learning_rate": 8.146238842754767e-07,
+      "logits/chosen": -0.6692488193511963,
+      "logits/rejected": -0.5590274930000305,
+      "logps/chosen": -4.661190986633301,
+      "logps/rejected": -5.144232273101807,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.661190986633301,
+      "rewards/margins": 0.4830414652824402,
+      "rewards/rejected": -5.144232273101807,
+      "sft_loss": 4.309359073638916,
+      "step": 1990
+    },
+    {
+      "epoch": 1.0677370797792274,
+      "grad_norm": 0.6839720555757945,
+      "learning_rate": 8.134119392211476e-07,
+      "logits/chosen": -0.5654734969139099,
+      "logits/rejected": -0.3846448063850403,
+      "logps/chosen": -4.629153251647949,
+      "logps/rejected": -5.220393180847168,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.629153251647949,
+      "rewards/margins": 0.5912397503852844,
+      "rewards/rejected": -5.220393180847168,
+      "sft_loss": 4.2791266441345215,
+      "step": 1995
+    },
+    {
+      "epoch": 1.0704131125606289,
+      "grad_norm": 0.33463115623057726,
+      "learning_rate": 8.121969534428094e-07,
+      "logits/chosen": -0.8162349462509155,
+      "logits/rejected": -0.6354729533195496,
+      "logps/chosen": -4.752208232879639,
+      "logps/rejected": -5.289694786071777,
+      "loss": 0.0533,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.752208232879639,
+      "rewards/margins": 0.537487268447876,
+      "rewards/rejected": -5.289694786071777,
+      "sft_loss": 4.394679546356201,
+      "step": 2000
+    },
+    {
+      "epoch": 1.0704131125606289,
+      "eval_logits/chosen": -0.11113026738166809,
+      "eval_logits/rejected": -0.025689993053674698,
+      "eval_logps/chosen": -4.431623458862305,
+      "eval_logps/rejected": -4.957681179046631,
+      "eval_loss": 0.05090980976819992,
+      "eval_rewards/accuracies": 0.6483679413795471,
+      "eval_rewards/chosen": -4.431623458862305,
+      "eval_rewards/margins": 0.5260574221611023,
+      "eval_rewards/rejected": -4.957681179046631,
+      "eval_runtime": 43.8103,
+      "eval_samples_per_second": 30.701,
+      "eval_sft_loss": 3.9533112049102783,
+      "eval_steps_per_second": 7.692,
+      "step": 2000
+    },
+    {
+      "epoch": 1.0730891453420304,
+      "grad_norm": 0.5058588176420259,
+      "learning_rate": 8.109789387282599e-07,
+      "logits/chosen": -0.6308828592300415,
+      "logits/rejected": -0.6025933623313904,
+      "logps/chosen": -4.522896766662598,
+      "logps/rejected": -4.889641284942627,
+      "loss": 0.0533,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.522896766662598,
+      "rewards/margins": 0.3667449951171875,
+      "rewards/rejected": -4.889641284942627,
+      "sft_loss": 4.171763896942139,
+      "step": 2005
+    },
+    {
+      "epoch": 1.075765178123432,
+      "grad_norm": 0.5743062543345286,
+      "learning_rate": 8.097579068946827e-07,
+      "logits/chosen": -0.742906391620636,
+      "logits/rejected": -0.6353303790092468,
+      "logps/chosen": -4.552280426025391,
+      "logps/rejected": -5.1396074295043945,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.552280426025391,
+      "rewards/margins": 0.5873273611068726,
+      "rewards/rejected": -5.1396074295043945,
+      "sft_loss": 4.1930928230285645,
+      "step": 2010
+    },
+    {
+      "epoch": 1.0784412109048336,
+      "grad_norm": 0.5980063663594536,
+      "learning_rate": 8.085338697885344e-07,
+      "logits/chosen": -0.6874445080757141,
+      "logits/rejected": -0.5575221180915833,
+      "logps/chosen": -4.699803352355957,
+      "logps/rejected": -5.109753608703613,
+      "loss": 0.053,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.699803352355957,
+      "rewards/margins": 0.4099506735801697,
+      "rewards/rejected": -5.109753608703613,
+      "sft_loss": 4.357962131500244,
+      "step": 2015
+    },
+    {
+      "epoch": 1.081117243686235,
+      "grad_norm": 0.5994974306382295,
+      "learning_rate": 8.073068392854282e-07,
+      "logits/chosen": -0.8651019930839539,
+      "logits/rejected": -0.6130464673042297,
+      "logps/chosen": -4.371953010559082,
+      "logps/rejected": -4.994250297546387,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.371953010559082,
+      "rewards/margins": 0.622296929359436,
+      "rewards/rejected": -4.994250297546387,
+      "sft_loss": 4.145905494689941,
+      "step": 2020
+    },
+    {
+      "epoch": 1.0837932764676368,
+      "grad_norm": 0.7043192940476449,
+      "learning_rate": 8.060768272900193e-07,
+      "logits/chosen": -0.5605143308639526,
+      "logits/rejected": -0.38041457533836365,
+      "logps/chosen": -4.351266860961914,
+      "logps/rejected": -4.961604595184326,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.351266860961914,
+      "rewards/margins": 0.6103374361991882,
+      "rewards/rejected": -4.961604595184326,
+      "sft_loss": 4.081421852111816,
+      "step": 2025
+    },
+    {
+      "epoch": 1.0864693092490383,
+      "grad_norm": 0.3620270131811262,
+      "learning_rate": 8.0484384573589e-07,
+      "logits/chosen": -0.5562458038330078,
+      "logits/rejected": -0.5688826441764832,
+      "logps/chosen": -4.398291110992432,
+      "logps/rejected": -4.868541717529297,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.398291110992432,
+      "rewards/margins": 0.4702511727809906,
+      "rewards/rejected": -4.868541717529297,
+      "sft_loss": 4.141671180725098,
+      "step": 2030
+    },
+    {
+      "epoch": 1.0891453420304398,
+      "grad_norm": 0.43748978323171417,
+      "learning_rate": 8.03607906585432e-07,
+      "logits/chosen": -0.5927788019180298,
+      "logits/rejected": -0.4347759783267975,
+      "logps/chosen": -4.721604347229004,
+      "logps/rejected": -5.251805305480957,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.721604347229004,
+      "rewards/margins": 0.53020179271698,
+      "rewards/rejected": -5.251805305480957,
+      "sft_loss": 4.432992935180664,
+      "step": 2035
+    },
+    {
+      "epoch": 1.0918213748118415,
+      "grad_norm": 0.47166444822286063,
+      "learning_rate": 8.023690218297329e-07,
+      "logits/chosen": -0.5683757066726685,
+      "logits/rejected": -0.5649135708808899,
+      "logps/chosen": -4.666032314300537,
+      "logps/rejected": -5.096746921539307,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.666032314300537,
+      "rewards/margins": 0.4307146668434143,
+      "rewards/rejected": -5.096746921539307,
+      "sft_loss": 4.308411598205566,
+      "step": 2040
+    },
+    {
+      "epoch": 1.094497407593243,
+      "grad_norm": 1.2425475829574255,
+      "learning_rate": 8.01127203488458e-07,
+      "logits/chosen": -0.49305790662765503,
+      "logits/rejected": -0.4600808024406433,
+      "logps/chosen": -4.512884616851807,
+      "logps/rejected": -5.031808376312256,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.512884616851807,
+      "rewards/margins": 0.5189234614372253,
+      "rewards/rejected": -5.031808376312256,
+      "sft_loss": 4.13416862487793,
+      "step": 2045
+    },
+    {
+      "epoch": 1.0971734403746445,
+      "grad_norm": 0.4847523769952426,
+      "learning_rate": 7.998824636097339e-07,
+      "logits/chosen": -0.6031547784805298,
+      "logits/rejected": -0.4596533179283142,
+      "logps/chosen": -4.451432228088379,
+      "logps/rejected": -4.968292713165283,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.451432228088379,
+      "rewards/margins": 0.5168604254722595,
+      "rewards/rejected": -4.968292713165283,
+      "sft_loss": 4.238602638244629,
+      "step": 2050
+    },
+    {
+      "epoch": 1.0998494731560462,
+      "grad_norm": 0.5885884487554622,
+      "learning_rate": 7.986348142700328e-07,
+      "logits/chosen": -0.7163101434707642,
+      "logits/rejected": -0.5847815275192261,
+      "logps/chosen": -4.645443916320801,
+      "logps/rejected": -5.1462788581848145,
+      "loss": 0.053,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.645443916320801,
+      "rewards/margins": 0.5008349418640137,
+      "rewards/rejected": -5.1462788581848145,
+      "sft_loss": 4.428071022033691,
+      "step": 2055
+    },
+    {
+      "epoch": 1.1025255059374477,
+      "grad_norm": 0.35783765779838983,
+      "learning_rate": 7.973842675740539e-07,
+      "logits/chosen": -0.6504713296890259,
+      "logits/rejected": -0.6073805689811707,
+      "logps/chosen": -4.593170166015625,
+      "logps/rejected": -5.16645622253418,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.593170166015625,
+      "rewards/margins": 0.5732863545417786,
+      "rewards/rejected": -5.16645622253418,
+      "sft_loss": 4.3151750564575195,
+      "step": 2060
+    },
+    {
+      "epoch": 1.1052015387188494,
+      "grad_norm": 0.5921086419726374,
+      "learning_rate": 7.961308356546066e-07,
+      "logits/chosen": -0.7081059217453003,
+      "logits/rejected": -0.5758619904518127,
+      "logps/chosen": -4.213842868804932,
+      "logps/rejected": -4.782872200012207,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.213842868804932,
+      "rewards/margins": 0.569028913974762,
+      "rewards/rejected": -4.782872200012207,
+      "sft_loss": 3.9818127155303955,
+      "step": 2065
+    },
+    {
+      "epoch": 1.107877571500251,
+      "grad_norm": 0.4200367833057126,
+      "learning_rate": 7.948745306724931e-07,
+      "logits/chosen": -0.6793488264083862,
+      "logits/rejected": -0.5395691990852356,
+      "logps/chosen": -4.421175479888916,
+      "logps/rejected": -5.0171284675598145,
+      "loss": 0.051,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.421175479888916,
+      "rewards/margins": 0.5959528088569641,
+      "rewards/rejected": -5.0171284675598145,
+      "sft_loss": 4.115272045135498,
+      "step": 2070
+    },
+    {
+      "epoch": 1.1105536042816524,
+      "grad_norm": 0.4506521379869919,
+      "learning_rate": 7.936153648163897e-07,
+      "logits/chosen": -0.78980952501297,
+      "logits/rejected": -0.6935015916824341,
+      "logps/chosen": -4.609810829162598,
+      "logps/rejected": -5.01607608795166,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.609810829162598,
+      "rewards/margins": 0.40626534819602966,
+      "rewards/rejected": -5.01607608795166,
+      "sft_loss": 4.334246635437012,
+      "step": 2075
+    },
+    {
+      "epoch": 1.1132296370630541,
+      "grad_norm": 0.38637977769852344,
+      "learning_rate": 7.92353350302729e-07,
+      "logits/chosen": -0.7808972001075745,
+      "logits/rejected": -0.6138529777526855,
+      "logps/chosen": -4.381281852722168,
+      "logps/rejected": -4.917135715484619,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.381281852722168,
+      "rewards/margins": 0.5358539819717407,
+      "rewards/rejected": -4.917135715484619,
+      "sft_loss": 4.087695121765137,
+      "step": 2080
+    },
+    {
+      "epoch": 1.1159056698444556,
+      "grad_norm": 0.6129264862444803,
+      "learning_rate": 7.910884993755816e-07,
+      "logits/chosen": -0.7120209336280823,
+      "logits/rejected": -0.647143542766571,
+      "logps/chosen": -4.497712135314941,
+      "logps/rejected": -5.308043003082275,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -4.497712135314941,
+      "rewards/margins": 0.8103300333023071,
+      "rewards/rejected": -5.308043003082275,
+      "sft_loss": 4.245053291320801,
+      "step": 2085
+    },
+    {
+      "epoch": 1.118581702625857,
+      "grad_norm": 0.9472551023205663,
+      "learning_rate": 7.898208243065367e-07,
+      "logits/chosen": -0.6690019369125366,
+      "logits/rejected": -0.7030475735664368,
+      "logps/chosen": -4.6517486572265625,
+      "logps/rejected": -5.016876697540283,
+      "loss": 0.053,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.6517486572265625,
+      "rewards/margins": 0.36512771248817444,
+      "rewards/rejected": -5.016876697540283,
+      "sft_loss": 4.246583938598633,
+      "step": 2090
+    },
+    {
+      "epoch": 1.1212577354072588,
+      "grad_norm": 0.4164060497069248,
+      "learning_rate": 7.88550337394583e-07,
+      "logits/chosen": -0.8282628059387207,
+      "logits/rejected": -0.6851271390914917,
+      "logps/chosen": -4.366282939910889,
+      "logps/rejected": -4.938212871551514,
+      "loss": 0.052,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.366282939910889,
+      "rewards/margins": 0.5719300508499146,
+      "rewards/rejected": -4.938212871551514,
+      "sft_loss": 4.170043468475342,
+      "step": 2095
+    },
+    {
+      "epoch": 1.1239337681886603,
+      "grad_norm": 0.44100357427268927,
+      "learning_rate": 7.872770509659905e-07,
+      "logits/chosen": -0.582910418510437,
+      "logits/rejected": -0.5721174478530884,
+      "logps/chosen": -4.609631061553955,
+      "logps/rejected": -5.009383201599121,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.609631061553955,
+      "rewards/margins": 0.39975276589393616,
+      "rewards/rejected": -5.009383201599121,
+      "sft_loss": 4.247095584869385,
+      "step": 2100
+    },
+    {
+      "epoch": 1.1266098009700618,
+      "grad_norm": 0.46519432301850133,
+      "learning_rate": 7.860009773741896e-07,
+      "logits/chosen": -0.5506517291069031,
+      "logits/rejected": -0.3862597644329071,
+      "logps/chosen": -4.553868770599365,
+      "logps/rejected": -5.183063983917236,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.553868770599365,
+      "rewards/margins": 0.6291950941085815,
+      "rewards/rejected": -5.183063983917236,
+      "sft_loss": 4.281840801239014,
+      "step": 2105
+    },
+    {
+      "epoch": 1.1292858337514635,
+      "grad_norm": 0.4942654454653144,
+      "learning_rate": 7.84722128999652e-07,
+      "logits/chosen": -0.5422680974006653,
+      "logits/rejected": -0.41944044828414917,
+      "logps/chosen": -4.61797571182251,
+      "logps/rejected": -5.248989105224609,
+      "loss": 0.0533,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.61797571182251,
+      "rewards/margins": 0.6310136914253235,
+      "rewards/rejected": -5.248989105224609,
+      "sft_loss": 4.303167343139648,
+      "step": 2110
+    },
+    {
+      "epoch": 1.131961866532865,
+      "grad_norm": 0.8090209327976161,
+      "learning_rate": 7.834405182497699e-07,
+      "logits/chosen": -0.4767027795314789,
+      "logits/rejected": -0.46281296014785767,
+      "logps/chosen": -4.557481288909912,
+      "logps/rejected": -5.096904754638672,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.557481288909912,
+      "rewards/margins": 0.5394233465194702,
+      "rewards/rejected": -5.096904754638672,
+      "sft_loss": 4.297233581542969,
+      "step": 2115
+    },
+    {
+      "epoch": 1.1346378993142665,
+      "grad_norm": 0.42648107064600993,
+      "learning_rate": 7.821561575587368e-07,
+      "logits/chosen": -0.648400068283081,
+      "logits/rejected": -0.6427134275436401,
+      "logps/chosen": -4.454525947570801,
+      "logps/rejected": -4.900279521942139,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.454525947570801,
+      "rewards/margins": 0.4457532465457916,
+      "rewards/rejected": -4.900279521942139,
+      "sft_loss": 4.263405799865723,
+      "step": 2120
+    },
+    {
+      "epoch": 1.1373139320956682,
+      "grad_norm": 0.38934184489413604,
+      "learning_rate": 7.808690593874254e-07,
+      "logits/chosen": -0.7590117454528809,
+      "logits/rejected": -0.666256844997406,
+      "logps/chosen": -4.5700836181640625,
+      "logps/rejected": -5.170238494873047,
+      "loss": 0.0536,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.5700836181640625,
+      "rewards/margins": 0.6001549363136292,
+      "rewards/rejected": -5.170238494873047,
+      "sft_loss": 4.343713283538818,
+      "step": 2125
+    },
+    {
+      "epoch": 1.1399899648770697,
+      "grad_norm": 0.5519627541956967,
+      "learning_rate": 7.79579236223268e-07,
+      "logits/chosen": -0.5651447772979736,
+      "logits/rejected": -0.30045801401138306,
+      "logps/chosen": -4.452510833740234,
+      "logps/rejected": -5.143182277679443,
+      "loss": 0.05,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.452510833740234,
+      "rewards/margins": 0.6906719207763672,
+      "rewards/rejected": -5.143182277679443,
+      "sft_loss": 4.1111884117126465,
+      "step": 2130
+    },
+    {
+      "epoch": 1.1426659976584714,
+      "grad_norm": 0.5613782212929518,
+      "learning_rate": 7.782867005801346e-07,
+      "logits/chosen": -0.6042733192443848,
+      "logits/rejected": -0.3875576853752136,
+      "logps/chosen": -4.237351417541504,
+      "logps/rejected": -4.909454345703125,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.237351417541504,
+      "rewards/margins": 0.6721026301383972,
+      "rewards/rejected": -4.909454345703125,
+      "sft_loss": 4.003586292266846,
+      "step": 2135
+    },
+    {
+      "epoch": 1.145342030439873,
+      "grad_norm": 0.5813948989883637,
+      "learning_rate": 7.769914649982117e-07,
+      "logits/chosen": -0.6354474425315857,
+      "logits/rejected": -0.4989188313484192,
+      "logps/chosen": -4.546786785125732,
+      "logps/rejected": -5.131110191345215,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.546786785125732,
+      "rewards/margins": 0.5843238830566406,
+      "rewards/rejected": -5.131110191345215,
+      "sft_loss": 4.286316394805908,
+      "step": 2140
+    },
+    {
+      "epoch": 1.1480180632212744,
+      "grad_norm": 0.6806839299359042,
+      "learning_rate": 7.756935420438803e-07,
+      "logits/chosen": -0.568804144859314,
+      "logits/rejected": -0.5142095685005188,
+      "logps/chosen": -4.532443046569824,
+      "logps/rejected": -5.189213752746582,
+      "loss": 0.0506,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.532443046569824,
+      "rewards/margins": 0.6567710041999817,
+      "rewards/rejected": -5.189213752746582,
+      "sft_loss": 4.172651767730713,
+      "step": 2145
+    },
+    {
+      "epoch": 1.1506940960026761,
+      "grad_norm": 0.7857281039646198,
+      "learning_rate": 7.743929443095951e-07,
+      "logits/chosen": -0.5835979580879211,
+      "logits/rejected": -0.5579145550727844,
+      "logps/chosen": -4.318539619445801,
+      "logps/rejected": -4.9528326988220215,
+      "loss": 0.053,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.318539619445801,
+      "rewards/margins": 0.6342941522598267,
+      "rewards/rejected": -4.9528326988220215,
+      "sft_loss": 4.007269382476807,
+      "step": 2150
+    },
+    {
+      "epoch": 1.1533701287840776,
+      "grad_norm": 0.7275614780443926,
+      "learning_rate": 7.730896844137609e-07,
+      "logits/chosen": -0.6391198635101318,
+      "logits/rejected": -0.549577534198761,
+      "logps/chosen": -4.828595161437988,
+      "logps/rejected": -5.2178263664245605,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.828595161437988,
+      "rewards/margins": 0.38923168182373047,
+      "rewards/rejected": -5.2178263664245605,
+      "sft_loss": 4.5139970779418945,
+      "step": 2155
+    },
+    {
+      "epoch": 1.1560461615654791,
+      "grad_norm": 0.4940992460164979,
+      "learning_rate": 7.717837750006106e-07,
+      "logits/chosen": -0.6821721792221069,
+      "logits/rejected": -0.6354080438613892,
+      "logps/chosen": -4.713136672973633,
+      "logps/rejected": -5.284125328063965,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.713136672973633,
+      "rewards/margins": 0.5709878206253052,
+      "rewards/rejected": -5.284125328063965,
+      "sft_loss": 4.511738300323486,
+      "step": 2160
+    },
+    {
+      "epoch": 1.1587221943468808,
+      "grad_norm": 0.7421895395176694,
+      "learning_rate": 7.704752287400832e-07,
+      "logits/chosen": -0.6947080492973328,
+      "logits/rejected": -0.48819500207901,
+      "logps/chosen": -4.358445167541504,
+      "logps/rejected": -4.839158535003662,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.358445167541504,
+      "rewards/margins": 0.48071417212486267,
+      "rewards/rejected": -4.839158535003662,
+      "sft_loss": 4.1708197593688965,
+      "step": 2165
+    },
+    {
+      "epoch": 1.1613982271282823,
+      "grad_norm": 0.44878522274791,
+      "learning_rate": 7.691640583277004e-07,
+      "logits/chosen": -0.6316672563552856,
+      "logits/rejected": -0.4783342480659485,
+      "logps/chosen": -4.266995906829834,
+      "logps/rejected": -4.869975566864014,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.266995906829834,
+      "rewards/margins": 0.6029794812202454,
+      "rewards/rejected": -4.869975566864014,
+      "sft_loss": 3.9994354248046875,
+      "step": 2170
+    },
+    {
+      "epoch": 1.1640742599096838,
+      "grad_norm": 0.548935259573157,
+      "learning_rate": 7.678502764844433e-07,
+      "logits/chosen": -0.7553048133850098,
+      "logits/rejected": -0.49919694662094116,
+      "logps/chosen": -4.464298248291016,
+      "logps/rejected": -4.863472938537598,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.464298248291016,
+      "rewards/margins": 0.3991743326187134,
+      "rewards/rejected": -4.863472938537598,
+      "sft_loss": 4.170223236083984,
+      "step": 2175
+    },
+    {
+      "epoch": 1.1667502926910855,
+      "grad_norm": 0.4851729321288679,
+      "learning_rate": 7.665338959566288e-07,
+      "logits/chosen": -0.7407499551773071,
+      "logits/rejected": -0.6942411661148071,
+      "logps/chosen": -4.578423023223877,
+      "logps/rejected": -5.188458442687988,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.578423023223877,
+      "rewards/margins": 0.61003577709198,
+      "rewards/rejected": -5.188458442687988,
+      "sft_loss": 4.262111186981201,
+      "step": 2180
+    },
+    {
+      "epoch": 1.169426325472487,
+      "grad_norm": 0.6447142613386584,
+      "learning_rate": 7.652149295157868e-07,
+      "logits/chosen": -0.6532949209213257,
+      "logits/rejected": -0.4770034849643707,
+      "logps/chosen": -4.581149578094482,
+      "logps/rejected": -4.976588249206543,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.581149578094482,
+      "rewards/margins": 0.3954388201236725,
+      "rewards/rejected": -4.976588249206543,
+      "sft_loss": 4.257651329040527,
+      "step": 2185
+    },
+    {
+      "epoch": 1.1721023582538885,
+      "grad_norm": 0.6132451344551202,
+      "learning_rate": 7.638933899585354e-07,
+      "logits/chosen": -0.4752999246120453,
+      "logits/rejected": -0.5158058404922485,
+      "logps/chosen": -4.38538932800293,
+      "logps/rejected": -5.0396928787231445,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.38538932800293,
+      "rewards/margins": 0.6543041467666626,
+      "rewards/rejected": -5.0396928787231445,
+      "sft_loss": 4.061063289642334,
+      "step": 2190
+    },
+    {
+      "epoch": 1.1747783910352902,
+      "grad_norm": 0.7387059160839864,
+      "learning_rate": 7.625692901064573e-07,
+      "logits/chosen": -0.6472002267837524,
+      "logits/rejected": -0.5495957136154175,
+      "logps/chosen": -4.54372501373291,
+      "logps/rejected": -5.037572860717773,
+      "loss": 0.055,
+      "rewards/accuracies": 0.581250011920929,
+      "rewards/chosen": -4.54372501373291,
+      "rewards/margins": 0.4938478469848633,
+      "rewards/rejected": -5.037572860717773,
+      "sft_loss": 4.294220447540283,
+      "step": 2195
+    },
+    {
+      "epoch": 1.1774544238166917,
+      "grad_norm": 0.5059293471702357,
+      "learning_rate": 7.61242642805975e-07,
+      "logits/chosen": -0.72947758436203,
+      "logits/rejected": -0.7847334742546082,
+      "logps/chosen": -4.876827716827393,
+      "logps/rejected": -5.208680152893066,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.876827716827393,
+      "rewards/margins": 0.3318529427051544,
+      "rewards/rejected": -5.208680152893066,
+      "sft_loss": 4.526150703430176,
+      "step": 2200
+    },
+    {
+      "epoch": 1.1801304565980932,
+      "grad_norm": 0.38361611712787774,
+      "learning_rate": 7.599134609282266e-07,
+      "logits/chosen": -0.7986024618148804,
+      "logits/rejected": -0.5711261034011841,
+      "logps/chosen": -4.482367515563965,
+      "logps/rejected": -5.0834221839904785,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.482367515563965,
+      "rewards/margins": 0.6010543704032898,
+      "rewards/rejected": -5.0834221839904785,
+      "sft_loss": 4.313359260559082,
+      "step": 2205
+    },
+    {
+      "epoch": 1.182806489379495,
+      "grad_norm": 0.5286962744432987,
+      "learning_rate": 7.585817573689402e-07,
+      "logits/chosen": -0.6769399642944336,
+      "logits/rejected": -0.5752993822097778,
+      "logps/chosen": -4.282863140106201,
+      "logps/rejected": -4.965208530426025,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.282863140106201,
+      "rewards/margins": 0.6823452711105347,
+      "rewards/rejected": -4.965208530426025,
+      "sft_loss": 4.119072914123535,
+      "step": 2210
+    },
+    {
+      "epoch": 1.1854825221608964,
+      "grad_norm": 0.5139603924787693,
+      "learning_rate": 7.572475450483098e-07,
+      "logits/chosen": -0.5727382898330688,
+      "logits/rejected": -0.4771631360054016,
+      "logps/chosen": -4.298099517822266,
+      "logps/rejected": -4.936698913574219,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.298099517822266,
+      "rewards/margins": 0.638599693775177,
+      "rewards/rejected": -4.936698913574219,
+      "sft_loss": 4.037377834320068,
+      "step": 2215
+    },
+    {
+      "epoch": 1.188158554942298,
+      "grad_norm": 0.5154609146458041,
+      "learning_rate": 7.559108369108689e-07,
+      "logits/chosen": -0.5601507425308228,
+      "logits/rejected": -0.445406049489975,
+      "logps/chosen": -4.282214164733887,
+      "logps/rejected": -4.886294364929199,
+      "loss": 0.0528,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.282214164733887,
+      "rewards/margins": 0.6040805578231812,
+      "rewards/rejected": -4.886294364929199,
+      "sft_loss": 4.023911476135254,
+      "step": 2220
+    },
+    {
+      "epoch": 1.1908345877236997,
+      "grad_norm": 0.5711027835530442,
+      "learning_rate": 7.54571645925366e-07,
+      "logits/chosen": -0.605337917804718,
+      "logits/rejected": -0.3317146897315979,
+      "logps/chosen": -4.7101922035217285,
+      "logps/rejected": -5.524775505065918,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.7101922035217285,
+      "rewards/margins": 0.8145831823348999,
+      "rewards/rejected": -5.524775505065918,
+      "sft_loss": 4.352533340454102,
+      "step": 2225
+    },
+    {
+      "epoch": 1.1935106205051011,
+      "grad_norm": 0.6611235187630187,
+      "learning_rate": 7.532299850846378e-07,
+      "logits/chosen": -0.5842553377151489,
+      "logits/rejected": -0.38650670647621155,
+      "logps/chosen": -4.717400550842285,
+      "logps/rejected": -5.565418243408203,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.717400550842285,
+      "rewards/margins": 0.8480178713798523,
+      "rewards/rejected": -5.565418243408203,
+      "sft_loss": 4.386633396148682,
+      "step": 2230
+    },
+    {
+      "epoch": 1.1961866532865026,
+      "grad_norm": 0.6642588343412641,
+      "learning_rate": 7.518858674054838e-07,
+      "logits/chosen": -0.546658992767334,
+      "logits/rejected": -0.3431429862976074,
+      "logps/chosen": -4.502182483673096,
+      "logps/rejected": -5.018511772155762,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.502182483673096,
+      "rewards/margins": 0.5163294076919556,
+      "rewards/rejected": -5.018511772155762,
+      "sft_loss": 4.152172088623047,
+      "step": 2235
+    },
+    {
+      "epoch": 1.1988626860679044,
+      "grad_norm": 0.4519687145218764,
+      "learning_rate": 7.505393059285394e-07,
+      "logits/chosen": -0.6633430123329163,
+      "logits/rejected": -0.4812769889831543,
+      "logps/chosen": -4.417510032653809,
+      "logps/rejected": -4.820703506469727,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.417510032653809,
+      "rewards/margins": 0.40319404006004333,
+      "rewards/rejected": -4.820703506469727,
+      "sft_loss": 4.104281902313232,
+      "step": 2240
+    },
+    {
+      "epoch": 1.2015387188493059,
+      "grad_norm": 0.4856591123771306,
+      "learning_rate": 7.491903137181501e-07,
+      "logits/chosen": -0.626348078250885,
+      "logits/rejected": -0.6236867904663086,
+      "logps/chosen": -4.471949100494385,
+      "logps/rejected": -4.941003322601318,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.471949100494385,
+      "rewards/margins": 0.46905431151390076,
+      "rewards/rejected": -4.941003322601318,
+      "sft_loss": 4.249975681304932,
+      "step": 2245
+    },
+    {
+      "epoch": 1.2042147516307076,
+      "grad_norm": 0.5777401710820074,
+      "learning_rate": 7.478389038622441e-07,
+      "logits/chosen": -0.5622078776359558,
+      "logits/rejected": -0.5374001264572144,
+      "logps/chosen": -4.538549900054932,
+      "logps/rejected": -5.063116550445557,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.538549900054932,
+      "rewards/margins": 0.5245659947395325,
+      "rewards/rejected": -5.063116550445557,
+      "sft_loss": 4.267182350158691,
+      "step": 2250
+    },
+    {
+      "epoch": 1.206890784412109,
+      "grad_norm": 0.7421281314697232,
+      "learning_rate": 7.46485089472206e-07,
+      "logits/chosen": -0.6193834543228149,
+      "logits/rejected": -0.5974622368812561,
+      "logps/chosen": -4.5142621994018555,
+      "logps/rejected": -4.9804887771606445,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.5142621994018555,
+      "rewards/margins": 0.46622681617736816,
+      "rewards/rejected": -4.9804887771606445,
+      "sft_loss": 4.24161434173584,
+      "step": 2255
+    },
+    {
+      "epoch": 1.2095668171935106,
+      "grad_norm": 0.4926243336718459,
+      "learning_rate": 7.451288836827487e-07,
+      "logits/chosen": -0.5476628541946411,
+      "logits/rejected": -0.5884389877319336,
+      "logps/chosen": -4.725428581237793,
+      "logps/rejected": -5.089140892028809,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.725428581237793,
+      "rewards/margins": 0.36371201276779175,
+      "rewards/rejected": -5.089140892028809,
+      "sft_loss": 4.442521572113037,
+      "step": 2260
+    },
+    {
+      "epoch": 1.2122428499749123,
+      "grad_norm": 0.6415635469256901,
+      "learning_rate": 7.437702996517869e-07,
+      "logits/chosen": -0.5639731884002686,
+      "logits/rejected": -0.4645105004310608,
+      "logps/chosen": -4.587918758392334,
+      "logps/rejected": -5.001802921295166,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.587918758392334,
+      "rewards/margins": 0.4138834476470947,
+      "rewards/rejected": -5.001802921295166,
+      "sft_loss": 4.319118976593018,
+      "step": 2265
+    },
+    {
+      "epoch": 1.2149188827563138,
+      "grad_norm": 0.4712570754741204,
+      "learning_rate": 7.424093505603087e-07,
+      "logits/chosen": -0.6785644888877869,
+      "logits/rejected": -0.4742724299430847,
+      "logps/chosen": -4.364579200744629,
+      "logps/rejected": -4.939398765563965,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.364579200744629,
+      "rewards/margins": 0.5748197436332703,
+      "rewards/rejected": -4.939398765563965,
+      "sft_loss": 4.10079288482666,
+      "step": 2270
+    },
+    {
+      "epoch": 1.2175949155377153,
+      "grad_norm": 0.5846752602767853,
+      "learning_rate": 7.410460496122482e-07,
+      "logits/chosen": -0.5660977363586426,
+      "logits/rejected": -0.42410382628440857,
+      "logps/chosen": -4.356973648071289,
+      "logps/rejected": -5.0322394371032715,
+      "loss": 0.0506,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.356973648071289,
+      "rewards/margins": 0.6752651333808899,
+      "rewards/rejected": -5.0322394371032715,
+      "sft_loss": 3.96946382522583,
+      "step": 2275
+    },
+    {
+      "epoch": 1.220270948319117,
+      "grad_norm": 0.5951245481805585,
+      "learning_rate": 7.396804100343572e-07,
+      "logits/chosen": -0.6611990928649902,
+      "logits/rejected": -0.46632593870162964,
+      "logps/chosen": -4.34031343460083,
+      "logps/rejected": -4.975114822387695,
+      "loss": 0.0504,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.34031343460083,
+      "rewards/margins": 0.6348011493682861,
+      "rewards/rejected": -4.975114822387695,
+      "sft_loss": 3.9688549041748047,
+      "step": 2280
+    },
+    {
+      "epoch": 1.2229469811005185,
+      "grad_norm": 0.45475296718893476,
+      "learning_rate": 7.383124450760768e-07,
+      "logits/chosen": -0.6614982485771179,
+      "logits/rejected": -0.4856722354888916,
+      "logps/chosen": -4.491690635681152,
+      "logps/rejected": -5.2794342041015625,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.491690635681152,
+      "rewards/margins": 0.7877441644668579,
+      "rewards/rejected": -5.2794342041015625,
+      "sft_loss": 4.181609153747559,
+      "step": 2285
+    },
+    {
+      "epoch": 1.22562301388192,
+      "grad_norm": 0.6018584647524708,
+      "learning_rate": 7.369421680094091e-07,
+      "logits/chosen": -0.8093674778938293,
+      "logits/rejected": -0.6580020189285278,
+      "logps/chosen": -4.6645588874816895,
+      "logps/rejected": -5.214623928070068,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.6645588874816895,
+      "rewards/margins": 0.5500649213790894,
+      "rewards/rejected": -5.214623928070068,
+      "sft_loss": 4.357946872711182,
+      "step": 2290
+    },
+    {
+      "epoch": 1.2282990466633217,
+      "grad_norm": 0.8161047546333619,
+      "learning_rate": 7.355695921287881e-07,
+      "logits/chosen": -0.7041813135147095,
+      "logits/rejected": -0.6303861141204834,
+      "logps/chosen": -4.555066108703613,
+      "logps/rejected": -5.077036380767822,
+      "loss": 0.053,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.555066108703613,
+      "rewards/margins": 0.5219703912734985,
+      "rewards/rejected": -5.077036380767822,
+      "sft_loss": 4.199130058288574,
+      "step": 2295
+    },
+    {
+      "epoch": 1.2309750794447232,
+      "grad_norm": 0.43306078189188285,
+      "learning_rate": 7.341947307509513e-07,
+      "logits/chosen": -0.6474259495735168,
+      "logits/rejected": -0.5673624873161316,
+      "logps/chosen": -4.532158851623535,
+      "logps/rejected": -5.043374061584473,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.532158851623535,
+      "rewards/margins": 0.5112149119377136,
+      "rewards/rejected": -5.043374061584473,
+      "sft_loss": 4.234317302703857,
+      "step": 2300
+    },
+    {
+      "epoch": 1.233651112226125,
+      "grad_norm": 0.4728830918294627,
+      "learning_rate": 7.328175972148094e-07,
+      "logits/chosen": -0.7278068661689758,
+      "logits/rejected": -0.6084392666816711,
+      "logps/chosen": -4.6190505027771,
+      "logps/rejected": -5.242172718048096,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.6190505027771,
+      "rewards/margins": 0.6231220364570618,
+      "rewards/rejected": -5.242172718048096,
+      "sft_loss": 4.327923774719238,
+      "step": 2305
+    },
+    {
+      "epoch": 1.2363271450075264,
+      "grad_norm": 0.7656650013875022,
+      "learning_rate": 7.314382048813185e-07,
+      "logits/chosen": -0.7033382654190063,
+      "logits/rejected": -0.4409807324409485,
+      "logps/chosen": -4.301620960235596,
+      "logps/rejected": -5.104963302612305,
+      "loss": 0.05,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -4.301620960235596,
+      "rewards/margins": 0.8033429384231567,
+      "rewards/rejected": -5.104963302612305,
+      "sft_loss": 4.030623435974121,
+      "step": 2310
+    },
+    {
+      "epoch": 1.2390031777889279,
+      "grad_norm": 0.7429383315365543,
+      "learning_rate": 7.300565671333486e-07,
+      "logits/chosen": -0.7581867575645447,
+      "logits/rejected": -0.5539118051528931,
+      "logps/chosen": -4.46927547454834,
+      "logps/rejected": -5.120739936828613,
+      "loss": 0.051,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.46927547454834,
+      "rewards/margins": 0.6514642834663391,
+      "rewards/rejected": -5.120739936828613,
+      "sft_loss": 4.156338691711426,
+      "step": 2315
+    },
+    {
+      "epoch": 1.2416792105703296,
+      "grad_norm": 0.32278258162480566,
+      "learning_rate": 7.286726973755554e-07,
+      "logits/chosen": -0.5927146077156067,
+      "logits/rejected": -0.5882304906845093,
+      "logps/chosen": -4.348783016204834,
+      "logps/rejected": -4.928504943847656,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.348783016204834,
+      "rewards/margins": 0.5797222852706909,
+      "rewards/rejected": -4.928504943847656,
+      "sft_loss": 3.991481065750122,
+      "step": 2320
+    },
+    {
+      "epoch": 1.244355243351731,
+      "grad_norm": 0.511211768593372,
+      "learning_rate": 7.272866090342493e-07,
+      "logits/chosen": -0.47499561309814453,
+      "logits/rejected": -0.4513750672340393,
+      "logps/chosen": -4.4810991287231445,
+      "logps/rejected": -5.18906307220459,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.4810991287231445,
+      "rewards/margins": 0.7079639434814453,
+      "rewards/rejected": -5.18906307220459,
+      "sft_loss": 4.083864212036133,
+      "step": 2325
+    },
+    {
+      "epoch": 1.2470312761331326,
+      "grad_norm": 0.6003732079575774,
+      "learning_rate": 7.258983155572656e-07,
+      "logits/chosen": -0.6991143822669983,
+      "logits/rejected": -0.634955108165741,
+      "logps/chosen": -4.761792182922363,
+      "logps/rejected": -5.33829402923584,
+      "loss": 0.052,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.761792182922363,
+      "rewards/margins": 0.5765026807785034,
+      "rewards/rejected": -5.33829402923584,
+      "sft_loss": 4.341994285583496,
+      "step": 2330
+    },
+    {
+      "epoch": 1.2497073089145343,
+      "grad_norm": 0.9922752441546293,
+      "learning_rate": 7.245078304138335e-07,
+      "logits/chosen": -0.6240192651748657,
+      "logits/rejected": -0.5747882127761841,
+      "logps/chosen": -4.513253211975098,
+      "logps/rejected": -5.224671363830566,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.513253211975098,
+      "rewards/margins": 0.7114177942276001,
+      "rewards/rejected": -5.224671363830566,
+      "sft_loss": 4.258262634277344,
+      "step": 2335
+    },
+    {
+      "epoch": 1.2523833416959358,
+      "grad_norm": 0.37104576148005725,
+      "learning_rate": 7.231151670944462e-07,
+      "logits/chosen": -0.8994210362434387,
+      "logits/rejected": -0.6443474888801575,
+      "logps/chosen": -4.498259544372559,
+      "logps/rejected": -5.11997127532959,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.498259544372559,
+      "rewards/margins": 0.6217113733291626,
+      "rewards/rejected": -5.11997127532959,
+      "sft_loss": 4.2076215744018555,
+      "step": 2340
+    },
+    {
+      "epoch": 1.2550593744773373,
+      "grad_norm": 0.4405812382355525,
+      "learning_rate": 7.217203391107291e-07,
+      "logits/chosen": -0.8214927911758423,
+      "logits/rejected": -0.6504918932914734,
+      "logps/chosen": -4.451966285705566,
+      "logps/rejected": -5.12908935546875,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.451966285705566,
+      "rewards/margins": 0.6771237254142761,
+      "rewards/rejected": -5.12908935546875,
+      "sft_loss": 4.1210761070251465,
+      "step": 2345
+    },
+    {
+      "epoch": 1.257735407258739,
+      "grad_norm": 0.691705223908103,
+      "learning_rate": 7.203233599953096e-07,
+      "logits/chosen": -0.8120743036270142,
+      "logits/rejected": -0.6662536263465881,
+      "logps/chosen": -4.297300338745117,
+      "logps/rejected": -4.907609462738037,
+      "loss": 0.052,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.297300338745117,
+      "rewards/margins": 0.6103091835975647,
+      "rewards/rejected": -4.907609462738037,
+      "sft_loss": 4.046349048614502,
+      "step": 2350
+    },
+    {
+      "epoch": 1.2604114400401405,
+      "grad_norm": 0.4580382761374032,
+      "learning_rate": 7.189242433016852e-07,
+      "logits/chosen": -0.7498005628585815,
+      "logits/rejected": -0.6268131136894226,
+      "logps/chosen": -4.600125312805176,
+      "logps/rejected": -5.273111820220947,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.600125312805176,
+      "rewards/margins": 0.6729866862297058,
+      "rewards/rejected": -5.273111820220947,
+      "sft_loss": 4.308657646179199,
+      "step": 2355
+    },
+    {
+      "epoch": 1.263087472821542,
+      "grad_norm": 0.5915020215955263,
+      "learning_rate": 7.17523002604092e-07,
+      "logits/chosen": -0.8824199438095093,
+      "logits/rejected": -0.6876664161682129,
+      "logps/chosen": -4.512190341949463,
+      "logps/rejected": -5.1109418869018555,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.512190341949463,
+      "rewards/margins": 0.5987512469291687,
+      "rewards/rejected": -5.1109418869018555,
+      "sft_loss": 4.2525835037231445,
+      "step": 2360
+    },
+    {
+      "epoch": 1.2657635056029437,
+      "grad_norm": 0.5690946194765862,
+      "learning_rate": 7.161196514973734e-07,
+      "logits/chosen": -0.6747918128967285,
+      "logits/rejected": -0.5300695300102234,
+      "logps/chosen": -4.3710174560546875,
+      "logps/rejected": -5.1945085525512695,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.3710174560546875,
+      "rewards/margins": 0.8234914541244507,
+      "rewards/rejected": -5.1945085525512695,
+      "sft_loss": 4.117964744567871,
+      "step": 2365
+    },
+    {
+      "epoch": 1.2684395383843452,
+      "grad_norm": 0.4351018604218048,
+      "learning_rate": 7.147142035968483e-07,
+      "logits/chosen": -0.6003702878952026,
+      "logits/rejected": -0.4170975089073181,
+      "logps/chosen": -4.435196876525879,
+      "logps/rejected": -5.031000137329102,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.435196876525879,
+      "rewards/margins": 0.5958040356636047,
+      "rewards/rejected": -5.031000137329102,
+      "sft_loss": 4.136528015136719,
+      "step": 2370
+    },
+    {
+      "epoch": 1.2711155711657467,
+      "grad_norm": 0.7303957463893005,
+      "learning_rate": 7.133066725381781e-07,
+      "logits/chosen": -0.7778987288475037,
+      "logits/rejected": -0.5770991444587708,
+      "logps/chosen": -4.4204912185668945,
+      "logps/rejected": -5.063147068023682,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.4204912185668945,
+      "rewards/margins": 0.6426565051078796,
+      "rewards/rejected": -5.063147068023682,
+      "sft_loss": 4.2004265785217285,
+      "step": 2375
+    },
+    {
+      "epoch": 1.2737916039471484,
+      "grad_norm": 0.485169085052069,
+      "learning_rate": 7.118970719772354e-07,
+      "logits/chosen": -0.7188619375228882,
+      "logits/rejected": -0.5356766581535339,
+      "logps/chosen": -4.620368003845215,
+      "logps/rejected": -5.3475141525268555,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.620368003845215,
+      "rewards/margins": 0.7271462678909302,
+      "rewards/rejected": -5.3475141525268555,
+      "sft_loss": 4.352916240692139,
+      "step": 2380
+    },
+    {
+      "epoch": 1.27646763672855,
+      "grad_norm": 0.4763305452143226,
+      "learning_rate": 7.104854155899711e-07,
+      "logits/chosen": -0.6405320167541504,
+      "logits/rejected": -0.5557799935340881,
+      "logps/chosen": -4.4577178955078125,
+      "logps/rejected": -5.03024435043335,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.4577178955078125,
+      "rewards/margins": 0.572527289390564,
+      "rewards/rejected": -5.03024435043335,
+      "sft_loss": 4.169253349304199,
+      "step": 2385
+    },
+    {
+      "epoch": 1.2791436695099514,
+      "grad_norm": 0.5990467306969168,
+      "learning_rate": 7.090717170722817e-07,
+      "logits/chosen": -0.5321937799453735,
+      "logits/rejected": -0.5410672426223755,
+      "logps/chosen": -4.411266326904297,
+      "logps/rejected": -5.152445316314697,
+      "loss": 0.0504,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.411266326904297,
+      "rewards/margins": 0.741179347038269,
+      "rewards/rejected": -5.152445316314697,
+      "sft_loss": 4.109254837036133,
+      "step": 2390
+    },
+    {
+      "epoch": 1.2818197022913531,
+      "grad_norm": 0.6012987235900056,
+      "learning_rate": 7.076559901398762e-07,
+      "logits/chosen": -0.6546708941459656,
+      "logits/rejected": -0.5062588453292847,
+      "logps/chosen": -4.278241157531738,
+      "logps/rejected": -4.8308024406433105,
+      "loss": 0.052,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.278241157531738,
+      "rewards/margins": 0.552561342716217,
+      "rewards/rejected": -4.8308024406433105,
+      "sft_loss": 4.041296005249023,
+      "step": 2395
+    },
+    {
+      "epoch": 1.2844957350727546,
+      "grad_norm": 0.4866774374056064,
+      "learning_rate": 7.062382485281436e-07,
+      "logits/chosen": -0.5467609167098999,
+      "logits/rejected": -0.42013612389564514,
+      "logps/chosen": -4.223976135253906,
+      "logps/rejected": -4.85884952545166,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.223976135253906,
+      "rewards/margins": 0.634873628616333,
+      "rewards/rejected": -4.85884952545166,
+      "sft_loss": 3.980189085006714,
+      "step": 2400
+    },
+    {
+      "epoch": 1.2844957350727546,
+      "eval_logits/chosen": -0.1385614424943924,
+      "eval_logits/rejected": -0.05511578917503357,
+      "eval_logps/chosen": -4.712944984436035,
+      "eval_logps/rejected": -5.373843669891357,
+      "eval_loss": 0.0507567897439003,
+      "eval_rewards/accuracies": 0.6609792113304138,
+      "eval_rewards/chosen": -4.712944984436035,
+      "eval_rewards/margins": 0.660898745059967,
+      "eval_rewards/rejected": -5.373843669891357,
+      "eval_runtime": 43.5868,
+      "eval_samples_per_second": 30.858,
+      "eval_sft_loss": 4.281847953796387,
+      "eval_steps_per_second": 7.732,
+      "step": 2400
+    },
+    {
+      "epoch": 1.287171767854156,
+      "grad_norm": 0.8751626388888276,
+      "learning_rate": 7.048185059920193e-07,
+      "logits/chosen": -0.5422787666320801,
+      "logits/rejected": -0.4219578206539154,
+      "logps/chosen": -4.8283233642578125,
+      "logps/rejected": -5.523282051086426,
+      "loss": 0.052,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.8283233642578125,
+      "rewards/margins": 0.6949586868286133,
+      "rewards/rejected": -5.523282051086426,
+      "sft_loss": 4.508396625518799,
+      "step": 2405
+    },
+    {
+      "epoch": 1.2898478006355578,
+      "grad_norm": 0.53586381052944,
+      "learning_rate": 7.033967763058516e-07,
+      "logits/chosen": -0.614895224571228,
+      "logits/rejected": -0.46858900785446167,
+      "logps/chosen": -4.68340539932251,
+      "logps/rejected": -5.169063568115234,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.68340539932251,
+      "rewards/margins": 0.4856584668159485,
+      "rewards/rejected": -5.169063568115234,
+      "sft_loss": 4.267060279846191,
+      "step": 2410
+    },
+    {
+      "epoch": 1.2925238334169593,
+      "grad_norm": 0.5770283777467645,
+      "learning_rate": 7.019730732632681e-07,
+      "logits/chosen": -0.4095725119113922,
+      "logits/rejected": -0.324873685836792,
+      "logps/chosen": -4.341700077056885,
+      "logps/rejected": -5.165848731994629,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.341700077056885,
+      "rewards/margins": 0.8241486549377441,
+      "rewards/rejected": -5.165848731994629,
+      "sft_loss": 4.013070583343506,
+      "step": 2415
+    },
+    {
+      "epoch": 1.2951998661983608,
+      "grad_norm": 0.6535497371525489,
+      "learning_rate": 7.005474106770418e-07,
+      "logits/chosen": -0.4207722544670105,
+      "logits/rejected": -0.3212054371833801,
+      "logps/chosen": -4.347165584564209,
+      "logps/rejected": -4.952691555023193,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.347165584564209,
+      "rewards/margins": 0.6055256128311157,
+      "rewards/rejected": -4.952691555023193,
+      "sft_loss": 3.9859261512756348,
+      "step": 2420
+    },
+    {
+      "epoch": 1.2978758989797625,
+      "grad_norm": 0.9515981981928276,
+      "learning_rate": 6.991198023789577e-07,
+      "logits/chosen": -0.4071879982948303,
+      "logits/rejected": -0.3304407000541687,
+      "logps/chosen": -4.39312744140625,
+      "logps/rejected": -4.891679763793945,
+      "loss": 0.0536,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.39312744140625,
+      "rewards/margins": 0.4985523819923401,
+      "rewards/rejected": -4.891679763793945,
+      "sft_loss": 4.153935432434082,
+      "step": 2425
+    },
+    {
+      "epoch": 1.300551931761164,
+      "grad_norm": 0.5250671890303608,
+      "learning_rate": 6.976902622196776e-07,
+      "logits/chosen": -0.49113330245018005,
+      "logits/rejected": -0.4230748116970062,
+      "logps/chosen": -4.6922197341918945,
+      "logps/rejected": -5.178853511810303,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.6922197341918945,
+      "rewards/margins": 0.48663386702537537,
+      "rewards/rejected": -5.178853511810303,
+      "sft_loss": 4.355113506317139,
+      "step": 2430
+    },
+    {
+      "epoch": 1.3032279645425655,
+      "grad_norm": 0.3705273880797938,
+      "learning_rate": 6.962588040686064e-07,
+      "logits/chosen": -0.5611008405685425,
+      "logits/rejected": -0.38462042808532715,
+      "logps/chosen": -4.640923023223877,
+      "logps/rejected": -5.155377388000488,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.640923023223877,
+      "rewards/margins": 0.5144541263580322,
+      "rewards/rejected": -5.155377388000488,
+      "sft_loss": 4.4788007736206055,
+      "step": 2435
+    },
+    {
+      "epoch": 1.3059039973239672,
+      "grad_norm": 0.500862424455962,
+      "learning_rate": 6.948254418137573e-07,
+      "logits/chosen": -0.46678757667541504,
+      "logits/rejected": -0.33529505133628845,
+      "logps/chosen": -4.563943862915039,
+      "logps/rejected": -5.159988880157471,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.563943862915039,
+      "rewards/margins": 0.596044659614563,
+      "rewards/rejected": -5.159988880157471,
+      "sft_loss": 4.286484718322754,
+      "step": 2440
+    },
+    {
+      "epoch": 1.3085800301053687,
+      "grad_norm": 0.44326807018661724,
+      "learning_rate": 6.933901893616174e-07,
+      "logits/chosen": -0.5239390134811401,
+      "logits/rejected": -0.3764905333518982,
+      "logps/chosen": -4.383409023284912,
+      "logps/rejected": -5.040538787841797,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.383409023284912,
+      "rewards/margins": 0.6571294069290161,
+      "rewards/rejected": -5.040538787841797,
+      "sft_loss": 4.0584516525268555,
+      "step": 2445
+    },
+    {
+      "epoch": 1.3112560628867704,
+      "grad_norm": 0.5170339642505193,
+      "learning_rate": 6.919530606370121e-07,
+      "logits/chosen": -0.4882650375366211,
+      "logits/rejected": -0.3291659951210022,
+      "logps/chosen": -4.313410758972168,
+      "logps/rejected": -4.961250305175781,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.313410758972168,
+      "rewards/margins": 0.6478389501571655,
+      "rewards/rejected": -4.961250305175781,
+      "sft_loss": 4.071700096130371,
+      "step": 2450
+    },
+    {
+      "epoch": 1.313932095668172,
+      "grad_norm": 0.43429799958723836,
+      "learning_rate": 6.905140695829706e-07,
+      "logits/chosen": -0.5654727816581726,
+      "logits/rejected": -0.278671532869339,
+      "logps/chosen": -4.373675346374512,
+      "logps/rejected": -5.146459102630615,
+      "loss": 0.0506,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.373675346374512,
+      "rewards/margins": 0.7727841734886169,
+      "rewards/rejected": -5.146459102630615,
+      "sft_loss": 4.107213020324707,
+      "step": 2455
+    },
+    {
+      "epoch": 1.3166081284495736,
+      "grad_norm": 0.6815478221410864,
+      "learning_rate": 6.890732301605904e-07,
+      "logits/chosen": -0.39504489302635193,
+      "logits/rejected": -0.30925464630126953,
+      "logps/chosen": -4.52138614654541,
+      "logps/rejected": -5.0054850578308105,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.52138614654541,
+      "rewards/margins": 0.48409897089004517,
+      "rewards/rejected": -5.0054850578308105,
+      "sft_loss": 4.16397762298584,
+      "step": 2460
+    },
+    {
+      "epoch": 1.3192841612309751,
+      "grad_norm": 0.5019498889032347,
+      "learning_rate": 6.876305563489021e-07,
+      "logits/chosen": -0.4539240896701813,
+      "logits/rejected": -0.4017064571380615,
+      "logps/chosen": -4.319275856018066,
+      "logps/rejected": -5.132537841796875,
+      "loss": 0.0499,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.319275856018066,
+      "rewards/margins": 0.8132628202438354,
+      "rewards/rejected": -5.132537841796875,
+      "sft_loss": 3.9580626487731934,
+      "step": 2465
+    },
+    {
+      "epoch": 1.3219601940123766,
+      "grad_norm": 0.6126362013577834,
+      "learning_rate": 6.861860621447331e-07,
+      "logits/chosen": -0.6290256381034851,
+      "logits/rejected": -0.5144913792610168,
+      "logps/chosen": -4.654995441436768,
+      "logps/rejected": -5.2134881019592285,
+      "loss": 0.053,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.654995441436768,
+      "rewards/margins": 0.558492124080658,
+      "rewards/rejected": -5.2134881019592285,
+      "sft_loss": 4.4049787521362305,
+      "step": 2470
+    },
+    {
+      "epoch": 1.3246362267937783,
+      "grad_norm": 1.1242885884848643,
+      "learning_rate": 6.847397615625725e-07,
+      "logits/chosen": -0.4887451231479645,
+      "logits/rejected": -0.46777215600013733,
+      "logps/chosen": -4.643002986907959,
+      "logps/rejected": -5.143282890319824,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.643002986907959,
+      "rewards/margins": 0.5002797842025757,
+      "rewards/rejected": -5.143282890319824,
+      "sft_loss": 4.291563034057617,
+      "step": 2475
+    },
+    {
+      "epoch": 1.3273122595751798,
+      "grad_norm": 0.6843958430147569,
+      "learning_rate": 6.83291668634435e-07,
+      "logits/chosen": -0.6661633849143982,
+      "logits/rejected": -0.4752282202243805,
+      "logps/chosen": -4.608720779418945,
+      "logps/rejected": -5.240445137023926,
+      "loss": 0.052,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.608720779418945,
+      "rewards/margins": 0.6317241787910461,
+      "rewards/rejected": -5.240445137023926,
+      "sft_loss": 4.33678674697876,
+      "step": 2480
+    },
+    {
+      "epoch": 1.3299882923565813,
+      "grad_norm": 0.6108029915362448,
+      "learning_rate": 6.818417974097246e-07,
+      "logits/chosen": -0.4230140149593353,
+      "logits/rejected": -0.2956461012363434,
+      "logps/chosen": -4.4645538330078125,
+      "logps/rejected": -5.2208757400512695,
+      "loss": 0.051,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.4645538330078125,
+      "rewards/margins": 0.7563215494155884,
+      "rewards/rejected": -5.2208757400512695,
+      "sft_loss": 4.190750598907471,
+      "step": 2485
+    },
+    {
+      "epoch": 1.332664325137983,
+      "grad_norm": 0.5206999138913958,
+      "learning_rate": 6.803901619550981e-07,
+      "logits/chosen": -0.5323997735977173,
+      "logits/rejected": -0.4850800931453705,
+      "logps/chosen": -4.313595771789551,
+      "logps/rejected": -4.872060775756836,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.313595771789551,
+      "rewards/margins": 0.5584650635719299,
+      "rewards/rejected": -4.872060775756836,
+      "sft_loss": 4.027804374694824,
+      "step": 2490
+    },
+    {
+      "epoch": 1.3353403579193845,
+      "grad_norm": 0.7606406487104825,
+      "learning_rate": 6.789367763543292e-07,
+      "logits/chosen": -0.4217154383659363,
+      "logits/rejected": -0.402413934469223,
+      "logps/chosen": -4.401356220245361,
+      "logps/rejected": -4.916018009185791,
+      "loss": 0.0543,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.401356220245361,
+      "rewards/margins": 0.5146620273590088,
+      "rewards/rejected": -4.916018009185791,
+      "sft_loss": 4.163581848144531,
+      "step": 2495
+    },
+    {
+      "epoch": 1.338016390700786,
+      "grad_norm": 0.8740369373885283,
+      "learning_rate": 6.774816547081714e-07,
+      "logits/chosen": -0.517784059047699,
+      "logits/rejected": -0.32167983055114746,
+      "logps/chosen": -4.534006595611572,
+      "logps/rejected": -5.119873046875,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.534006595611572,
+      "rewards/margins": 0.5858662724494934,
+      "rewards/rejected": -5.119873046875,
+      "sft_loss": 4.325592994689941,
+      "step": 2500
+    },
+    {
+      "epoch": 1.3406924234821878,
+      "grad_norm": 0.37955249063988084,
+      "learning_rate": 6.760248111342211e-07,
+      "logits/chosen": -0.5520938634872437,
+      "logits/rejected": -0.37044256925582886,
+      "logps/chosen": -4.716580390930176,
+      "logps/rejected": -5.31876277923584,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.716580390930176,
+      "rewards/margins": 0.6021825075149536,
+      "rewards/rejected": -5.31876277923584,
+      "sft_loss": 4.433178901672363,
+      "step": 2505
+    },
+    {
+      "epoch": 1.3433684562635893,
+      "grad_norm": 0.5526291475670098,
+      "learning_rate": 6.745662597667813e-07,
+      "logits/chosen": -0.5777138471603394,
+      "logits/rejected": -0.42819347977638245,
+      "logps/chosen": -4.469142436981201,
+      "logps/rejected": -5.03638219833374,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.469142436981201,
+      "rewards/margins": 0.56723952293396,
+      "rewards/rejected": -5.03638219833374,
+      "sft_loss": 4.1554365158081055,
+      "step": 2510
+    },
+    {
+      "epoch": 1.3460444890449907,
+      "grad_norm": 0.5123609275560559,
+      "learning_rate": 6.731060147567236e-07,
+      "logits/chosen": -0.42814022302627563,
+      "logits/rejected": -0.3708665668964386,
+      "logps/chosen": -4.259950637817383,
+      "logps/rejected": -4.8878889083862305,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.259950637817383,
+      "rewards/margins": 0.6279383301734924,
+      "rewards/rejected": -4.8878889083862305,
+      "sft_loss": 3.9868011474609375,
+      "step": 2515
+    },
+    {
+      "epoch": 1.3487205218263925,
+      "grad_norm": 1.043578134549379,
+      "learning_rate": 6.716440902713515e-07,
+      "logits/chosen": -0.5802567601203918,
+      "logits/rejected": -0.5306234955787659,
+      "logps/chosen": -4.2891154289245605,
+      "logps/rejected": -4.734459400177002,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.2891154289245605,
+      "rewards/margins": 0.4453437924385071,
+      "rewards/rejected": -4.734459400177002,
+      "sft_loss": 3.9280121326446533,
+      "step": 2520
+    },
+    {
+      "epoch": 1.351396554607794,
+      "grad_norm": 0.4368528274403268,
+      "learning_rate": 6.701805004942627e-07,
+      "logits/chosen": -0.5906983613967896,
+      "logits/rejected": -0.5404535531997681,
+      "logps/chosen": -4.676029205322266,
+      "logps/rejected": -5.317166805267334,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.676029205322266,
+      "rewards/margins": 0.6411380767822266,
+      "rewards/rejected": -5.317166805267334,
+      "sft_loss": 4.461281776428223,
+      "step": 2525
+    },
+    {
+      "epoch": 1.3540725873891954,
+      "grad_norm": 0.7322308576700994,
+      "learning_rate": 6.687152596252119e-07,
+      "logits/chosen": -0.7153034210205078,
+      "logits/rejected": -0.6650447249412537,
+      "logps/chosen": -4.918754577636719,
+      "logps/rejected": -5.285243511199951,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.918754577636719,
+      "rewards/margins": 0.3664887249469757,
+      "rewards/rejected": -5.285243511199951,
+      "sft_loss": 4.644976615905762,
+      "step": 2530
+    },
+    {
+      "epoch": 1.3567486201705972,
+      "grad_norm": 0.44941070367181873,
+      "learning_rate": 6.672483818799722e-07,
+      "logits/chosen": -0.7034205794334412,
+      "logits/rejected": -0.5504308938980103,
+      "logps/chosen": -4.57156229019165,
+      "logps/rejected": -5.094230651855469,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.57156229019165,
+      "rewards/margins": 0.5226688385009766,
+      "rewards/rejected": -5.094230651855469,
+      "sft_loss": 4.343292236328125,
+      "step": 2535
+    },
+    {
+      "epoch": 1.3594246529519987,
+      "grad_norm": 0.4046531817450191,
+      "learning_rate": 6.657798814901978e-07,
+      "logits/chosen": -0.6097856760025024,
+      "logits/rejected": -0.4016978144645691,
+      "logps/chosen": -4.473921775817871,
+      "logps/rejected": -4.938872337341309,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.473921775817871,
+      "rewards/margins": 0.46495094895362854,
+      "rewards/rejected": -4.938872337341309,
+      "sft_loss": 4.1183295249938965,
+      "step": 2540
+    },
+    {
+      "epoch": 1.3621006857334002,
+      "grad_norm": 0.5055596445159121,
+      "learning_rate": 6.643097727032863e-07,
+      "logits/chosen": -0.5995045900344849,
+      "logits/rejected": -0.38927775621414185,
+      "logps/chosen": -4.236147880554199,
+      "logps/rejected": -4.9611687660217285,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.236147880554199,
+      "rewards/margins": 0.7250210642814636,
+      "rewards/rejected": -4.9611687660217285,
+      "sft_loss": 4.008961200714111,
+      "step": 2545
+    },
+    {
+      "epoch": 1.3647767185148019,
+      "grad_norm": 0.5850481177585032,
+      "learning_rate": 6.628380697822392e-07,
+      "logits/chosen": -0.5949841737747192,
+      "logits/rejected": -0.43908077478408813,
+      "logps/chosen": -4.427455902099609,
+      "logps/rejected": -4.932312965393066,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.427455902099609,
+      "rewards/margins": 0.5048567056655884,
+      "rewards/rejected": -4.932312965393066,
+      "sft_loss": 4.125482082366943,
+      "step": 2550
+    },
+    {
+      "epoch": 1.3674527512962034,
+      "grad_norm": 0.5054164459036671,
+      "learning_rate": 6.61364787005525e-07,
+      "logits/chosen": -0.5829646587371826,
+      "logits/rejected": -0.4393673837184906,
+      "logps/chosen": -4.557315349578857,
+      "logps/rejected": -5.199041843414307,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.557315349578857,
+      "rewards/margins": 0.6417262554168701,
+      "rewards/rejected": -5.199041843414307,
+      "sft_loss": 4.364875793457031,
+      "step": 2555
+    },
+    {
+      "epoch": 1.3701287840776049,
+      "grad_norm": 0.5769754442294773,
+      "learning_rate": 6.598899386669395e-07,
+      "logits/chosen": -0.6286736130714417,
+      "logits/rejected": -0.5021859407424927,
+      "logps/chosen": -4.56746768951416,
+      "logps/rejected": -5.148749351501465,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.56746768951416,
+      "rewards/margins": 0.5812815427780151,
+      "rewards/rejected": -5.148749351501465,
+      "sft_loss": 4.27559757232666,
+      "step": 2560
+    },
+    {
+      "epoch": 1.3728048168590066,
+      "grad_norm": 0.5198057081298884,
+      "learning_rate": 6.584135390754679e-07,
+      "logits/chosen": -0.6246506571769714,
+      "logits/rejected": -0.4918225407600403,
+      "logps/chosen": -4.391090393066406,
+      "logps/rejected": -5.208021640777588,
+      "loss": 0.051,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.391090393066406,
+      "rewards/margins": 0.8169304132461548,
+      "rewards/rejected": -5.208021640777588,
+      "sft_loss": 4.124699592590332,
+      "step": 2565
+    },
+    {
+      "epoch": 1.375480849640408,
+      "grad_norm": 0.3904175802781019,
+      "learning_rate": 6.569356025551454e-07,
+      "logits/chosen": -0.569642186164856,
+      "logits/rejected": -0.5206794738769531,
+      "logps/chosen": -4.505235195159912,
+      "logps/rejected": -5.000717639923096,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.505235195159912,
+      "rewards/margins": 0.49548235535621643,
+      "rewards/rejected": -5.000717639923096,
+      "sft_loss": 4.084939479827881,
+      "step": 2570
+    },
+    {
+      "epoch": 1.3781568824218096,
+      "grad_norm": 0.4284126301502038,
+      "learning_rate": 6.554561434449186e-07,
+      "logits/chosen": -0.6713020205497742,
+      "logits/rejected": -0.4908788800239563,
+      "logps/chosen": -4.553044319152832,
+      "logps/rejected": -5.149251461029053,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.553044319152832,
+      "rewards/margins": 0.5962072610855103,
+      "rewards/rejected": -5.149251461029053,
+      "sft_loss": 4.29677677154541,
+      "step": 2575
+    },
+    {
+      "epoch": 1.3808329152032113,
+      "grad_norm": 0.43693773117100526,
+      "learning_rate": 6.539751760985063e-07,
+      "logits/chosen": -0.5928459763526917,
+      "logits/rejected": -0.518842339515686,
+      "logps/chosen": -4.526194095611572,
+      "logps/rejected": -4.862375259399414,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.526194095611572,
+      "rewards/margins": 0.33618125319480896,
+      "rewards/rejected": -4.862375259399414,
+      "sft_loss": 4.17771577835083,
+      "step": 2580
+    },
+    {
+      "epoch": 1.3835089479846128,
+      "grad_norm": 0.4272368591632599,
+      "learning_rate": 6.524927148842602e-07,
+      "logits/chosen": -0.49407991766929626,
+      "logits/rejected": -0.31041932106018066,
+      "logps/chosen": -4.6627631187438965,
+      "logps/rejected": -5.185405731201172,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.6627631187438965,
+      "rewards/margins": 0.5226420164108276,
+      "rewards/rejected": -5.185405731201172,
+      "sft_loss": 4.263367176055908,
+      "step": 2585
+    },
+    {
+      "epoch": 1.3861849807660143,
+      "grad_norm": 0.6017253938081369,
+      "learning_rate": 6.510087741850254e-07,
+      "logits/chosen": -0.5680667161941528,
+      "logits/rejected": -0.45448827743530273,
+      "logps/chosen": -4.5009260177612305,
+      "logps/rejected": -5.0340142250061035,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.5009260177612305,
+      "rewards/margins": 0.5330880880355835,
+      "rewards/rejected": -5.0340142250061035,
+      "sft_loss": 4.204409599304199,
+      "step": 2590
+    },
+    {
+      "epoch": 1.388861013547416,
+      "grad_norm": 0.3834732299898269,
+      "learning_rate": 6.495233683980012e-07,
+      "logits/chosen": -0.6274980306625366,
+      "logits/rejected": -0.5894747376441956,
+      "logps/chosen": -4.352082252502441,
+      "logps/rejected": -4.913810729980469,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.352082252502441,
+      "rewards/margins": 0.561728835105896,
+      "rewards/rejected": -4.913810729980469,
+      "sft_loss": 4.131616592407227,
+      "step": 2595
+    },
+    {
+      "epoch": 1.3915370463288175,
+      "grad_norm": 0.7113553800524897,
+      "learning_rate": 6.480365119346011e-07,
+      "logits/chosen": -0.5005598068237305,
+      "logits/rejected": -0.3732762336730957,
+      "logps/chosen": -4.533462047576904,
+      "logps/rejected": -4.987781524658203,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.533462047576904,
+      "rewards/margins": 0.45431938767433167,
+      "rewards/rejected": -4.987781524658203,
+      "sft_loss": 4.211251258850098,
+      "step": 2600
+    },
+    {
+      "epoch": 1.394213079110219,
+      "grad_norm": 0.46171155185512186,
+      "learning_rate": 6.465482192203129e-07,
+      "logits/chosen": -0.5044493079185486,
+      "logits/rejected": -0.43740129470825195,
+      "logps/chosen": -4.586564064025879,
+      "logps/rejected": -5.1160783767700195,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.586564064025879,
+      "rewards/margins": 0.5295146107673645,
+      "rewards/rejected": -5.1160783767700195,
+      "sft_loss": 4.3244524002075195,
+      "step": 2605
+    },
+    {
+      "epoch": 1.3968891118916207,
+      "grad_norm": 0.4123811680795034,
+      "learning_rate": 6.45058504694559e-07,
+      "logits/chosen": -0.4199046194553375,
+      "logits/rejected": -0.4002401828765869,
+      "logps/chosen": -4.472055435180664,
+      "logps/rejected": -5.099357604980469,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.472055435180664,
+      "rewards/margins": 0.627302348613739,
+      "rewards/rejected": -5.099357604980469,
+      "sft_loss": 4.206655502319336,
+      "step": 2610
+    },
+    {
+      "epoch": 1.3995651446730222,
+      "grad_norm": 0.6033441928029727,
+      "learning_rate": 6.435673828105564e-07,
+      "logits/chosen": -0.536944568157196,
+      "logits/rejected": -0.3742099106311798,
+      "logps/chosen": -4.438148498535156,
+      "logps/rejected": -5.0379743576049805,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.438148498535156,
+      "rewards/margins": 0.5998249053955078,
+      "rewards/rejected": -5.0379743576049805,
+      "sft_loss": 4.172540187835693,
+      "step": 2615
+    },
+    {
+      "epoch": 1.402241177454424,
+      "grad_norm": 0.6677651851881835,
+      "learning_rate": 6.420748680351763e-07,
+      "logits/chosen": -0.4909477233886719,
+      "logits/rejected": -0.5832905173301697,
+      "logps/chosen": -4.574097156524658,
+      "logps/rejected": -5.0403947830200195,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.574097156524658,
+      "rewards/margins": 0.46629753708839417,
+      "rewards/rejected": -5.0403947830200195,
+      "sft_loss": 4.372440338134766,
+      "step": 2620
+    },
+    {
+      "epoch": 1.4049172102358254,
+      "grad_norm": 0.41337147810438973,
+      "learning_rate": 6.405809748488032e-07,
+      "logits/chosen": -0.6795511245727539,
+      "logits/rejected": -0.5350240468978882,
+      "logps/chosen": -4.659113883972168,
+      "logps/rejected": -5.271306991577148,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.659113883972168,
+      "rewards/margins": 0.6121935248374939,
+      "rewards/rejected": -5.271306991577148,
+      "sft_loss": 4.381941318511963,
+      "step": 2625
+    },
+    {
+      "epoch": 1.4075932430172269,
+      "grad_norm": 0.4699970379173335,
+      "learning_rate": 6.390857177451956e-07,
+      "logits/chosen": -0.7591134309768677,
+      "logits/rejected": -0.5330101847648621,
+      "logps/chosen": -4.584787845611572,
+      "logps/rejected": -5.0374627113342285,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.584787845611572,
+      "rewards/margins": 0.4526752531528473,
+      "rewards/rejected": -5.0374627113342285,
+      "sft_loss": 4.36737585067749,
+      "step": 2630
+    },
+    {
+      "epoch": 1.4102692757986286,
+      "grad_norm": 0.48120506353502984,
+      "learning_rate": 6.375891112313445e-07,
+      "logits/chosen": -0.7231135368347168,
+      "logits/rejected": -0.6274951100349426,
+      "logps/chosen": -4.365334510803223,
+      "logps/rejected": -4.895984649658203,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.365334510803223,
+      "rewards/margins": 0.530649721622467,
+      "rewards/rejected": -4.895984649658203,
+      "sft_loss": 4.074070453643799,
+      "step": 2635
+    },
+    {
+      "epoch": 1.41294530858003,
+      "grad_norm": 0.7025950418032688,
+      "learning_rate": 6.360911698273326e-07,
+      "logits/chosen": -0.5732952952384949,
+      "logits/rejected": -0.4492496848106384,
+      "logps/chosen": -4.392951011657715,
+      "logps/rejected": -4.779691219329834,
+      "loss": 0.0541,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.392951011657715,
+      "rewards/margins": 0.3867405951023102,
+      "rewards/rejected": -4.779691219329834,
+      "sft_loss": 4.139547348022461,
+      "step": 2640
+    },
+    {
+      "epoch": 1.4156213413614318,
+      "grad_norm": 0.3817804751124299,
+      "learning_rate": 6.345919080661944e-07,
+      "logits/chosen": -0.6146822571754456,
+      "logits/rejected": -0.5575979351997375,
+      "logps/chosen": -4.378892421722412,
+      "logps/rejected": -5.082538604736328,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.378892421722412,
+      "rewards/margins": 0.7036463022232056,
+      "rewards/rejected": -5.082538604736328,
+      "sft_loss": 4.1927947998046875,
+      "step": 2645
+    },
+    {
+      "epoch": 1.4182973741428333,
+      "grad_norm": 0.3484067347185367,
+      "learning_rate": 6.330913404937737e-07,
+      "logits/chosen": -0.6765519380569458,
+      "logits/rejected": -0.5006201863288879,
+      "logps/chosen": -4.664497375488281,
+      "logps/rejected": -5.329741477966309,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.664497375488281,
+      "rewards/margins": 0.6652437448501587,
+      "rewards/rejected": -5.329741477966309,
+      "sft_loss": 4.3615312576293945,
+      "step": 2650
+    },
+    {
+      "epoch": 1.4209734069242348,
+      "grad_norm": 0.5359963623152867,
+      "learning_rate": 6.315894816685838e-07,
+      "logits/chosen": -0.507836639881134,
+      "logits/rejected": -0.3151811957359314,
+      "logps/chosen": -4.616525650024414,
+      "logps/rejected": -4.9871506690979,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.616525650024414,
+      "rewards/margins": 0.37062522768974304,
+      "rewards/rejected": -4.9871506690979,
+      "sft_loss": 4.264617443084717,
+      "step": 2655
+    },
+    {
+      "epoch": 1.4236494397056365,
+      "grad_norm": 0.5569788457008981,
+      "learning_rate": 6.300863461616657e-07,
+      "logits/chosen": -0.4941771924495697,
+      "logits/rejected": -0.404980331659317,
+      "logps/chosen": -4.432186603546143,
+      "logps/rejected": -4.979880332946777,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.432186603546143,
+      "rewards/margins": 0.5476942658424377,
+      "rewards/rejected": -4.979880332946777,
+      "sft_loss": 4.154915809631348,
+      "step": 2660
+    },
+    {
+      "epoch": 1.426325472487038,
+      "grad_norm": 0.5557224282438943,
+      "learning_rate": 6.285819485564465e-07,
+      "logits/chosen": -0.6572912931442261,
+      "logits/rejected": -0.4884692132472992,
+      "logps/chosen": -4.340250492095947,
+      "logps/rejected": -4.960097789764404,
+      "loss": 0.052,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.340250492095947,
+      "rewards/margins": 0.6198478937149048,
+      "rewards/rejected": -4.960097789764404,
+      "sft_loss": 4.011279106140137,
+      "step": 2665
+    },
+    {
+      "epoch": 1.4290015052684395,
+      "grad_norm": 0.4966264028029257,
+      "learning_rate": 6.270763034485986e-07,
+      "logits/chosen": -0.5745595693588257,
+      "logits/rejected": -0.45156174898147583,
+      "logps/chosen": -4.528615474700928,
+      "logps/rejected": -5.070132255554199,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.528615474700928,
+      "rewards/margins": 0.5415170788764954,
+      "rewards/rejected": -5.070132255554199,
+      "sft_loss": 4.147845268249512,
+      "step": 2670
+    },
+    {
+      "epoch": 1.4316775380498412,
+      "grad_norm": 0.5516956159885005,
+      "learning_rate": 6.255694254458972e-07,
+      "logits/chosen": -0.7048081159591675,
+      "logits/rejected": -0.5410698056221008,
+      "logps/chosen": -4.632177352905273,
+      "logps/rejected": -5.155903339385986,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.632177352905273,
+      "rewards/margins": 0.5237262845039368,
+      "rewards/rejected": -5.155903339385986,
+      "sft_loss": 4.193863391876221,
+      "step": 2675
+    },
+    {
+      "epoch": 1.4343535708312427,
+      "grad_norm": 0.6798152772865235,
+      "learning_rate": 6.240613291680795e-07,
+      "logits/chosen": -0.6842337846755981,
+      "logits/rejected": -0.5142254829406738,
+      "logps/chosen": -4.667544364929199,
+      "logps/rejected": -5.1955060958862305,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.667544364929199,
+      "rewards/margins": 0.5279615521430969,
+      "rewards/rejected": -5.1955060958862305,
+      "sft_loss": 4.322264194488525,
+      "step": 2680
+    },
+    {
+      "epoch": 1.4370296036126442,
+      "grad_norm": 0.4794517877865857,
+      "learning_rate": 6.225520292467021e-07,
+      "logits/chosen": -0.6852758526802063,
+      "logits/rejected": -0.47948360443115234,
+      "logps/chosen": -4.298248291015625,
+      "logps/rejected": -5.128951072692871,
+      "loss": 0.0492,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.298248291015625,
+      "rewards/margins": 0.8307035565376282,
+      "rewards/rejected": -5.128951072692871,
+      "sft_loss": 4.007875919342041,
+      "step": 2685
+    },
+    {
+      "epoch": 1.439705636394046,
+      "grad_norm": 0.5931639917236166,
+      "learning_rate": 6.210415403249993e-07,
+      "logits/chosen": -0.7698257565498352,
+      "logits/rejected": -0.49045735597610474,
+      "logps/chosen": -4.419869899749756,
+      "logps/rejected": -5.26506233215332,
+      "loss": 0.051,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.419869899749756,
+      "rewards/margins": 0.8451923131942749,
+      "rewards/rejected": -5.26506233215332,
+      "sft_loss": 3.9987094402313232,
+      "step": 2690
+    },
+    {
+      "epoch": 1.4423816691754474,
+      "grad_norm": 0.6680417881685654,
+      "learning_rate": 6.195298770577415e-07,
+      "logits/chosen": -0.6223223209381104,
+      "logits/rejected": -0.6436036825180054,
+      "logps/chosen": -4.467902183532715,
+      "logps/rejected": -5.064675331115723,
+      "loss": 0.0528,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.467902183532715,
+      "rewards/margins": 0.5967725515365601,
+      "rewards/rejected": -5.064675331115723,
+      "sft_loss": 4.223404407501221,
+      "step": 2695
+    },
+    {
+      "epoch": 1.445057701956849,
+      "grad_norm": 0.41813825541952365,
+      "learning_rate": 6.180170541110923e-07,
+      "logits/chosen": -0.6707671880722046,
+      "logits/rejected": -0.47789353132247925,
+      "logps/chosen": -4.537481307983398,
+      "logps/rejected": -5.07298469543457,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.537481307983398,
+      "rewards/margins": 0.5355033874511719,
+      "rewards/rejected": -5.07298469543457,
+      "sft_loss": 4.25797176361084,
+      "step": 2700
+    },
+    {
+      "epoch": 1.4477337347382506,
+      "grad_norm": 0.4045049853183389,
+      "learning_rate": 6.165030861624663e-07,
+      "logits/chosen": -0.8904841542243958,
+      "logits/rejected": -0.5857111215591431,
+      "logps/chosen": -4.475926399230957,
+      "logps/rejected": -5.331136703491211,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.475926399230957,
+      "rewards/margins": 0.8552104830741882,
+      "rewards/rejected": -5.331136703491211,
+      "sft_loss": 4.307584762573242,
+      "step": 2705
+    },
+    {
+      "epoch": 1.4504097675196521,
+      "grad_norm": 0.4023569815722522,
+      "learning_rate": 6.149879879003876e-07,
+      "logits/chosen": -0.6515690088272095,
+      "logits/rejected": -0.6833658814430237,
+      "logps/chosen": -4.548512935638428,
+      "logps/rejected": -4.988827705383301,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.548512935638428,
+      "rewards/margins": 0.44031429290771484,
+      "rewards/rejected": -4.988827705383301,
+      "sft_loss": 4.278894901275635,
+      "step": 2710
+    },
+    {
+      "epoch": 1.4530858003010536,
+      "grad_norm": 0.397208497229481,
+      "learning_rate": 6.13471774024346e-07,
+      "logits/chosen": -0.8046046495437622,
+      "logits/rejected": -0.6881554126739502,
+      "logps/chosen": -4.304449081420898,
+      "logps/rejected": -4.92133903503418,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.304449081420898,
+      "rewards/margins": 0.6168895363807678,
+      "rewards/rejected": -4.92133903503418,
+      "sft_loss": 4.068329811096191,
+      "step": 2715
+    },
+    {
+      "epoch": 1.4557618330824553,
+      "grad_norm": 0.5095693467130826,
+      "learning_rate": 6.119544592446551e-07,
+      "logits/chosen": -0.6910004019737244,
+      "logits/rejected": -0.5953119993209839,
+      "logps/chosen": -4.223155498504639,
+      "logps/rejected": -4.724904537200928,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.223155498504639,
+      "rewards/margins": 0.5017491579055786,
+      "rewards/rejected": -4.724904537200928,
+      "sft_loss": 3.9544272422790527,
+      "step": 2720
+    },
+    {
+      "epoch": 1.4584378658638568,
+      "grad_norm": 0.4349881324413229,
+      "learning_rate": 6.104360582823096e-07,
+      "logits/chosen": -0.7427471876144409,
+      "logits/rejected": -0.6419012546539307,
+      "logps/chosen": -4.591288089752197,
+      "logps/rejected": -5.109165191650391,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.591288089752197,
+      "rewards/margins": 0.5178766846656799,
+      "rewards/rejected": -5.109165191650391,
+      "sft_loss": 4.29121208190918,
+      "step": 2725
+    },
+    {
+      "epoch": 1.4611138986452583,
+      "grad_norm": 0.5332361503745241,
+      "learning_rate": 6.089165858688423e-07,
+      "logits/chosen": -0.6411378383636475,
+      "logits/rejected": -0.4586389660835266,
+      "logps/chosen": -4.49960994720459,
+      "logps/rejected": -5.278151988983154,
+      "loss": 0.0502,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.49960994720459,
+      "rewards/margins": 0.7785428762435913,
+      "rewards/rejected": -5.278151988983154,
+      "sft_loss": 4.1990766525268555,
+      "step": 2730
+    },
+    {
+      "epoch": 1.46378993142666,
+      "grad_norm": 0.6425686695108819,
+      "learning_rate": 6.073960567461811e-07,
+      "logits/chosen": -0.638020932674408,
+      "logits/rejected": -0.4485744535923004,
+      "logps/chosen": -4.348964691162109,
+      "logps/rejected": -5.104625225067139,
+      "loss": 0.0502,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.348964691162109,
+      "rewards/margins": 0.7556604146957397,
+      "rewards/rejected": -5.104625225067139,
+      "sft_loss": 4.003968715667725,
+      "step": 2735
+    },
+    {
+      "epoch": 1.4664659642080615,
+      "grad_norm": 0.3893109326659487,
+      "learning_rate": 6.058744856665065e-07,
+      "logits/chosen": -0.576491117477417,
+      "logits/rejected": -0.5205652117729187,
+      "logps/chosen": -4.520781517028809,
+      "logps/rejected": -5.457547187805176,
+      "loss": 0.0501,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.520781517028809,
+      "rewards/margins": 0.9367659687995911,
+      "rewards/rejected": -5.457547187805176,
+      "sft_loss": 4.14150333404541,
+      "step": 2740
+    },
+    {
+      "epoch": 1.469141996989463,
+      "grad_norm": 0.4766405887480856,
+      "learning_rate": 6.043518873921074e-07,
+      "logits/chosen": -0.6036868691444397,
+      "logits/rejected": -0.4955274164676666,
+      "logps/chosen": -4.2526655197143555,
+      "logps/rejected": -4.9464311599731445,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.2526655197143555,
+      "rewards/margins": 0.6937664151191711,
+      "rewards/rejected": -4.9464311599731445,
+      "sft_loss": 3.8363234996795654,
+      "step": 2745
+    },
+    {
+      "epoch": 1.4718180297708647,
+      "grad_norm": 0.5286565847045684,
+      "learning_rate": 6.028282766952393e-07,
+      "logits/chosen": -0.5392686128616333,
+      "logits/rejected": -0.47638773918151855,
+      "logps/chosen": -4.556574821472168,
+      "logps/rejected": -5.3537187576293945,
+      "loss": 0.0499,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.556574821472168,
+      "rewards/margins": 0.7971442937850952,
+      "rewards/rejected": -5.3537187576293945,
+      "sft_loss": 4.128913879394531,
+      "step": 2750
+    },
+    {
+      "epoch": 1.4744940625522662,
+      "grad_norm": 0.5894915869003183,
+      "learning_rate": 6.013036683579798e-07,
+      "logits/chosen": -0.533528745174408,
+      "logits/rejected": -0.45111075043678284,
+      "logps/chosen": -4.42080020904541,
+      "logps/rejected": -5.1628546714782715,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.42080020904541,
+      "rewards/margins": 0.7420543432235718,
+      "rewards/rejected": -5.1628546714782715,
+      "sft_loss": 4.113458156585693,
+      "step": 2755
+    },
+    {
+      "epoch": 1.4771700953336677,
+      "grad_norm": 0.494258977667678,
+      "learning_rate": 5.997780771720854e-07,
+      "logits/chosen": -0.696727991104126,
+      "logits/rejected": -0.4603763520717621,
+      "logps/chosen": -4.480790138244629,
+      "logps/rejected": -5.2772603034973145,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.480790138244629,
+      "rewards/margins": 0.7964698076248169,
+      "rewards/rejected": -5.2772603034973145,
+      "sft_loss": 4.252973556518555,
+      "step": 2760
+    },
+    {
+      "epoch": 1.4798461281150694,
+      "grad_norm": 0.5824907531945315,
+      "learning_rate": 5.982515179388486e-07,
+      "logits/chosen": -0.5312920808792114,
+      "logits/rejected": -0.42109164595603943,
+      "logps/chosen": -4.590191841125488,
+      "logps/rejected": -5.211472988128662,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.590191841125488,
+      "rewards/margins": 0.6212812662124634,
+      "rewards/rejected": -5.211472988128662,
+      "sft_loss": 4.298916816711426,
+      "step": 2765
+    },
+    {
+      "epoch": 1.482522160896471,
+      "grad_norm": 0.39799397306324386,
+      "learning_rate": 5.967240054689541e-07,
+      "logits/chosen": -0.6680682301521301,
+      "logits/rejected": -0.6447493433952332,
+      "logps/chosen": -4.263683319091797,
+      "logps/rejected": -4.877932548522949,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.263683319091797,
+      "rewards/margins": 0.6142495274543762,
+      "rewards/rejected": -4.877932548522949,
+      "sft_loss": 4.076264381408691,
+      "step": 2770
+    },
+    {
+      "epoch": 1.4851981936778724,
+      "grad_norm": 0.38749152187510244,
+      "learning_rate": 5.951955545823342e-07,
+      "logits/chosen": -0.6154682040214539,
+      "logits/rejected": -0.5455372929573059,
+      "logps/chosen": -4.43137788772583,
+      "logps/rejected": -5.089561462402344,
+      "loss": 0.053,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.43137788772583,
+      "rewards/margins": 0.6581836938858032,
+      "rewards/rejected": -5.089561462402344,
+      "sft_loss": 4.259114742279053,
+      "step": 2775
+    },
+    {
+      "epoch": 1.4878742264592741,
+      "grad_norm": 0.3419474438030561,
+      "learning_rate": 5.936661801080263e-07,
+      "logits/chosen": -0.5836928486824036,
+      "logits/rejected": -0.5123255848884583,
+      "logps/chosen": -4.479122638702393,
+      "logps/rejected": -5.086029052734375,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.479122638702393,
+      "rewards/margins": 0.6069061160087585,
+      "rewards/rejected": -5.086029052734375,
+      "sft_loss": 4.2064971923828125,
+      "step": 2780
+    },
+    {
+      "epoch": 1.4905502592406756,
+      "grad_norm": 0.46022415654201376,
+      "learning_rate": 5.92135896884028e-07,
+      "logits/chosen": -0.7167954444885254,
+      "logits/rejected": -0.5608044862747192,
+      "logps/chosen": -4.5941009521484375,
+      "logps/rejected": -5.4228925704956055,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.5941009521484375,
+      "rewards/margins": 0.828791618347168,
+      "rewards/rejected": -5.4228925704956055,
+      "sft_loss": 4.3245368003845215,
+      "step": 2785
+    },
+    {
+      "epoch": 1.4932262920220774,
+      "grad_norm": 0.5843407203787223,
+      "learning_rate": 5.906047197571541e-07,
+      "logits/chosen": -0.5601950287818909,
+      "logits/rejected": -0.6335816979408264,
+      "logps/chosen": -4.622066020965576,
+      "logps/rejected": -5.09460973739624,
+      "loss": 0.054,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": -4.622066020965576,
+      "rewards/margins": 0.4725441336631775,
+      "rewards/rejected": -5.09460973739624,
+      "sft_loss": 4.3805437088012695,
+      "step": 2790
+    },
+    {
+      "epoch": 1.4959023248034788,
+      "grad_norm": 0.5774938255693773,
+      "learning_rate": 5.890726635828919e-07,
+      "logits/chosen": -0.5170835256576538,
+      "logits/rejected": -0.5625238418579102,
+      "logps/chosen": -4.416979789733887,
+      "logps/rejected": -4.956758975982666,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.416979789733887,
+      "rewards/margins": 0.539779007434845,
+      "rewards/rejected": -4.956758975982666,
+      "sft_loss": 4.20320463180542,
+      "step": 2795
+    },
+    {
+      "epoch": 1.4985783575848803,
+      "grad_norm": 0.5595353471210672,
+      "learning_rate": 5.875397432252569e-07,
+      "logits/chosen": -0.6986640691757202,
+      "logits/rejected": -0.6899408102035522,
+      "logps/chosen": -4.487471580505371,
+      "logps/rejected": -5.051347255706787,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.487471580505371,
+      "rewards/margins": 0.5638760328292847,
+      "rewards/rejected": -5.051347255706787,
+      "sft_loss": 4.179348468780518,
+      "step": 2800
+    },
+    {
+      "epoch": 1.4985783575848803,
+      "eval_logits/chosen": -0.2576724588871002,
+      "eval_logits/rejected": -0.17293211817741394,
+      "eval_logps/chosen": -4.4932732582092285,
+      "eval_logps/rejected": -5.13569974899292,
+      "eval_loss": 0.05055845528841019,
+      "eval_rewards/accuracies": 0.6817507147789001,
+      "eval_rewards/chosen": -4.4932732582092285,
+      "eval_rewards/margins": 0.6424273252487183,
+      "eval_rewards/rejected": -5.13569974899292,
+      "eval_runtime": 43.9283,
+      "eval_samples_per_second": 30.618,
+      "eval_sft_loss": 4.150156497955322,
+      "eval_steps_per_second": 7.672,
+      "step": 2800
+    },
+    {
+      "epoch": 1.5012543903662818,
+      "grad_norm": 0.768983230997841,
+      "learning_rate": 5.860059735566491e-07,
+      "logits/chosen": -0.9295442700386047,
+      "logits/rejected": -0.7197269201278687,
+      "logps/chosen": -4.4977641105651855,
+      "logps/rejected": -5.085012435913086,
+      "loss": 0.052,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.4977641105651855,
+      "rewards/margins": 0.587248682975769,
+      "rewards/rejected": -5.085012435913086,
+      "sft_loss": 4.269857406616211,
+      "step": 2805
+    },
+    {
+      "epoch": 1.5039304231476835,
+      "grad_norm": 0.6637208426197561,
+      "learning_rate": 5.844713694577087e-07,
+      "logits/chosen": -0.66325843334198,
+      "logits/rejected": -0.5970256924629211,
+      "logps/chosen": -4.535934925079346,
+      "logps/rejected": -5.195251941680908,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.535934925079346,
+      "rewards/margins": 0.6593173146247864,
+      "rewards/rejected": -5.195251941680908,
+      "sft_loss": 4.343783378601074,
+      "step": 2810
+    },
+    {
+      "epoch": 1.5066064559290853,
+      "grad_norm": 0.8015889363748395,
+      "learning_rate": 5.829359458171714e-07,
+      "logits/chosen": -0.6388770937919617,
+      "logits/rejected": -0.5585208535194397,
+      "logps/chosen": -4.257367134094238,
+      "logps/rejected": -5.059530735015869,
+      "loss": 0.0505,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -4.257367134094238,
+      "rewards/margins": 0.8021625280380249,
+      "rewards/rejected": -5.059530735015869,
+      "sft_loss": 4.100651741027832,
+      "step": 2815
+    },
+    {
+      "epoch": 1.5092824887104868,
+      "grad_norm": 0.821817257528352,
+      "learning_rate": 5.81399717531724e-07,
+      "logits/chosen": -0.691882312297821,
+      "logits/rejected": -0.5029860734939575,
+      "logps/chosen": -4.201731204986572,
+      "logps/rejected": -4.800837516784668,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.201731204986572,
+      "rewards/margins": 0.5991066694259644,
+      "rewards/rejected": -4.800837516784668,
+      "sft_loss": 4.004908084869385,
+      "step": 2820
+    },
+    {
+      "epoch": 1.5119585214918883,
+      "grad_norm": 0.5884059335614251,
+      "learning_rate": 5.798626995058602e-07,
+      "logits/chosen": -0.8955677151679993,
+      "logits/rejected": -0.6793142557144165,
+      "logps/chosen": -4.418914794921875,
+      "logps/rejected": -5.227511405944824,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.418914794921875,
+      "rewards/margins": 0.808597207069397,
+      "rewards/rejected": -5.227511405944824,
+      "sft_loss": 4.196443557739258,
+      "step": 2825
+    },
+    {
+      "epoch": 1.51463455427329,
+      "grad_norm": 0.5937357282897382,
+      "learning_rate": 5.783249066517354e-07,
+      "logits/chosen": -0.7414668798446655,
+      "logits/rejected": -0.7135878801345825,
+      "logps/chosen": -4.4636054039001465,
+      "logps/rejected": -5.124371528625488,
+      "loss": 0.0495,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.4636054039001465,
+      "rewards/margins": 0.6607660055160522,
+      "rewards/rejected": -5.124371528625488,
+      "sft_loss": 4.119414329528809,
+      "step": 2830
+    },
+    {
+      "epoch": 1.5173105870546915,
+      "grad_norm": 0.44498445784467033,
+      "learning_rate": 5.767863538890228e-07,
+      "logits/chosen": -0.7979896664619446,
+      "logits/rejected": -0.6582103967666626,
+      "logps/chosen": -4.36227560043335,
+      "logps/rejected": -5.178773880004883,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.36227560043335,
+      "rewards/margins": 0.8164981603622437,
+      "rewards/rejected": -5.178773880004883,
+      "sft_loss": 4.110637187957764,
+      "step": 2835
+    },
+    {
+      "epoch": 1.519986619836093,
+      "grad_norm": 0.5440672665662433,
+      "learning_rate": 5.75247056144768e-07,
+      "logits/chosen": -0.7303460240364075,
+      "logits/rejected": -0.7098056674003601,
+      "logps/chosen": -4.405373573303223,
+      "logps/rejected": -4.896122455596924,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.405373573303223,
+      "rewards/margins": 0.49074921011924744,
+      "rewards/rejected": -4.896122455596924,
+      "sft_loss": 4.130322456359863,
+      "step": 2840
+    },
+    {
+      "epoch": 1.5226626526174947,
+      "grad_norm": 0.4417864165021903,
+      "learning_rate": 5.737070283532444e-07,
+      "logits/chosen": -0.8585020899772644,
+      "logits/rejected": -0.7397746443748474,
+      "logps/chosen": -4.672549247741699,
+      "logps/rejected": -5.209013938903809,
+      "loss": 0.053,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.672549247741699,
+      "rewards/margins": 0.5364635586738586,
+      "rewards/rejected": -5.209013938903809,
+      "sft_loss": 4.422049522399902,
+      "step": 2845
+    },
+    {
+      "epoch": 1.5253386853988962,
+      "grad_norm": 0.6708033380316883,
+      "learning_rate": 5.721662854558084e-07,
+      "logits/chosen": -0.8504046201705933,
+      "logits/rejected": -0.803429901599884,
+      "logps/chosen": -4.534087181091309,
+      "logps/rejected": -5.290635585784912,
+      "loss": 0.0503,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -4.534087181091309,
+      "rewards/margins": 0.7565481066703796,
+      "rewards/rejected": -5.290635585784912,
+      "sft_loss": 4.170504570007324,
+      "step": 2850
+    },
+    {
+      "epoch": 1.5280147181802977,
+      "grad_norm": 0.5434085857536741,
+      "learning_rate": 5.706248424007545e-07,
+      "logits/chosen": -0.8644043207168579,
+      "logits/rejected": -0.653121829032898,
+      "logps/chosen": -4.300144672393799,
+      "logps/rejected": -4.864175796508789,
+      "loss": 0.052,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.300144672393799,
+      "rewards/margins": 0.5640307664871216,
+      "rewards/rejected": -4.864175796508789,
+      "sft_loss": 4.084044456481934,
+      "step": 2855
+    },
+    {
+      "epoch": 1.5306907509616994,
+      "grad_norm": 0.4363012859825485,
+      "learning_rate": 5.690827141431699e-07,
+      "logits/chosen": -0.903763473033905,
+      "logits/rejected": -0.6830427050590515,
+      "logps/chosen": -4.534507751464844,
+      "logps/rejected": -5.026754379272461,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.534507751464844,
+      "rewards/margins": 0.4922458529472351,
+      "rewards/rejected": -5.026754379272461,
+      "sft_loss": 4.220981121063232,
+      "step": 2860
+    },
+    {
+      "epoch": 1.5333667837431009,
+      "grad_norm": 0.7916839888433597,
+      "learning_rate": 5.675399156447897e-07,
+      "logits/chosen": -0.8549512028694153,
+      "logits/rejected": -0.7020593881607056,
+      "logps/chosen": -4.545232772827148,
+      "logps/rejected": -5.0495147705078125,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.545232772827148,
+      "rewards/margins": 0.5042813420295715,
+      "rewards/rejected": -5.0495147705078125,
+      "sft_loss": 4.266694068908691,
+      "step": 2865
+    },
+    {
+      "epoch": 1.5360428165245024,
+      "grad_norm": 0.5390968937670435,
+      "learning_rate": 5.659964618738515e-07,
+      "logits/chosen": -0.7600046396255493,
+      "logits/rejected": -0.6426360011100769,
+      "logps/chosen": -4.455992698669434,
+      "logps/rejected": -5.1160993576049805,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.455992698669434,
+      "rewards/margins": 0.6601068377494812,
+      "rewards/rejected": -5.1160993576049805,
+      "sft_loss": 4.049184322357178,
+      "step": 2870
+    },
+    {
+      "epoch": 1.538718849305904,
+      "grad_norm": 0.5207081742722265,
+      "learning_rate": 5.644523678049509e-07,
+      "logits/chosen": -0.7210767269134521,
+      "logits/rejected": -0.6467476487159729,
+      "logps/chosen": -4.41653299331665,
+      "logps/rejected": -4.965178489685059,
+      "loss": 0.0504,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.41653299331665,
+      "rewards/margins": 0.5486454963684082,
+      "rewards/rejected": -4.965178489685059,
+      "sft_loss": 3.9591877460479736,
+      "step": 2875
+    },
+    {
+      "epoch": 1.5413948820873056,
+      "grad_norm": 0.754243559002919,
+      "learning_rate": 5.629076484188952e-07,
+      "logits/chosen": -0.5206449031829834,
+      "logits/rejected": -0.43926820158958435,
+      "logps/chosen": -4.605841636657715,
+      "logps/rejected": -5.2476396560668945,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.605841636657715,
+      "rewards/margins": 0.6417974233627319,
+      "rewards/rejected": -5.2476396560668945,
+      "sft_loss": 4.207988262176514,
+      "step": 2880
+    },
+    {
+      "epoch": 1.544070914868707,
+      "grad_norm": 0.4468744859117235,
+      "learning_rate": 5.613623187025587e-07,
+      "logits/chosen": -0.7043927311897278,
+      "logits/rejected": -0.6246447563171387,
+      "logps/chosen": -4.531125545501709,
+      "logps/rejected": -5.215548992156982,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.531125545501709,
+      "rewards/margins": 0.68442302942276,
+      "rewards/rejected": -5.215548992156982,
+      "sft_loss": 4.1478352546691895,
+      "step": 2885
+    },
+    {
+      "epoch": 1.5467469476501088,
+      "grad_norm": 0.5475211187345611,
+      "learning_rate": 5.598163936487369e-07,
+      "logits/chosen": -0.6709910035133362,
+      "logits/rejected": -0.5131962299346924,
+      "logps/chosen": -4.403019428253174,
+      "logps/rejected": -5.154913425445557,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.403019428253174,
+      "rewards/margins": 0.7518946528434753,
+      "rewards/rejected": -5.154913425445557,
+      "sft_loss": 4.077244758605957,
+      "step": 2890
+    },
+    {
+      "epoch": 1.5494229804315103,
+      "grad_norm": 0.48649288414648467,
+      "learning_rate": 5.582698882560017e-07,
+      "logits/chosen": -0.5866044759750366,
+      "logits/rejected": -0.4063977301120758,
+      "logps/chosen": -4.6435089111328125,
+      "logps/rejected": -5.319693565368652,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.6435089111328125,
+      "rewards/margins": 0.6761849522590637,
+      "rewards/rejected": -5.319693565368652,
+      "sft_loss": 4.325295448303223,
+      "step": 2895
+    },
+    {
+      "epoch": 1.5520990132129118,
+      "grad_norm": 0.6173505156011603,
+      "learning_rate": 5.567228175285549e-07,
+      "logits/chosen": -0.477353036403656,
+      "logits/rejected": -0.4176778793334961,
+      "logps/chosen": -4.330788612365723,
+      "logps/rejected": -5.041531562805176,
+      "loss": 0.0492,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.330788612365723,
+      "rewards/margins": 0.7107427716255188,
+      "rewards/rejected": -5.041531562805176,
+      "sft_loss": 3.901872158050537,
+      "step": 2900
+    },
+    {
+      "epoch": 1.5547750459943135,
+      "grad_norm": 0.6301059988922334,
+      "learning_rate": 5.551751964760838e-07,
+      "logits/chosen": -0.4368586540222168,
+      "logits/rejected": -0.4734298288822174,
+      "logps/chosen": -4.526708602905273,
+      "logps/rejected": -5.055941581726074,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.526708602905273,
+      "rewards/margins": 0.529233455657959,
+      "rewards/rejected": -5.055941581726074,
+      "sft_loss": 4.226334571838379,
+      "step": 2905
+    },
+    {
+      "epoch": 1.557451078775715,
+      "grad_norm": 0.4399652694603445,
+      "learning_rate": 5.536270401136145e-07,
+      "logits/chosen": -0.5751160383224487,
+      "logits/rejected": -0.4912167191505432,
+      "logps/chosen": -4.352970600128174,
+      "logps/rejected": -4.974688529968262,
+      "loss": 0.0505,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.352970600128174,
+      "rewards/margins": 0.621717095375061,
+      "rewards/rejected": -4.974688529968262,
+      "sft_loss": 4.020804405212402,
+      "step": 2910
+    },
+    {
+      "epoch": 1.5601271115571165,
+      "grad_norm": 0.530012826816097,
+      "learning_rate": 5.520783634613667e-07,
+      "logits/chosen": -0.45896950364112854,
+      "logits/rejected": -0.2448553591966629,
+      "logps/chosen": -4.597033500671387,
+      "logps/rejected": -5.2001471519470215,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.597033500671387,
+      "rewards/margins": 0.6031134724617004,
+      "rewards/rejected": -5.2001471519470215,
+      "sft_loss": 4.241999626159668,
+      "step": 2915
+    },
+    {
+      "epoch": 1.5628031443385182,
+      "grad_norm": 1.465870477887042,
+      "learning_rate": 5.505291815446082e-07,
+      "logits/chosen": -0.5133945345878601,
+      "logits/rejected": -0.40424662828445435,
+      "logps/chosen": -4.378600120544434,
+      "logps/rejected": -5.0904035568237305,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.378600120544434,
+      "rewards/margins": 0.71180260181427,
+      "rewards/rejected": -5.0904035568237305,
+      "sft_loss": 4.154856204986572,
+      "step": 2920
+    },
+    {
+      "epoch": 1.5654791771199197,
+      "grad_norm": 0.5803605093266383,
+      "learning_rate": 5.489795093935089e-07,
+      "logits/chosen": -0.46809881925582886,
+      "logits/rejected": -0.38210025429725647,
+      "logps/chosen": -4.535780429840088,
+      "logps/rejected": -5.164534568786621,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.535780429840088,
+      "rewards/margins": 0.6287539601325989,
+      "rewards/rejected": -5.164534568786621,
+      "sft_loss": 4.194882392883301,
+      "step": 2925
+    },
+    {
+      "epoch": 1.5681552099013212,
+      "grad_norm": 0.4478023247588819,
+      "learning_rate": 5.474293620429946e-07,
+      "logits/chosen": -0.6563531160354614,
+      "logits/rejected": -0.49145808815956116,
+      "logps/chosen": -4.50363302230835,
+      "logps/rejected": -5.435848236083984,
+      "loss": 0.0501,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.50363302230835,
+      "rewards/margins": 0.9322155117988586,
+      "rewards/rejected": -5.435848236083984,
+      "sft_loss": 4.1722307205200195,
+      "step": 2930
+    },
+    {
+      "epoch": 1.570831242682723,
+      "grad_norm": 0.5590239470097637,
+      "learning_rate": 5.458787545326018e-07,
+      "logits/chosen": -0.6394512057304382,
+      "logits/rejected": -0.5258924961090088,
+      "logps/chosen": -4.511829376220703,
+      "logps/rejected": -5.0560808181762695,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.511829376220703,
+      "rewards/margins": 0.5442512035369873,
+      "rewards/rejected": -5.0560808181762695,
+      "sft_loss": 4.22859001159668,
+      "step": 2935
+    },
+    {
+      "epoch": 1.5735072754641244,
+      "grad_norm": 0.5089043258820021,
+      "learning_rate": 5.443277019063311e-07,
+      "logits/chosen": -0.623425304889679,
+      "logits/rejected": -0.485008180141449,
+      "logps/chosen": -4.466963768005371,
+      "logps/rejected": -5.209366321563721,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.466963768005371,
+      "rewards/margins": 0.7424025535583496,
+      "rewards/rejected": -5.209366321563721,
+      "sft_loss": 4.249908447265625,
+      "step": 2940
+    },
+    {
+      "epoch": 1.5761833082455259,
+      "grad_norm": 1.0036333777326483,
+      "learning_rate": 5.427762192125023e-07,
+      "logits/chosen": -0.5547440052032471,
+      "logits/rejected": -0.46942177414894104,
+      "logps/chosen": -4.215888023376465,
+      "logps/rejected": -4.870157718658447,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.215888023376465,
+      "rewards/margins": 0.6542699337005615,
+      "rewards/rejected": -4.870157718658447,
+      "sft_loss": 3.8963356018066406,
+      "step": 2945
+    },
+    {
+      "epoch": 1.5788593410269276,
+      "grad_norm": 0.6082716686519534,
+      "learning_rate": 5.41224321503607e-07,
+      "logits/chosen": -0.615654468536377,
+      "logits/rejected": -0.34824177622795105,
+      "logps/chosen": -4.432827949523926,
+      "logps/rejected": -5.264912128448486,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.432827949523926,
+      "rewards/margins": 0.8320847749710083,
+      "rewards/rejected": -5.264912128448486,
+      "sft_loss": 4.204735279083252,
+      "step": 2950
+    },
+    {
+      "epoch": 1.5815353738083293,
+      "grad_norm": 0.790082231714183,
+      "learning_rate": 5.396720238361637e-07,
+      "logits/chosen": -0.581564724445343,
+      "logits/rejected": -0.4562462270259857,
+      "logps/chosen": -4.6167497634887695,
+      "logps/rejected": -5.168881893157959,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.6167497634887695,
+      "rewards/margins": 0.5521320700645447,
+      "rewards/rejected": -5.168881893157959,
+      "sft_loss": 4.3020734786987305,
+      "step": 2955
+    },
+    {
+      "epoch": 1.5842114065897306,
+      "grad_norm": 0.7409971306533258,
+      "learning_rate": 5.381193412705711e-07,
+      "logits/chosen": -0.6942346692085266,
+      "logits/rejected": -0.5436898469924927,
+      "logps/chosen": -4.458543300628662,
+      "logps/rejected": -5.094224452972412,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.458543300628662,
+      "rewards/margins": 0.63568115234375,
+      "rewards/rejected": -5.094224452972412,
+      "sft_loss": 4.1265668869018555,
+      "step": 2960
+    },
+    {
+      "epoch": 1.5868874393711323,
+      "grad_norm": 0.43937386822640034,
+      "learning_rate": 5.365662888709622e-07,
+      "logits/chosen": -0.6356379985809326,
+      "logits/rejected": -0.48853403329849243,
+      "logps/chosen": -4.4219865798950195,
+      "logps/rejected": -5.067595958709717,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.4219865798950195,
+      "rewards/margins": 0.6456089615821838,
+      "rewards/rejected": -5.067595958709717,
+      "sft_loss": 4.14740514755249,
+      "step": 2965
+    },
+    {
+      "epoch": 1.589563472152534,
+      "grad_norm": 0.4882088431216468,
+      "learning_rate": 5.350128817050585e-07,
+      "logits/chosen": -0.6768869161605835,
+      "logits/rejected": -0.49296027421951294,
+      "logps/chosen": -4.4894700050354,
+      "logps/rejected": -5.249065399169922,
+      "loss": 0.0506,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.4894700050354,
+      "rewards/margins": 0.7595952749252319,
+      "rewards/rejected": -5.249065399169922,
+      "sft_loss": 4.124958515167236,
+      "step": 2970
+    },
+    {
+      "epoch": 1.5922395049339353,
+      "grad_norm": 0.7357666157407255,
+      "learning_rate": 5.334591348440229e-07,
+      "logits/chosen": -0.6499765515327454,
+      "logits/rejected": -0.49098148941993713,
+      "logps/chosen": -4.480090141296387,
+      "logps/rejected": -5.1628313064575195,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.480090141296387,
+      "rewards/margins": 0.6827414035797119,
+      "rewards/rejected": -5.1628313064575195,
+      "sft_loss": 4.206170558929443,
+      "step": 2975
+    },
+    {
+      "epoch": 1.594915537715337,
+      "grad_norm": 0.46193921490301953,
+      "learning_rate": 5.319050633623141e-07,
+      "logits/chosen": -0.7076265215873718,
+      "logits/rejected": -0.5210967659950256,
+      "logps/chosen": -4.467806816101074,
+      "logps/rejected": -5.050066947937012,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.467806816101074,
+      "rewards/margins": 0.5822607278823853,
+      "rewards/rejected": -5.050066947937012,
+      "sft_loss": 4.146100044250488,
+      "step": 2980
+    },
+    {
+      "epoch": 1.5975915704967387,
+      "grad_norm": 0.4814929534898333,
+      "learning_rate": 5.303506823375409e-07,
+      "logits/chosen": -0.7259056568145752,
+      "logits/rejected": -0.5022546648979187,
+      "logps/chosen": -4.393611907958984,
+      "logps/rejected": -5.1992363929748535,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.393611907958984,
+      "rewards/margins": 0.8056240081787109,
+      "rewards/rejected": -5.1992363929748535,
+      "sft_loss": 4.143006801605225,
+      "step": 2985
+    },
+    {
+      "epoch": 1.60026760327814,
+      "grad_norm": 0.4446825637324848,
+      "learning_rate": 5.287960068503143e-07,
+      "logits/chosen": -0.715182363986969,
+      "logits/rejected": -0.5113447904586792,
+      "logps/chosen": -4.382565021514893,
+      "logps/rejected": -5.109246253967285,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.382565021514893,
+      "rewards/margins": 0.726681113243103,
+      "rewards/rejected": -5.109246253967285,
+      "sft_loss": 4.164192199707031,
+      "step": 2990
+    },
+    {
+      "epoch": 1.6029436360595417,
+      "grad_norm": 0.5996440568914445,
+      "learning_rate": 5.272410519841032e-07,
+      "logits/chosen": -0.5031524896621704,
+      "logits/rejected": -0.4284706711769104,
+      "logps/chosen": -4.4230217933654785,
+      "logps/rejected": -5.265072345733643,
+      "loss": 0.0498,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.4230217933654785,
+      "rewards/margins": 0.8420506715774536,
+      "rewards/rejected": -5.265072345733643,
+      "sft_loss": 4.037542343139648,
+      "step": 2995
+    },
+    {
+      "epoch": 1.6056196688409434,
+      "grad_norm": 0.5391392676271535,
+      "learning_rate": 5.256858328250861e-07,
+      "logits/chosen": -0.6340783834457397,
+      "logits/rejected": -0.423819363117218,
+      "logps/chosen": -4.520924091339111,
+      "logps/rejected": -5.115809440612793,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.520924091339111,
+      "rewards/margins": 0.5948851704597473,
+      "rewards/rejected": -5.115809440612793,
+      "sft_loss": 4.174069881439209,
+      "step": 3000
+    },
+    {
+      "epoch": 1.608295701622345,
+      "grad_norm": 0.8379306755066839,
+      "learning_rate": 5.241303644620063e-07,
+      "logits/chosen": -0.7196758985519409,
+      "logits/rejected": -0.48901844024658203,
+      "logps/chosen": -4.500253200531006,
+      "logps/rejected": -4.98700475692749,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.500253200531006,
+      "rewards/margins": 0.4867522120475769,
+      "rewards/rejected": -4.98700475692749,
+      "sft_loss": 4.1780219078063965,
+      "step": 3005
+    },
+    {
+      "epoch": 1.6109717344037464,
+      "grad_norm": 0.4655577368767824,
+      "learning_rate": 5.225746619860248e-07,
+      "logits/chosen": -0.7154421806335449,
+      "logits/rejected": -0.5863774418830872,
+      "logps/chosen": -4.440865516662598,
+      "logps/rejected": -5.217991828918457,
+      "loss": 0.053,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.440865516662598,
+      "rewards/margins": 0.7771264314651489,
+      "rewards/rejected": -5.217991828918457,
+      "sft_loss": 4.147049903869629,
+      "step": 3010
+    },
+    {
+      "epoch": 1.6136477671851481,
+      "grad_norm": 0.4660757476109834,
+      "learning_rate": 5.210187404905735e-07,
+      "logits/chosen": -0.5504111647605896,
+      "logits/rejected": -0.48893022537231445,
+      "logps/chosen": -4.7645158767700195,
+      "logps/rejected": -5.19139289855957,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.7645158767700195,
+      "rewards/margins": 0.4268765449523926,
+      "rewards/rejected": -5.19139289855957,
+      "sft_loss": 4.44781494140625,
+      "step": 3015
+    },
+    {
+      "epoch": 1.6163237999665496,
+      "grad_norm": 0.3871637628506248,
+      "learning_rate": 5.194626150712098e-07,
+      "logits/chosen": -0.7384278774261475,
+      "logits/rejected": -0.6122447848320007,
+      "logps/chosen": -4.537661075592041,
+      "logps/rejected": -5.120093822479248,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.537661075592041,
+      "rewards/margins": 0.5824323892593384,
+      "rewards/rejected": -5.120093822479248,
+      "sft_loss": 4.368290901184082,
+      "step": 3020
+    },
+    {
+      "epoch": 1.6189998327479511,
+      "grad_norm": 0.4037726105537447,
+      "learning_rate": 5.179063008254695e-07,
+      "logits/chosen": -0.6896153092384338,
+      "logits/rejected": -0.4879343509674072,
+      "logps/chosen": -4.333171367645264,
+      "logps/rejected": -4.875120639801025,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.333171367645264,
+      "rewards/margins": 0.5419489145278931,
+      "rewards/rejected": -4.875120639801025,
+      "sft_loss": 4.054774761199951,
+      "step": 3025
+    },
+    {
+      "epoch": 1.6216758655293528,
+      "grad_norm": 0.6217082596751843,
+      "learning_rate": 5.163498128527199e-07,
+      "logits/chosen": -0.6020374298095703,
+      "logits/rejected": -0.46034756302833557,
+      "logps/chosen": -4.333188056945801,
+      "logps/rejected": -4.944770812988281,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.333188056945801,
+      "rewards/margins": 0.611582338809967,
+      "rewards/rejected": -4.944770812988281,
+      "sft_loss": 4.070063591003418,
+      "step": 3030
+    },
+    {
+      "epoch": 1.6243518983107543,
+      "grad_norm": 0.5262751016496414,
+      "learning_rate": 5.147931662540144e-07,
+      "logits/chosen": -0.5743936896324158,
+      "logits/rejected": -0.4724980294704437,
+      "logps/chosen": -4.352665901184082,
+      "logps/rejected": -4.921462535858154,
+      "loss": 0.0505,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.352665901184082,
+      "rewards/margins": 0.5687968134880066,
+      "rewards/rejected": -4.921462535858154,
+      "sft_loss": 4.013000965118408,
+      "step": 3035
+    },
+    {
+      "epoch": 1.6270279310921558,
+      "grad_norm": 0.6059579771244825,
+      "learning_rate": 5.132363761319449e-07,
+      "logits/chosen": -0.7480147480964661,
+      "logits/rejected": -0.6982057690620422,
+      "logps/chosen": -4.525022506713867,
+      "logps/rejected": -5.196837425231934,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.525022506713867,
+      "rewards/margins": 0.6718145608901978,
+      "rewards/rejected": -5.196837425231934,
+      "sft_loss": 4.182358741760254,
+      "step": 3040
+    },
+    {
+      "epoch": 1.6297039638735575,
+      "grad_norm": 0.7560128571308733,
+      "learning_rate": 5.116794575904962e-07,
+      "logits/chosen": -0.5923138856887817,
+      "logits/rejected": -0.5394676923751831,
+      "logps/chosen": -4.500862121582031,
+      "logps/rejected": -5.164424419403076,
+      "loss": 0.0506,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.500862121582031,
+      "rewards/margins": 0.6635624170303345,
+      "rewards/rejected": -5.164424419403076,
+      "sft_loss": 4.127222061157227,
+      "step": 3045
+    },
+    {
+      "epoch": 1.632379996654959,
+      "grad_norm": 0.39703076849796204,
+      "learning_rate": 5.101224257348987e-07,
+      "logits/chosen": -0.6253441572189331,
+      "logits/rejected": -0.5251750946044922,
+      "logps/chosen": -4.559195518493652,
+      "logps/rejected": -5.241268157958984,
+      "loss": 0.0506,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.559195518493652,
+      "rewards/margins": 0.6820730566978455,
+      "rewards/rejected": -5.241268157958984,
+      "sft_loss": 4.102574348449707,
+      "step": 3050
+    },
+    {
+      "epoch": 1.6350560294363605,
+      "grad_norm": 0.43989357842306076,
+      "learning_rate": 5.085652956714823e-07,
+      "logits/chosen": -0.7432211637496948,
+      "logits/rejected": -0.5417572855949402,
+      "logps/chosen": -4.619542598724365,
+      "logps/rejected": -5.228615760803223,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.619542598724365,
+      "rewards/margins": 0.6090730428695679,
+      "rewards/rejected": -5.228615760803223,
+      "sft_loss": 4.360207557678223,
+      "step": 3055
+    },
+    {
+      "epoch": 1.6377320622177622,
+      "grad_norm": 0.44075806779111804,
+      "learning_rate": 5.070080825075298e-07,
+      "logits/chosen": -0.5573010444641113,
+      "logits/rejected": -0.37539201974868774,
+      "logps/chosen": -4.312080383300781,
+      "logps/rejected": -5.0412750244140625,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.312080383300781,
+      "rewards/margins": 0.7291947603225708,
+      "rewards/rejected": -5.0412750244140625,
+      "sft_loss": 4.037957191467285,
+      "step": 3060
+    },
+    {
+      "epoch": 1.6404080949991637,
+      "grad_norm": 0.5272575555335578,
+      "learning_rate": 5.0545080135113e-07,
+      "logits/chosen": -0.5241907238960266,
+      "logits/rejected": -0.4427323341369629,
+      "logps/chosen": -4.395872592926025,
+      "logps/rejected": -5.07017707824707,
+      "loss": 0.0528,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.395872592926025,
+      "rewards/margins": 0.6743049621582031,
+      "rewards/rejected": -5.07017707824707,
+      "sft_loss": 4.139717102050781,
+      "step": 3065
+    },
+    {
+      "epoch": 1.6430841277805652,
+      "grad_norm": 0.5559871804749301,
+      "learning_rate": 5.038934673110316e-07,
+      "logits/chosen": -0.6505641937255859,
+      "logits/rejected": -0.5425621271133423,
+      "logps/chosen": -4.486371040344238,
+      "logps/rejected": -5.178048610687256,
+      "loss": 0.052,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.486371040344238,
+      "rewards/margins": 0.6916787028312683,
+      "rewards/rejected": -5.178048610687256,
+      "sft_loss": 4.203831672668457,
+      "step": 3070
+    },
+    {
+      "epoch": 1.645760160561967,
+      "grad_norm": 0.4182529948232291,
+      "learning_rate": 5.023360954964963e-07,
+      "logits/chosen": -0.7273457050323486,
+      "logits/rejected": -0.6802915930747986,
+      "logps/chosen": -4.684045314788818,
+      "logps/rejected": -5.266082286834717,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.684045314788818,
+      "rewards/margins": 0.5820373296737671,
+      "rewards/rejected": -5.266082286834717,
+      "sft_loss": 4.290963649749756,
+      "step": 3075
+    },
+    {
+      "epoch": 1.6484361933433684,
+      "grad_norm": 0.4147006496703341,
+      "learning_rate": 5.007787010171524e-07,
+      "logits/chosen": -0.775445282459259,
+      "logits/rejected": -0.5694190263748169,
+      "logps/chosen": -4.3551249504089355,
+      "logps/rejected": -5.173240661621094,
+      "loss": 0.0504,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -4.3551249504089355,
+      "rewards/margins": 0.8181155920028687,
+      "rewards/rejected": -5.173240661621094,
+      "sft_loss": 4.108549118041992,
+      "step": 3080
+    },
+    {
+      "epoch": 1.65111222612477,
+      "grad_norm": 0.3045952783005783,
+      "learning_rate": 4.992212989828477e-07,
+      "logits/chosen": -0.6025485992431641,
+      "logits/rejected": -0.5876261591911316,
+      "logps/chosen": -4.600819110870361,
+      "logps/rejected": -5.143906593322754,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.600819110870361,
+      "rewards/margins": 0.5430876612663269,
+      "rewards/rejected": -5.143906593322754,
+      "sft_loss": 4.310132026672363,
+      "step": 3085
+    },
+    {
+      "epoch": 1.6537882589061716,
+      "grad_norm": 0.540673347551131,
+      "learning_rate": 4.976639045035036e-07,
+      "logits/chosen": -0.5635827779769897,
+      "logits/rejected": -0.5256134271621704,
+      "logps/chosen": -4.392816543579102,
+      "logps/rejected": -4.900942802429199,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.392816543579102,
+      "rewards/margins": 0.5081263780593872,
+      "rewards/rejected": -4.900942802429199,
+      "sft_loss": 4.163025856018066,
+      "step": 3090
+    },
+    {
+      "epoch": 1.6564642916875731,
+      "grad_norm": 0.7200415092163559,
+      "learning_rate": 4.961065326889683e-07,
+      "logits/chosen": -0.6170357465744019,
+      "logits/rejected": -0.45310425758361816,
+      "logps/chosen": -4.459501266479492,
+      "logps/rejected": -5.003754138946533,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.459501266479492,
+      "rewards/margins": 0.544252872467041,
+      "rewards/rejected": -5.003754138946533,
+      "sft_loss": 4.138365268707275,
+      "step": 3095
+    },
+    {
+      "epoch": 1.6591403244689746,
+      "grad_norm": 0.45647313442948984,
+      "learning_rate": 4.9454919864887e-07,
+      "logits/chosen": -0.7191554307937622,
+      "logits/rejected": -0.6085395812988281,
+      "logps/chosen": -4.476665496826172,
+      "logps/rejected": -5.1111249923706055,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.476665496826172,
+      "rewards/margins": 0.634459376335144,
+      "rewards/rejected": -5.1111249923706055,
+      "sft_loss": 4.225938320159912,
+      "step": 3100
+    },
+    {
+      "epoch": 1.6618163572503764,
+      "grad_norm": 0.5788266893138243,
+      "learning_rate": 4.929919174924701e-07,
+      "logits/chosen": -0.7438724637031555,
+      "logits/rejected": -0.49867621064186096,
+      "logps/chosen": -4.484989166259766,
+      "logps/rejected": -5.13234281539917,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.484989166259766,
+      "rewards/margins": 0.6473537087440491,
+      "rewards/rejected": -5.13234281539917,
+      "sft_loss": 4.27896785736084,
+      "step": 3105
+    },
+    {
+      "epoch": 1.6644923900317778,
+      "grad_norm": 0.4575082397926408,
+      "learning_rate": 4.914347043285177e-07,
+      "logits/chosen": -0.5735903978347778,
+      "logits/rejected": -0.4313809275627136,
+      "logps/chosen": -4.549923419952393,
+      "logps/rejected": -5.196960926055908,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.549923419952393,
+      "rewards/margins": 0.6470370292663574,
+      "rewards/rejected": -5.196960926055908,
+      "sft_loss": 4.106003761291504,
+      "step": 3110
+    },
+    {
+      "epoch": 1.6671684228131793,
+      "grad_norm": 0.5661296376240132,
+      "learning_rate": 4.898775742651013e-07,
+      "logits/chosen": -0.49922671914100647,
+      "logits/rejected": -0.3821621835231781,
+      "logps/chosen": -4.460130214691162,
+      "logps/rejected": -5.07846212387085,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.460130214691162,
+      "rewards/margins": 0.6183321475982666,
+      "rewards/rejected": -5.07846212387085,
+      "sft_loss": 4.051510810852051,
+      "step": 3115
+    },
+    {
+      "epoch": 1.669844455594581,
+      "grad_norm": 0.5038668419054537,
+      "learning_rate": 4.883205424095037e-07,
+      "logits/chosen": -0.6256593465805054,
+      "logits/rejected": -0.4619271159172058,
+      "logps/chosen": -4.252190589904785,
+      "logps/rejected": -5.054263114929199,
+      "loss": 0.051,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.252190589904785,
+      "rewards/margins": 0.8020722270011902,
+      "rewards/rejected": -5.054263114929199,
+      "sft_loss": 3.9893879890441895,
+      "step": 3120
+    },
+    {
+      "epoch": 1.6725204883759828,
+      "grad_norm": 0.5153425298726638,
+      "learning_rate": 4.86763623868055e-07,
+      "logits/chosen": -0.551916241645813,
+      "logits/rejected": -0.42506498098373413,
+      "logps/chosen": -4.627806186676025,
+      "logps/rejected": -5.251055717468262,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.627806186676025,
+      "rewards/margins": 0.6232494115829468,
+      "rewards/rejected": -5.251055717468262,
+      "sft_loss": 4.246230602264404,
+      "step": 3125
+    },
+    {
+      "epoch": 1.675196521157384,
+      "grad_norm": 0.3997125689536002,
+      "learning_rate": 4.852068337459856e-07,
+      "logits/chosen": -0.459419310092926,
+      "logits/rejected": -0.2802503705024719,
+      "logps/chosen": -4.408881664276123,
+      "logps/rejected": -5.176243782043457,
+      "loss": 0.0504,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.408881664276123,
+      "rewards/margins": 0.7673624157905579,
+      "rewards/rejected": -5.176243782043457,
+      "sft_loss": 4.061155796051025,
+      "step": 3130
+    },
+    {
+      "epoch": 1.6778725539387858,
+      "grad_norm": 0.46776480880751287,
+      "learning_rate": 4.8365018714728e-07,
+      "logits/chosen": -0.3843730092048645,
+      "logits/rejected": -0.3154175877571106,
+      "logps/chosen": -4.531125068664551,
+      "logps/rejected": -5.120407581329346,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.531125068664551,
+      "rewards/margins": 0.5892825722694397,
+      "rewards/rejected": -5.120407581329346,
+      "sft_loss": 4.253077030181885,
+      "step": 3135
+    },
+    {
+      "epoch": 1.6805485867201875,
+      "grad_norm": 0.39289055029028164,
+      "learning_rate": 4.820936991745304e-07,
+      "logits/chosen": -0.7061978578567505,
+      "logits/rejected": -0.5787612795829773,
+      "logps/chosen": -4.551207065582275,
+      "logps/rejected": -5.057315349578857,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.551207065582275,
+      "rewards/margins": 0.5061080455780029,
+      "rewards/rejected": -5.057315349578857,
+      "sft_loss": 4.190797328948975,
+      "step": 3140
+    },
+    {
+      "epoch": 1.6832246195015887,
+      "grad_norm": 0.5248974986121218,
+      "learning_rate": 4.8053738492879e-07,
+      "logits/chosen": -0.49673542380332947,
+      "logits/rejected": -0.3729138970375061,
+      "logps/chosen": -4.485753059387207,
+      "logps/rejected": -5.2956743240356445,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.485753059387207,
+      "rewards/margins": 0.809921145439148,
+      "rewards/rejected": -5.2956743240356445,
+      "sft_loss": 4.1620683670043945,
+      "step": 3145
+    },
+    {
+      "epoch": 1.6859006522829905,
+      "grad_norm": 0.5275843311190732,
+      "learning_rate": 4.789812595094265e-07,
+      "logits/chosen": -0.6816089153289795,
+      "logits/rejected": -0.5871399641036987,
+      "logps/chosen": -4.438056945800781,
+      "logps/rejected": -5.106205940246582,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.438056945800781,
+      "rewards/margins": 0.6681481599807739,
+      "rewards/rejected": -5.106205940246582,
+      "sft_loss": 4.150810241699219,
+      "step": 3150
+    },
+    {
+      "epoch": 1.6885766850643922,
+      "grad_norm": 0.4727466138483916,
+      "learning_rate": 4.774253380139752e-07,
+      "logits/chosen": -0.6856822967529297,
+      "logits/rejected": -0.5773698091506958,
+      "logps/chosen": -4.567780494689941,
+      "logps/rejected": -5.124373435974121,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.567780494689941,
+      "rewards/margins": 0.5565928816795349,
+      "rewards/rejected": -5.124373435974121,
+      "sft_loss": 4.247193813323975,
+      "step": 3155
+    },
+    {
+      "epoch": 1.6912527178457935,
+      "grad_norm": 0.5162834128478235,
+      "learning_rate": 4.758696355379936e-07,
+      "logits/chosen": -0.4406503140926361,
+      "logits/rejected": -0.557584822177887,
+      "logps/chosen": -4.5678558349609375,
+      "logps/rejected": -5.084294319152832,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.5678558349609375,
+      "rewards/margins": 0.5164377093315125,
+      "rewards/rejected": -5.084294319152832,
+      "sft_loss": 4.25069522857666,
+      "step": 3160
+    },
+    {
+      "epoch": 1.6939287506271952,
+      "grad_norm": 0.7013298937578976,
+      "learning_rate": 4.743141671749138e-07,
+      "logits/chosen": -0.7435869574546814,
+      "logits/rejected": -0.5817897319793701,
+      "logps/chosen": -4.2917985916137695,
+      "logps/rejected": -4.787955284118652,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.2917985916137695,
+      "rewards/margins": 0.496157169342041,
+      "rewards/rejected": -4.787955284118652,
+      "sft_loss": 4.076413154602051,
+      "step": 3165
+    },
+    {
+      "epoch": 1.6966047834085969,
+      "grad_norm": 0.43754853602093785,
+      "learning_rate": 4.727589480158968e-07,
+      "logits/chosen": -0.6555178761482239,
+      "logits/rejected": -0.5711929798126221,
+      "logps/chosen": -4.529534339904785,
+      "logps/rejected": -5.278780460357666,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.529534339904785,
+      "rewards/margins": 0.7492465972900391,
+      "rewards/rejected": -5.278780460357666,
+      "sft_loss": 4.292778491973877,
+      "step": 3170
+    },
+    {
+      "epoch": 1.6992808161899984,
+      "grad_norm": 0.4647068491991218,
+      "learning_rate": 4.712039931496855e-07,
+      "logits/chosen": -0.6414646506309509,
+      "logits/rejected": -0.4491817355155945,
+      "logps/chosen": -4.58360481262207,
+      "logps/rejected": -5.076839447021484,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.58360481262207,
+      "rewards/margins": 0.49323463439941406,
+      "rewards/rejected": -5.076839447021484,
+      "sft_loss": 4.294300556182861,
+      "step": 3175
+    },
+    {
+      "epoch": 1.7019568489713999,
+      "grad_norm": 0.4432376622568674,
+      "learning_rate": 4.6964931766245905e-07,
+      "logits/chosen": -0.5140138268470764,
+      "logits/rejected": -0.48328647017478943,
+      "logps/chosen": -4.412505626678467,
+      "logps/rejected": -5.062849044799805,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.412505626678467,
+      "rewards/margins": 0.650343120098114,
+      "rewards/rejected": -5.062849044799805,
+      "sft_loss": 4.1344075202941895,
+      "step": 3180
+    },
+    {
+      "epoch": 1.7046328817528016,
+      "grad_norm": 0.4461820119109125,
+      "learning_rate": 4.6809493663768575e-07,
+      "logits/chosen": -0.6357397437095642,
+      "logits/rejected": -0.6165580749511719,
+      "logps/chosen": -4.6179914474487305,
+      "logps/rejected": -4.951748847961426,
+      "loss": 0.052,
+      "rewards/accuracies": 0.5687500238418579,
+      "rewards/chosen": -4.6179914474487305,
+      "rewards/margins": 0.33375757932662964,
+      "rewards/rejected": -4.951748847961426,
+      "sft_loss": 4.208844184875488,
+      "step": 3185
+    },
+    {
+      "epoch": 1.707308914534203,
+      "grad_norm": 0.4226026064147668,
+      "learning_rate": 4.6654086515597716e-07,
+      "logits/chosen": -0.8228461146354675,
+      "logits/rejected": -0.61443030834198,
+      "logps/chosen": -4.3584394454956055,
+      "logps/rejected": -5.0693278312683105,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.3584394454956055,
+      "rewards/margins": 0.7108885645866394,
+      "rewards/rejected": -5.0693278312683105,
+      "sft_loss": 4.128300666809082,
+      "step": 3190
+    },
+    {
+      "epoch": 1.7099849473156046,
+      "grad_norm": 0.4939402773038487,
+      "learning_rate": 4.6498711829494154e-07,
+      "logits/chosen": -0.8291040658950806,
+      "logits/rejected": -0.7133563756942749,
+      "logps/chosen": -4.560546875,
+      "logps/rejected": -5.0296454429626465,
+      "loss": 0.0536,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.560546875,
+      "rewards/margins": 0.46909865736961365,
+      "rewards/rejected": -5.0296454429626465,
+      "sft_loss": 4.325606822967529,
+      "step": 3195
+    },
+    {
+      "epoch": 1.7126609800970063,
+      "grad_norm": 0.5429553430237601,
+      "learning_rate": 4.6343371112903777e-07,
+      "logits/chosen": -0.691290020942688,
+      "logits/rejected": -0.48237770795822144,
+      "logps/chosen": -4.563403129577637,
+      "logps/rejected": -5.202398777008057,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.563403129577637,
+      "rewards/margins": 0.6389960050582886,
+      "rewards/rejected": -5.202398777008057,
+      "sft_loss": 4.276124477386475,
+      "step": 3200
+    },
+    {
+      "epoch": 1.7126609800970063,
+      "eval_logits/chosen": -0.15210114419460297,
+      "eval_logits/rejected": -0.0613933689892292,
+      "eval_logps/chosen": -4.472229480743408,
+      "eval_logps/rejected": -5.117489814758301,
+      "eval_loss": 0.05049404874444008,
+      "eval_rewards/accuracies": 0.674332320690155,
+      "eval_rewards/chosen": -4.472229480743408,
+      "eval_rewards/margins": 0.6452605128288269,
+      "eval_rewards/rejected": -5.117489814758301,
+      "eval_runtime": 43.5886,
+      "eval_samples_per_second": 30.857,
+      "eval_sft_loss": 4.108198642730713,
+      "eval_steps_per_second": 7.731,
+      "step": 3200
+    },
+    {
+      "epoch": 1.7153370128784078,
+      "grad_norm": 0.5192944808895836,
+      "learning_rate": 4.618806587294291e-07,
+      "logits/chosen": -0.7633549571037292,
+      "logits/rejected": -0.6807888150215149,
+      "logps/chosen": -4.452502727508545,
+      "logps/rejected": -5.129948616027832,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.452502727508545,
+      "rewards/margins": 0.6774452924728394,
+      "rewards/rejected": -5.129948616027832,
+      "sft_loss": 4.249197959899902,
+      "step": 3205
+    },
+    {
+      "epoch": 1.7180130456598093,
+      "grad_norm": 0.4037862112128149,
+      "learning_rate": 4.603279761638365e-07,
+      "logits/chosen": -0.8041302561759949,
+      "logits/rejected": -0.6776725053787231,
+      "logps/chosen": -4.4214606285095215,
+      "logps/rejected": -5.059202194213867,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.4214606285095215,
+      "rewards/margins": 0.6377416849136353,
+      "rewards/rejected": -5.059202194213867,
+      "sft_loss": 4.173728942871094,
+      "step": 3210
+    },
+    {
+      "epoch": 1.720689078441211,
+      "grad_norm": 0.38702959064108305,
+      "learning_rate": 4.5877567849639315e-07,
+      "logits/chosen": -0.6881679892539978,
+      "logits/rejected": -0.6041558980941772,
+      "logps/chosen": -4.4578046798706055,
+      "logps/rejected": -5.0373406410217285,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.4578046798706055,
+      "rewards/margins": 0.5795360803604126,
+      "rewards/rejected": -5.0373406410217285,
+      "sft_loss": 4.222383975982666,
+      "step": 3215
+    },
+    {
+      "epoch": 1.7233651112226125,
+      "grad_norm": 0.5192188486985571,
+      "learning_rate": 4.572237807874979e-07,
+      "logits/chosen": -0.8102655410766602,
+      "logits/rejected": -0.4698098301887512,
+      "logps/chosen": -4.444644927978516,
+      "logps/rejected": -5.1222944259643555,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.444644927978516,
+      "rewards/margins": 0.6776494979858398,
+      "rewards/rejected": -5.1222944259643555,
+      "sft_loss": 4.182662010192871,
+      "step": 3220
+    },
+    {
+      "epoch": 1.726041144004014,
+      "grad_norm": 0.4902322890035671,
+      "learning_rate": 4.5567229809366895e-07,
+      "logits/chosen": -0.6690499186515808,
+      "logits/rejected": -0.5294996500015259,
+      "logps/chosen": -4.391629219055176,
+      "logps/rejected": -4.9917378425598145,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.391629219055176,
+      "rewards/margins": 0.6001091599464417,
+      "rewards/rejected": -4.9917378425598145,
+      "sft_loss": 4.090245723724365,
+      "step": 3225
+    },
+    {
+      "epoch": 1.7287171767854157,
+      "grad_norm": 0.4975080015555503,
+      "learning_rate": 4.541212454673984e-07,
+      "logits/chosen": -0.7487164735794067,
+      "logits/rejected": -0.5614835023880005,
+      "logps/chosen": -4.513947486877441,
+      "logps/rejected": -5.419942855834961,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.513947486877441,
+      "rewards/margins": 0.9059955477714539,
+      "rewards/rejected": -5.419942855834961,
+      "sft_loss": 4.26330041885376,
+      "step": 3230
+    },
+    {
+      "epoch": 1.7313932095668172,
+      "grad_norm": 0.5103254937323408,
+      "learning_rate": 4.525706379570055e-07,
+      "logits/chosen": -0.7287732362747192,
+      "logits/rejected": -0.6540551781654358,
+      "logps/chosen": -4.286916255950928,
+      "logps/rejected": -4.951821327209473,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -4.286916255950928,
+      "rewards/margins": 0.664903998374939,
+      "rewards/rejected": -4.951821327209473,
+      "sft_loss": 4.128801345825195,
+      "step": 3235
+    },
+    {
+      "epoch": 1.7340692423482187,
+      "grad_norm": 0.5500963923007846,
+      "learning_rate": 4.510204906064911e-07,
+      "logits/chosen": -0.6376282572746277,
+      "logits/rejected": -0.4842913746833801,
+      "logps/chosen": -4.394224166870117,
+      "logps/rejected": -5.104223728179932,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.394224166870117,
+      "rewards/margins": 0.709998607635498,
+      "rewards/rejected": -5.104223728179932,
+      "sft_loss": 4.103116035461426,
+      "step": 3240
+    },
+    {
+      "epoch": 1.7367452751296204,
+      "grad_norm": 1.3227037437294287,
+      "learning_rate": 4.4947081845539177e-07,
+      "logits/chosen": -0.7193289399147034,
+      "logits/rejected": -0.6123319268226624,
+      "logps/chosen": -4.325865268707275,
+      "logps/rejected": -5.033795356750488,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.325865268707275,
+      "rewards/margins": 0.7079305052757263,
+      "rewards/rejected": -5.033795356750488,
+      "sft_loss": 4.012349605560303,
+      "step": 3245
+    },
+    {
+      "epoch": 1.739421307911022,
+      "grad_norm": 0.48991792894292036,
+      "learning_rate": 4.479216365386333e-07,
+      "logits/chosen": -0.5226157307624817,
+      "logits/rejected": -0.3413340449333191,
+      "logps/chosen": -4.1164045333862305,
+      "logps/rejected": -5.005209445953369,
+      "loss": 0.0498,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.1164045333862305,
+      "rewards/margins": 0.8888049125671387,
+      "rewards/rejected": -5.005209445953369,
+      "sft_loss": 3.805973768234253,
+      "step": 3250
+    },
+    {
+      "epoch": 1.7420973406924234,
+      "grad_norm": 0.504866696897772,
+      "learning_rate": 4.4637295988638555e-07,
+      "logits/chosen": -0.6154767870903015,
+      "logits/rejected": -0.5357116460800171,
+      "logps/chosen": -4.35190486907959,
+      "logps/rejected": -5.079216480255127,
+      "loss": 0.051,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.35190486907959,
+      "rewards/margins": 0.7273120880126953,
+      "rewards/rejected": -5.079216480255127,
+      "sft_loss": 4.064234733581543,
+      "step": 3255
+    },
+    {
+      "epoch": 1.744773373473825,
+      "grad_norm": 0.7218864401426324,
+      "learning_rate": 4.4482480352391623e-07,
+      "logits/chosen": -0.6496624946594238,
+      "logits/rejected": -0.5319706797599792,
+      "logps/chosen": -4.618185520172119,
+      "logps/rejected": -5.271091461181641,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.618185520172119,
+      "rewards/margins": 0.6529065370559692,
+      "rewards/rejected": -5.271091461181641,
+      "sft_loss": 4.362316131591797,
+      "step": 3260
+    },
+    {
+      "epoch": 1.7474494062552266,
+      "grad_norm": 0.7384956872787122,
+      "learning_rate": 4.4327718247144507e-07,
+      "logits/chosen": -0.5355771780014038,
+      "logits/rejected": -0.3831943869590759,
+      "logps/chosen": -4.874344825744629,
+      "logps/rejected": -5.482754707336426,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.874344825744629,
+      "rewards/margins": 0.6084098219871521,
+      "rewards/rejected": -5.482754707336426,
+      "sft_loss": 4.577004909515381,
+      "step": 3265
+    },
+    {
+      "epoch": 1.750125439036628,
+      "grad_norm": 0.4852721808398797,
+      "learning_rate": 4.417301117439984e-07,
+      "logits/chosen": -0.675261378288269,
+      "logits/rejected": -0.531201183795929,
+      "logps/chosen": -4.468598365783691,
+      "logps/rejected": -5.062955379486084,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.468598365783691,
+      "rewards/margins": 0.594357430934906,
+      "rewards/rejected": -5.062955379486084,
+      "sft_loss": 4.193814277648926,
+      "step": 3270
+    },
+    {
+      "epoch": 1.7528014718180298,
+      "grad_norm": 0.44552316816909177,
+      "learning_rate": 4.401836063512631e-07,
+      "logits/chosen": -0.7346017360687256,
+      "logits/rejected": -0.3838501572608948,
+      "logps/chosen": -4.396548271179199,
+      "logps/rejected": -5.152390956878662,
+      "loss": 0.0504,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.396548271179199,
+      "rewards/margins": 0.7558425068855286,
+      "rewards/rejected": -5.152390956878662,
+      "sft_loss": 4.075710296630859,
+      "step": 3275
+    },
+    {
+      "epoch": 1.7554775045994313,
+      "grad_norm": 0.69846171922734,
+      "learning_rate": 4.386376812974413e-07,
+      "logits/chosen": -0.645746111869812,
+      "logits/rejected": -0.635964572429657,
+      "logps/chosen": -4.513228416442871,
+      "logps/rejected": -5.130314826965332,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.513228416442871,
+      "rewards/margins": 0.6170862913131714,
+      "rewards/rejected": -5.130314826965332,
+      "sft_loss": 4.255279064178467,
+      "step": 3280
+    },
+    {
+      "epoch": 1.7581535373808328,
+      "grad_norm": 0.41749008448426816,
+      "learning_rate": 4.370923515811048e-07,
+      "logits/chosen": -0.6760843992233276,
+      "logits/rejected": -0.4505118727684021,
+      "logps/chosen": -4.28780460357666,
+      "logps/rejected": -4.993500709533691,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.28780460357666,
+      "rewards/margins": 0.7056947946548462,
+      "rewards/rejected": -4.993500709533691,
+      "sft_loss": 3.97005033493042,
+      "step": 3285
+    },
+    {
+      "epoch": 1.7608295701622345,
+      "grad_norm": 0.41402381522478005,
+      "learning_rate": 4.35547632195049e-07,
+      "logits/chosen": -0.6169165372848511,
+      "logits/rejected": -0.5249618291854858,
+      "logps/chosen": -4.62155294418335,
+      "logps/rejected": -5.299553394317627,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.62155294418335,
+      "rewards/margins": 0.6780003309249878,
+      "rewards/rejected": -5.299553394317627,
+      "sft_loss": 4.248922348022461,
+      "step": 3290
+    },
+    {
+      "epoch": 1.763505602943636,
+      "grad_norm": 0.594226689061985,
+      "learning_rate": 4.340035381261484e-07,
+      "logits/chosen": -0.6368000507354736,
+      "logits/rejected": -0.5852474570274353,
+      "logps/chosen": -4.4297966957092285,
+      "logps/rejected": -5.215722560882568,
+      "loss": 0.0502,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.4297966957092285,
+      "rewards/margins": 0.7859266400337219,
+      "rewards/rejected": -5.215722560882568,
+      "sft_loss": 4.1008195877075195,
+      "step": 3295
+    },
+    {
+      "epoch": 1.7661816357250375,
+      "grad_norm": 0.4261141203167247,
+      "learning_rate": 4.324600843552104e-07,
+      "logits/chosen": -0.773404598236084,
+      "logits/rejected": -0.6615234613418579,
+      "logps/chosen": -4.449922561645508,
+      "logps/rejected": -5.212803840637207,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.449922561645508,
+      "rewards/margins": 0.7628811001777649,
+      "rewards/rejected": -5.212803840637207,
+      "sft_loss": 4.144711494445801,
+      "step": 3300
+    },
+    {
+      "epoch": 1.7688576685064392,
+      "grad_norm": 0.4033807179932179,
+      "learning_rate": 4.309172858568302e-07,
+      "logits/chosen": -0.7160771489143372,
+      "logits/rejected": -0.5191089510917664,
+      "logps/chosen": -4.4391279220581055,
+      "logps/rejected": -5.147273063659668,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.4391279220581055,
+      "rewards/margins": 0.7081449627876282,
+      "rewards/rejected": -5.147273063659668,
+      "sft_loss": 4.143710136413574,
+      "step": 3305
+    },
+    {
+      "epoch": 1.771533701287841,
+      "grad_norm": 0.6842510296567842,
+      "learning_rate": 4.293751575992455e-07,
+      "logits/chosen": -0.5255097150802612,
+      "logits/rejected": -0.517221987247467,
+      "logps/chosen": -4.5699992179870605,
+      "logps/rejected": -5.127143383026123,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.5699992179870605,
+      "rewards/margins": 0.5571444630622864,
+      "rewards/rejected": -5.127143383026123,
+      "sft_loss": 4.320311546325684,
+      "step": 3310
+    },
+    {
+      "epoch": 1.7742097340692422,
+      "grad_norm": 0.3413623804505247,
+      "learning_rate": 4.278337145441916e-07,
+      "logits/chosen": -0.6041097044944763,
+      "logits/rejected": -0.41528400778770447,
+      "logps/chosen": -4.342318534851074,
+      "logps/rejected": -5.069688320159912,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.342318534851074,
+      "rewards/margins": 0.7273694276809692,
+      "rewards/rejected": -5.069688320159912,
+      "sft_loss": 4.100344657897949,
+      "step": 3315
+    },
+    {
+      "epoch": 1.776885766850644,
+      "grad_norm": 0.44520283256192456,
+      "learning_rate": 4.262929716467556e-07,
+      "logits/chosen": -0.5153679847717285,
+      "logits/rejected": -0.2379719465970993,
+      "logps/chosen": -4.302934169769287,
+      "logps/rejected": -5.132648944854736,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.302934169769287,
+      "rewards/margins": 0.8297148942947388,
+      "rewards/rejected": -5.132648944854736,
+      "sft_loss": 4.078235626220703,
+      "step": 3320
+    },
+    {
+      "epoch": 1.7795617996320456,
+      "grad_norm": 0.42105726517447006,
+      "learning_rate": 4.247529438552321e-07,
+      "logits/chosen": -0.6211832165718079,
+      "logits/rejected": -0.38394981622695923,
+      "logps/chosen": -4.535687446594238,
+      "logps/rejected": -5.059223175048828,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.535687446594238,
+      "rewards/margins": 0.5235356092453003,
+      "rewards/rejected": -5.059223175048828,
+      "sft_loss": 4.289804935455322,
+      "step": 3325
+    },
+    {
+      "epoch": 1.782237832413447,
+      "grad_norm": 0.5285680264653614,
+      "learning_rate": 4.232136461109773e-07,
+      "logits/chosen": -0.5854800939559937,
+      "logits/rejected": -0.48620110750198364,
+      "logps/chosen": -4.470789909362793,
+      "logps/rejected": -5.155722618103027,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.470789909362793,
+      "rewards/margins": 0.6849329471588135,
+      "rewards/rejected": -5.155722618103027,
+      "sft_loss": 4.172421455383301,
+      "step": 3330
+    },
+    {
+      "epoch": 1.7849138651948486,
+      "grad_norm": 0.4396477416091562,
+      "learning_rate": 4.216750933482646e-07,
+      "logits/chosen": -0.5841988325119019,
+      "logits/rejected": -0.4032633304595947,
+      "logps/chosen": -4.558676242828369,
+      "logps/rejected": -5.077760696411133,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.558676242828369,
+      "rewards/margins": 0.5190838575363159,
+      "rewards/rejected": -5.077760696411133,
+      "sft_loss": 4.182391166687012,
+      "step": 3335
+    },
+    {
+      "epoch": 1.7875898979762503,
+      "grad_norm": 0.640109669762743,
+      "learning_rate": 4.2013730049413986e-07,
+      "logits/chosen": -0.5710462331771851,
+      "logits/rejected": -0.37740975618362427,
+      "logps/chosen": -4.278960227966309,
+      "logps/rejected": -5.2915873527526855,
+      "loss": 0.05,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -4.278960227966309,
+      "rewards/margins": 1.0126279592514038,
+      "rewards/rejected": -5.2915873527526855,
+      "sft_loss": 4.033893585205078,
+      "step": 3340
+    },
+    {
+      "epoch": 1.7902659307576518,
+      "grad_norm": 0.4039240436873603,
+      "learning_rate": 4.1860028246827594e-07,
+      "logits/chosen": -0.5724278688430786,
+      "logits/rejected": -0.32125282287597656,
+      "logps/chosen": -4.434798240661621,
+      "logps/rejected": -5.011025905609131,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.434798240661621,
+      "rewards/margins": 0.5762277841567993,
+      "rewards/rejected": -5.011025905609131,
+      "sft_loss": 4.190737724304199,
+      "step": 3345
+    },
+    {
+      "epoch": 1.7929419635390533,
+      "grad_norm": 0.5223364436039674,
+      "learning_rate": 4.170640541828285e-07,
+      "logits/chosen": -0.7454400062561035,
+      "logits/rejected": -0.6011232137680054,
+      "logps/chosen": -4.344622611999512,
+      "logps/rejected": -4.949635982513428,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.344622611999512,
+      "rewards/margins": 0.6050136089324951,
+      "rewards/rejected": -4.949635982513428,
+      "sft_loss": 4.138838768005371,
+      "step": 3350
+    },
+    {
+      "epoch": 1.795617996320455,
+      "grad_norm": 0.4545793741818451,
+      "learning_rate": 4.1552863054229116e-07,
+      "logits/chosen": -0.5223687887191772,
+      "logits/rejected": -0.48168906569480896,
+      "logps/chosen": -4.478524684906006,
+      "logps/rejected": -5.133755683898926,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.478524684906006,
+      "rewards/margins": 0.6552313566207886,
+      "rewards/rejected": -5.133755683898926,
+      "sft_loss": 4.1638383865356445,
+      "step": 3355
+    },
+    {
+      "epoch": 1.7982940291018565,
+      "grad_norm": 0.6727654772149192,
+      "learning_rate": 4.139940264433508e-07,
+      "logits/chosen": -0.6394672989845276,
+      "logits/rejected": -0.43455833196640015,
+      "logps/chosen": -4.370895862579346,
+      "logps/rejected": -5.10739803314209,
+      "loss": 0.051,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.370895862579346,
+      "rewards/margins": 0.7365025877952576,
+      "rewards/rejected": -5.10739803314209,
+      "sft_loss": 4.044856548309326,
+      "step": 3360
+    },
+    {
+      "epoch": 1.800970061883258,
+      "grad_norm": 0.3589502265745943,
+      "learning_rate": 4.1246025677474303e-07,
+      "logits/chosen": -0.7238172888755798,
+      "logits/rejected": -0.47152742743492126,
+      "logps/chosen": -4.547296047210693,
+      "logps/rejected": -5.125532627105713,
+      "loss": 0.053,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.547296047210693,
+      "rewards/margins": 0.5782366991043091,
+      "rewards/rejected": -5.125532627105713,
+      "sft_loss": 4.311211585998535,
+      "step": 3365
+    },
+    {
+      "epoch": 1.8036460946646597,
+      "grad_norm": 0.40595128955105286,
+      "learning_rate": 4.10927336417108e-07,
+      "logits/chosen": -0.6705687642097473,
+      "logits/rejected": -0.4331951141357422,
+      "logps/chosen": -4.5498223304748535,
+      "logps/rejected": -4.988067626953125,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.5498223304748535,
+      "rewards/margins": 0.4382452070713043,
+      "rewards/rejected": -4.988067626953125,
+      "sft_loss": 4.224373817443848,
+      "step": 3370
+    },
+    {
+      "epoch": 1.8063221274460612,
+      "grad_norm": 0.6417927401398111,
+      "learning_rate": 4.093952802428457e-07,
+      "logits/chosen": -0.5054597854614258,
+      "logits/rejected": -0.49510788917541504,
+      "logps/chosen": -4.723372459411621,
+      "logps/rejected": -5.253734111785889,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.723372459411621,
+      "rewards/margins": 0.5303624272346497,
+      "rewards/rejected": -5.253734111785889,
+      "sft_loss": 4.410287857055664,
+      "step": 3375
+    },
+    {
+      "epoch": 1.8089981602274627,
+      "grad_norm": 0.6132712392051819,
+      "learning_rate": 4.0786410311597184e-07,
+      "logits/chosen": -0.7825726270675659,
+      "logits/rejected": -0.5462926030158997,
+      "logps/chosen": -4.558102607727051,
+      "logps/rejected": -5.160484790802002,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.558102607727051,
+      "rewards/margins": 0.602383017539978,
+      "rewards/rejected": -5.160484790802002,
+      "sft_loss": 4.206010818481445,
+      "step": 3380
+    },
+    {
+      "epoch": 1.8116741930088645,
+      "grad_norm": 0.385967798232852,
+      "learning_rate": 4.063338198919737e-07,
+      "logits/chosen": -0.6903921961784363,
+      "logits/rejected": -0.7371004223823547,
+      "logps/chosen": -4.396435737609863,
+      "logps/rejected": -4.971739768981934,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.396435737609863,
+      "rewards/margins": 0.5753039121627808,
+      "rewards/rejected": -4.971739768981934,
+      "sft_loss": 4.139103889465332,
+      "step": 3385
+    },
+    {
+      "epoch": 1.814350225790266,
+      "grad_norm": 0.39338215064487586,
+      "learning_rate": 4.0480444541766575e-07,
+      "logits/chosen": -0.7332966327667236,
+      "logits/rejected": -0.5903289914131165,
+      "logps/chosen": -4.715156078338623,
+      "logps/rejected": -5.369782447814941,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.715156078338623,
+      "rewards/margins": 0.6546260714530945,
+      "rewards/rejected": -5.369782447814941,
+      "sft_loss": 4.377230167388916,
+      "step": 3390
+    },
+    {
+      "epoch": 1.8170262585716674,
+      "grad_norm": 0.6030105319663668,
+      "learning_rate": 4.0327599453104606e-07,
+      "logits/chosen": -0.8181564211845398,
+      "logits/rejected": -0.6402052640914917,
+      "logps/chosen": -4.497122764587402,
+      "logps/rejected": -5.083517551422119,
+      "loss": 0.0506,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.497122764587402,
+      "rewards/margins": 0.5863949656486511,
+      "rewards/rejected": -5.083517551422119,
+      "sft_loss": 4.141504764556885,
+      "step": 3395
+    },
+    {
+      "epoch": 1.8197022913530692,
+      "grad_norm": 0.5048991888888651,
+      "learning_rate": 4.017484820611514e-07,
+      "logits/chosen": -0.645340085029602,
+      "logits/rejected": -0.5028203129768372,
+      "logps/chosen": -4.37846565246582,
+      "logps/rejected": -5.11088228225708,
+      "loss": 0.0504,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.37846565246582,
+      "rewards/margins": 0.7324164509773254,
+      "rewards/rejected": -5.11088228225708,
+      "sft_loss": 4.0104265213012695,
+      "step": 3400
+    },
+    {
+      "epoch": 1.8223783241344707,
+      "grad_norm": 1.0651671122689967,
+      "learning_rate": 4.002219228279148e-07,
+      "logits/chosen": -0.7175520658493042,
+      "logits/rejected": -0.5956283211708069,
+      "logps/chosen": -4.275979518890381,
+      "logps/rejected": -4.8901238441467285,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.275979518890381,
+      "rewards/margins": 0.6141440868377686,
+      "rewards/rejected": -4.8901238441467285,
+      "sft_loss": 3.992905855178833,
+      "step": 3405
+    },
+    {
+      "epoch": 1.8250543569158721,
+      "grad_norm": 0.5449755198054754,
+      "learning_rate": 3.9869633164202045e-07,
+      "logits/chosen": -0.7337294220924377,
+      "logits/rejected": -0.4620814323425293,
+      "logps/chosen": -4.322390556335449,
+      "logps/rejected": -5.124471664428711,
+      "loss": 0.0504,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.322390556335449,
+      "rewards/margins": 0.8020809888839722,
+      "rewards/rejected": -5.124471664428711,
+      "sft_loss": 4.025493621826172,
+      "step": 3410
+    },
+    {
+      "epoch": 1.8277303896972739,
+      "grad_norm": 0.7352016662381,
+      "learning_rate": 3.9717172330476077e-07,
+      "logits/chosen": -0.8215198516845703,
+      "logits/rejected": -0.7087420225143433,
+      "logps/chosen": -4.538886070251465,
+      "logps/rejected": -5.233793258666992,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.538886070251465,
+      "rewards/margins": 0.6949065327644348,
+      "rewards/rejected": -5.233793258666992,
+      "sft_loss": 4.277106285095215,
+      "step": 3415
+    },
+    {
+      "epoch": 1.8304064224786754,
+      "grad_norm": 0.5604027114659111,
+      "learning_rate": 3.956481126078927e-07,
+      "logits/chosen": -0.5984781980514526,
+      "logits/rejected": -0.49451103806495667,
+      "logps/chosen": -4.805777072906494,
+      "logps/rejected": -5.480899333953857,
+      "loss": 0.0543,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.805777072906494,
+      "rewards/margins": 0.6751219034194946,
+      "rewards/rejected": -5.480899333953857,
+      "sft_loss": 4.538644313812256,
+      "step": 3420
+    },
+    {
+      "epoch": 1.8330824552600768,
+      "grad_norm": 0.5545760801706651,
+      "learning_rate": 3.941255143334937e-07,
+      "logits/chosen": -0.7788869142532349,
+      "logits/rejected": -0.7783342599868774,
+      "logps/chosen": -4.51423454284668,
+      "logps/rejected": -5.1121721267700195,
+      "loss": 0.051,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.51423454284668,
+      "rewards/margins": 0.5979379415512085,
+      "rewards/rejected": -5.1121721267700195,
+      "sft_loss": 4.199769020080566,
+      "step": 3425
+    },
+    {
+      "epoch": 1.8357584880414786,
+      "grad_norm": 0.370388145556135,
+      "learning_rate": 3.9260394325381895e-07,
+      "logits/chosen": -0.7272287011146545,
+      "logits/rejected": -0.6226170659065247,
+      "logps/chosen": -4.263165473937988,
+      "logps/rejected": -5.2529520988464355,
+      "loss": 0.0496,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.263165473937988,
+      "rewards/margins": 0.9897869825363159,
+      "rewards/rejected": -5.2529520988464355,
+      "sft_loss": 3.947901487350464,
+      "step": 3430
+    },
+    {
+      "epoch": 1.83843452082288,
+      "grad_norm": 0.5459081295394673,
+      "learning_rate": 3.9108341413115784e-07,
+      "logits/chosen": -0.647996723651886,
+      "logits/rejected": -0.5486742258071899,
+      "logps/chosen": -4.266923427581787,
+      "logps/rejected": -4.853913307189941,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.266923427581787,
+      "rewards/margins": 0.5869892239570618,
+      "rewards/rejected": -4.853913307189941,
+      "sft_loss": 3.897509813308716,
+      "step": 3435
+    },
+    {
+      "epoch": 1.8411105536042816,
+      "grad_norm": 0.5711559005897949,
+      "learning_rate": 3.895639417176905e-07,
+      "logits/chosen": -0.7516553997993469,
+      "logits/rejected": -0.6304369568824768,
+      "logps/chosen": -4.520443916320801,
+      "logps/rejected": -5.215183258056641,
+      "loss": 0.0543,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.520443916320801,
+      "rewards/margins": 0.6947392225265503,
+      "rewards/rejected": -5.215183258056641,
+      "sft_loss": 4.249512672424316,
+      "step": 3440
+    },
+    {
+      "epoch": 1.8437865863856833,
+      "grad_norm": 0.5234020888177435,
+      "learning_rate": 3.8804554075534497e-07,
+      "logits/chosen": -0.7366546392440796,
+      "logits/rejected": -0.4990767538547516,
+      "logps/chosen": -4.357216835021973,
+      "logps/rejected": -5.187743663787842,
+      "loss": 0.051,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.357216835021973,
+      "rewards/margins": 0.8305273056030273,
+      "rewards/rejected": -5.187743663787842,
+      "sft_loss": 4.121443748474121,
+      "step": 3445
+    },
+    {
+      "epoch": 1.8464626191670848,
+      "grad_norm": 0.5833773670559123,
+      "learning_rate": 3.8652822597565403e-07,
+      "logits/chosen": -0.7845597267150879,
+      "logits/rejected": -0.5876230001449585,
+      "logps/chosen": -4.595117568969727,
+      "logps/rejected": -5.286187648773193,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.595117568969727,
+      "rewards/margins": 0.6910702586174011,
+      "rewards/rejected": -5.286187648773193,
+      "sft_loss": 4.322709560394287,
+      "step": 3450
+    },
+    {
+      "epoch": 1.8491386519484863,
+      "grad_norm": 0.5746118595998752,
+      "learning_rate": 3.850120120996123e-07,
+      "logits/chosen": -0.738675057888031,
+      "logits/rejected": -0.4819965362548828,
+      "logps/chosen": -4.569746017456055,
+      "logps/rejected": -5.286129951477051,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.569746017456055,
+      "rewards/margins": 0.7163844108581543,
+      "rewards/rejected": -5.286129951477051,
+      "sft_loss": 4.353842735290527,
+      "step": 3455
+    },
+    {
+      "epoch": 1.851814684729888,
+      "grad_norm": 0.44553629713062876,
+      "learning_rate": 3.8349691383753356e-07,
+      "logits/chosen": -0.5867315530776978,
+      "logits/rejected": -0.46066465973854065,
+      "logps/chosen": -4.479857921600342,
+      "logps/rejected": -5.148479461669922,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.479857921600342,
+      "rewards/margins": 0.6686211824417114,
+      "rewards/rejected": -5.148479461669922,
+      "sft_loss": 4.108327388763428,
+      "step": 3460
+    },
+    {
+      "epoch": 1.8544907175112895,
+      "grad_norm": 0.4407831570890799,
+      "learning_rate": 3.819829458889078e-07,
+      "logits/chosen": -0.6969954371452332,
+      "logits/rejected": -0.6196625828742981,
+      "logps/chosen": -4.214973449707031,
+      "logps/rejected": -4.744257926940918,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.214973449707031,
+      "rewards/margins": 0.529284656047821,
+      "rewards/rejected": -4.744257926940918,
+      "sft_loss": 3.9413580894470215,
+      "step": 3465
+    },
+    {
+      "epoch": 1.857166750292691,
+      "grad_norm": 0.6022218359082767,
+      "learning_rate": 3.804701229422585e-07,
+      "logits/chosen": -0.7823773622512817,
+      "logits/rejected": -0.719424843788147,
+      "logps/chosen": -4.537754058837891,
+      "logps/rejected": -5.0080060958862305,
+      "loss": 0.054,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.537754058837891,
+      "rewards/margins": 0.4702521860599518,
+      "rewards/rejected": -5.0080060958862305,
+      "sft_loss": 4.1914873123168945,
+      "step": 3470
+    },
+    {
+      "epoch": 1.8598427830740927,
+      "grad_norm": 0.5513510287659418,
+      "learning_rate": 3.789584596750007e-07,
+      "logits/chosen": -0.8201929330825806,
+      "logits/rejected": -0.8097238540649414,
+      "logps/chosen": -4.460206985473633,
+      "logps/rejected": -5.128048419952393,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.460206985473633,
+      "rewards/margins": 0.667841911315918,
+      "rewards/rejected": -5.128048419952393,
+      "sft_loss": 4.25015115737915,
+      "step": 3475
+    },
+    {
+      "epoch": 1.8625188158554944,
+      "grad_norm": 0.44224086507204363,
+      "learning_rate": 3.77447970753298e-07,
+      "logits/chosen": -0.7006460428237915,
+      "logits/rejected": -0.7202643156051636,
+      "logps/chosen": -4.563055992126465,
+      "logps/rejected": -5.094748497009277,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.563055992126465,
+      "rewards/margins": 0.5316920876502991,
+      "rewards/rejected": -5.094748497009277,
+      "sft_loss": 4.338143348693848,
+      "step": 3480
+    },
+    {
+      "epoch": 1.8651948486368957,
+      "grad_norm": 0.5070280678544562,
+      "learning_rate": 3.7593867083192057e-07,
+      "logits/chosen": -0.802207350730896,
+      "logits/rejected": -0.6420674324035645,
+      "logps/chosen": -4.551300048828125,
+      "logps/rejected": -5.105351448059082,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.551300048828125,
+      "rewards/margins": 0.55405193567276,
+      "rewards/rejected": -5.105351448059082,
+      "sft_loss": 4.313631534576416,
+      "step": 3485
+    },
+    {
+      "epoch": 1.8678708814182974,
+      "grad_norm": 0.4769604352959686,
+      "learning_rate": 3.7443057455410276e-07,
+      "logits/chosen": -0.669545590877533,
+      "logits/rejected": -0.6156834363937378,
+      "logps/chosen": -4.158803462982178,
+      "logps/rejected": -5.050396919250488,
+      "loss": 0.0502,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.158803462982178,
+      "rewards/margins": 0.891592800617218,
+      "rewards/rejected": -5.050396919250488,
+      "sft_loss": 3.9954910278320312,
+      "step": 3490
+    },
+    {
+      "epoch": 1.870546914199699,
+      "grad_norm": 0.4221363119339456,
+      "learning_rate": 3.7292369655140145e-07,
+      "logits/chosen": -0.8769127130508423,
+      "logits/rejected": -0.6703230738639832,
+      "logps/chosen": -4.447329521179199,
+      "logps/rejected": -4.99340295791626,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.447329521179199,
+      "rewards/margins": 0.5460734367370605,
+      "rewards/rejected": -4.99340295791626,
+      "sft_loss": 4.285516738891602,
+      "step": 3495
+    },
+    {
+      "epoch": 1.8732229469811004,
+      "grad_norm": 0.5576566812784183,
+      "learning_rate": 3.714180514435534e-07,
+      "logits/chosen": -0.6373888254165649,
+      "logits/rejected": -0.459861695766449,
+      "logps/chosen": -4.473584175109863,
+      "logps/rejected": -5.074347496032715,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.473584175109863,
+      "rewards/margins": 0.6007635593414307,
+      "rewards/rejected": -5.074347496032715,
+      "sft_loss": 4.14390754699707,
+      "step": 3500
+    },
+    {
+      "epoch": 1.875898979762502,
+      "grad_norm": 0.44593624965935286,
+      "learning_rate": 3.6991365383833426e-07,
+      "logits/chosen": -0.7124999761581421,
+      "logits/rejected": -0.5442683100700378,
+      "logps/chosen": -4.463021278381348,
+      "logps/rejected": -5.113253593444824,
+      "loss": 0.051,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.463021278381348,
+      "rewards/margins": 0.6502333879470825,
+      "rewards/rejected": -5.113253593444824,
+      "sft_loss": 4.128976821899414,
+      "step": 3505
+    },
+    {
+      "epoch": 1.8785750125439038,
+      "grad_norm": 0.4701856300423086,
+      "learning_rate": 3.684105183314162e-07,
+      "logits/chosen": -0.712171196937561,
+      "logits/rejected": -0.6478809118270874,
+      "logps/chosen": -4.455292701721191,
+      "logps/rejected": -5.007076263427734,
+      "loss": 0.052,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.455292701721191,
+      "rewards/margins": 0.5517837405204773,
+      "rewards/rejected": -5.007076263427734,
+      "sft_loss": 4.192362308502197,
+      "step": 3510
+    },
+    {
+      "epoch": 1.881251045325305,
+      "grad_norm": 0.5246272963662074,
+      "learning_rate": 3.669086595062263e-07,
+      "logits/chosen": -0.7043023109436035,
+      "logits/rejected": -0.49362897872924805,
+      "logps/chosen": -4.398072719573975,
+      "logps/rejected": -5.120244979858398,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.398072719573975,
+      "rewards/margins": 0.7221720814704895,
+      "rewards/rejected": -5.120244979858398,
+      "sft_loss": 4.181361675262451,
+      "step": 3515
+    },
+    {
+      "epoch": 1.8839270781067068,
+      "grad_norm": 0.40623000940235376,
+      "learning_rate": 3.654080919338056e-07,
+      "logits/chosen": -0.8072683215141296,
+      "logits/rejected": -0.647091805934906,
+      "logps/chosen": -4.426050662994385,
+      "logps/rejected": -5.1923828125,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.426050662994385,
+      "rewards/margins": 0.7663322687149048,
+      "rewards/rejected": -5.1923828125,
+      "sft_loss": 4.176932334899902,
+      "step": 3520
+    },
+    {
+      "epoch": 1.8866031108881085,
+      "grad_norm": 0.9447657909729972,
+      "learning_rate": 3.639088301726673e-07,
+      "logits/chosen": -0.6944276094436646,
+      "logits/rejected": -0.48604708909988403,
+      "logps/chosen": -4.413082122802734,
+      "logps/rejected": -5.082810401916504,
+      "loss": 0.052,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.413082122802734,
+      "rewards/margins": 0.6697279214859009,
+      "rewards/rejected": -5.082810401916504,
+      "sft_loss": 4.1303391456604,
+      "step": 3525
+    },
+    {
+      "epoch": 1.88927914366951,
+      "grad_norm": 0.35499926674614773,
+      "learning_rate": 3.624108887686556e-07,
+      "logits/chosen": -0.7390032410621643,
+      "logits/rejected": -0.6901714205741882,
+      "logps/chosen": -4.537540435791016,
+      "logps/rejected": -4.95667028427124,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.537540435791016,
+      "rewards/margins": 0.41912975907325745,
+      "rewards/rejected": -4.95667028427124,
+      "sft_loss": 4.238809108734131,
+      "step": 3530
+    },
+    {
+      "epoch": 1.8919551764509115,
+      "grad_norm": 0.37725035427472214,
+      "learning_rate": 3.6091428225480433e-07,
+      "logits/chosen": -0.8973050117492676,
+      "logits/rejected": -0.7953372001647949,
+      "logps/chosen": -4.429739475250244,
+      "logps/rejected": -5.134444236755371,
+      "loss": 0.0505,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.429739475250244,
+      "rewards/margins": 0.7047051191329956,
+      "rewards/rejected": -5.134444236755371,
+      "sft_loss": 4.123137474060059,
+      "step": 3535
+    },
+    {
+      "epoch": 1.8946312092323132,
+      "grad_norm": 0.3983297125217811,
+      "learning_rate": 3.5941902515119674e-07,
+      "logits/chosen": -0.8561373949050903,
+      "logits/rejected": -0.589727520942688,
+      "logps/chosen": -4.494597434997559,
+      "logps/rejected": -5.031756401062012,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.494597434997559,
+      "rewards/margins": 0.5371583104133606,
+      "rewards/rejected": -5.031756401062012,
+      "sft_loss": 4.228826522827148,
+      "step": 3540
+    },
+    {
+      "epoch": 1.8973072420137147,
+      "grad_norm": 0.9925780359947863,
+      "learning_rate": 3.5792513196482373e-07,
+      "logits/chosen": -0.9617146253585815,
+      "logits/rejected": -0.6414197683334351,
+      "logps/chosen": -4.428873538970947,
+      "logps/rejected": -5.051154136657715,
+      "loss": 0.0501,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.428873538970947,
+      "rewards/margins": 0.6222794055938721,
+      "rewards/rejected": -5.051154136657715,
+      "sft_loss": 4.073060035705566,
+      "step": 3545
+    },
+    {
+      "epoch": 1.8999832747951162,
+      "grad_norm": 0.464740691584323,
+      "learning_rate": 3.5643261718944346e-07,
+      "logits/chosen": -0.66382896900177,
+      "logits/rejected": -0.5829827785491943,
+      "logps/chosen": -4.471495628356934,
+      "logps/rejected": -4.948642730712891,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.471495628356934,
+      "rewards/margins": 0.47714701294898987,
+      "rewards/rejected": -4.948642730712891,
+      "sft_loss": 4.171401023864746,
+      "step": 3550
+    },
+    {
+      "epoch": 1.902659307576518,
+      "grad_norm": 0.54188043505912,
+      "learning_rate": 3.5494149530544087e-07,
+      "logits/chosen": -0.8052138090133667,
+      "logits/rejected": -0.6735678315162659,
+      "logps/chosen": -4.545689582824707,
+      "logps/rejected": -5.245549201965332,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.545689582824707,
+      "rewards/margins": 0.6998597383499146,
+      "rewards/rejected": -5.245549201965332,
+      "sft_loss": 4.238376140594482,
+      "step": 3555
+    },
+    {
+      "epoch": 1.9053353403579194,
+      "grad_norm": 0.5862093022972775,
+      "learning_rate": 3.534517807796871e-07,
+      "logits/chosen": -0.7284679412841797,
+      "logits/rejected": -0.6327691078186035,
+      "logps/chosen": -4.589996337890625,
+      "logps/rejected": -5.222556114196777,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.589996337890625,
+      "rewards/margins": 0.6325600147247314,
+      "rewards/rejected": -5.222556114196777,
+      "sft_loss": 4.320279121398926,
+      "step": 3560
+    },
+    {
+      "epoch": 1.908011373139321,
+      "grad_norm": 0.49984158564641323,
+      "learning_rate": 3.519634880653988e-07,
+      "logits/chosen": -0.5908458232879639,
+      "logits/rejected": -0.5253081321716309,
+      "logps/chosen": -4.507323265075684,
+      "logps/rejected": -5.1625776290893555,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.507323265075684,
+      "rewards/margins": 0.6552544236183167,
+      "rewards/rejected": -5.1625776290893555,
+      "sft_loss": 4.136566162109375,
+      "step": 3565
+    },
+    {
+      "epoch": 1.9106874059207226,
+      "grad_norm": 0.4584107855223247,
+      "learning_rate": 3.504766316019987e-07,
+      "logits/chosen": -0.6161302328109741,
+      "logits/rejected": -0.4110661447048187,
+      "logps/chosen": -4.216422080993652,
+      "logps/rejected": -4.989528656005859,
+      "loss": 0.0493,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -4.216422080993652,
+      "rewards/margins": 0.7731070518493652,
+      "rewards/rejected": -4.989528656005859,
+      "sft_loss": 3.8478825092315674,
+      "step": 3570
+    },
+    {
+      "epoch": 1.913363438702124,
+      "grad_norm": 0.4589982714769067,
+      "learning_rate": 3.489912258149745e-07,
+      "logits/chosen": -0.5121530294418335,
+      "logits/rejected": -0.40415287017822266,
+      "logps/chosen": -4.210015773773193,
+      "logps/rejected": -4.940587997436523,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.210015773773193,
+      "rewards/margins": 0.730571985244751,
+      "rewards/rejected": -4.940587997436523,
+      "sft_loss": 3.851905345916748,
+      "step": 3575
+    },
+    {
+      "epoch": 1.9160394714835256,
+      "grad_norm": 0.467352378766595,
+      "learning_rate": 3.475072851157397e-07,
+      "logits/chosen": -0.5266348123550415,
+      "logits/rejected": -0.5173133611679077,
+      "logps/chosen": -4.477862358093262,
+      "logps/rejected": -5.177495002746582,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.477862358093262,
+      "rewards/margins": 0.6996321082115173,
+      "rewards/rejected": -5.177495002746582,
+      "sft_loss": 4.106453895568848,
+      "step": 3580
+    },
+    {
+      "epoch": 1.9187155042649273,
+      "grad_norm": 0.5608432730857718,
+      "learning_rate": 3.460248239014936e-07,
+      "logits/chosen": -0.48529133200645447,
+      "logits/rejected": -0.4915315508842468,
+      "logps/chosen": -4.587717533111572,
+      "logps/rejected": -5.344851970672607,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.587717533111572,
+      "rewards/margins": 0.7571350932121277,
+      "rewards/rejected": -5.344851970672607,
+      "sft_loss": 4.331247806549072,
+      "step": 3585
+    },
+    {
+      "epoch": 1.9213915370463288,
+      "grad_norm": 1.2350605271613377,
+      "learning_rate": 3.4454385655508134e-07,
+      "logits/chosen": -0.5099002718925476,
+      "logits/rejected": -0.5103076696395874,
+      "logps/chosen": -4.546607494354248,
+      "logps/rejected": -5.049306392669678,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.546607494354248,
+      "rewards/margins": 0.5026994943618774,
+      "rewards/rejected": -5.049306392669678,
+      "sft_loss": 4.312752723693848,
+      "step": 3590
+    },
+    {
+      "epoch": 1.9240675698277303,
+      "grad_norm": 0.6963753893867877,
+      "learning_rate": 3.4306439744485447e-07,
+      "logits/chosen": -0.708784818649292,
+      "logits/rejected": -0.51127690076828,
+      "logps/chosen": -4.459948539733887,
+      "logps/rejected": -5.253852844238281,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.459948539733887,
+      "rewards/margins": 0.7939046025276184,
+      "rewards/rejected": -5.253852844238281,
+      "sft_loss": 4.155204772949219,
+      "step": 3595
+    },
+    {
+      "epoch": 1.926743602609132,
+      "grad_norm": 0.6993822451522304,
+      "learning_rate": 3.415864609245322e-07,
+      "logits/chosen": -0.6285854578018188,
+      "logits/rejected": -0.39469218254089355,
+      "logps/chosen": -4.612381935119629,
+      "logps/rejected": -5.276145935058594,
+      "loss": 0.0538,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.612381935119629,
+      "rewards/margins": 0.6637641787528992,
+      "rewards/rejected": -5.276145935058594,
+      "sft_loss": 4.32518196105957,
+      "step": 3600
+    },
+    {
+      "epoch": 1.926743602609132,
+      "eval_logits/chosen": -0.19392162561416626,
+      "eval_logits/rejected": -0.11848235875368118,
+      "eval_logps/chosen": -4.392752170562744,
+      "eval_logps/rejected": -5.105625152587891,
+      "eval_loss": 0.050199832767248154,
+      "eval_rewards/accuracies": 0.6706231236457825,
+      "eval_rewards/chosen": -4.392752170562744,
+      "eval_rewards/margins": 0.7128726840019226,
+      "eval_rewards/rejected": -5.105625152587891,
+      "eval_runtime": 43.6821,
+      "eval_samples_per_second": 30.791,
+      "eval_sft_loss": 4.002551555633545,
+      "eval_steps_per_second": 7.715,
+      "step": 3600
+    },
+    {
+      "epoch": 1.9294196353905335,
+      "grad_norm": 0.6257521040940824,
+      "learning_rate": 3.401100613330605e-07,
+      "logits/chosen": -0.659002423286438,
+      "logits/rejected": -0.6791312098503113,
+      "logps/chosen": -4.4525041580200195,
+      "logps/rejected": -4.979853630065918,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.4525041580200195,
+      "rewards/margins": 0.5273498296737671,
+      "rewards/rejected": -4.979853630065918,
+      "sft_loss": 4.170391082763672,
+      "step": 3605
+    },
+    {
+      "epoch": 1.932095668171935,
+      "grad_norm": 0.4089140471882675,
+      "learning_rate": 3.3863521299447514e-07,
+      "logits/chosen": -0.7611128091812134,
+      "logits/rejected": -0.5972421765327454,
+      "logps/chosen": -4.496156692504883,
+      "logps/rejected": -5.160966396331787,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.496156692504883,
+      "rewards/margins": 0.6648092269897461,
+      "rewards/rejected": -5.160966396331787,
+      "sft_loss": 4.231772422790527,
+      "step": 3610
+    },
+    {
+      "epoch": 1.9347717009533367,
+      "grad_norm": 0.48322369126586207,
+      "learning_rate": 3.371619302177609e-07,
+      "logits/chosen": -0.5951191186904907,
+      "logits/rejected": -0.4748874306678772,
+      "logps/chosen": -4.238120079040527,
+      "logps/rejected": -4.906714916229248,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.238120079040527,
+      "rewards/margins": 0.6685947179794312,
+      "rewards/rejected": -4.906714916229248,
+      "sft_loss": 4.043089866638184,
+      "step": 3615
+    },
+    {
+      "epoch": 1.9374477337347382,
+      "grad_norm": 0.5794409196041558,
+      "learning_rate": 3.3569022729671393e-07,
+      "logits/chosen": -0.6748926043510437,
+      "logits/rejected": -0.5976932644844055,
+      "logps/chosen": -4.524143695831299,
+      "logps/rejected": -5.054104804992676,
+      "loss": 0.0535,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.524143695831299,
+      "rewards/margins": 0.5299606323242188,
+      "rewards/rejected": -5.054104804992676,
+      "sft_loss": 4.340511322021484,
+      "step": 3620
+    },
+    {
+      "epoch": 1.9401237665161397,
+      "grad_norm": 0.5279893079245322,
+      "learning_rate": 3.342201185098024e-07,
+      "logits/chosen": -0.6123486757278442,
+      "logits/rejected": -0.6996939182281494,
+      "logps/chosen": -4.531851768493652,
+      "logps/rejected": -4.982184410095215,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.531851768493652,
+      "rewards/margins": 0.45033249258995056,
+      "rewards/rejected": -4.982184410095215,
+      "sft_loss": 4.264678001403809,
+      "step": 3625
+    },
+    {
+      "epoch": 1.9427997992975414,
+      "grad_norm": 0.37947357656290986,
+      "learning_rate": 3.3275161812002807e-07,
+      "logits/chosen": -0.7515096068382263,
+      "logits/rejected": -0.7556384801864624,
+      "logps/chosen": -4.489173889160156,
+      "logps/rejected": -5.211699485778809,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.489173889160156,
+      "rewards/margins": 0.7225257754325867,
+      "rewards/rejected": -5.211699485778809,
+      "sft_loss": 4.294496536254883,
+      "step": 3630
+    },
+    {
+      "epoch": 1.945475832078943,
+      "grad_norm": 0.4762230882406768,
+      "learning_rate": 3.312847403747883e-07,
+      "logits/chosen": -0.8362226486206055,
+      "logits/rejected": -0.7179775238037109,
+      "logps/chosen": -4.3791608810424805,
+      "logps/rejected": -5.017988204956055,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.3791608810424805,
+      "rewards/margins": 0.6388277411460876,
+      "rewards/rejected": -5.017988204956055,
+      "sft_loss": 4.1786041259765625,
+      "step": 3635
+    },
+    {
+      "epoch": 1.9481518648603444,
+      "grad_norm": 0.3912746197909673,
+      "learning_rate": 3.2981949950573733e-07,
+      "logits/chosen": -0.798759400844574,
+      "logits/rejected": -0.7608662843704224,
+      "logps/chosen": -4.474329948425293,
+      "logps/rejected": -4.8702874183654785,
+      "loss": 0.0528,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.474329948425293,
+      "rewards/margins": 0.39595749974250793,
+      "rewards/rejected": -4.8702874183654785,
+      "sft_loss": 4.2373576164245605,
+      "step": 3640
+    },
+    {
+      "epoch": 1.9508278976417461,
+      "grad_norm": 0.35840613224127876,
+      "learning_rate": 3.283559097286486e-07,
+      "logits/chosen": -0.808552086353302,
+      "logits/rejected": -0.7010661363601685,
+      "logps/chosen": -4.502739906311035,
+      "logps/rejected": -5.010723114013672,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.502739906311035,
+      "rewards/margins": 0.5079833269119263,
+      "rewards/rejected": -5.010723114013672,
+      "sft_loss": 4.265407562255859,
+      "step": 3645
+    },
+    {
+      "epoch": 1.9535039304231478,
+      "grad_norm": 0.5624166775638073,
+      "learning_rate": 3.268939852432765e-07,
+      "logits/chosen": -0.8713280558586121,
+      "logits/rejected": -0.7609729766845703,
+      "logps/chosen": -4.547420501708984,
+      "logps/rejected": -4.945748805999756,
+      "loss": 0.0537,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.547420501708984,
+      "rewards/margins": 0.39832815527915955,
+      "rewards/rejected": -4.945748805999756,
+      "sft_loss": 4.321547031402588,
+      "step": 3650
+    },
+    {
+      "epoch": 1.9561799632045491,
+      "grad_norm": 0.640464388594747,
+      "learning_rate": 3.254337402332187e-07,
+      "logits/chosen": -0.7530469298362732,
+      "logits/rejected": -0.6614114046096802,
+      "logps/chosen": -4.3991498947143555,
+      "logps/rejected": -4.974169731140137,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.3991498947143555,
+      "rewards/margins": 0.5750197768211365,
+      "rewards/rejected": -4.974169731140137,
+      "sft_loss": 4.073417663574219,
+      "step": 3655
+    },
+    {
+      "epoch": 1.9588559959859508,
+      "grad_norm": 0.7826149378891815,
+      "learning_rate": 3.239751888657788e-07,
+      "logits/chosen": -0.8150887489318848,
+      "logits/rejected": -0.6764761209487915,
+      "logps/chosen": -4.439477443695068,
+      "logps/rejected": -5.054062843322754,
+      "loss": 0.052,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.439477443695068,
+      "rewards/margins": 0.6145855188369751,
+      "rewards/rejected": -5.054062843322754,
+      "sft_loss": 4.1504130363464355,
+      "step": 3660
+    },
+    {
+      "epoch": 1.9615320287673526,
+      "grad_norm": 0.5243116461810982,
+      "learning_rate": 3.2251834529182856e-07,
+      "logits/chosen": -0.811129093170166,
+      "logits/rejected": -0.741647481918335,
+      "logps/chosen": -4.62777042388916,
+      "logps/rejected": -5.247431755065918,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.62777042388916,
+      "rewards/margins": 0.619661271572113,
+      "rewards/rejected": -5.247431755065918,
+      "sft_loss": 4.312007904052734,
+      "step": 3665
+    },
+    {
+      "epoch": 1.9642080615487538,
+      "grad_norm": 0.6932294843418075,
+      "learning_rate": 3.2106322364567075e-07,
+      "logits/chosen": -0.8636505007743835,
+      "logits/rejected": -0.7221062779426575,
+      "logps/chosen": -4.473482131958008,
+      "logps/rejected": -5.16109561920166,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.473482131958008,
+      "rewards/margins": 0.687613844871521,
+      "rewards/rejected": -5.16109561920166,
+      "sft_loss": 4.3453369140625,
+      "step": 3670
+    },
+    {
+      "epoch": 1.9668840943301555,
+      "grad_norm": 0.48803374595992904,
+      "learning_rate": 3.1960983804490183e-07,
+      "logits/chosen": -0.7714400887489319,
+      "logits/rejected": -0.6253121495246887,
+      "logps/chosen": -4.469629764556885,
+      "logps/rejected": -5.101070880889893,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.469629764556885,
+      "rewards/margins": 0.6314407587051392,
+      "rewards/rejected": -5.101070880889893,
+      "sft_loss": 4.1811909675598145,
+      "step": 3675
+    },
+    {
+      "epoch": 1.9695601271115573,
+      "grad_norm": 0.49570477577166744,
+      "learning_rate": 3.1815820259027537e-07,
+      "logits/chosen": -0.7076514959335327,
+      "logits/rejected": -0.6141311526298523,
+      "logps/chosen": -4.3212409019470215,
+      "logps/rejected": -4.94888973236084,
+      "loss": 0.0505,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.3212409019470215,
+      "rewards/margins": 0.6276490092277527,
+      "rewards/rejected": -4.94888973236084,
+      "sft_loss": 3.9617087841033936,
+      "step": 3680
+    },
+    {
+      "epoch": 1.9722361598929585,
+      "grad_norm": 0.4735982835000428,
+      "learning_rate": 3.16708331365565e-07,
+      "logits/chosen": -0.7425435781478882,
+      "logits/rejected": -0.6649722456932068,
+      "logps/chosen": -4.505172252655029,
+      "logps/rejected": -5.1141815185546875,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.505172252655029,
+      "rewards/margins": 0.6090092062950134,
+      "rewards/rejected": -5.1141815185546875,
+      "sft_loss": 4.269981384277344,
+      "step": 3685
+    },
+    {
+      "epoch": 1.9749121926743602,
+      "grad_norm": 0.4522049169058261,
+      "learning_rate": 3.152602384374275e-07,
+      "logits/chosen": -0.762370228767395,
+      "logits/rejected": -0.5487732887268066,
+      "logps/chosen": -4.422214508056641,
+      "logps/rejected": -5.175688743591309,
+      "loss": 0.051,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.422214508056641,
+      "rewards/margins": 0.7534740567207336,
+      "rewards/rejected": -5.175688743591309,
+      "sft_loss": 4.079712867736816,
+      "step": 3690
+    },
+    {
+      "epoch": 1.977588225455762,
+      "grad_norm": 0.4854058110443599,
+      "learning_rate": 3.1381393785526697e-07,
+      "logits/chosen": -0.7188117504119873,
+      "logits/rejected": -0.650273859500885,
+      "logps/chosen": -4.508002281188965,
+      "logps/rejected": -5.148285865783691,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.508002281188965,
+      "rewards/margins": 0.6402831077575684,
+      "rewards/rejected": -5.148285865783691,
+      "sft_loss": 4.219584941864014,
+      "step": 3695
+    },
+    {
+      "epoch": 1.9802642582371635,
+      "grad_norm": 0.47726523208034904,
+      "learning_rate": 3.123694436510979e-07,
+      "logits/chosen": -0.7229140996932983,
+      "logits/rejected": -0.5779396295547485,
+      "logps/chosen": -4.463196754455566,
+      "logps/rejected": -5.073942184448242,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.463196754455566,
+      "rewards/margins": 0.6107456684112549,
+      "rewards/rejected": -5.073942184448242,
+      "sft_loss": 4.222643852233887,
+      "step": 3700
+    },
+    {
+      "epoch": 1.982940291018565,
+      "grad_norm": 0.4677889584155691,
+      "learning_rate": 3.1092676983940946e-07,
+      "logits/chosen": -0.6963552832603455,
+      "logits/rejected": -0.6475489735603333,
+      "logps/chosen": -4.44657564163208,
+      "logps/rejected": -5.048283576965332,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.44657564163208,
+      "rewards/margins": 0.6017080545425415,
+      "rewards/rejected": -5.048283576965332,
+      "sft_loss": 4.102275848388672,
+      "step": 3705
+    },
+    {
+      "epoch": 1.9856163237999667,
+      "grad_norm": 0.37810938228783636,
+      "learning_rate": 3.094859304170293e-07,
+      "logits/chosen": -0.49164676666259766,
+      "logits/rejected": -0.5156913995742798,
+      "logps/chosen": -4.413491249084473,
+      "logps/rejected": -5.021629333496094,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.413491249084473,
+      "rewards/margins": 0.6081380248069763,
+      "rewards/rejected": -5.021629333496094,
+      "sft_loss": 4.1239399909973145,
+      "step": 3710
+    },
+    {
+      "epoch": 1.9882923565813682,
+      "grad_norm": 0.4315476171180668,
+      "learning_rate": 3.0804693936298795e-07,
+      "logits/chosen": -0.6968036890029907,
+      "logits/rejected": -0.6801964640617371,
+      "logps/chosen": -4.3998332023620605,
+      "logps/rejected": -5.117763519287109,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.3998332023620605,
+      "rewards/margins": 0.7179301381111145,
+      "rewards/rejected": -5.117763519287109,
+      "sft_loss": 4.231060028076172,
+      "step": 3715
+    },
+    {
+      "epoch": 1.9909683893627697,
+      "grad_norm": 0.5569686698493027,
+      "learning_rate": 3.066098106383826e-07,
+      "logits/chosen": -0.6866349577903748,
+      "logits/rejected": -0.5895728468894958,
+      "logps/chosen": -4.407230854034424,
+      "logps/rejected": -5.047307014465332,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.407230854034424,
+      "rewards/margins": 0.6400759816169739,
+      "rewards/rejected": -5.047307014465332,
+      "sft_loss": 4.018852710723877,
+      "step": 3720
+    },
+    {
+      "epoch": 1.9936444221441714,
+      "grad_norm": 0.4670996934775318,
+      "learning_rate": 3.0517455818624263e-07,
+      "logits/chosen": -0.7536731958389282,
+      "logits/rejected": -0.6881700754165649,
+      "logps/chosen": -4.4293742179870605,
+      "logps/rejected": -5.185140609741211,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.4293742179870605,
+      "rewards/margins": 0.7557664513587952,
+      "rewards/rejected": -5.185140609741211,
+      "sft_loss": 4.174285411834717,
+      "step": 3725
+    },
+    {
+      "epoch": 1.9963204549255729,
+      "grad_norm": 0.5546489639055374,
+      "learning_rate": 3.037411959313936e-07,
+      "logits/chosen": -0.6394184827804565,
+      "logits/rejected": -0.49718838930130005,
+      "logps/chosen": -4.460624694824219,
+      "logps/rejected": -5.078179359436035,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.460624694824219,
+      "rewards/margins": 0.6175544261932373,
+      "rewards/rejected": -5.078179359436035,
+      "sft_loss": 4.196967124938965,
+      "step": 3730
+    },
+    {
+      "epoch": 1.9989964877069744,
+      "grad_norm": 1.0187071463911035,
+      "learning_rate": 3.023097377803224e-07,
+      "logits/chosen": -0.593445897102356,
+      "logits/rejected": -0.5435713529586792,
+      "logps/chosen": -4.4680399894714355,
+      "logps/rejected": -5.042878150939941,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.4680399894714355,
+      "rewards/margins": 0.5748375058174133,
+      "rewards/rejected": -5.042878150939941,
+      "sft_loss": 4.189925193786621,
+      "step": 3735
+    },
+    {
+      "epoch": 2.001672520488376,
+      "grad_norm": 0.5385102791360227,
+      "learning_rate": 3.008801976210423e-07,
+      "logits/chosen": -0.6135790348052979,
+      "logits/rejected": -0.5923279523849487,
+      "logps/chosen": -4.367274284362793,
+      "logps/rejected": -4.868128776550293,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.367274284362793,
+      "rewards/margins": 0.5008544325828552,
+      "rewards/rejected": -4.868128776550293,
+      "sft_loss": 4.043814659118652,
+      "step": 3740
+    },
+    {
+      "epoch": 2.0043485532697773,
+      "grad_norm": 0.42762656945134886,
+      "learning_rate": 2.994525893229581e-07,
+      "logits/chosen": -0.6263781189918518,
+      "logits/rejected": -0.5807645320892334,
+      "logps/chosen": -4.394901275634766,
+      "logps/rejected": -5.174359321594238,
+      "loss": 0.0494,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.394901275634766,
+      "rewards/margins": 0.7794585227966309,
+      "rewards/rejected": -5.174359321594238,
+      "sft_loss": 4.081959247589111,
+      "step": 3745
+    },
+    {
+      "epoch": 2.007024586051179,
+      "grad_norm": 0.543376451163454,
+      "learning_rate": 2.98026926736732e-07,
+      "logits/chosen": -0.7248175740242004,
+      "logits/rejected": -0.6078269481658936,
+      "logps/chosen": -4.447038650512695,
+      "logps/rejected": -5.048276424407959,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.447038650512695,
+      "rewards/margins": 0.6012377738952637,
+      "rewards/rejected": -5.048276424407959,
+      "sft_loss": 4.144289970397949,
+      "step": 3750
+    },
+    {
+      "epoch": 2.0097006188325808,
+      "grad_norm": 0.52379641957468,
+      "learning_rate": 2.9660322369414846e-07,
+      "logits/chosen": -0.6346576809883118,
+      "logits/rejected": -0.4830950200557709,
+      "logps/chosen": -4.529818534851074,
+      "logps/rejected": -5.26165771484375,
+      "loss": 0.052,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.529818534851074,
+      "rewards/margins": 0.7318395376205444,
+      "rewards/rejected": -5.26165771484375,
+      "sft_loss": 4.303341865539551,
+      "step": 3755
+    },
+    {
+      "epoch": 2.0123766516139825,
+      "grad_norm": 0.3017257682052039,
+      "learning_rate": 2.9518149400798063e-07,
+      "logits/chosen": -0.6983194947242737,
+      "logits/rejected": -0.6876857876777649,
+      "logps/chosen": -4.475781440734863,
+      "logps/rejected": -5.232280731201172,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.475781440734863,
+      "rewards/margins": 0.7564995884895325,
+      "rewards/rejected": -5.232280731201172,
+      "sft_loss": 4.246873378753662,
+      "step": 3760
+    },
+    {
+      "epoch": 2.0150526843953838,
+      "grad_norm": 0.49335157540079083,
+      "learning_rate": 2.9376175147185633e-07,
+      "logits/chosen": -0.7001711130142212,
+      "logits/rejected": -0.46811890602111816,
+      "logps/chosen": -4.475892066955566,
+      "logps/rejected": -5.218499660491943,
+      "loss": 0.0503,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.475892066955566,
+      "rewards/margins": 0.742607831954956,
+      "rewards/rejected": -5.218499660491943,
+      "sft_loss": 4.178433418273926,
+      "step": 3765
+    },
+    {
+      "epoch": 2.0177287171767855,
+      "grad_norm": 0.4934910914071619,
+      "learning_rate": 2.9234400986012376e-07,
+      "logits/chosen": -0.6719244122505188,
+      "logits/rejected": -0.46986284852027893,
+      "logps/chosen": -4.391491413116455,
+      "logps/rejected": -5.1664628982543945,
+      "loss": 0.0505,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.391491413116455,
+      "rewards/margins": 0.7749711275100708,
+      "rewards/rejected": -5.1664628982543945,
+      "sft_loss": 4.1042280197143555,
+      "step": 3770
+    },
+    {
+      "epoch": 2.020404749958187,
+      "grad_norm": 0.47750762194874763,
+      "learning_rate": 2.9092828292771817e-07,
+      "logits/chosen": -0.5587464570999146,
+      "logits/rejected": -0.5431086421012878,
+      "logps/chosen": -4.312157154083252,
+      "logps/rejected": -5.090926170349121,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.312157154083252,
+      "rewards/margins": 0.7787691354751587,
+      "rewards/rejected": -5.090926170349121,
+      "sft_loss": 4.055374622344971,
+      "step": 3775
+    },
+    {
+      "epoch": 2.0230807827395885,
+      "grad_norm": 0.4971383253803019,
+      "learning_rate": 2.8951458441002875e-07,
+      "logits/chosen": -0.5055549144744873,
+      "logits/rejected": -0.5332974195480347,
+      "logps/chosen": -4.194530487060547,
+      "logps/rejected": -4.8951096534729,
+      "loss": 0.0505,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.194530487060547,
+      "rewards/margins": 0.7005791068077087,
+      "rewards/rejected": -4.8951096534729,
+      "sft_loss": 3.8925559520721436,
+      "step": 3780
+    },
+    {
+      "epoch": 2.02575681552099,
+      "grad_norm": 0.6609434060993625,
+      "learning_rate": 2.881029280227643e-07,
+      "logits/chosen": -0.6029443740844727,
+      "logits/rejected": -0.4277864098548889,
+      "logps/chosen": -4.241337776184082,
+      "logps/rejected": -5.0103559494018555,
+      "loss": 0.0505,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.241337776184082,
+      "rewards/margins": 0.7690183520317078,
+      "rewards/rejected": -5.0103559494018555,
+      "sft_loss": 3.975001573562622,
+      "step": 3785
+    },
+    {
+      "epoch": 2.028432848302392,
+      "grad_norm": 0.5220314563570935,
+      "learning_rate": 2.8669332746182177e-07,
+      "logits/chosen": -0.6425802111625671,
+      "logits/rejected": -0.4697285294532776,
+      "logps/chosen": -4.399476051330566,
+      "logps/rejected": -5.2246599197387695,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.399476051330566,
+      "rewards/margins": 0.8251844644546509,
+      "rewards/rejected": -5.2246599197387695,
+      "sft_loss": 4.189549446105957,
+      "step": 3790
+    },
+    {
+      "epoch": 2.031108881083793,
+      "grad_norm": 0.5237960313740657,
+      "learning_rate": 2.8528579640315156e-07,
+      "logits/chosen": -0.5389834046363831,
+      "logits/rejected": -0.5551042556762695,
+      "logps/chosen": -4.534903526306152,
+      "logps/rejected": -5.066584587097168,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.534903526306152,
+      "rewards/margins": 0.5316810607910156,
+      "rewards/rejected": -5.066584587097168,
+      "sft_loss": 4.209554672241211,
+      "step": 3795
+    },
+    {
+      "epoch": 2.033784913865195,
+      "grad_norm": 0.8509289539923249,
+      "learning_rate": 2.8388034850262646e-07,
+      "logits/chosen": -0.6247214674949646,
+      "logits/rejected": -0.4796470105648041,
+      "logps/chosen": -4.471749305725098,
+      "logps/rejected": -5.176628589630127,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.471749305725098,
+      "rewards/margins": 0.7048790454864502,
+      "rewards/rejected": -5.176628589630127,
+      "sft_loss": 4.188010215759277,
+      "step": 3800
+    },
+    {
+      "epoch": 2.0364609466465966,
+      "grad_norm": 0.49482388890694134,
+      "learning_rate": 2.824769973959079e-07,
+      "logits/chosen": -0.5981681942939758,
+      "logits/rejected": -0.46991783380508423,
+      "logps/chosen": -4.412623405456543,
+      "logps/rejected": -5.10774040222168,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.412623405456543,
+      "rewards/margins": 0.6951168775558472,
+      "rewards/rejected": -5.10774040222168,
+      "sft_loss": 4.103912830352783,
+      "step": 3805
+    },
+    {
+      "epoch": 2.039136979427998,
+      "grad_norm": 0.4973281897641397,
+      "learning_rate": 2.81075756698315e-07,
+      "logits/chosen": -0.419619083404541,
+      "logits/rejected": -0.36085718870162964,
+      "logps/chosen": -4.509213447570801,
+      "logps/rejected": -5.325683116912842,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.509213447570801,
+      "rewards/margins": 0.8164698481559753,
+      "rewards/rejected": -5.325683116912842,
+      "sft_loss": 4.111828804016113,
+      "step": 3810
+    },
+    {
+      "epoch": 2.0418130122093996,
+      "grad_norm": 0.483257312769065,
+      "learning_rate": 2.7967664000469035e-07,
+      "logits/chosen": -0.643826961517334,
+      "logits/rejected": -0.571256160736084,
+      "logps/chosen": -4.353998184204102,
+      "logps/rejected": -5.058455944061279,
+      "loss": 0.0497,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.353998184204102,
+      "rewards/margins": 0.7044575810432434,
+      "rewards/rejected": -5.058455944061279,
+      "sft_loss": 4.034123420715332,
+      "step": 3815
+    },
+    {
+      "epoch": 2.0444890449908013,
+      "grad_norm": 0.5003139679448172,
+      "learning_rate": 2.7827966088927095e-07,
+      "logits/chosen": -0.6849731206893921,
+      "logits/rejected": -0.45091620087623596,
+      "logps/chosen": -4.401778221130371,
+      "logps/rejected": -5.4099321365356445,
+      "loss": 0.0501,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.401778221130371,
+      "rewards/margins": 1.0081536769866943,
+      "rewards/rejected": -5.4099321365356445,
+      "sft_loss": 4.158273220062256,
+      "step": 3820
+    },
+    {
+      "epoch": 2.0471650777722026,
+      "grad_norm": 0.62368546467921,
+      "learning_rate": 2.768848329055538e-07,
+      "logits/chosen": -0.5754817724227905,
+      "logits/rejected": -0.5407590270042419,
+      "logps/chosen": -4.350062370300293,
+      "logps/rejected": -5.087815284729004,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.350062370300293,
+      "rewards/margins": 0.7377529740333557,
+      "rewards/rejected": -5.087815284729004,
+      "sft_loss": 4.1428022384643555,
+      "step": 3825
+    },
+    {
+      "epoch": 2.0498411105536043,
+      "grad_norm": 0.6755483572701749,
+      "learning_rate": 2.7549216958616657e-07,
+      "logits/chosen": -0.6843256950378418,
+      "logits/rejected": -0.5472108125686646,
+      "logps/chosen": -4.316120147705078,
+      "logps/rejected": -5.0950846672058105,
+      "loss": 0.0501,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.316120147705078,
+      "rewards/margins": 0.778964638710022,
+      "rewards/rejected": -5.0950846672058105,
+      "sft_loss": 3.9356772899627686,
+      "step": 3830
+    },
+    {
+      "epoch": 2.052517143335006,
+      "grad_norm": 0.5183086582560419,
+      "learning_rate": 2.741016844427344e-07,
+      "logits/chosen": -0.5944709181785583,
+      "logits/rejected": -0.42749032378196716,
+      "logps/chosen": -4.316996097564697,
+      "logps/rejected": -5.204145908355713,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.316996097564697,
+      "rewards/margins": 0.8871499300003052,
+      "rewards/rejected": -5.204145908355713,
+      "sft_loss": 4.075043201446533,
+      "step": 3835
+    },
+    {
+      "epoch": 2.0551931761164073,
+      "grad_norm": 0.6822860631658114,
+      "learning_rate": 2.7271339096575073e-07,
+      "logits/chosen": -0.48182693123817444,
+      "logits/rejected": -0.332283079624176,
+      "logps/chosen": -4.471286773681641,
+      "logps/rejected": -5.244685649871826,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.471286773681641,
+      "rewards/margins": 0.7733985781669617,
+      "rewards/rejected": -5.244685649871826,
+      "sft_loss": 4.19659948348999,
+      "step": 3840
+    },
+    {
+      "epoch": 2.057869208897809,
+      "grad_norm": 0.5230422649929072,
+      "learning_rate": 2.713273026244446e-07,
+      "logits/chosen": -0.6334673762321472,
+      "logits/rejected": -0.4097228944301605,
+      "logps/chosen": -4.314341068267822,
+      "logps/rejected": -5.308169364929199,
+      "loss": 0.0496,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -4.314341068267822,
+      "rewards/margins": 0.9938287734985352,
+      "rewards/rejected": -5.308169364929199,
+      "sft_loss": 4.097606182098389,
+      "step": 3845
+    },
+    {
+      "epoch": 2.0605452416792107,
+      "grad_norm": 0.5414559580682413,
+      "learning_rate": 2.6994343286665156e-07,
+      "logits/chosen": -0.5962134599685669,
+      "logits/rejected": -0.40147629380226135,
+      "logps/chosen": -4.273350715637207,
+      "logps/rejected": -5.072994232177734,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.273350715637207,
+      "rewards/margins": 0.7996436357498169,
+      "rewards/rejected": -5.072994232177734,
+      "sft_loss": 4.124640941619873,
+      "step": 3850
+    },
+    {
+      "epoch": 2.063221274460612,
+      "grad_norm": 0.5728560322778394,
+      "learning_rate": 2.6856179511868156e-07,
+      "logits/chosen": -0.536620020866394,
+      "logits/rejected": -0.3343598246574402,
+      "logps/chosen": -4.325739860534668,
+      "logps/rejected": -5.295504093170166,
+      "loss": 0.0504,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.325739860534668,
+      "rewards/margins": 0.9697640538215637,
+      "rewards/rejected": -5.295504093170166,
+      "sft_loss": 4.102086067199707,
+      "step": 3855
+    },
+    {
+      "epoch": 2.0658973072420137,
+      "grad_norm": 1.2383843978128164,
+      "learning_rate": 2.6718240278519056e-07,
+      "logits/chosen": -0.4679384231567383,
+      "logits/rejected": -0.3461325466632843,
+      "logps/chosen": -4.238699913024902,
+      "logps/rejected": -5.298345565795898,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.238699913024902,
+      "rewards/margins": 1.059645652770996,
+      "rewards/rejected": -5.298345565795898,
+      "sft_loss": 4.039071083068848,
+      "step": 3860
+    },
+    {
+      "epoch": 2.0685733400234154,
+      "grad_norm": 0.5339414572143288,
+      "learning_rate": 2.6580526924904866e-07,
+      "logits/chosen": -0.591120719909668,
+      "logits/rejected": -0.44554227590560913,
+      "logps/chosen": -4.540846347808838,
+      "logps/rejected": -5.21270751953125,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.540846347808838,
+      "rewards/margins": 0.6718612909317017,
+      "rewards/rejected": -5.21270751953125,
+      "sft_loss": 4.238110065460205,
+      "step": 3865
+    },
+    {
+      "epoch": 2.0712493728048167,
+      "grad_norm": 0.7737575710376245,
+      "learning_rate": 2.6443040787121186e-07,
+      "logits/chosen": -0.6046738028526306,
+      "logits/rejected": -0.5335342884063721,
+      "logps/chosen": -4.3026251792907715,
+      "logps/rejected": -5.078793525695801,
+      "loss": 0.0506,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.3026251792907715,
+      "rewards/margins": 0.776168942451477,
+      "rewards/rejected": -5.078793525695801,
+      "sft_loss": 4.051812171936035,
+      "step": 3870
+    },
+    {
+      "epoch": 2.0739254055862184,
+      "grad_norm": 0.6711585502238905,
+      "learning_rate": 2.6305783199059084e-07,
+      "logits/chosen": -0.506575345993042,
+      "logits/rejected": -0.43495336174964905,
+      "logps/chosen": -4.321440696716309,
+      "logps/rejected": -4.975607395172119,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.321440696716309,
+      "rewards/margins": 0.6541663408279419,
+      "rewards/rejected": -4.975607395172119,
+      "sft_loss": 3.996849536895752,
+      "step": 3875
+    },
+    {
+      "epoch": 2.07660143836762,
+      "grad_norm": 0.5671498478920758,
+      "learning_rate": 2.6168755492392324e-07,
+      "logits/chosen": -0.5049850344657898,
+      "logits/rejected": -0.3581571578979492,
+      "logps/chosen": -4.16782808303833,
+      "logps/rejected": -5.161293029785156,
+      "loss": 0.0478,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -4.16782808303833,
+      "rewards/margins": 0.9934650659561157,
+      "rewards/rejected": -5.161293029785156,
+      "sft_loss": 3.852004289627075,
+      "step": 3880
+    },
+    {
+      "epoch": 2.0792774711490214,
+      "grad_norm": 0.6388184892624315,
+      "learning_rate": 2.6031958996564274e-07,
+      "logits/chosen": -0.5345726013183594,
+      "logits/rejected": -0.4473126530647278,
+      "logps/chosen": -4.377729892730713,
+      "logps/rejected": -5.321441173553467,
+      "loss": 0.052,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.377729892730713,
+      "rewards/margins": 0.9437112808227539,
+      "rewards/rejected": -5.321441173553467,
+      "sft_loss": 4.083974361419678,
+      "step": 3885
+    },
+    {
+      "epoch": 2.081953503930423,
+      "grad_norm": 0.5796454245935159,
+      "learning_rate": 2.589539503877518e-07,
+      "logits/chosen": -0.49546051025390625,
+      "logits/rejected": -0.4058174192905426,
+      "logps/chosen": -4.520596504211426,
+      "logps/rejected": -5.284416675567627,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.520596504211426,
+      "rewards/margins": 0.7638201713562012,
+      "rewards/rejected": -5.284416675567627,
+      "sft_loss": 4.191913604736328,
+      "step": 3890
+    },
+    {
+      "epoch": 2.084629536711825,
+      "grad_norm": 0.6308232585959354,
+      "learning_rate": 2.5759064943969125e-07,
+      "logits/chosen": -0.6473763585090637,
+      "logits/rejected": -0.4293528199195862,
+      "logps/chosen": -4.480085372924805,
+      "logps/rejected": -5.307429313659668,
+      "loss": 0.0496,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.480085372924805,
+      "rewards/margins": 0.8273438215255737,
+      "rewards/rejected": -5.307429313659668,
+      "sft_loss": 4.166953086853027,
+      "step": 3895
+    },
+    {
+      "epoch": 2.087305569493226,
+      "grad_norm": 0.41789748875310195,
+      "learning_rate": 2.562297003482131e-07,
+      "logits/chosen": -0.4854046404361725,
+      "logits/rejected": -0.5152328014373779,
+      "logps/chosen": -4.265536308288574,
+      "logps/rejected": -5.083382606506348,
+      "loss": 0.05,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -4.265536308288574,
+      "rewards/margins": 0.8178454637527466,
+      "rewards/rejected": -5.083382606506348,
+      "sft_loss": 4.026615142822266,
+      "step": 3900
+    },
+    {
+      "epoch": 2.089981602274628,
+      "grad_norm": 0.50281439414106,
+      "learning_rate": 2.548711163172512e-07,
+      "logits/chosen": -0.508335292339325,
+      "logits/rejected": -0.45731106400489807,
+      "logps/chosen": -4.417618751525879,
+      "logps/rejected": -4.980834484100342,
+      "loss": 0.0529,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.417618751525879,
+      "rewards/margins": 0.5632158517837524,
+      "rewards/rejected": -4.980834484100342,
+      "sft_loss": 4.1282525062561035,
+      "step": 3905
+    },
+    {
+      "epoch": 2.0926576350560295,
+      "grad_norm": 0.45251177117536173,
+      "learning_rate": 2.53514910527794e-07,
+      "logits/chosen": -0.5585693120956421,
+      "logits/rejected": -0.45644134283065796,
+      "logps/chosen": -4.4364118576049805,
+      "logps/rejected": -5.043321132659912,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.4364118576049805,
+      "rewards/margins": 0.6069098711013794,
+      "rewards/rejected": -5.043321132659912,
+      "sft_loss": 4.154627799987793,
+      "step": 3910
+    },
+    {
+      "epoch": 2.095333667837431,
+      "grad_norm": 0.40511189458381863,
+      "learning_rate": 2.5216109613775573e-07,
+      "logits/chosen": -0.6440985202789307,
+      "logits/rejected": -0.4846329689025879,
+      "logps/chosen": -4.497926712036133,
+      "logps/rejected": -5.19435977935791,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.497926712036133,
+      "rewards/margins": 0.6964332461357117,
+      "rewards/rejected": -5.19435977935791,
+      "sft_loss": 4.278082847595215,
+      "step": 3915
+    },
+    {
+      "epoch": 2.0980097006188325,
+      "grad_norm": 0.4670713965434905,
+      "learning_rate": 2.5080968628184993e-07,
+      "logits/chosen": -0.6064902544021606,
+      "logits/rejected": -0.45147815346717834,
+      "logps/chosen": -4.359574317932129,
+      "logps/rejected": -5.300755500793457,
+      "loss": 0.0499,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.359574317932129,
+      "rewards/margins": 0.9411810040473938,
+      "rewards/rejected": -5.300755500793457,
+      "sft_loss": 4.128987789154053,
+      "step": 3920
+    },
+    {
+      "epoch": 2.1006857334002342,
+      "grad_norm": 0.5730415890834424,
+      "learning_rate": 2.494606940714605e-07,
+      "logits/chosen": -0.5470038652420044,
+      "logits/rejected": -0.5130370855331421,
+      "logps/chosen": -4.274531364440918,
+      "logps/rejected": -5.124925136566162,
+      "loss": 0.0501,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.274531364440918,
+      "rewards/margins": 0.8503939509391785,
+      "rewards/rejected": -5.124925136566162,
+      "sft_loss": 4.024096488952637,
+      "step": 3925
+    },
+    {
+      "epoch": 2.103361766181636,
+      "grad_norm": 0.6418488771852127,
+      "learning_rate": 2.4811413259451625e-07,
+      "logits/chosen": -0.6766397953033447,
+      "logits/rejected": -0.5185081362724304,
+      "logps/chosen": -4.179051876068115,
+      "logps/rejected": -5.043671131134033,
+      "loss": 0.0503,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.179051876068115,
+      "rewards/margins": 0.8646184802055359,
+      "rewards/rejected": -5.043671131134033,
+      "sft_loss": 3.994061231613159,
+      "step": 3930
+    },
+    {
+      "epoch": 2.106037798963037,
+      "grad_norm": 0.8344092727300619,
+      "learning_rate": 2.46770014915362e-07,
+      "logits/chosen": -0.5785678029060364,
+      "logits/rejected": -0.5231860280036926,
+      "logps/chosen": -4.328639507293701,
+      "logps/rejected": -5.083830833435059,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.328639507293701,
+      "rewards/margins": 0.7551919221878052,
+      "rewards/rejected": -5.083830833435059,
+      "sft_loss": 4.054540634155273,
+      "step": 3935
+    },
+    {
+      "epoch": 2.108713831744439,
+      "grad_norm": 0.572911342683585,
+      "learning_rate": 2.45428354074634e-07,
+      "logits/chosen": -0.5792239904403687,
+      "logits/rejected": -0.5249115228652954,
+      "logps/chosen": -4.367335319519043,
+      "logps/rejected": -5.200791358947754,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.367335319519043,
+      "rewards/margins": 0.8334552645683289,
+      "rewards/rejected": -5.200791358947754,
+      "sft_loss": 4.094059467315674,
+      "step": 3940
+    },
+    {
+      "epoch": 2.1113898645258407,
+      "grad_norm": 0.7828143017853226,
+      "learning_rate": 2.4408916308913105e-07,
+      "logits/chosen": -0.627918004989624,
+      "logits/rejected": -0.43884220719337463,
+      "logps/chosen": -4.496485233306885,
+      "logps/rejected": -5.016319274902344,
+      "loss": 0.0542,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.496485233306885,
+      "rewards/margins": 0.5198333263397217,
+      "rewards/rejected": -5.016319274902344,
+      "sft_loss": 4.241923809051514,
+      "step": 3945
+    },
+    {
+      "epoch": 2.114065897307242,
+      "grad_norm": 0.4412923182404486,
+      "learning_rate": 2.4275245495169025e-07,
+      "logits/chosen": -0.46839475631713867,
+      "logits/rejected": -0.3399216830730438,
+      "logps/chosen": -4.385307788848877,
+      "logps/rejected": -5.189986228942871,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.385307788848877,
+      "rewards/margins": 0.8046786189079285,
+      "rewards/rejected": -5.189986228942871,
+      "sft_loss": 4.065909385681152,
+      "step": 3950
+    },
+    {
+      "epoch": 2.1167419300886436,
+      "grad_norm": 0.5664224161497169,
+      "learning_rate": 2.414182426310597e-07,
+      "logits/chosen": -0.5879368782043457,
+      "logits/rejected": -0.5610846877098083,
+      "logps/chosen": -4.477884769439697,
+      "logps/rejected": -5.357515811920166,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.477884769439697,
+      "rewards/margins": 0.8796316385269165,
+      "rewards/rejected": -5.357515811920166,
+      "sft_loss": 4.248618125915527,
+      "step": 3955
+    },
+    {
+      "epoch": 2.1194179628700454,
+      "grad_norm": 0.7459652480812919,
+      "learning_rate": 2.400865390717734e-07,
+      "logits/chosen": -0.5339723825454712,
+      "logits/rejected": -0.45431438088417053,
+      "logps/chosen": -4.340689659118652,
+      "logps/rejected": -5.388014793395996,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.340689659118652,
+      "rewards/margins": 1.0473250150680542,
+      "rewards/rejected": -5.388014793395996,
+      "sft_loss": 4.13299036026001,
+      "step": 3960
+    },
+    {
+      "epoch": 2.1220939956514466,
+      "grad_norm": 0.6963469828066498,
+      "learning_rate": 2.3875735719402475e-07,
+      "logits/chosen": -0.5947713851928711,
+      "logits/rejected": -0.46804705262184143,
+      "logps/chosen": -4.319635391235352,
+      "logps/rejected": -5.16073751449585,
+      "loss": 0.0503,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.319635391235352,
+      "rewards/margins": 0.8411022424697876,
+      "rewards/rejected": -5.16073751449585,
+      "sft_loss": 4.115952491760254,
+      "step": 3965
+    },
+    {
+      "epoch": 2.1247700284328483,
+      "grad_norm": 0.6912742586775039,
+      "learning_rate": 2.3743070989354258e-07,
+      "logits/chosen": -0.49584469199180603,
+      "logits/rejected": -0.43432727456092834,
+      "logps/chosen": -4.370326042175293,
+      "logps/rejected": -5.163401126861572,
+      "loss": 0.0501,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.370326042175293,
+      "rewards/margins": 0.7930749654769897,
+      "rewards/rejected": -5.163401126861572,
+      "sft_loss": 4.079747200012207,
+      "step": 3970
+    },
+    {
+      "epoch": 2.12744606121425,
+      "grad_norm": 0.8168965575173682,
+      "learning_rate": 2.3610661004146454e-07,
+      "logits/chosen": -0.4701816439628601,
+      "logits/rejected": -0.37462377548217773,
+      "logps/chosen": -4.166764736175537,
+      "logps/rejected": -4.933764457702637,
+      "loss": 0.0492,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.166764736175537,
+      "rewards/margins": 0.7669999599456787,
+      "rewards/rejected": -4.933764457702637,
+      "sft_loss": 3.8806662559509277,
+      "step": 3975
+    },
+    {
+      "epoch": 2.1301220939956513,
+      "grad_norm": 0.5062705554333068,
+      "learning_rate": 2.3478507048421314e-07,
+      "logits/chosen": -0.5912007093429565,
+      "logits/rejected": -0.4949302077293396,
+      "logps/chosen": -4.348996162414551,
+      "logps/rejected": -5.174300193786621,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.348996162414551,
+      "rewards/margins": 0.8253036737442017,
+      "rewards/rejected": -5.174300193786621,
+      "sft_loss": 4.080183982849121,
+      "step": 3980
+    },
+    {
+      "epoch": 2.132798126777053,
+      "grad_norm": 0.4939655496191941,
+      "learning_rate": 2.334661040433713e-07,
+      "logits/chosen": -0.6417457461357117,
+      "logits/rejected": -0.5208768844604492,
+      "logps/chosen": -4.459622383117676,
+      "logps/rejected": -5.17165470123291,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.459622383117676,
+      "rewards/margins": 0.712032675743103,
+      "rewards/rejected": -5.17165470123291,
+      "sft_loss": 4.136000633239746,
+      "step": 3985
+    },
+    {
+      "epoch": 2.1354741595584548,
+      "grad_norm": 0.5756338757986629,
+      "learning_rate": 2.321497235155568e-07,
+      "logits/chosen": -0.6245895624160767,
+      "logits/rejected": -0.48283690214157104,
+      "logps/chosen": -4.137881278991699,
+      "logps/rejected": -4.993329048156738,
+      "loss": 0.051,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -4.137881278991699,
+      "rewards/margins": 0.8554474115371704,
+      "rewards/rejected": -4.993329048156738,
+      "sft_loss": 3.891702175140381,
+      "step": 3990
+    },
+    {
+      "epoch": 2.138150192339856,
+      "grad_norm": 0.44803672826678165,
+      "learning_rate": 2.3083594167229965e-07,
+      "logits/chosen": -0.6495420336723328,
+      "logits/rejected": -0.3557949364185333,
+      "logps/chosen": -4.3771562576293945,
+      "logps/rejected": -5.219357967376709,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.3771562576293945,
+      "rewards/margins": 0.8422020077705383,
+      "rewards/rejected": -5.219357967376709,
+      "sft_loss": 4.094968318939209,
+      "step": 3995
+    },
+    {
+      "epoch": 2.1408262251212578,
+      "grad_norm": 0.49845593248591585,
+      "learning_rate": 2.295247712599167e-07,
+      "logits/chosen": -0.47788986563682556,
+      "logits/rejected": -0.4273671507835388,
+      "logps/chosen": -4.382460594177246,
+      "logps/rejected": -5.201376914978027,
+      "loss": 0.0495,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.382460594177246,
+      "rewards/margins": 0.8189162015914917,
+      "rewards/rejected": -5.201376914978027,
+      "sft_loss": 4.001456260681152,
+      "step": 4000
+    },
+    {
+      "epoch": 2.1408262251212578,
+      "eval_logits/chosen": -0.1283819079399109,
+      "eval_logits/rejected": -0.04881530627608299,
+      "eval_logps/chosen": -4.425081253051758,
+      "eval_logps/rejected": -5.172266006469727,
+      "eval_loss": 0.05017642304301262,
+      "eval_rewards/accuracies": 0.6824925541877747,
+      "eval_rewards/chosen": -4.425081253051758,
+      "eval_rewards/margins": 0.7471847534179688,
+      "eval_rewards/rejected": -5.172266006469727,
+      "eval_runtime": 43.6658,
+      "eval_samples_per_second": 30.802,
+      "eval_sft_loss": 4.0304131507873535,
+      "eval_steps_per_second": 7.718,
+      "step": 4000
+    },
+    {
+      "epoch": 2.1435022579026595,
+      "grad_norm": 0.7882371901975319,
+      "learning_rate": 2.2821622499938948e-07,
+      "logits/chosen": -0.4993532598018646,
+      "logits/rejected": -0.27499154210090637,
+      "logps/chosen": -4.4369096755981445,
+      "logps/rejected": -5.1286773681640625,
+      "loss": 0.0505,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.4369096755981445,
+      "rewards/margins": 0.6917680501937866,
+      "rewards/rejected": -5.1286773681640625,
+      "sft_loss": 4.113725185394287,
+      "step": 4005
+    },
+    {
+      "epoch": 2.1461782906840607,
+      "grad_norm": 0.43338654645231023,
+      "learning_rate": 2.269103155862391e-07,
+      "logits/chosen": -0.5864294767379761,
+      "logits/rejected": -0.4958516061306,
+      "logps/chosen": -4.39797830581665,
+      "logps/rejected": -5.171106815338135,
+      "loss": 0.0492,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.39797830581665,
+      "rewards/margins": 0.7731285691261292,
+      "rewards/rejected": -5.171106815338135,
+      "sft_loss": 4.035377502441406,
+      "step": 4010
+    },
+    {
+      "epoch": 2.1488543234654625,
+      "grad_norm": 0.6139850564200314,
+      "learning_rate": 2.2560705569040483e-07,
+      "logits/chosen": -0.5838503241539001,
+      "logits/rejected": -0.29924023151397705,
+      "logps/chosen": -4.422863483428955,
+      "logps/rejected": -5.126218318939209,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.422863483428955,
+      "rewards/margins": 0.7033548951148987,
+      "rewards/rejected": -5.126218318939209,
+      "sft_loss": 4.190646648406982,
+      "step": 4015
+    },
+    {
+      "epoch": 2.151530356246864,
+      "grad_norm": 0.38126355663852096,
+      "learning_rate": 2.2430645795611963e-07,
+      "logits/chosen": -0.6254383325576782,
+      "logits/rejected": -0.5177367329597473,
+      "logps/chosen": -4.241569995880127,
+      "logps/rejected": -5.178333282470703,
+      "loss": 0.0492,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.241569995880127,
+      "rewards/margins": 0.9367631077766418,
+      "rewards/rejected": -5.178333282470703,
+      "sft_loss": 4.025848388671875,
+      "step": 4020
+    },
+    {
+      "epoch": 2.1542063890282654,
+      "grad_norm": 0.6022080952404661,
+      "learning_rate": 2.230085350017884e-07,
+      "logits/chosen": -0.4921099543571472,
+      "logits/rejected": -0.4022812843322754,
+      "logps/chosen": -4.403489112854004,
+      "logps/rejected": -5.125845909118652,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.403489112854004,
+      "rewards/margins": 0.7223564386367798,
+      "rewards/rejected": -5.125845909118652,
+      "sft_loss": 4.141639232635498,
+      "step": 4025
+    },
+    {
+      "epoch": 2.156882421809667,
+      "grad_norm": 0.41144679514794413,
+      "learning_rate": 2.2171329941986554e-07,
+      "logits/chosen": -0.567827582359314,
+      "logits/rejected": -0.5381026864051819,
+      "logps/chosen": -4.448178768157959,
+      "logps/rejected": -5.184752464294434,
+      "loss": 0.0487,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.448178768157959,
+      "rewards/margins": 0.7365739345550537,
+      "rewards/rejected": -5.184752464294434,
+      "sft_loss": 3.9973690509796143,
+      "step": 4030
+    },
+    {
+      "epoch": 2.159558454591069,
+      "grad_norm": 0.36913164131639964,
+      "learning_rate": 2.2042076377673202e-07,
+      "logits/chosen": -0.4624423086643219,
+      "logits/rejected": -0.5211446285247803,
+      "logps/chosen": -4.310798645019531,
+      "logps/rejected": -4.899218559265137,
+      "loss": 0.051,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.310798645019531,
+      "rewards/margins": 0.5884201526641846,
+      "rewards/rejected": -4.899218559265137,
+      "sft_loss": 3.971083879470825,
+      "step": 4035
+    },
+    {
+      "epoch": 2.16223448737247,
+      "grad_norm": 0.4706814819040189,
+      "learning_rate": 2.1913094061257476e-07,
+      "logits/chosen": -0.48843860626220703,
+      "logits/rejected": -0.5447765588760376,
+      "logps/chosen": -4.407719612121582,
+      "logps/rejected": -5.136025428771973,
+      "loss": 0.0503,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.407719612121582,
+      "rewards/margins": 0.7283057570457458,
+      "rewards/rejected": -5.136025428771973,
+      "sft_loss": 4.117656707763672,
+      "step": 4040
+    },
+    {
+      "epoch": 2.164910520153872,
+      "grad_norm": 0.5224363116699678,
+      "learning_rate": 2.178438424412633e-07,
+      "logits/chosen": -0.48264041543006897,
+      "logits/rejected": -0.35010209679603577,
+      "logps/chosen": -4.316220283508301,
+      "logps/rejected": -5.103582859039307,
+      "loss": 0.0506,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.316220283508301,
+      "rewards/margins": 0.7873626351356506,
+      "rewards/rejected": -5.103582859039307,
+      "sft_loss": 4.044312953948975,
+      "step": 4045
+    },
+    {
+      "epoch": 2.1675865529352736,
+      "grad_norm": 0.5005902988702267,
+      "learning_rate": 2.165594817502302e-07,
+      "logits/chosen": -0.606683611869812,
+      "logits/rejected": -0.4693070948123932,
+      "logps/chosen": -4.577552318572998,
+      "logps/rejected": -5.201845645904541,
+      "loss": 0.0543,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.577552318572998,
+      "rewards/margins": 0.6242929697036743,
+      "rewards/rejected": -5.201845645904541,
+      "sft_loss": 4.395869255065918,
+      "step": 4050
+    },
+    {
+      "epoch": 2.170262585716675,
+      "grad_norm": 0.8616687130483774,
+      "learning_rate": 2.1527787100034806e-07,
+      "logits/chosen": -0.4424276351928711,
+      "logits/rejected": -0.3929523825645447,
+      "logps/chosen": -4.442006587982178,
+      "logps/rejected": -5.012038707733154,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.442006587982178,
+      "rewards/margins": 0.5700324773788452,
+      "rewards/rejected": -5.012038707733154,
+      "sft_loss": 4.1983137130737305,
+      "step": 4055
+    },
+    {
+      "epoch": 2.1729386184980766,
+      "grad_norm": 0.5607644795062077,
+      "learning_rate": 2.1399902262581037e-07,
+      "logits/chosen": -0.47108763456344604,
+      "logits/rejected": -0.2918552756309509,
+      "logps/chosen": -4.58528995513916,
+      "logps/rejected": -5.079229354858398,
+      "loss": 0.0536,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.58528995513916,
+      "rewards/margins": 0.4939393997192383,
+      "rewards/rejected": -5.079229354858398,
+      "sft_loss": 4.271696090698242,
+      "step": 4060
+    },
+    {
+      "epoch": 2.1756146512794783,
+      "grad_norm": 0.43632048260347456,
+      "learning_rate": 2.127229490340094e-07,
+      "logits/chosen": -0.6190832257270813,
+      "logits/rejected": -0.5601855516433716,
+      "logps/chosen": -4.535567283630371,
+      "logps/rejected": -5.2590484619140625,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.535567283630371,
+      "rewards/margins": 0.7234816551208496,
+      "rewards/rejected": -5.2590484619140625,
+      "sft_loss": 4.2203569412231445,
+      "step": 4065
+    },
+    {
+      "epoch": 2.1782906840608796,
+      "grad_norm": 0.6195928853464827,
+      "learning_rate": 2.1144966260541698e-07,
+      "logits/chosen": -0.47305387258529663,
+      "logits/rejected": -0.2560986578464508,
+      "logps/chosen": -4.438614845275879,
+      "logps/rejected": -5.22701358795166,
+      "loss": 0.0504,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.438614845275879,
+      "rewards/margins": 0.788398802280426,
+      "rewards/rejected": -5.22701358795166,
+      "sft_loss": 4.108904838562012,
+      "step": 4070
+    },
+    {
+      "epoch": 2.1809667168422813,
+      "grad_norm": 0.6276113445477532,
+      "learning_rate": 2.1017917569346332e-07,
+      "logits/chosen": -0.5066081285476685,
+      "logits/rejected": -0.33284324407577515,
+      "logps/chosen": -4.2841796875,
+      "logps/rejected": -5.065493583679199,
+      "loss": 0.0488,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.2841796875,
+      "rewards/margins": 0.781313955783844,
+      "rewards/rejected": -5.065493583679199,
+      "sft_loss": 3.9601006507873535,
+      "step": 4075
+    },
+    {
+      "epoch": 2.183642749623683,
+      "grad_norm": 0.4990160075189289,
+      "learning_rate": 2.0891150062441837e-07,
+      "logits/chosen": -0.5469022989273071,
+      "logits/rejected": -0.43186479806900024,
+      "logps/chosen": -4.410531044006348,
+      "logps/rejected": -5.272799491882324,
+      "loss": 0.0501,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.410531044006348,
+      "rewards/margins": 0.862269401550293,
+      "rewards/rejected": -5.272799491882324,
+      "sft_loss": 4.101141929626465,
+      "step": 4080
+    },
+    {
+      "epoch": 2.1863187824050843,
+      "grad_norm": 0.5608444030127769,
+      "learning_rate": 2.0764664969727086e-07,
+      "logits/chosen": -0.4370051920413971,
+      "logits/rejected": -0.4030459523200989,
+      "logps/chosen": -4.343447685241699,
+      "logps/rejected": -5.117300033569336,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.343447685241699,
+      "rewards/margins": 0.7738524675369263,
+      "rewards/rejected": -5.117300033569336,
+      "sft_loss": 4.0537004470825195,
+      "step": 4085
+    },
+    {
+      "epoch": 2.188994815186486,
+      "grad_norm": 0.4616541721561178,
+      "learning_rate": 2.0638463518361033e-07,
+      "logits/chosen": -0.5887266397476196,
+      "logits/rejected": -0.40685874223709106,
+      "logps/chosen": -4.335751533508301,
+      "logps/rejected": -5.148130893707275,
+      "loss": 0.0497,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.335751533508301,
+      "rewards/margins": 0.8123795390129089,
+      "rewards/rejected": -5.148130893707275,
+      "sft_loss": 4.002298355102539,
+      "step": 4090
+    },
+    {
+      "epoch": 2.1916708479678877,
+      "grad_norm": 0.48487732652880655,
+      "learning_rate": 2.0512546932750702e-07,
+      "logits/chosen": -0.6202269792556763,
+      "logits/rejected": -0.5550543069839478,
+      "logps/chosen": -4.4029717445373535,
+      "logps/rejected": -5.139110565185547,
+      "loss": 0.05,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.4029717445373535,
+      "rewards/margins": 0.7361379861831665,
+      "rewards/rejected": -5.139110565185547,
+      "sft_loss": 4.068113327026367,
+      "step": 4095
+    },
+    {
+      "epoch": 2.194346880749289,
+      "grad_norm": 0.6573744914370229,
+      "learning_rate": 2.0386916434539343e-07,
+      "logits/chosen": -0.5262848138809204,
+      "logits/rejected": -0.36401256918907166,
+      "logps/chosen": -4.441623210906982,
+      "logps/rejected": -5.173140525817871,
+      "loss": 0.0504,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.441623210906982,
+      "rewards/margins": 0.7315176129341125,
+      "rewards/rejected": -5.173140525817871,
+      "sft_loss": 4.165780544281006,
+      "step": 4100
+    },
+    {
+      "epoch": 2.1970229135306907,
+      "grad_norm": 0.3888934141845449,
+      "learning_rate": 2.0261573242594627e-07,
+      "logits/chosen": -0.5840184092521667,
+      "logits/rejected": -0.37567561864852905,
+      "logps/chosen": -4.419126033782959,
+      "logps/rejected": -5.130116939544678,
+      "loss": 0.0503,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.419126033782959,
+      "rewards/margins": 0.710990309715271,
+      "rewards/rejected": -5.130116939544678,
+      "sft_loss": 4.077830791473389,
+      "step": 4105
+    },
+    {
+      "epoch": 2.1996989463120924,
+      "grad_norm": 0.456644627783221,
+      "learning_rate": 2.0136518572996724e-07,
+      "logits/chosen": -0.5277017951011658,
+      "logits/rejected": -0.3363708555698395,
+      "logps/chosen": -4.343173503875732,
+      "logps/rejected": -5.2076520919799805,
+      "loss": 0.0498,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.343173503875732,
+      "rewards/margins": 0.864478588104248,
+      "rewards/rejected": -5.2076520919799805,
+      "sft_loss": 4.059588432312012,
+      "step": 4110
+    },
+    {
+      "epoch": 2.202374979093494,
+      "grad_norm": 0.8814258415453387,
+      "learning_rate": 2.0011753639026617e-07,
+      "logits/chosen": -0.5193076133728027,
+      "logits/rejected": -0.4687017500400543,
+      "logps/chosen": -4.271040916442871,
+      "logps/rejected": -5.097018241882324,
+      "loss": 0.0506,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.271040916442871,
+      "rewards/margins": 0.8259771466255188,
+      "rewards/rejected": -5.097018241882324,
+      "sft_loss": 4.0006866455078125,
+      "step": 4115
+    },
+    {
+      "epoch": 2.2050510118748954,
+      "grad_norm": 1.0062502081971152,
+      "learning_rate": 1.988727965115421e-07,
+      "logits/chosen": -0.5313402414321899,
+      "logits/rejected": -0.4530180096626282,
+      "logps/chosen": -4.26535177230835,
+      "logps/rejected": -4.9814958572387695,
+      "loss": 0.051,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.26535177230835,
+      "rewards/margins": 0.7161434888839722,
+      "rewards/rejected": -4.9814958572387695,
+      "sft_loss": 3.927295207977295,
+      "step": 4120
+    },
+    {
+      "epoch": 2.207727044656297,
+      "grad_norm": 0.5130358179198009,
+      "learning_rate": 1.9763097817026713e-07,
+      "logits/chosen": -0.6455596685409546,
+      "logits/rejected": -0.4709509015083313,
+      "logps/chosen": -4.263947486877441,
+      "logps/rejected": -5.230985164642334,
+      "loss": 0.0494,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.263947486877441,
+      "rewards/margins": 0.9670373797416687,
+      "rewards/rejected": -5.230985164642334,
+      "sft_loss": 4.01256799697876,
+      "step": 4125
+    },
+    {
+      "epoch": 2.210403077437699,
+      "grad_norm": 0.5308083984882473,
+      "learning_rate": 1.9639209341456796e-07,
+      "logits/chosen": -0.48948168754577637,
+      "logits/rejected": -0.4048754572868347,
+      "logps/chosen": -4.374345302581787,
+      "logps/rejected": -5.212045669555664,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.374345302581787,
+      "rewards/margins": 0.8377002477645874,
+      "rewards/rejected": -5.212045669555664,
+      "sft_loss": 4.143846035003662,
+      "step": 4130
+    },
+    {
+      "epoch": 2.2130791102191,
+      "grad_norm": 0.6579927947500825,
+      "learning_rate": 1.951561542641102e-07,
+      "logits/chosen": -0.43879157304763794,
+      "logits/rejected": -0.47929683327674866,
+      "logps/chosen": -4.64826774597168,
+      "logps/rejected": -5.293527126312256,
+      "loss": 0.0528,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.64826774597168,
+      "rewards/margins": 0.6452595591545105,
+      "rewards/rejected": -5.293527126312256,
+      "sft_loss": 4.384035587310791,
+      "step": 4135
+    },
+    {
+      "epoch": 2.215755143000502,
+      "grad_norm": 0.47203788257132495,
+      "learning_rate": 1.939231727099806e-07,
+      "logits/chosen": -0.7212651968002319,
+      "logits/rejected": -0.6945225596427917,
+      "logps/chosen": -4.426300525665283,
+      "logps/rejected": -5.091549873352051,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.426300525665283,
+      "rewards/margins": 0.6652489900588989,
+      "rewards/rejected": -5.091549873352051,
+      "sft_loss": 4.201467037200928,
+      "step": 4140
+    },
+    {
+      "epoch": 2.2184311757819035,
+      "grad_norm": 0.5210625073348275,
+      "learning_rate": 1.926931607145719e-07,
+      "logits/chosen": -0.48468074202537537,
+      "logits/rejected": -0.34400853514671326,
+      "logps/chosen": -4.48496150970459,
+      "logps/rejected": -5.186110496520996,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.48496150970459,
+      "rewards/margins": 0.7011483311653137,
+      "rewards/rejected": -5.186110496520996,
+      "sft_loss": 4.240595817565918,
+      "step": 4145
+    },
+    {
+      "epoch": 2.221107208563305,
+      "grad_norm": 0.5057793732411678,
+      "learning_rate": 1.9146613021146564e-07,
+      "logits/chosen": -0.5926092863082886,
+      "logits/rejected": -0.5084186792373657,
+      "logps/chosen": -4.342706203460693,
+      "logps/rejected": -5.126007080078125,
+      "loss": 0.051,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.342706203460693,
+      "rewards/margins": 0.7833006381988525,
+      "rewards/rejected": -5.126007080078125,
+      "sft_loss": 4.122036933898926,
+      "step": 4150
+    },
+    {
+      "epoch": 2.2237832413447065,
+      "grad_norm": 0.5561533289544062,
+      "learning_rate": 1.9024209310531736e-07,
+      "logits/chosen": -0.53857421875,
+      "logits/rejected": -0.564507007598877,
+      "logps/chosen": -4.204834938049316,
+      "logps/rejected": -4.915310859680176,
+      "loss": 0.0498,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.204834938049316,
+      "rewards/margins": 0.7104752063751221,
+      "rewards/rejected": -4.915310859680176,
+      "sft_loss": 3.8613171577453613,
+      "step": 4155
+    },
+    {
+      "epoch": 2.2264592741261082,
+      "grad_norm": 0.5253662073311304,
+      "learning_rate": 1.890210612717401e-07,
+      "logits/chosen": -0.5691213607788086,
+      "logits/rejected": -0.46318894624710083,
+      "logps/chosen": -4.2865753173828125,
+      "logps/rejected": -5.037796974182129,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.2865753173828125,
+      "rewards/margins": 0.7512213587760925,
+      "rewards/rejected": -5.037796974182129,
+      "sft_loss": 3.989332914352417,
+      "step": 4160
+    },
+    {
+      "epoch": 2.2291353069075095,
+      "grad_norm": 0.8607988910539622,
+      "learning_rate": 1.8780304655719054e-07,
+      "logits/chosen": -0.5708852410316467,
+      "logits/rejected": -0.4509698748588562,
+      "logps/chosen": -4.365791320800781,
+      "logps/rejected": -5.173333168029785,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.365791320800781,
+      "rewards/margins": 0.8075419664382935,
+      "rewards/rejected": -5.173333168029785,
+      "sft_loss": 4.056873798370361,
+      "step": 4165
+    },
+    {
+      "epoch": 2.231811339688911,
+      "grad_norm": 0.5134203809896304,
+      "learning_rate": 1.865880607788523e-07,
+      "logits/chosen": -0.41466793417930603,
+      "logits/rejected": -0.3671296536922455,
+      "logps/chosen": -4.496591567993164,
+      "logps/rejected": -5.2434611320495605,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.496591567993164,
+      "rewards/margins": 0.7468695640563965,
+      "rewards/rejected": -5.2434611320495605,
+      "sft_loss": 4.2933125495910645,
+      "step": 4170
+    },
+    {
+      "epoch": 2.234487372470313,
+      "grad_norm": 0.8386477494904435,
+      "learning_rate": 1.8537611572452316e-07,
+      "logits/chosen": -0.5454100370407104,
+      "logits/rejected": -0.489818274974823,
+      "logps/chosen": -4.303423881530762,
+      "logps/rejected": -4.978426456451416,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.303423881530762,
+      "rewards/margins": 0.6750025153160095,
+      "rewards/rejected": -4.978426456451416,
+      "sft_loss": 4.052245140075684,
+      "step": 4175
+    },
+    {
+      "epoch": 2.237163405251714,
+      "grad_norm": 0.5292166862327569,
+      "learning_rate": 1.84167223152499e-07,
+      "logits/chosen": -0.5939118266105652,
+      "logits/rejected": -0.3603335916996002,
+      "logps/chosen": -4.377429008483887,
+      "logps/rejected": -5.199066638946533,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.377429008483887,
+      "rewards/margins": 0.8216379284858704,
+      "rewards/rejected": -5.199066638946533,
+      "sft_loss": 4.115596294403076,
+      "step": 4180
+    },
+    {
+      "epoch": 2.239839438033116,
+      "grad_norm": 0.46250808555766243,
+      "learning_rate": 1.8296139479146112e-07,
+      "logits/chosen": -0.5663339495658875,
+      "logits/rejected": -0.5795478224754333,
+      "logps/chosen": -4.320345401763916,
+      "logps/rejected": -4.9951090812683105,
+      "loss": 0.0496,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.320345401763916,
+      "rewards/margins": 0.6747641563415527,
+      "rewards/rejected": -4.9951090812683105,
+      "sft_loss": 3.985612392425537,
+      "step": 4185
+    },
+    {
+      "epoch": 2.2425154708145176,
+      "grad_norm": 0.5679127795921808,
+      "learning_rate": 1.8175864234036132e-07,
+      "logits/chosen": -0.38469135761260986,
+      "logits/rejected": -0.32198047637939453,
+      "logps/chosen": -4.41266393661499,
+      "logps/rejected": -5.213027477264404,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.41266393661499,
+      "rewards/margins": 0.8003634214401245,
+      "rewards/rejected": -5.213027477264404,
+      "sft_loss": 4.142411708831787,
+      "step": 4190
+    },
+    {
+      "epoch": 2.245191503595919,
+      "grad_norm": 0.5275047258385396,
+      "learning_rate": 1.805589774683094e-07,
+      "logits/chosen": -0.6430431008338928,
+      "logits/rejected": -0.5319651365280151,
+      "logps/chosen": -4.51082706451416,
+      "logps/rejected": -5.1332502365112305,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.51082706451416,
+      "rewards/margins": 0.6224239468574524,
+      "rewards/rejected": -5.1332502365112305,
+      "sft_loss": 4.149545192718506,
+      "step": 4195
+    },
+    {
+      "epoch": 2.2478675363773206,
+      "grad_norm": 0.9553414854876922,
+      "learning_rate": 1.79362411814459e-07,
+      "logits/chosen": -0.4228852689266205,
+      "logits/rejected": -0.48189735412597656,
+      "logps/chosen": -4.54064416885376,
+      "logps/rejected": -5.114188194274902,
+      "loss": 0.0533,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.54064416885376,
+      "rewards/margins": 0.5735443234443665,
+      "rewards/rejected": -5.114188194274902,
+      "sft_loss": 4.239465236663818,
+      "step": 4200
+    },
+    {
+      "epoch": 2.2505435691587223,
+      "grad_norm": 0.5301697919356788,
+      "learning_rate": 1.7816895698789552e-07,
+      "logits/chosen": -0.6335599422454834,
+      "logits/rejected": -0.5704717040061951,
+      "logps/chosen": -4.397948265075684,
+      "logps/rejected": -5.013107776641846,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.397948265075684,
+      "rewards/margins": 0.6151598691940308,
+      "rewards/rejected": -5.013107776641846,
+      "sft_loss": 4.019219398498535,
+      "step": 4205
+    },
+    {
+      "epoch": 2.2532196019401236,
+      "grad_norm": 0.5965623207104415,
+      "learning_rate": 1.7697862456752271e-07,
+      "logits/chosen": -0.6049562692642212,
+      "logits/rejected": -0.5032862424850464,
+      "logps/chosen": -4.441383361816406,
+      "logps/rejected": -5.372580051422119,
+      "loss": 0.0504,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -4.441383361816406,
+      "rewards/margins": 0.9311968088150024,
+      "rewards/rejected": -5.372580051422119,
+      "sft_loss": 4.193260192871094,
+      "step": 4210
+    },
+    {
+      "epoch": 2.2558956347215253,
+      "grad_norm": 0.5692373231128439,
+      "learning_rate": 1.7579142610195124e-07,
+      "logits/chosen": -0.5579692721366882,
+      "logits/rejected": -0.41172391176223755,
+      "logps/chosen": -4.479569435119629,
+      "logps/rejected": -5.320120334625244,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.479569435119629,
+      "rewards/margins": 0.8405508995056152,
+      "rewards/rejected": -5.320120334625244,
+      "sft_loss": 4.19023323059082,
+      "step": 4215
+    },
+    {
+      "epoch": 2.258571667502927,
+      "grad_norm": 0.4625400570301139,
+      "learning_rate": 1.7460737310938568e-07,
+      "logits/chosen": -0.6162181496620178,
+      "logits/rejected": -0.4331566393375397,
+      "logps/chosen": -4.258967399597168,
+      "logps/rejected": -5.199514865875244,
+      "loss": 0.0489,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -4.258967399597168,
+      "rewards/margins": 0.9405480623245239,
+      "rewards/rejected": -5.199514865875244,
+      "sft_loss": 4.05273962020874,
+      "step": 4220
+    },
+    {
+      "epoch": 2.2612477002843283,
+      "grad_norm": 0.43115961904462236,
+      "learning_rate": 1.734264770775133e-07,
+      "logits/chosen": -0.607951283454895,
+      "logits/rejected": -0.33437463641166687,
+      "logps/chosen": -4.299620151519775,
+      "logps/rejected": -5.036246299743652,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.299620151519775,
+      "rewards/margins": 0.7366262078285217,
+      "rewards/rejected": -5.036246299743652,
+      "sft_loss": 4.027946949005127,
+      "step": 4225
+    },
+    {
+      "epoch": 2.26392373306573,
+      "grad_norm": 0.48013732174352947,
+      "learning_rate": 1.7224874946339241e-07,
+      "logits/chosen": -0.6221799850463867,
+      "logits/rejected": -0.5745702385902405,
+      "logps/chosen": -4.352818012237549,
+      "logps/rejected": -5.210912704467773,
+      "loss": 0.0496,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.352818012237549,
+      "rewards/margins": 0.8580950498580933,
+      "rewards/rejected": -5.210912704467773,
+      "sft_loss": 3.973453998565674,
+      "step": 4230
+    },
+    {
+      "epoch": 2.2665997658471317,
+      "grad_norm": 0.7057461663028228,
+      "learning_rate": 1.7107420169334186e-07,
+      "logits/chosen": -0.580518364906311,
+      "logits/rejected": -0.5512189269065857,
+      "logps/chosen": -4.463744640350342,
+      "logps/rejected": -5.126254081726074,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.463744640350342,
+      "rewards/margins": 0.6625092029571533,
+      "rewards/rejected": -5.126254081726074,
+      "sft_loss": 4.1474785804748535,
+      "step": 4235
+    },
+    {
+      "epoch": 2.269275798628533,
+      "grad_norm": 0.507618517027458,
+      "learning_rate": 1.6990284516282893e-07,
+      "logits/chosen": -0.6491494178771973,
+      "logits/rejected": -0.6031497120857239,
+      "logps/chosen": -4.328498840332031,
+      "logps/rejected": -5.156960964202881,
+      "loss": 0.0492,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.328498840332031,
+      "rewards/margins": 0.8284620046615601,
+      "rewards/rejected": -5.156960964202881,
+      "sft_loss": 3.988081455230713,
+      "step": 4240
+    },
+    {
+      "epoch": 2.2719518314099347,
+      "grad_norm": 0.5354500570840905,
+      "learning_rate": 1.687346912363602e-07,
+      "logits/chosen": -0.649688184261322,
+      "logits/rejected": -0.5386859178543091,
+      "logps/chosen": -4.2751312255859375,
+      "logps/rejected": -5.127442359924316,
+      "loss": 0.0494,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -4.2751312255859375,
+      "rewards/margins": 0.8523112535476685,
+      "rewards/rejected": -5.127442359924316,
+      "sft_loss": 3.931128740310669,
+      "step": 4245
+    },
+    {
+      "epoch": 2.2746278641913364,
+      "grad_norm": 0.45037905004230455,
+      "learning_rate": 1.675697512473697e-07,
+      "logits/chosen": -0.6232026219367981,
+      "logits/rejected": -0.4436233639717102,
+      "logps/chosen": -4.427603244781494,
+      "logps/rejected": -5.331129550933838,
+      "loss": 0.0489,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -4.427603244781494,
+      "rewards/margins": 0.9035263061523438,
+      "rewards/rejected": -5.331129550933838,
+      "sft_loss": 4.014822959899902,
+      "step": 4250
+    },
+    {
+      "epoch": 2.2773038969727377,
+      "grad_norm": 0.47836813814529966,
+      "learning_rate": 1.6640803649811087e-07,
+      "logits/chosen": -0.6468229293823242,
+      "logits/rejected": -0.3750753402709961,
+      "logps/chosen": -4.370386600494385,
+      "logps/rejected": -5.446681976318359,
+      "loss": 0.0493,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.370386600494385,
+      "rewards/margins": 1.0762957334518433,
+      "rewards/rejected": -5.446681976318359,
+      "sft_loss": 3.9946656227111816,
+      "step": 4255
+    },
+    {
+      "epoch": 2.2799799297541394,
+      "grad_norm": 0.5266132700479489,
+      "learning_rate": 1.6524955825954472e-07,
+      "logits/chosen": -0.5416319966316223,
+      "logits/rejected": -0.4869862496852875,
+      "logps/chosen": -4.454476833343506,
+      "logps/rejected": -5.066670894622803,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.454476833343506,
+      "rewards/margins": 0.6121941208839417,
+      "rewards/rejected": -5.066670894622803,
+      "sft_loss": 4.193382263183594,
+      "step": 4260
+    },
+    {
+      "epoch": 2.282655962535541,
+      "grad_norm": 0.4791585184216119,
+      "learning_rate": 1.6409432777123277e-07,
+      "logits/chosen": -0.6844125986099243,
+      "logits/rejected": -0.5556963682174683,
+      "logps/chosen": -4.241826057434082,
+      "logps/rejected": -5.29033899307251,
+      "loss": 0.0483,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.241826057434082,
+      "rewards/margins": 1.0485130548477173,
+      "rewards/rejected": -5.29033899307251,
+      "sft_loss": 3.9664692878723145,
+      "step": 4265
+    },
+    {
+      "epoch": 2.285331995316943,
+      "grad_norm": 0.7294224331857638,
+      "learning_rate": 1.6294235624122577e-07,
+      "logits/chosen": -0.5351814031600952,
+      "logits/rejected": -0.28361183404922485,
+      "logps/chosen": -4.251220703125,
+      "logps/rejected": -5.081398963928223,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.251220703125,
+      "rewards/margins": 0.8301783800125122,
+      "rewards/rejected": -5.081398963928223,
+      "sft_loss": 3.9799981117248535,
+      "step": 4270
+    },
+    {
+      "epoch": 2.288008028098344,
+      "grad_norm": 0.4378161663138991,
+      "learning_rate": 1.6179365484595697e-07,
+      "logits/chosen": -0.588020920753479,
+      "logits/rejected": -0.5045811533927917,
+      "logps/chosen": -4.37821626663208,
+      "logps/rejected": -5.1824727058410645,
+      "loss": 0.051,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.37821626663208,
+      "rewards/margins": 0.8042561411857605,
+      "rewards/rejected": -5.1824727058410645,
+      "sft_loss": 4.135249137878418,
+      "step": 4275
+    },
+    {
+      "epoch": 2.290684060879746,
+      "grad_norm": 0.8272843307460743,
+      "learning_rate": 1.60648234730132e-07,
+      "logits/chosen": -0.6302292943000793,
+      "logits/rejected": -0.5782641172409058,
+      "logps/chosen": -4.302960395812988,
+      "logps/rejected": -5.206368446350098,
+      "loss": 0.0492,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -4.302960395812988,
+      "rewards/margins": 0.9034079313278198,
+      "rewards/rejected": -5.206368446350098,
+      "sft_loss": 4.0424885749816895,
+      "step": 4280
+    },
+    {
+      "epoch": 2.293360093661147,
+      "grad_norm": 0.5946705762583467,
+      "learning_rate": 1.595061070066222e-07,
+      "logits/chosen": -0.5625177621841431,
+      "logits/rejected": -0.6391332745552063,
+      "logps/chosen": -4.442146301269531,
+      "logps/rejected": -5.200289249420166,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.442146301269531,
+      "rewards/margins": 0.758142352104187,
+      "rewards/rejected": -5.200289249420166,
+      "sft_loss": 4.180250644683838,
+      "step": 4285
+    },
+    {
+      "epoch": 2.296036126442549,
+      "grad_norm": 0.5675158330734196,
+      "learning_rate": 1.5836728275635542e-07,
+      "logits/chosen": -0.6432707905769348,
+      "logits/rejected": -0.4786619544029236,
+      "logps/chosen": -4.437047958374023,
+      "logps/rejected": -5.1985273361206055,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.437047958374023,
+      "rewards/margins": 0.7614789009094238,
+      "rewards/rejected": -5.1985273361206055,
+      "sft_loss": 4.190291404724121,
+      "step": 4290
+    },
+    {
+      "epoch": 2.2987121592239506,
+      "grad_norm": 0.45583570819956776,
+      "learning_rate": 1.5723177302820984e-07,
+      "logits/chosen": -0.6301642060279846,
+      "logits/rejected": -0.5716463327407837,
+      "logps/chosen": -4.472960472106934,
+      "logps/rejected": -5.121406078338623,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.472960472106934,
+      "rewards/margins": 0.648445725440979,
+      "rewards/rejected": -5.121406078338623,
+      "sft_loss": 4.243009567260742,
+      "step": 4295
+    },
+    {
+      "epoch": 2.3013881920053523,
+      "grad_norm": 0.450362064409949,
+      "learning_rate": 1.5609958883890544e-07,
+      "logits/chosen": -0.5362349152565002,
+      "logits/rejected": -0.44398775696754456,
+      "logps/chosen": -4.322482109069824,
+      "logps/rejected": -5.034620761871338,
+      "loss": 0.0498,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.322482109069824,
+      "rewards/margins": 0.7121386528015137,
+      "rewards/rejected": -5.034620761871338,
+      "sft_loss": 3.9561564922332764,
+      "step": 4300
+    },
+    {
+      "epoch": 2.3040642247867535,
+      "grad_norm": 0.558868189353547,
+      "learning_rate": 1.5497074117289865e-07,
+      "logits/chosen": -0.5846543908119202,
+      "logits/rejected": -0.47276654839515686,
+      "logps/chosen": -4.40239953994751,
+      "logps/rejected": -5.194138526916504,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.40239953994751,
+      "rewards/margins": 0.791739284992218,
+      "rewards/rejected": -5.194138526916504,
+      "sft_loss": 4.112706184387207,
+      "step": 4305
+    },
+    {
+      "epoch": 2.3067402575681553,
+      "grad_norm": 0.5265103632451326,
+      "learning_rate": 1.5384524098227402e-07,
+      "logits/chosen": -0.6137844324111938,
+      "logits/rejected": -0.4249110817909241,
+      "logps/chosen": -4.234074592590332,
+      "logps/rejected": -5.159043312072754,
+      "loss": 0.0501,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.234074592590332,
+      "rewards/margins": 0.9249696731567383,
+      "rewards/rejected": -5.159043312072754,
+      "sft_loss": 3.974463939666748,
+      "step": 4310
+    },
+    {
+      "epoch": 2.3094162903495565,
+      "grad_norm": 0.6579420988527105,
+      "learning_rate": 1.5272309918663974e-07,
+      "logits/chosen": -0.5823651552200317,
+      "logits/rejected": -0.42209291458129883,
+      "logps/chosen": -4.454043388366699,
+      "logps/rejected": -5.022500038146973,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.454043388366699,
+      "rewards/margins": 0.5684566497802734,
+      "rewards/rejected": -5.022500038146973,
+      "sft_loss": 4.171141624450684,
+      "step": 4315
+    },
+    {
+      "epoch": 2.3120923231309582,
+      "grad_norm": 0.6385276744701301,
+      "learning_rate": 1.516043266730201e-07,
+      "logits/chosen": -0.5919250249862671,
+      "logits/rejected": -0.47442251443862915,
+      "logps/chosen": -4.313601493835449,
+      "logps/rejected": -5.097935676574707,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.313601493835449,
+      "rewards/margins": 0.7843343615531921,
+      "rewards/rejected": -5.097935676574707,
+      "sft_loss": 3.97279691696167,
+      "step": 4320
+    },
+    {
+      "epoch": 2.31476835591236,
+      "grad_norm": 0.672619298253807,
+      "learning_rate": 1.504889342957512e-07,
+      "logits/chosen": -0.5901211500167847,
+      "logits/rejected": -0.4074975550174713,
+      "logps/chosen": -4.464235782623291,
+      "logps/rejected": -5.2786359786987305,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.464235782623291,
+      "rewards/margins": 0.8144004940986633,
+      "rewards/rejected": -5.2786359786987305,
+      "sft_loss": 4.189356803894043,
+      "step": 4325
+    },
+    {
+      "epoch": 2.3174443886937617,
+      "grad_norm": 0.481965232331999,
+      "learning_rate": 1.4937693287637453e-07,
+      "logits/chosen": -0.6231690645217896,
+      "logits/rejected": -0.5151903033256531,
+      "logps/chosen": -4.453746318817139,
+      "logps/rejected": -5.190335273742676,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.453746318817139,
+      "rewards/margins": 0.7365895509719849,
+      "rewards/rejected": -5.190335273742676,
+      "sft_loss": 4.131800174713135,
+      "step": 4330
+    },
+    {
+      "epoch": 2.320120421475163,
+      "grad_norm": 0.42983353223855325,
+      "learning_rate": 1.4826833320353305e-07,
+      "logits/chosen": -0.6066884994506836,
+      "logits/rejected": -0.531694769859314,
+      "logps/chosen": -4.546232223510742,
+      "logps/rejected": -5.299452781677246,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.546232223510742,
+      "rewards/margins": 0.7532203793525696,
+      "rewards/rejected": -5.299452781677246,
+      "sft_loss": 4.088216781616211,
+      "step": 4335
+    },
+    {
+      "epoch": 2.3227964542565647,
+      "grad_norm": 0.5382694437763403,
+      "learning_rate": 1.4716314603286528e-07,
+      "logits/chosen": -0.6575163006782532,
+      "logits/rejected": -0.44630640745162964,
+      "logps/chosen": -4.471531867980957,
+      "logps/rejected": -5.292723655700684,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.471531867980957,
+      "rewards/margins": 0.8211923837661743,
+      "rewards/rejected": -5.292723655700684,
+      "sft_loss": 4.151998043060303,
+      "step": 4340
+    },
+    {
+      "epoch": 2.3254724870379664,
+      "grad_norm": 0.6269565005608625,
+      "learning_rate": 1.4606138208690233e-07,
+      "logits/chosen": -0.6339553594589233,
+      "logits/rejected": -0.5867515802383423,
+      "logps/chosen": -4.533459663391113,
+      "logps/rejected": -5.065715789794922,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.533459663391113,
+      "rewards/margins": 0.5322555303573608,
+      "rewards/rejected": -5.065715789794922,
+      "sft_loss": 4.233628273010254,
+      "step": 4345
+    },
+    {
+      "epoch": 2.3281485198193677,
+      "grad_norm": 0.3691865416373609,
+      "learning_rate": 1.4496305205496251e-07,
+      "logits/chosen": -0.6191138625144958,
+      "logits/rejected": -0.5620366334915161,
+      "logps/chosen": -4.4623332023620605,
+      "logps/rejected": -5.251715660095215,
+      "loss": 0.0506,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.4623332023620605,
+      "rewards/margins": 0.7893827557563782,
+      "rewards/rejected": -5.251715660095215,
+      "sft_loss": 4.253348350524902,
+      "step": 4350
+    },
+    {
+      "epoch": 2.3308245526007694,
+      "grad_norm": 0.48834830175558125,
+      "learning_rate": 1.4386816659304895e-07,
+      "logits/chosen": -0.7415441274642944,
+      "logits/rejected": -0.6099624633789062,
+      "logps/chosen": -4.442951202392578,
+      "logps/rejected": -5.09112024307251,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.442951202392578,
+      "rewards/margins": 0.6481693387031555,
+      "rewards/rejected": -5.09112024307251,
+      "sft_loss": 4.204721450805664,
+      "step": 4355
+    },
+    {
+      "epoch": 2.333500585382171,
+      "grad_norm": 0.5293653139535832,
+      "learning_rate": 1.4277673632374492e-07,
+      "logits/chosen": -0.6939634084701538,
+      "logits/rejected": -0.48100510239601135,
+      "logps/chosen": -4.262767314910889,
+      "logps/rejected": -5.095149040222168,
+      "loss": 0.0496,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.262767314910889,
+      "rewards/margins": 0.832382321357727,
+      "rewards/rejected": -5.095149040222168,
+      "sft_loss": 3.9535045623779297,
+      "step": 4360
+    },
+    {
+      "epoch": 2.3361766181635724,
+      "grad_norm": 0.4819722906143891,
+      "learning_rate": 1.416887718361119e-07,
+      "logits/chosen": -0.5756624937057495,
+      "logits/rejected": -0.5869953036308289,
+      "logps/chosen": -4.386821269989014,
+      "logps/rejected": -5.003874778747559,
+      "loss": 0.0534,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.386821269989014,
+      "rewards/margins": 0.6170530915260315,
+      "rewards/rejected": -5.003874778747559,
+      "sft_loss": 4.190293788909912,
+      "step": 4365
+    },
+    {
+      "epoch": 2.338852650944974,
+      "grad_norm": 0.5317980105881874,
+      "learning_rate": 1.406042836855859e-07,
+      "logits/chosen": -0.5243075489997864,
+      "logits/rejected": -0.4126424789428711,
+      "logps/chosen": -4.490993499755859,
+      "logps/rejected": -5.309933662414551,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.490993499755859,
+      "rewards/margins": 0.8189405202865601,
+      "rewards/rejected": -5.309933662414551,
+      "sft_loss": 4.259095668792725,
+      "step": 4370
+    },
+    {
+      "epoch": 2.341528683726376,
+      "grad_norm": 0.39211317835042,
+      "learning_rate": 1.3952328239387595e-07,
+      "logits/chosen": -0.7038995623588562,
+      "logits/rejected": -0.4603196978569031,
+      "logps/chosen": -4.370800971984863,
+      "logps/rejected": -5.199505805969238,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.370800971984863,
+      "rewards/margins": 0.828704833984375,
+      "rewards/rejected": -5.199505805969238,
+      "sft_loss": 4.199721336364746,
+      "step": 4375
+    },
+    {
+      "epoch": 2.344204716507777,
+      "grad_norm": 0.48830217049670294,
+      "learning_rate": 1.3844577844886109e-07,
+      "logits/chosen": -0.6748576760292053,
+      "logits/rejected": -0.45836204290390015,
+      "logps/chosen": -4.092423915863037,
+      "logps/rejected": -5.01913595199585,
+      "loss": 0.0492,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.092423915863037,
+      "rewards/margins": 0.926711916923523,
+      "rewards/rejected": -5.01913595199585,
+      "sft_loss": 3.9437003135681152,
+      "step": 4380
+    },
+    {
+      "epoch": 2.346880749289179,
+      "grad_norm": 0.5593406133053682,
+      "learning_rate": 1.3737178230448955e-07,
+      "logits/chosen": -0.7443893551826477,
+      "logits/rejected": -0.636515736579895,
+      "logps/chosen": -4.410309791564941,
+      "logps/rejected": -4.988183498382568,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.410309791564941,
+      "rewards/margins": 0.5778740644454956,
+      "rewards/rejected": -4.988183498382568,
+      "sft_loss": 4.044222354888916,
+      "step": 4385
+    },
+    {
+      "epoch": 2.3495567820705805,
+      "grad_norm": 0.5088906146317755,
+      "learning_rate": 1.363013043806764e-07,
+      "logits/chosen": -0.6782569885253906,
+      "logits/rejected": -0.5861415266990662,
+      "logps/chosen": -4.158144474029541,
+      "logps/rejected": -4.948266506195068,
+      "loss": 0.0502,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -4.158144474029541,
+      "rewards/margins": 0.7901225090026855,
+      "rewards/rejected": -4.948266506195068,
+      "sft_loss": 3.9805991649627686,
+      "step": 4390
+    },
+    {
+      "epoch": 2.3522328148519818,
+      "grad_norm": 0.8022888937468455,
+      "learning_rate": 1.352343550632034e-07,
+      "logits/chosen": -0.6014167070388794,
+      "logits/rejected": -0.4426348805427551,
+      "logps/chosen": -4.211198329925537,
+      "logps/rejected": -5.17777156829834,
+      "loss": 0.0505,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.211198329925537,
+      "rewards/margins": 0.9665730595588684,
+      "rewards/rejected": -5.17777156829834,
+      "sft_loss": 4.008025169372559,
+      "step": 4395
+    },
+    {
+      "epoch": 2.3549088476333835,
+      "grad_norm": 0.3827257494103996,
+      "learning_rate": 1.3417094470361722e-07,
+      "logits/chosen": -0.6943827271461487,
+      "logits/rejected": -0.5535784959793091,
+      "logps/chosen": -4.440021514892578,
+      "logps/rejected": -5.1550188064575195,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.440021514892578,
+      "rewards/margins": 0.7149966955184937,
+      "rewards/rejected": -5.1550188064575195,
+      "sft_loss": 4.156125068664551,
+      "step": 4400
+    },
+    {
+      "epoch": 2.3549088476333835,
+      "eval_logits/chosen": -0.2170492559671402,
+      "eval_logits/rejected": -0.14492151141166687,
+      "eval_logps/chosen": -4.375136375427246,
+      "eval_logps/rejected": -5.111087799072266,
+      "eval_loss": 0.050139155238866806,
+      "eval_rewards/accuracies": 0.6750741600990295,
+      "eval_rewards/chosen": -4.375136375427246,
+      "eval_rewards/margins": 0.735951840877533,
+      "eval_rewards/rejected": -5.111087799072266,
+      "eval_runtime": 43.9025,
+      "eval_samples_per_second": 30.636,
+      "eval_sft_loss": 3.9711320400238037,
+      "eval_steps_per_second": 7.676,
+      "step": 4400
+    },
+    {
+      "epoch": 2.357584880414785,
+      "grad_norm": 0.5371627811606488,
+      "learning_rate": 1.3311108361913015e-07,
+      "logits/chosen": -0.674767792224884,
+      "logits/rejected": -0.7053124308586121,
+      "logps/chosen": -4.369377136230469,
+      "logps/rejected": -5.01291561126709,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.369377136230469,
+      "rewards/margins": 0.6435383558273315,
+      "rewards/rejected": -5.01291561126709,
+      "sft_loss": 4.148804664611816,
+      "step": 4405
+    },
+    {
+      "epoch": 2.3602609131961865,
+      "grad_norm": 0.4706449572439089,
+      "learning_rate": 1.3205478209251874e-07,
+      "logits/chosen": -0.5489293932914734,
+      "logits/rejected": -0.47727441787719727,
+      "logps/chosen": -4.477015972137451,
+      "logps/rejected": -5.244737148284912,
+      "loss": 0.051,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.477015972137451,
+      "rewards/margins": 0.76772141456604,
+      "rewards/rejected": -5.244737148284912,
+      "sft_loss": 4.224416732788086,
+      "step": 4410
+    },
+    {
+      "epoch": 2.362936945977588,
+      "grad_norm": 0.6388482260299206,
+      "learning_rate": 1.310020503720254e-07,
+      "logits/chosen": -0.6039767265319824,
+      "logits/rejected": -0.45032182335853577,
+      "logps/chosen": -4.325869560241699,
+      "logps/rejected": -5.107138156890869,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.325869560241699,
+      "rewards/margins": 0.7812689542770386,
+      "rewards/rejected": -5.107138156890869,
+      "sft_loss": 4.050150394439697,
+      "step": 4415
+    },
+    {
+      "epoch": 2.36561297875899,
+      "grad_norm": 0.47161791903708744,
+      "learning_rate": 1.2995289867125752e-07,
+      "logits/chosen": -0.625280499458313,
+      "logits/rejected": -0.5440713763237,
+      "logps/chosen": -4.447295188903809,
+      "logps/rejected": -5.070778846740723,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.447295188903809,
+      "rewards/margins": 0.6234831809997559,
+      "rewards/rejected": -5.070778846740723,
+      "sft_loss": 4.171244144439697,
+      "step": 4420
+    },
+    {
+      "epoch": 2.368289011540391,
+      "grad_norm": 0.4148074088478754,
+      "learning_rate": 1.2890733716908986e-07,
+      "logits/chosen": -0.584781289100647,
+      "logits/rejected": -0.5371939539909363,
+      "logps/chosen": -4.193655490875244,
+      "logps/rejected": -4.905811786651611,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.193655490875244,
+      "rewards/margins": 0.7121565341949463,
+      "rewards/rejected": -4.905811786651611,
+      "sft_loss": 3.9834952354431152,
+      "step": 4425
+    },
+    {
+      "epoch": 2.370965044321793,
+      "grad_norm": 0.4365230404795349,
+      "learning_rate": 1.2786537600956454e-07,
+      "logits/chosen": -0.645124077796936,
+      "logits/rejected": -0.49594640731811523,
+      "logps/chosen": -4.184140205383301,
+      "logps/rejected": -5.194951057434082,
+      "loss": 0.0477,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.184140205383301,
+      "rewards/margins": 1.0108121633529663,
+      "rewards/rejected": -5.194951057434082,
+      "sft_loss": 3.886601686477661,
+      "step": 4430
+    },
+    {
+      "epoch": 2.3736410771031946,
+      "grad_norm": 0.5059973444461836,
+      "learning_rate": 1.268270253017933e-07,
+      "logits/chosen": -0.6193794012069702,
+      "logits/rejected": -0.422700971364975,
+      "logps/chosen": -4.3496222496032715,
+      "logps/rejected": -5.10540771484375,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.3496222496032715,
+      "rewards/margins": 0.7557852864265442,
+      "rewards/rejected": -5.10540771484375,
+      "sft_loss": 4.064044952392578,
+      "step": 4435
+    },
+    {
+      "epoch": 2.376317109884596,
+      "grad_norm": 0.5279541393164932,
+      "learning_rate": 1.257922951198591e-07,
+      "logits/chosen": -0.7423588037490845,
+      "logits/rejected": -0.4645689129829407,
+      "logps/chosen": -4.233798503875732,
+      "logps/rejected": -5.062592506408691,
+      "loss": 0.0499,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -4.233798503875732,
+      "rewards/margins": 0.8287937045097351,
+      "rewards/rejected": -5.062592506408691,
+      "sft_loss": 4.0026044845581055,
+      "step": 4440
+    },
+    {
+      "epoch": 2.3789931426659976,
+      "grad_norm": 0.5913806951717187,
+      "learning_rate": 1.24761195502719e-07,
+      "logits/chosen": -0.6655135750770569,
+      "logits/rejected": -0.4213865399360657,
+      "logps/chosen": -4.549498558044434,
+      "logps/rejected": -5.189752101898193,
+      "loss": 0.0503,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.549498558044434,
+      "rewards/margins": 0.6402536630630493,
+      "rewards/rejected": -5.189752101898193,
+      "sft_loss": 4.175745964050293,
+      "step": 4445
+    },
+    {
+      "epoch": 2.3816691754473993,
+      "grad_norm": 0.7887475660632963,
+      "learning_rate": 1.2373373645410573e-07,
+      "logits/chosen": -0.6140779256820679,
+      "logits/rejected": -0.4921630322933197,
+      "logps/chosen": -4.5451741218566895,
+      "logps/rejected": -5.283486843109131,
+      "loss": 0.051,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.5451741218566895,
+      "rewards/margins": 0.7383132576942444,
+      "rewards/rejected": -5.283486843109131,
+      "sft_loss": 4.2206878662109375,
+      "step": 4450
+    },
+    {
+      "epoch": 2.384345208228801,
+      "grad_norm": 0.5208211747006889,
+      "learning_rate": 1.2270992794243175e-07,
+      "logits/chosen": -0.7170608043670654,
+      "logits/rejected": -0.623271644115448,
+      "logps/chosen": -4.37496280670166,
+      "logps/rejected": -5.164194583892822,
+      "loss": 0.0502,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.37496280670166,
+      "rewards/margins": 0.7892317175865173,
+      "rewards/rejected": -5.164194583892822,
+      "sft_loss": 4.082230567932129,
+      "step": 4455
+    },
+    {
+      "epoch": 2.3870212410102023,
+      "grad_norm": 0.7702519071760379,
+      "learning_rate": 1.2168977990069147e-07,
+      "logits/chosen": -0.6991745233535767,
+      "logits/rejected": -0.4843880534172058,
+      "logps/chosen": -4.240236759185791,
+      "logps/rejected": -5.103304862976074,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.240236759185791,
+      "rewards/margins": 0.8630678057670593,
+      "rewards/rejected": -5.103304862976074,
+      "sft_loss": 3.94366455078125,
+      "step": 4460
+    },
+    {
+      "epoch": 2.389697273791604,
+      "grad_norm": 0.44010205165526484,
+      "learning_rate": 1.206733022263659e-07,
+      "logits/chosen": -0.7043512463569641,
+      "logits/rejected": -0.5454774498939514,
+      "logps/chosen": -4.243316650390625,
+      "logps/rejected": -5.067329406738281,
+      "loss": 0.0493,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.243316650390625,
+      "rewards/margins": 0.8240129351615906,
+      "rewards/rejected": -5.067329406738281,
+      "sft_loss": 3.9276695251464844,
+      "step": 4465
+    },
+    {
+      "epoch": 2.3923733065730053,
+      "grad_norm": 0.6088262421773382,
+      "learning_rate": 1.1966050478132572e-07,
+      "logits/chosen": -0.6027519702911377,
+      "logits/rejected": -0.5432054400444031,
+      "logps/chosen": -4.4336090087890625,
+      "logps/rejected": -5.110543251037598,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.4336090087890625,
+      "rewards/margins": 0.6769340634346008,
+      "rewards/rejected": -5.110543251037598,
+      "sft_loss": 4.138341903686523,
+      "step": 4470
+    },
+    {
+      "epoch": 2.395049339354407,
+      "grad_norm": 0.493704583831731,
+      "learning_rate": 1.1865139739173635e-07,
+      "logits/chosen": -0.6488873958587646,
+      "logits/rejected": -0.44093388319015503,
+      "logps/chosen": -4.528680801391602,
+      "logps/rejected": -5.203161716461182,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.528680801391602,
+      "rewards/margins": 0.6744807958602905,
+      "rewards/rejected": -5.203161716461182,
+      "sft_loss": 4.175751209259033,
+      "step": 4475
+    },
+    {
+      "epoch": 2.3977253721358087,
+      "grad_norm": 0.45757520906311516,
+      "learning_rate": 1.1764598984796187e-07,
+      "logits/chosen": -0.7642940878868103,
+      "logits/rejected": -0.658028244972229,
+      "logps/chosen": -4.318887710571289,
+      "logps/rejected": -5.101247310638428,
+      "loss": 0.0495,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.318887710571289,
+      "rewards/margins": 0.7823596596717834,
+      "rewards/rejected": -5.101247310638428,
+      "sft_loss": 4.026830673217773,
+      "step": 4480
+    },
+    {
+      "epoch": 2.4004014049172104,
+      "grad_norm": 0.44848305279358136,
+      "learning_rate": 1.1664429190447095e-07,
+      "logits/chosen": -0.5974345803260803,
+      "logits/rejected": -0.5365961790084839,
+      "logps/chosen": -4.444516181945801,
+      "logps/rejected": -5.3155717849731445,
+      "loss": 0.0506,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -4.444516181945801,
+      "rewards/margins": 0.8710557222366333,
+      "rewards/rejected": -5.3155717849731445,
+      "sft_loss": 4.178521156311035,
+      "step": 4485
+    },
+    {
+      "epoch": 2.4030774376986117,
+      "grad_norm": 0.5057411082521135,
+      "learning_rate": 1.1564631327974122e-07,
+      "logits/chosen": -0.6875180006027222,
+      "logits/rejected": -0.4622613787651062,
+      "logps/chosen": -4.509605407714844,
+      "logps/rejected": -5.245260715484619,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.509605407714844,
+      "rewards/margins": 0.7356553673744202,
+      "rewards/rejected": -5.245260715484619,
+      "sft_loss": 4.180906772613525,
+      "step": 4490
+    },
+    {
+      "epoch": 2.4057534704800134,
+      "grad_norm": 0.6087213090441006,
+      "learning_rate": 1.1465206365616587e-07,
+      "logits/chosen": -0.738355815410614,
+      "logits/rejected": -0.5236421823501587,
+      "logps/chosen": -4.449094772338867,
+      "logps/rejected": -5.266437530517578,
+      "loss": 0.0498,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.449094772338867,
+      "rewards/margins": 0.8173427581787109,
+      "rewards/rejected": -5.266437530517578,
+      "sft_loss": 4.1014814376831055,
+      "step": 4495
+    },
+    {
+      "epoch": 2.408429503261415,
+      "grad_norm": 0.5663632798386813,
+      "learning_rate": 1.1366155267995887e-07,
+      "logits/chosen": -0.5962930917739868,
+      "logits/rejected": -0.598239541053772,
+      "logps/chosen": -4.435351371765137,
+      "logps/rejected": -5.083191394805908,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.435351371765137,
+      "rewards/margins": 0.6478403806686401,
+      "rewards/rejected": -5.083191394805908,
+      "sft_loss": 4.089267253875732,
+      "step": 4500
+    },
+    {
+      "epoch": 2.4111055360428164,
+      "grad_norm": 0.5251827978242332,
+      "learning_rate": 1.1267478996106228e-07,
+      "logits/chosen": -0.6230689287185669,
+      "logits/rejected": -0.42026257514953613,
+      "logps/chosen": -4.326894760131836,
+      "logps/rejected": -5.141059398651123,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.326894760131836,
+      "rewards/margins": 0.8141642808914185,
+      "rewards/rejected": -5.141059398651123,
+      "sft_loss": 3.996236801147461,
+      "step": 4505
+    },
+    {
+      "epoch": 2.413781568824218,
+      "grad_norm": 0.5542921157378735,
+      "learning_rate": 1.116917850730521e-07,
+      "logits/chosen": -0.7194280028343201,
+      "logits/rejected": -0.5925047993659973,
+      "logps/chosen": -4.487764358520508,
+      "logps/rejected": -5.054713249206543,
+      "loss": 0.0527,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.487764358520508,
+      "rewards/margins": 0.5669490098953247,
+      "rewards/rejected": -5.054713249206543,
+      "sft_loss": 4.162412166595459,
+      "step": 4510
+    },
+    {
+      "epoch": 2.41645760160562,
+      "grad_norm": 0.5588662136617383,
+      "learning_rate": 1.1071254755304637e-07,
+      "logits/chosen": -0.6750051975250244,
+      "logits/rejected": -0.6528328061103821,
+      "logps/chosen": -4.215541362762451,
+      "logps/rejected": -5.034451484680176,
+      "loss": 0.0484,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.215541362762451,
+      "rewards/margins": 0.8189102411270142,
+      "rewards/rejected": -5.034451484680176,
+      "sft_loss": 3.8783822059631348,
+      "step": 4515
+    },
+    {
+      "epoch": 2.419133634387021,
+      "grad_norm": 0.5712178134665697,
+      "learning_rate": 1.0973708690161143e-07,
+      "logits/chosen": -0.6738842725753784,
+      "logits/rejected": -0.580504298210144,
+      "logps/chosen": -4.435823917388916,
+      "logps/rejected": -5.125463962554932,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.435823917388916,
+      "rewards/margins": 0.6896399259567261,
+      "rewards/rejected": -5.125463962554932,
+      "sft_loss": 4.147543907165527,
+      "step": 4520
+    },
+    {
+      "epoch": 2.421809667168423,
+      "grad_norm": 0.489469378710008,
+      "learning_rate": 1.0876541258267119e-07,
+      "logits/chosen": -0.6761329770088196,
+      "logits/rejected": -0.5104591846466064,
+      "logps/chosen": -4.297384262084961,
+      "logps/rejected": -5.181668281555176,
+      "loss": 0.0495,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.297384262084961,
+      "rewards/margins": 0.8842836618423462,
+      "rewards/rejected": -5.181668281555176,
+      "sft_loss": 4.0386152267456055,
+      "step": 4525
+    },
+    {
+      "epoch": 2.4244856999498245,
+      "grad_norm": 1.0066257451866902,
+      "learning_rate": 1.0779753402341379e-07,
+      "logits/chosen": -0.720579206943512,
+      "logits/rejected": -0.6526123285293579,
+      "logps/chosen": -4.2700324058532715,
+      "logps/rejected": -5.035819053649902,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.2700324058532715,
+      "rewards/margins": 0.7657860517501831,
+      "rewards/rejected": -5.035819053649902,
+      "sft_loss": 4.025691032409668,
+      "step": 4530
+    },
+    {
+      "epoch": 2.427161732731226,
+      "grad_norm": 0.612543003798058,
+      "learning_rate": 1.0683346061420157e-07,
+      "logits/chosen": -0.5280240774154663,
+      "logits/rejected": -0.4593251347541809,
+      "logps/chosen": -4.427683353424072,
+      "logps/rejected": -5.2557573318481445,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.427683353424072,
+      "rewards/margins": 0.8280740976333618,
+      "rewards/rejected": -5.2557573318481445,
+      "sft_loss": 4.217066287994385,
+      "step": 4535
+    },
+    {
+      "epoch": 2.4298377655126275,
+      "grad_norm": 0.45545476112264677,
+      "learning_rate": 1.0587320170847874e-07,
+      "logits/chosen": -0.6139400005340576,
+      "logits/rejected": -0.5337790250778198,
+      "logps/chosen": -4.44398307800293,
+      "logps/rejected": -5.079217433929443,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.44398307800293,
+      "rewards/margins": 0.6352342367172241,
+      "rewards/rejected": -5.079217433929443,
+      "sft_loss": 4.190762996673584,
+      "step": 4540
+    },
+    {
+      "epoch": 2.4325137982940293,
+      "grad_norm": 0.3840213133651746,
+      "learning_rate": 1.0491676662268156e-07,
+      "logits/chosen": -0.5504549741744995,
+      "logits/rejected": -0.4786248803138733,
+      "logps/chosen": -4.464652061462402,
+      "logps/rejected": -5.272549152374268,
+      "loss": 0.0499,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.464652061462402,
+      "rewards/margins": 0.8078978657722473,
+      "rewards/rejected": -5.272549152374268,
+      "sft_loss": 4.128154754638672,
+      "step": 4545
+    },
+    {
+      "epoch": 2.4351898310754305,
+      "grad_norm": 0.4929618860870753,
+      "learning_rate": 1.0396416463614732e-07,
+      "logits/chosen": -0.703862190246582,
+      "logits/rejected": -0.6290112733840942,
+      "logps/chosen": -4.399338722229004,
+      "logps/rejected": -5.126931667327881,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.399338722229004,
+      "rewards/margins": 0.7275932431221008,
+      "rewards/rejected": -5.126931667327881,
+      "sft_loss": 4.119030952453613,
+      "step": 4550
+    },
+    {
+      "epoch": 2.4378658638568322,
+      "grad_norm": 0.9834360924021707,
+      "learning_rate": 1.0301540499102479e-07,
+      "logits/chosen": -0.6297124624252319,
+      "logits/rejected": -0.5710808634757996,
+      "logps/chosen": -4.40581750869751,
+      "logps/rejected": -4.969916820526123,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.40581750869751,
+      "rewards/margins": 0.5640992522239685,
+      "rewards/rejected": -4.969916820526123,
+      "sft_loss": 4.104538440704346,
+      "step": 4555
+    },
+    {
+      "epoch": 2.440541896638234,
+      "grad_norm": 0.5676263622267435,
+      "learning_rate": 1.0207049689218405e-07,
+      "logits/chosen": -0.7151520848274231,
+      "logits/rejected": -0.46953868865966797,
+      "logps/chosen": -4.364294052124023,
+      "logps/rejected": -5.133480072021484,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.364294052124023,
+      "rewards/margins": 0.76918625831604,
+      "rewards/rejected": -5.133480072021484,
+      "sft_loss": 4.03622579574585,
+      "step": 4560
+    },
+    {
+      "epoch": 2.4432179294196352,
+      "grad_norm": 0.47840507535960825,
+      "learning_rate": 1.0112944950712782e-07,
+      "logits/chosen": -0.6493829488754272,
+      "logits/rejected": -0.5766769647598267,
+      "logps/chosen": -4.331072807312012,
+      "logps/rejected": -5.191370487213135,
+      "loss": 0.0493,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.331072807312012,
+      "rewards/margins": 0.860297679901123,
+      "rewards/rejected": -5.191370487213135,
+      "sft_loss": 4.016242027282715,
+      "step": 4565
+    },
+    {
+      "epoch": 2.445893962201037,
+      "grad_norm": 0.7214769143395418,
+      "learning_rate": 1.0019227196590174e-07,
+      "logits/chosen": -0.6021589636802673,
+      "logits/rejected": -0.4612697660923004,
+      "logps/chosen": -4.441576957702637,
+      "logps/rejected": -5.250128746032715,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.441576957702637,
+      "rewards/margins": 0.8085517883300781,
+      "rewards/rejected": -5.250128746032715,
+      "sft_loss": 4.175487518310547,
+      "step": 4570
+    },
+    {
+      "epoch": 2.4485699949824387,
+      "grad_norm": 0.45379625662291867,
+      "learning_rate": 9.925897336100664e-08,
+      "logits/chosen": -0.591006338596344,
+      "logits/rejected": -0.5572066307067871,
+      "logps/chosen": -4.436382293701172,
+      "logps/rejected": -5.219793796539307,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.436382293701172,
+      "rewards/margins": 0.7834123969078064,
+      "rewards/rejected": -5.219793796539307,
+      "sft_loss": 4.143486976623535,
+      "step": 4575
+    },
+    {
+      "epoch": 2.45124602776384,
+      "grad_norm": 0.5710558027302429,
+      "learning_rate": 9.832956274730946e-08,
+      "logits/chosen": -0.5513227581977844,
+      "logits/rejected": -0.5147534012794495,
+      "logps/chosen": -4.613712787628174,
+      "logps/rejected": -5.120513439178467,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": -4.613712787628174,
+      "rewards/margins": 0.5068010091781616,
+      "rewards/rejected": -5.120513439178467,
+      "sft_loss": 4.222799301147461,
+      "step": 4580
+    },
+    {
+      "epoch": 2.4539220605452416,
+      "grad_norm": 0.8789274019827504,
+      "learning_rate": 9.740404914195633e-08,
+      "logits/chosen": -0.5852376222610474,
+      "logits/rejected": -0.4495258331298828,
+      "logps/chosen": -4.300984859466553,
+      "logps/rejected": -5.110598564147949,
+      "loss": 0.0498,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.300984859466553,
+      "rewards/margins": 0.8096135854721069,
+      "rewards/rejected": -5.110598564147949,
+      "sft_loss": 4.0735554695129395,
+      "step": 4585
+    },
+    {
+      "epoch": 2.4565980933266434,
+      "grad_norm": 0.6694953406110393,
+      "learning_rate": 9.648244152428392e-08,
+      "logits/chosen": -0.6197008490562439,
+      "logits/rejected": -0.4967406690120697,
+      "logps/chosen": -4.40405797958374,
+      "logps/rejected": -5.028562545776367,
+      "loss": 0.052,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.40405797958374,
+      "rewards/margins": 0.6245037317276001,
+      "rewards/rejected": -5.028562545776367,
+      "sft_loss": 4.14216947555542,
+      "step": 4590
+    },
+    {
+      "epoch": 2.4592741261080446,
+      "grad_norm": 0.42467234591310443,
+      "learning_rate": 9.556474883573379e-08,
+      "logits/chosen": -0.6408802270889282,
+      "logits/rejected": -0.5117521286010742,
+      "logps/chosen": -4.2645416259765625,
+      "logps/rejected": -5.111584663391113,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.2645416259765625,
+      "rewards/margins": 0.8470427393913269,
+      "rewards/rejected": -5.111584663391113,
+      "sft_loss": 3.9881603717803955,
+      "step": 4595
+    },
+    {
+      "epoch": 2.4619501588894463,
+      "grad_norm": 0.7016944720380158,
+      "learning_rate": 9.465097997976412e-08,
+      "logits/chosen": -0.6521051526069641,
+      "logits/rejected": -0.447486937046051,
+      "logps/chosen": -4.24138879776001,
+      "logps/rejected": -5.111659049987793,
+      "loss": 0.05,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.24138879776001,
+      "rewards/margins": 0.8702705502510071,
+      "rewards/rejected": -5.111659049987793,
+      "sft_loss": 3.971604824066162,
+      "step": 4600
+    },
+    {
+      "epoch": 2.464626191670848,
+      "grad_norm": 0.6495939536123898,
+      "learning_rate": 9.374114382176457e-08,
+      "logits/chosen": -0.6396633982658386,
+      "logits/rejected": -0.47933229804039,
+      "logps/chosen": -4.426239967346191,
+      "logps/rejected": -5.367574214935303,
+      "loss": 0.0492,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.426239967346191,
+      "rewards/margins": 0.9413352012634277,
+      "rewards/rejected": -5.367574214935303,
+      "sft_loss": 4.09249210357666,
+      "step": 4605
+    },
+    {
+      "epoch": 2.46730222445225,
+      "grad_norm": 0.565516701473944,
+      "learning_rate": 9.283524918896945e-08,
+      "logits/chosen": -0.6594346761703491,
+      "logits/rejected": -0.5122818946838379,
+      "logps/chosen": -4.475986957550049,
+      "logps/rejected": -5.197885036468506,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.475986957550049,
+      "rewards/margins": 0.7218977808952332,
+      "rewards/rejected": -5.197885036468506,
+      "sft_loss": 4.181154727935791,
+      "step": 4610
+    },
+    {
+      "epoch": 2.469978257233651,
+      "grad_norm": 0.5327390683140417,
+      "learning_rate": 9.193330487037232e-08,
+      "logits/chosen": -0.6300376653671265,
+      "logits/rejected": -0.4747852683067322,
+      "logps/chosen": -4.382430076599121,
+      "logps/rejected": -5.273916721343994,
+      "loss": 0.051,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.382430076599121,
+      "rewards/margins": 0.8914863467216492,
+      "rewards/rejected": -5.273916721343994,
+      "sft_loss": 4.206415176391602,
+      "step": 4615
+    },
+    {
+      "epoch": 2.4726542900150528,
+      "grad_norm": 0.49682984116494505,
+      "learning_rate": 9.103531961664118e-08,
+      "logits/chosen": -0.5900342464447021,
+      "logits/rejected": -0.4282127022743225,
+      "logps/chosen": -4.40287446975708,
+      "logps/rejected": -5.209569931030273,
+      "loss": 0.049,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.40287446975708,
+      "rewards/margins": 0.8066957592964172,
+      "rewards/rejected": -5.209569931030273,
+      "sft_loss": 4.044670581817627,
+      "step": 4620
+    },
+    {
+      "epoch": 2.475330322796454,
+      "grad_norm": 0.5317156147552861,
+      "learning_rate": 9.014130214003269e-08,
+      "logits/chosen": -0.6556288599967957,
+      "logits/rejected": -0.7031276822090149,
+      "logps/chosen": -4.211493015289307,
+      "logps/rejected": -4.99152135848999,
+      "loss": 0.0501,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.211493015289307,
+      "rewards/margins": 0.7800289392471313,
+      "rewards/rejected": -4.99152135848999,
+      "sft_loss": 3.9011688232421875,
+      "step": 4625
+    },
+    {
+      "epoch": 2.4780063555778558,
+      "grad_norm": 0.43211617277524894,
+      "learning_rate": 8.925126111430848e-08,
+      "logits/chosen": -0.54551762342453,
+      "logits/rejected": -0.494556725025177,
+      "logps/chosen": -4.364080429077148,
+      "logps/rejected": -5.020254611968994,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.364080429077148,
+      "rewards/margins": 0.6561740636825562,
+      "rewards/rejected": -5.020254611968994,
+      "sft_loss": 3.972407579421997,
+      "step": 4630
+    },
+    {
+      "epoch": 2.4806823883592575,
+      "grad_norm": 0.5794124856061074,
+      "learning_rate": 8.83652051746504e-08,
+      "logits/chosen": -0.47839322686195374,
+      "logits/rejected": -0.31888797879219055,
+      "logps/chosen": -4.368432521820068,
+      "logps/rejected": -5.375771999359131,
+      "loss": 0.0499,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.368432521820068,
+      "rewards/margins": 1.007339596748352,
+      "rewards/rejected": -5.375771999359131,
+      "sft_loss": 4.1021881103515625,
+      "step": 4635
+    },
+    {
+      "epoch": 2.483358421140659,
+      "grad_norm": 0.4014260737708445,
+      "learning_rate": 8.748314291757696e-08,
+      "logits/chosen": -0.56255704164505,
+      "logits/rejected": -0.4708307683467865,
+      "logps/chosen": -4.459442138671875,
+      "logps/rejected": -5.198812007904053,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.459442138671875,
+      "rewards/margins": 0.7393695116043091,
+      "rewards/rejected": -5.198812007904053,
+      "sft_loss": 4.105051517486572,
+      "step": 4640
+    },
+    {
+      "epoch": 2.4860344539220605,
+      "grad_norm": 0.6113352658624956,
+      "learning_rate": 8.660508290086032e-08,
+      "logits/chosen": -0.5913625955581665,
+      "logits/rejected": -0.45196396112442017,
+      "logps/chosen": -4.2816596031188965,
+      "logps/rejected": -5.070683002471924,
+      "loss": 0.0506,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.2816596031188965,
+      "rewards/margins": 0.7890230417251587,
+      "rewards/rejected": -5.070683002471924,
+      "sft_loss": 4.0688581466674805,
+      "step": 4645
+    },
+    {
+      "epoch": 2.488710486703462,
+      "grad_norm": 0.6715225905082456,
+      "learning_rate": 8.573103364344231e-08,
+      "logits/chosen": -0.6570698022842407,
+      "logits/rejected": -0.4290023446083069,
+      "logps/chosen": -4.31870698928833,
+      "logps/rejected": -5.184931755065918,
+      "loss": 0.0496,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.31870698928833,
+      "rewards/margins": 0.8662254214286804,
+      "rewards/rejected": -5.184931755065918,
+      "sft_loss": 3.9577102661132812,
+      "step": 4650
+    },
+    {
+      "epoch": 2.4913865194848634,
+      "grad_norm": 0.4482321236034095,
+      "learning_rate": 8.486100362535292e-08,
+      "logits/chosen": -0.6576000452041626,
+      "logits/rejected": -0.5152201056480408,
+      "logps/chosen": -4.459080696105957,
+      "logps/rejected": -5.089616298675537,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.459080696105957,
+      "rewards/margins": 0.6305348873138428,
+      "rewards/rejected": -5.089616298675537,
+      "sft_loss": 4.153569221496582,
+      "step": 4655
+    },
+    {
+      "epoch": 2.494062552266265,
+      "grad_norm": 0.5202924530786741,
+      "learning_rate": 8.399500128762693e-08,
+      "logits/chosen": -0.6455222368240356,
+      "logits/rejected": -0.5461894273757935,
+      "logps/chosen": -4.488771915435791,
+      "logps/rejected": -5.1508049964904785,
+      "loss": 0.0505,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.488771915435791,
+      "rewards/margins": 0.6620326638221741,
+      "rewards/rejected": -5.1508049964904785,
+      "sft_loss": 4.204336166381836,
+      "step": 4660
+    },
+    {
+      "epoch": 2.496738585047667,
+      "grad_norm": 0.4384076226247688,
+      "learning_rate": 8.313303503222313e-08,
+      "logits/chosen": -0.60161954164505,
+      "logits/rejected": -0.5591880083084106,
+      "logps/chosen": -4.493898391723633,
+      "logps/rejected": -5.113542079925537,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.493898391723633,
+      "rewards/margins": 0.6196438074111938,
+      "rewards/rejected": -5.113542079925537,
+      "sft_loss": 4.1594672203063965,
+      "step": 4665
+    },
+    {
+      "epoch": 2.4994146178290686,
+      "grad_norm": 0.46184563897584546,
+      "learning_rate": 8.227511322194164e-08,
+      "logits/chosen": -0.5961467623710632,
+      "logits/rejected": -0.48461294174194336,
+      "logps/chosen": -4.278865814208984,
+      "logps/rejected": -4.975838661193848,
+      "loss": 0.052,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.278865814208984,
+      "rewards/margins": 0.6969722509384155,
+      "rewards/rejected": -4.975838661193848,
+      "sft_loss": 4.005204677581787,
+      "step": 4670
+    },
+    {
+      "epoch": 2.50209065061047,
+      "grad_norm": 0.45426527025887287,
+      "learning_rate": 8.142124418034385e-08,
+      "logits/chosen": -0.538938045501709,
+      "logits/rejected": -0.340365469455719,
+      "logps/chosen": -4.515494346618652,
+      "logps/rejected": -5.184680938720703,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.515494346618652,
+      "rewards/margins": 0.6691871285438538,
+      "rewards/rejected": -5.184680938720703,
+      "sft_loss": 4.164722442626953,
+      "step": 4675
+    },
+    {
+      "epoch": 2.5047666833918716,
+      "grad_norm": 0.4938296533440213,
+      "learning_rate": 8.057143619167073e-08,
+      "logits/chosen": -0.5143073201179504,
+      "logits/rejected": -0.39615967869758606,
+      "logps/chosen": -4.439083576202393,
+      "logps/rejected": -5.253573894500732,
+      "loss": 0.051,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.439083576202393,
+      "rewards/margins": 0.8144906759262085,
+      "rewards/rejected": -5.253573894500732,
+      "sft_loss": 4.184497833251953,
+      "step": 4680
+    },
+    {
+      "epoch": 2.507442716173273,
+      "grad_norm": 0.753016347667016,
+      "learning_rate": 7.97256975007633e-08,
+      "logits/chosen": -0.5837646722793579,
+      "logits/rejected": -0.3747573494911194,
+      "logps/chosen": -4.154690265655518,
+      "logps/rejected": -5.060885906219482,
+      "loss": 0.0498,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -4.154690265655518,
+      "rewards/margins": 0.9061959981918335,
+      "rewards/rejected": -5.060885906219482,
+      "sft_loss": 3.9000186920166016,
+      "step": 4685
+    },
+    {
+      "epoch": 2.5101187489546746,
+      "grad_norm": 0.5019050668936905,
+      "learning_rate": 7.888403631298186e-08,
+      "logits/chosen": -0.5046710968017578,
+      "logits/rejected": -0.4844224452972412,
+      "logps/chosen": -4.427618980407715,
+      "logps/rejected": -5.084200859069824,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.427618980407715,
+      "rewards/margins": 0.6565815806388855,
+      "rewards/rejected": -5.084200859069824,
+      "sft_loss": 4.106476783752441,
+      "step": 4690
+    },
+    {
+      "epoch": 2.5127947817360763,
+      "grad_norm": 0.5192681787532761,
+      "learning_rate": 7.804646079412719e-08,
+      "logits/chosen": -0.558775007724762,
+      "logits/rejected": -0.3784492015838623,
+      "logps/chosen": -4.472651958465576,
+      "logps/rejected": -5.213320732116699,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.472651958465576,
+      "rewards/margins": 0.7406686544418335,
+      "rewards/rejected": -5.213320732116699,
+      "sft_loss": 4.204833030700684,
+      "step": 4695
+    },
+    {
+      "epoch": 2.515470814517478,
+      "grad_norm": 0.48365443890957205,
+      "learning_rate": 7.72129790703604e-08,
+      "logits/chosen": -0.6363487243652344,
+      "logits/rejected": -0.5025444626808167,
+      "logps/chosen": -4.454331874847412,
+      "logps/rejected": -5.120041847229004,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.454331874847412,
+      "rewards/margins": 0.6657102704048157,
+      "rewards/rejected": -5.120041847229004,
+      "sft_loss": 4.133848667144775,
+      "step": 4700
+    },
+    {
+      "epoch": 2.5181468472988793,
+      "grad_norm": 0.48668584008364435,
+      "learning_rate": 7.638359922812504e-08,
+      "logits/chosen": -0.5567082762718201,
+      "logits/rejected": -0.5363680720329285,
+      "logps/chosen": -4.455945014953613,
+      "logps/rejected": -5.264870643615723,
+      "loss": 0.0495,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.455945014953613,
+      "rewards/margins": 0.8089253306388855,
+      "rewards/rejected": -5.264870643615723,
+      "sft_loss": 4.1262640953063965,
+      "step": 4705
+    },
+    {
+      "epoch": 2.520822880080281,
+      "grad_norm": 0.6004591352306471,
+      "learning_rate": 7.555832931406774e-08,
+      "logits/chosen": -0.6345446109771729,
+      "logits/rejected": -0.42940983176231384,
+      "logps/chosen": -4.445584297180176,
+      "logps/rejected": -5.25039005279541,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.445584297180176,
+      "rewards/margins": 0.8048050999641418,
+      "rewards/rejected": -5.25039005279541,
+      "sft_loss": 4.166441917419434,
+      "step": 4710
+    },
+    {
+      "epoch": 2.5234989128616827,
+      "grad_norm": 0.47765780021957177,
+      "learning_rate": 7.47371773349611e-08,
+      "logits/chosen": -0.5504805445671082,
+      "logits/rejected": -0.5885323286056519,
+      "logps/chosen": -4.370347023010254,
+      "logps/rejected": -5.237016201019287,
+      "loss": 0.0489,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.370347023010254,
+      "rewards/margins": 0.8666695356369019,
+      "rewards/rejected": -5.237016201019287,
+      "sft_loss": 4.009383678436279,
+      "step": 4715
+    },
+    {
+      "epoch": 2.526174945643084,
+      "grad_norm": 0.4459515797629196,
+      "learning_rate": 7.392015125762496e-08,
+      "logits/chosen": -0.5894617438316345,
+      "logits/rejected": -0.4673500061035156,
+      "logps/chosen": -4.369865894317627,
+      "logps/rejected": -5.101893424987793,
+      "loss": 0.0497,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.369865894317627,
+      "rewards/margins": 0.7320277094841003,
+      "rewards/rejected": -5.101893424987793,
+      "sft_loss": 4.020104885101318,
+      "step": 4720
+    },
+    {
+      "epoch": 2.5288509784244857,
+      "grad_norm": 0.5575338394491988,
+      "learning_rate": 7.310725900885018e-08,
+      "logits/chosen": -0.6002328991889954,
+      "logits/rejected": -0.58238685131073,
+      "logps/chosen": -4.3686418533325195,
+      "logps/rejected": -5.014484405517578,
+      "loss": 0.053,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.3686418533325195,
+      "rewards/margins": 0.6458426713943481,
+      "rewards/rejected": -5.014484405517578,
+      "sft_loss": 4.113285541534424,
+      "step": 4725
+    },
+    {
+      "epoch": 2.5315270112058874,
+      "grad_norm": 0.5484216940416137,
+      "learning_rate": 7.229850847532076e-08,
+      "logits/chosen": -0.5159083008766174,
+      "logits/rejected": -0.3510948717594147,
+      "logps/chosen": -4.445266246795654,
+      "logps/rejected": -5.217179298400879,
+      "loss": 0.0501,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.445266246795654,
+      "rewards/margins": 0.7719129323959351,
+      "rewards/rejected": -5.217179298400879,
+      "sft_loss": 4.128899574279785,
+      "step": 4730
+    },
+    {
+      "epoch": 2.5342030439872887,
+      "grad_norm": 0.9673779363020051,
+      "learning_rate": 7.149390750353779e-08,
+      "logits/chosen": -0.4700179696083069,
+      "logits/rejected": -0.602042019367218,
+      "logps/chosen": -4.330955505371094,
+      "logps/rejected": -4.948132514953613,
+      "loss": 0.0505,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.330955505371094,
+      "rewards/margins": 0.617177426815033,
+      "rewards/rejected": -4.948132514953613,
+      "sft_loss": 3.9751884937286377,
+      "step": 4735
+    },
+    {
+      "epoch": 2.5368790767686904,
+      "grad_norm": 0.4711804549323274,
+      "learning_rate": 7.069346389974374e-08,
+      "logits/chosen": -0.606751561164856,
+      "logits/rejected": -0.44376516342163086,
+      "logps/chosen": -4.376944541931152,
+      "logps/rejected": -5.113620758056641,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -4.376944541931152,
+      "rewards/margins": 0.7366760969161987,
+      "rewards/rejected": -5.113620758056641,
+      "sft_loss": 4.143258571624756,
+      "step": 4740
+    },
+    {
+      "epoch": 2.539555109550092,
+      "grad_norm": 0.513288016157318,
+      "learning_rate": 6.989718542984563e-08,
+      "logits/chosen": -0.6091118454933167,
+      "logits/rejected": -0.6088340878486633,
+      "logps/chosen": -4.455063819885254,
+      "logps/rejected": -5.0955891609191895,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.455063819885254,
+      "rewards/margins": 0.6405249238014221,
+      "rewards/rejected": -5.0955891609191895,
+      "sft_loss": 4.216749668121338,
+      "step": 4745
+    },
+    {
+      "epoch": 2.5422311423314934,
+      "grad_norm": 0.680159533383111,
+      "learning_rate": 6.9105079819341e-08,
+      "logits/chosen": -0.5284208059310913,
+      "logits/rejected": -0.306267648935318,
+      "logps/chosen": -4.190617561340332,
+      "logps/rejected": -5.343145370483398,
+      "loss": 0.0486,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -4.190617561340332,
+      "rewards/margins": 1.1525280475616455,
+      "rewards/rejected": -5.343145370483398,
+      "sft_loss": 4.004500389099121,
+      "step": 4750
+    },
+    {
+      "epoch": 2.544907175112895,
+      "grad_norm": 0.41912317781250796,
+      "learning_rate": 6.831715475324163e-08,
+      "logits/chosen": -0.6683145761489868,
+      "logits/rejected": -0.4936584532260895,
+      "logps/chosen": -4.370855331420898,
+      "logps/rejected": -5.237181186676025,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.370855331420898,
+      "rewards/margins": 0.866325855255127,
+      "rewards/rejected": -5.237181186676025,
+      "sft_loss": 4.171796798706055,
+      "step": 4755
+    },
+    {
+      "epoch": 2.547583207894297,
+      "grad_norm": 1.3034826658354273,
+      "learning_rate": 6.753341787600026e-08,
+      "logits/chosen": -0.6106042861938477,
+      "logits/rejected": -0.5314139723777771,
+      "logps/chosen": -4.351446628570557,
+      "logps/rejected": -5.160747528076172,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -4.351446628570557,
+      "rewards/margins": 0.8093011975288391,
+      "rewards/rejected": -5.160747528076172,
+      "sft_loss": 4.110096454620361,
+      "step": 4760
+    },
+    {
+      "epoch": 2.5502592406756985,
+      "grad_norm": 0.5572878486921042,
+      "learning_rate": 6.67538767914353e-08,
+      "logits/chosen": -0.5988022089004517,
+      "logits/rejected": -0.39046013355255127,
+      "logps/chosen": -4.455069541931152,
+      "logps/rejected": -5.206971645355225,
+      "loss": 0.054,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.455069541931152,
+      "rewards/margins": 0.7519021034240723,
+      "rewards/rejected": -5.206971645355225,
+      "sft_loss": 4.30315637588501,
+      "step": 4765
+    },
+    {
+      "epoch": 2.5529352734571,
+      "grad_norm": 0.9063695912980203,
+      "learning_rate": 6.597853906265793e-08,
+      "logits/chosen": -0.5196844339370728,
+      "logits/rejected": -0.39652159810066223,
+      "logps/chosen": -4.335490703582764,
+      "logps/rejected": -5.287980556488037,
+      "loss": 0.0501,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.335490703582764,
+      "rewards/margins": 0.9524902105331421,
+      "rewards/rejected": -5.287980556488037,
+      "sft_loss": 4.02547550201416,
+      "step": 4770
+    },
+    {
+      "epoch": 2.5556113062385015,
+      "grad_norm": 0.42531457572646897,
+      "learning_rate": 6.5207412211998e-08,
+      "logits/chosen": -0.5043855905532837,
+      "logits/rejected": -0.42247647047042847,
+      "logps/chosen": -4.347072601318359,
+      "logps/rejected": -5.087707996368408,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.347072601318359,
+      "rewards/margins": 0.7406353950500488,
+      "rewards/rejected": -5.087707996368408,
+      "sft_loss": 4.018216609954834,
+      "step": 4775
+    },
+    {
+      "epoch": 2.558287339019903,
+      "grad_norm": 0.43493742878107877,
+      "learning_rate": 6.444050372093186e-08,
+      "logits/chosen": -0.6051164269447327,
+      "logits/rejected": -0.520149290561676,
+      "logps/chosen": -4.302689552307129,
+      "logps/rejected": -5.056397914886475,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.302689552307129,
+      "rewards/margins": 0.7537086606025696,
+      "rewards/rejected": -5.056397914886475,
+      "sft_loss": 4.09941291809082,
+      "step": 4780
+    },
+    {
+      "epoch": 2.5609633718013045,
+      "grad_norm": 0.5267775816777619,
+      "learning_rate": 6.367782103000873e-08,
+      "logits/chosen": -0.5131546258926392,
+      "logits/rejected": -0.4773063659667969,
+      "logps/chosen": -4.355741500854492,
+      "logps/rejected": -4.989833354949951,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.355741500854492,
+      "rewards/margins": 0.6340919733047485,
+      "rewards/rejected": -4.989833354949951,
+      "sft_loss": 4.04448938369751,
+      "step": 4785
+    },
+    {
+      "epoch": 2.5636394045827062,
+      "grad_norm": 0.5428781055586078,
+      "learning_rate": 6.29193715387798e-08,
+      "logits/chosen": -0.5961020588874817,
+      "logits/rejected": -0.49942970275878906,
+      "logps/chosen": -4.322155475616455,
+      "logps/rejected": -5.115612506866455,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.322155475616455,
+      "rewards/margins": 0.7934563755989075,
+      "rewards/rejected": -5.115612506866455,
+      "sft_loss": 4.066105842590332,
+      "step": 4790
+    },
+    {
+      "epoch": 2.566315437364108,
+      "grad_norm": 0.756164246620467,
+      "learning_rate": 6.216516260572502e-08,
+      "logits/chosen": -0.5418619513511658,
+      "logits/rejected": -0.4399585723876953,
+      "logps/chosen": -4.446822643280029,
+      "logps/rejected": -5.173761367797852,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.446822643280029,
+      "rewards/margins": 0.7269390225410461,
+      "rewards/rejected": -5.173761367797852,
+      "sft_loss": 4.22568416595459,
+      "step": 4795
+    },
+    {
+      "epoch": 2.568991470145509,
+      "grad_norm": 0.6286540593204735,
+      "learning_rate": 6.141520154818297e-08,
+      "logits/chosen": -0.5734771490097046,
+      "logits/rejected": -0.4777015745639801,
+      "logps/chosen": -4.429290294647217,
+      "logps/rejected": -5.125835418701172,
+      "loss": 0.0517,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.429290294647217,
+      "rewards/margins": 0.6965449452400208,
+      "rewards/rejected": -5.125835418701172,
+      "sft_loss": 4.193821907043457,
+      "step": 4800
+    },
+    {
+      "epoch": 2.568991470145509,
+      "eval_logits/chosen": -0.1310480684041977,
+      "eval_logits/rejected": -0.05083135515451431,
+      "eval_logps/chosen": -4.3976359367370605,
+      "eval_logps/rejected": -5.14287805557251,
+      "eval_loss": 0.05011750012636185,
+      "eval_rewards/accuracies": 0.6832343935966492,
+      "eval_rewards/chosen": -4.3976359367370605,
+      "eval_rewards/margins": 0.745241641998291,
+      "eval_rewards/rejected": -5.14287805557251,
+      "eval_runtime": 44.1527,
+      "eval_samples_per_second": 30.462,
+      "eval_sft_loss": 4.0092549324035645,
+      "eval_steps_per_second": 7.633,
+      "step": 4800
+    },
+    {
+      "epoch": 2.571667502926911,
+      "grad_norm": 0.4279108131541974,
+      "learning_rate": 6.066949564227897e-08,
+      "logits/chosen": -0.627518892288208,
+      "logits/rejected": -0.557266354560852,
+      "logps/chosen": -4.316287517547607,
+      "logps/rejected": -5.085277080535889,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.316287517547607,
+      "rewards/margins": 0.7689894437789917,
+      "rewards/rejected": -5.085277080535889,
+      "sft_loss": 4.034054756164551,
+      "step": 4805
+    },
+    {
+      "epoch": 2.574343535708312,
+      "grad_norm": 0.8316214754736503,
+      "learning_rate": 5.992805212285523e-08,
+      "logits/chosen": -0.5568079352378845,
+      "logits/rejected": -0.49940842390060425,
+      "logps/chosen": -4.266221046447754,
+      "logps/rejected": -5.169581413269043,
+      "loss": 0.0502,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.266221046447754,
+      "rewards/margins": 0.9033603668212891,
+      "rewards/rejected": -5.169581413269043,
+      "sft_loss": 4.01610803604126,
+      "step": 4810
+    },
+    {
+      "epoch": 2.577019568489714,
+      "grad_norm": 0.6341903842542911,
+      "learning_rate": 5.9190878183399684e-08,
+      "logits/chosen": -0.5636864304542542,
+      "logits/rejected": -0.4387081563472748,
+      "logps/chosen": -4.344354152679443,
+      "logps/rejected": -5.193105220794678,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.344354152679443,
+      "rewards/margins": 0.8487511873245239,
+      "rewards/rejected": -5.193105220794678,
+      "sft_loss": 4.155457973480225,
+      "step": 4815
+    },
+    {
+      "epoch": 2.5796956012711156,
+      "grad_norm": 0.49812032593786665,
+      "learning_rate": 5.845798097597748e-08,
+      "logits/chosen": -0.6051997542381287,
+      "logits/rejected": -0.5160627365112305,
+      "logps/chosen": -4.519273281097412,
+      "logps/rejected": -5.107385635375977,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.519273281097412,
+      "rewards/margins": 0.588112473487854,
+      "rewards/rejected": -5.107385635375977,
+      "sft_loss": 4.226346492767334,
+      "step": 4820
+    },
+    {
+      "epoch": 2.5823716340525174,
+      "grad_norm": 0.5137356088294925,
+      "learning_rate": 5.772936761116026e-08,
+      "logits/chosen": -0.5411312580108643,
+      "logits/rejected": -0.4147259294986725,
+      "logps/chosen": -4.3093461990356445,
+      "logps/rejected": -5.115912437438965,
+      "loss": 0.0499,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.3093461990356445,
+      "rewards/margins": 0.8065663576126099,
+      "rewards/rejected": -5.115912437438965,
+      "sft_loss": 4.040058135986328,
+      "step": 4825
+    },
+    {
+      "epoch": 2.5850476668339186,
+      "grad_norm": 0.5679993951727613,
+      "learning_rate": 5.700504515795829e-08,
+      "logits/chosen": -0.6407551765441895,
+      "logits/rejected": -0.49152690172195435,
+      "logps/chosen": -4.370004177093506,
+      "logps/rejected": -5.13353967666626,
+      "loss": 0.051,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.370004177093506,
+      "rewards/margins": 0.7635352611541748,
+      "rewards/rejected": -5.13353967666626,
+      "sft_loss": 4.191131591796875,
+      "step": 4830
+    },
+    {
+      "epoch": 2.5877236996153203,
+      "grad_norm": 0.6046695743005422,
+      "learning_rate": 5.628502064375101e-08,
+      "logits/chosen": -0.6635502576828003,
+      "logits/rejected": -0.47964462637901306,
+      "logps/chosen": -4.40742826461792,
+      "logps/rejected": -5.276705741882324,
+      "loss": 0.05,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -4.40742826461792,
+      "rewards/margins": 0.8692773580551147,
+      "rewards/rejected": -5.276705741882324,
+      "sft_loss": 4.135905742645264,
+      "step": 4835
+    },
+    {
+      "epoch": 2.5903997323967216,
+      "grad_norm": 0.6479165373198206,
+      "learning_rate": 5.55693010542197e-08,
+      "logits/chosen": -0.631353497505188,
+      "logits/rejected": -0.42470064759254456,
+      "logps/chosen": -4.271271228790283,
+      "logps/rejected": -5.198349475860596,
+      "loss": 0.048,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.271271228790283,
+      "rewards/margins": 0.9270779490470886,
+      "rewards/rejected": -5.198349475860596,
+      "sft_loss": 3.866617202758789,
+      "step": 4840
+    },
+    {
+      "epoch": 2.5930757651781233,
+      "grad_norm": 1.1174918581803077,
+      "learning_rate": 5.485789333327856e-08,
+      "logits/chosen": -0.572890043258667,
+      "logits/rejected": -0.5420287847518921,
+      "logps/chosen": -4.4225053787231445,
+      "logps/rejected": -5.114369869232178,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.4225053787231445,
+      "rewards/margins": 0.6918636560440063,
+      "rewards/rejected": -5.114369869232178,
+      "sft_loss": 4.175688743591309,
+      "step": 4845
+    },
+    {
+      "epoch": 2.595751797959525,
+      "grad_norm": 0.5237835591030086,
+      "learning_rate": 5.4150804383008675e-08,
+      "logits/chosen": -0.7328628301620483,
+      "logits/rejected": -0.5686417818069458,
+      "logps/chosen": -4.425593376159668,
+      "logps/rejected": -5.209748268127441,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.425593376159668,
+      "rewards/margins": 0.7841545939445496,
+      "rewards/rejected": -5.209748268127441,
+      "sft_loss": 4.164927005767822,
+      "step": 4850
+    },
+    {
+      "epoch": 2.5984278307409268,
+      "grad_norm": 0.6504617493007205,
+      "learning_rate": 5.344804106359002e-08,
+      "logits/chosen": -0.5078977942466736,
+      "logits/rejected": -0.3700367510318756,
+      "logps/chosen": -4.396625995635986,
+      "logps/rejected": -5.096922397613525,
+      "loss": 0.0519,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.396625995635986,
+      "rewards/margins": 0.7002967000007629,
+      "rewards/rejected": -5.096922397613525,
+      "sft_loss": 4.137091159820557,
+      "step": 4855
+    },
+    {
+      "epoch": 2.601103863522328,
+      "grad_norm": 0.6180206142159782,
+      "learning_rate": 5.274961019323559e-08,
+      "logits/chosen": -0.6101963520050049,
+      "logits/rejected": -0.5082840323448181,
+      "logps/chosen": -4.349689483642578,
+      "logps/rejected": -5.105837345123291,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.349689483642578,
+      "rewards/margins": 0.7561479806900024,
+      "rewards/rejected": -5.105837345123291,
+      "sft_loss": 4.168674468994141,
+      "step": 4860
+    },
+    {
+      "epoch": 2.6037798963037297,
+      "grad_norm": 0.5145625450475846,
+      "learning_rate": 5.205551854812451e-08,
+      "logits/chosen": -0.7037869095802307,
+      "logits/rejected": -0.6382130980491638,
+      "logps/chosen": -4.2961835861206055,
+      "logps/rejected": -5.145020484924316,
+      "loss": 0.0496,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.2961835861206055,
+      "rewards/margins": 0.8488362431526184,
+      "rewards/rejected": -5.145020484924316,
+      "sft_loss": 4.0863213539123535,
+      "step": 4865
+    },
+    {
+      "epoch": 2.606455929085131,
+      "grad_norm": 0.8816851602158896,
+      "learning_rate": 5.1365772862337177e-08,
+      "logits/chosen": -0.5311302542686462,
+      "logits/rejected": -0.4471372961997986,
+      "logps/chosen": -4.2873992919921875,
+      "logps/rejected": -5.161667823791504,
+      "loss": 0.0502,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.2873992919921875,
+      "rewards/margins": 0.8742688298225403,
+      "rewards/rejected": -5.161667823791504,
+      "sft_loss": 3.99345064163208,
+      "step": 4870
+    },
+    {
+      "epoch": 2.6091319618665327,
+      "grad_norm": 0.7715063836777378,
+      "learning_rate": 5.068037982778905e-08,
+      "logits/chosen": -0.5043749213218689,
+      "logits/rejected": -0.420607328414917,
+      "logps/chosen": -4.412718296051025,
+      "logps/rejected": -5.225282669067383,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.412718296051025,
+      "rewards/margins": 0.8125641942024231,
+      "rewards/rejected": -5.225282669067383,
+      "sft_loss": 4.112118721008301,
+      "step": 4875
+    },
+    {
+      "epoch": 2.6118079946479344,
+      "grad_norm": 0.5655215067235567,
+      "learning_rate": 4.999934609416656e-08,
+      "logits/chosen": -0.5136893391609192,
+      "logits/rejected": -0.41715818643569946,
+      "logps/chosen": -4.247698783874512,
+      "logps/rejected": -5.231947422027588,
+      "loss": 0.0501,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.247698783874512,
+      "rewards/margins": 0.9842487573623657,
+      "rewards/rejected": -5.231947422027588,
+      "sft_loss": 4.044260501861572,
+      "step": 4880
+    },
+    {
+      "epoch": 2.614484027429336,
+      "grad_norm": 0.6190399890836108,
+      "learning_rate": 4.932267826886183e-08,
+      "logits/chosen": -0.4868631958961487,
+      "logits/rejected": -0.44767284393310547,
+      "logps/chosen": -4.37949275970459,
+      "logps/rejected": -5.1437883377075195,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.37949275970459,
+      "rewards/margins": 0.764295220375061,
+      "rewards/rejected": -5.1437883377075195,
+      "sft_loss": 4.194809436798096,
+      "step": 4885
+    },
+    {
+      "epoch": 2.6171600602107374,
+      "grad_norm": 0.45602480638058607,
+      "learning_rate": 4.8650382916909206e-08,
+      "logits/chosen": -0.6891030073165894,
+      "logits/rejected": -0.5076113939285278,
+      "logps/chosen": -4.308724403381348,
+      "logps/rejected": -5.062498569488525,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.308724403381348,
+      "rewards/margins": 0.7537741661071777,
+      "rewards/rejected": -5.062498569488525,
+      "sft_loss": 4.0498762130737305,
+      "step": 4890
+    },
+    {
+      "epoch": 2.619836092992139,
+      "grad_norm": 0.6204571382318878,
+      "learning_rate": 4.7982466560920976e-08,
+      "logits/chosen": -0.5606731176376343,
+      "logits/rejected": -0.533047080039978,
+      "logps/chosen": -4.373346328735352,
+      "logps/rejected": -5.0479230880737305,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.373346328735352,
+      "rewards/margins": 0.6745761036872864,
+      "rewards/rejected": -5.0479230880737305,
+      "sft_loss": 4.139103889465332,
+      "step": 4895
+    },
+    {
+      "epoch": 2.622512125773541,
+      "grad_norm": 0.842422852967741,
+      "learning_rate": 4.7318935681024685e-08,
+      "logits/chosen": -0.5427811741828918,
+      "logits/rejected": -0.3881460130214691,
+      "logps/chosen": -4.327120304107666,
+      "logps/rejected": -5.237512588500977,
+      "loss": 0.0502,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.327120304107666,
+      "rewards/margins": 0.9103924036026001,
+      "rewards/rejected": -5.237512588500977,
+      "sft_loss": 4.039536476135254,
+      "step": 4900
+    },
+    {
+      "epoch": 2.625188158554942,
+      "grad_norm": 0.6175658328831507,
+      "learning_rate": 4.6659796714799745e-08,
+      "logits/chosen": -0.5474620461463928,
+      "logits/rejected": -0.4273964762687683,
+      "logps/chosen": -4.311333656311035,
+      "logps/rejected": -5.33859395980835,
+      "loss": 0.0484,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.311333656311035,
+      "rewards/margins": 1.0272610187530518,
+      "rewards/rejected": -5.33859395980835,
+      "sft_loss": 4.0710930824279785,
+      "step": 4905
+    },
+    {
+      "epoch": 2.627864191336344,
+      "grad_norm": 0.7129519719182232,
+      "learning_rate": 4.60050560572155e-08,
+      "logits/chosen": -0.5638284087181091,
+      "logits/rejected": -0.641061007976532,
+      "logps/chosen": -4.43626070022583,
+      "logps/rejected": -5.271731376647949,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.43626070022583,
+      "rewards/margins": 0.8354707956314087,
+      "rewards/rejected": -5.271731376647949,
+      "sft_loss": 4.2041120529174805,
+      "step": 4910
+    },
+    {
+      "epoch": 2.6305402241177456,
+      "grad_norm": 0.48846546570812444,
+      "learning_rate": 4.535472006056834e-08,
+      "logits/chosen": -0.5889512300491333,
+      "logits/rejected": -0.4078417420387268,
+      "logps/chosen": -4.3628435134887695,
+      "logps/rejected": -5.040680408477783,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.3628435134887695,
+      "rewards/margins": 0.6778360605239868,
+      "rewards/rejected": -5.040680408477783,
+      "sft_loss": 4.057613849639893,
+      "step": 4915
+    },
+    {
+      "epoch": 2.6332162568991473,
+      "grad_norm": 0.5698611802896947,
+      "learning_rate": 4.470879503442132e-08,
+      "logits/chosen": -0.6120142936706543,
+      "logits/rejected": -0.5297871828079224,
+      "logps/chosen": -4.30316162109375,
+      "logps/rejected": -5.0614094734191895,
+      "loss": 0.0503,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.30316162109375,
+      "rewards/margins": 0.7582475543022156,
+      "rewards/rejected": -5.0614094734191895,
+      "sft_loss": 4.057737350463867,
+      "step": 4920
+    },
+    {
+      "epoch": 2.6358922896805486,
+      "grad_norm": 0.5486338559058009,
+      "learning_rate": 4.406728724554154e-08,
+      "logits/chosen": -0.7255766987800598,
+      "logits/rejected": -0.454000324010849,
+      "logps/chosen": -4.289834976196289,
+      "logps/rejected": -5.203850746154785,
+      "loss": 0.0496,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.289834976196289,
+      "rewards/margins": 0.9140159487724304,
+      "rewards/rejected": -5.203850746154785,
+      "sft_loss": 4.062923908233643,
+      "step": 4925
+    },
+    {
+      "epoch": 2.6385683224619503,
+      "grad_norm": 0.5063843785218863,
+      "learning_rate": 4.3430202917840664e-08,
+      "logits/chosen": -0.5567297339439392,
+      "logits/rejected": -0.3618205189704895,
+      "logps/chosen": -4.474720478057861,
+      "logps/rejected": -5.34441614151001,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.474720478057861,
+      "rewards/margins": 0.8696953654289246,
+      "rewards/rejected": -5.34441614151001,
+      "sft_loss": 4.205283164978027,
+      "step": 4930
+    },
+    {
+      "epoch": 2.6412443552433515,
+      "grad_norm": 0.5616850632105653,
+      "learning_rate": 4.279754823231346e-08,
+      "logits/chosen": -0.6103255152702332,
+      "logits/rejected": -0.4230746328830719,
+      "logps/chosen": -4.202220916748047,
+      "logps/rejected": -5.077874660491943,
+      "loss": 0.0498,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.202220916748047,
+      "rewards/margins": 0.8756539225578308,
+      "rewards/rejected": -5.077874660491943,
+      "sft_loss": 3.887943983078003,
+      "step": 4935
+    },
+    {
+      "epoch": 2.6439203880247533,
+      "grad_norm": 0.509725536427103,
+      "learning_rate": 4.216932932697859e-08,
+      "logits/chosen": -0.6400852799415588,
+      "logits/rejected": -0.5801583528518677,
+      "logps/chosen": -4.237285614013672,
+      "logps/rejected": -4.9107465744018555,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.237285614013672,
+      "rewards/margins": 0.6734606027603149,
+      "rewards/rejected": -4.9107465744018555,
+      "sft_loss": 3.98652982711792,
+      "step": 4940
+    },
+    {
+      "epoch": 2.646596420806155,
+      "grad_norm": 0.6003025487895531,
+      "learning_rate": 4.154555229681844e-08,
+      "logits/chosen": -0.6006342172622681,
+      "logits/rejected": -0.3996833860874176,
+      "logps/chosen": -4.317721843719482,
+      "logps/rejected": -5.138119697570801,
+      "loss": 0.05,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.317721843719482,
+      "rewards/margins": 0.8203983306884766,
+      "rewards/rejected": -5.138119697570801,
+      "sft_loss": 3.9243438243865967,
+      "step": 4945
+    },
+    {
+      "epoch": 2.6492724535875567,
+      "grad_norm": 0.7337346570788273,
+      "learning_rate": 4.092622319372069e-08,
+      "logits/chosen": -0.5690840482711792,
+      "logits/rejected": -0.3954506814479828,
+      "logps/chosen": -4.2830095291137695,
+      "logps/rejected": -5.03558874130249,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.2830095291137695,
+      "rewards/margins": 0.752579391002655,
+      "rewards/rejected": -5.03558874130249,
+      "sft_loss": 3.9829344749450684,
+      "step": 4950
+    },
+    {
+      "epoch": 2.651948486368958,
+      "grad_norm": 0.4895325217640353,
+      "learning_rate": 4.031134802641889e-08,
+      "logits/chosen": -0.5619646906852722,
+      "logits/rejected": -0.5575190782546997,
+      "logps/chosen": -4.584791660308838,
+      "logps/rejected": -5.15277099609375,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.584791660308838,
+      "rewards/margins": 0.5679799318313599,
+      "rewards/rejected": -5.15277099609375,
+      "sft_loss": 4.250580787658691,
+      "step": 4955
+    },
+    {
+      "epoch": 2.6546245191503597,
+      "grad_norm": 0.5452690088539321,
+      "learning_rate": 3.970093276043468e-08,
+      "logits/chosen": -0.535243809223175,
+      "logits/rejected": -0.47107964754104614,
+      "logps/chosen": -4.297226905822754,
+      "logps/rejected": -5.147265911102295,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.297226905822754,
+      "rewards/margins": 0.8500385284423828,
+      "rewards/rejected": -5.147265911102295,
+      "sft_loss": 4.075446128845215,
+      "step": 4960
+    },
+    {
+      "epoch": 2.657300551931761,
+      "grad_norm": 0.4787818394131923,
+      "learning_rate": 3.9094983318019584e-08,
+      "logits/chosen": -0.67536860704422,
+      "logits/rejected": -0.5395069718360901,
+      "logps/chosen": -4.3160905838012695,
+      "logps/rejected": -5.104388236999512,
+      "loss": 0.051,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.3160905838012695,
+      "rewards/margins": 0.7882981300354004,
+      "rewards/rejected": -5.104388236999512,
+      "sft_loss": 4.1181511878967285,
+      "step": 4965
+    },
+    {
+      "epoch": 2.6599765847131627,
+      "grad_norm": 0.5490272062690024,
+      "learning_rate": 3.849350557809789e-08,
+      "logits/chosen": -0.4833317697048187,
+      "logits/rejected": -0.4706900119781494,
+      "logps/chosen": -4.4363179206848145,
+      "logps/rejected": -5.109228134155273,
+      "loss": 0.0495,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.4363179206848145,
+      "rewards/margins": 0.672910213470459,
+      "rewards/rejected": -5.109228134155273,
+      "sft_loss": 3.9787001609802246,
+      "step": 4970
+    },
+    {
+      "epoch": 2.6626526174945644,
+      "grad_norm": 0.7210787168562746,
+      "learning_rate": 3.789650537620903e-08,
+      "logits/chosen": -0.5201641321182251,
+      "logits/rejected": -0.5272789001464844,
+      "logps/chosen": -4.41770601272583,
+      "logps/rejected": -5.237369537353516,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.41770601272583,
+      "rewards/margins": 0.819663405418396,
+      "rewards/rejected": -5.237369537353516,
+      "sft_loss": 4.144536018371582,
+      "step": 4975
+    },
+    {
+      "epoch": 2.665328650275966,
+      "grad_norm": 0.5031437826574756,
+      "learning_rate": 3.730398850445182e-08,
+      "logits/chosen": -0.42731088399887085,
+      "logits/rejected": -0.4264354109764099,
+      "logps/chosen": -4.396569728851318,
+      "logps/rejected": -5.177783012390137,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.396569728851318,
+      "rewards/margins": 0.7812132239341736,
+      "rewards/rejected": -5.177783012390137,
+      "sft_loss": 4.042810916900635,
+      "step": 4980
+    },
+    {
+      "epoch": 2.6680046830573674,
+      "grad_norm": 0.5440838703218333,
+      "learning_rate": 3.671596071142735e-08,
+      "logits/chosen": -0.519400954246521,
+      "logits/rejected": -0.36543017625808716,
+      "logps/chosen": -4.404606819152832,
+      "logps/rejected": -5.322443008422852,
+      "loss": 0.05,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.404606819152832,
+      "rewards/margins": 0.9178358316421509,
+      "rewards/rejected": -5.322443008422852,
+      "sft_loss": 4.077485084533691,
+      "step": 4985
+    },
+    {
+      "epoch": 2.670680715838769,
+      "grad_norm": 0.547836377113304,
+      "learning_rate": 3.6132427702183996e-08,
+      "logits/chosen": -0.7032692432403564,
+      "logits/rejected": -0.547438383102417,
+      "logps/chosen": -4.2701215744018555,
+      "logps/rejected": -5.307445049285889,
+      "loss": 0.0494,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.2701215744018555,
+      "rewards/margins": 1.0373241901397705,
+      "rewards/rejected": -5.307445049285889,
+      "sft_loss": 4.049527168273926,
+      "step": 4990
+    },
+    {
+      "epoch": 2.6733567486201704,
+      "grad_norm": 0.6747221769683753,
+      "learning_rate": 3.555339513816147e-08,
+      "logits/chosen": -0.6259239315986633,
+      "logits/rejected": -0.6647960543632507,
+      "logps/chosen": -4.4503607749938965,
+      "logps/rejected": -5.176076889038086,
+      "loss": 0.0522,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.4503607749938965,
+      "rewards/margins": 0.7257159352302551,
+      "rewards/rejected": -5.176076889038086,
+      "sft_loss": 4.2231340408325195,
+      "step": 4995
+    },
+    {
+      "epoch": 2.676032781401572,
+      "grad_norm": 0.4978709876574727,
+      "learning_rate": 3.497886863713639e-08,
+      "logits/chosen": -0.639491617679596,
+      "logits/rejected": -0.6323713064193726,
+      "logps/chosen": -4.537841320037842,
+      "logps/rejected": -5.226479530334473,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.537841320037842,
+      "rewards/margins": 0.6886383295059204,
+      "rewards/rejected": -5.226479530334473,
+      "sft_loss": 4.242206573486328,
+      "step": 5000
+    },
+    {
+      "epoch": 2.678708814182974,
+      "grad_norm": 0.5407039474835489,
+      "learning_rate": 3.440885377316721e-08,
+      "logits/chosen": -0.5516177415847778,
+      "logits/rejected": -0.5085643529891968,
+      "logps/chosen": -4.365184307098389,
+      "logps/rejected": -4.952767372131348,
+      "loss": 0.051,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.365184307098389,
+      "rewards/margins": 0.5875837802886963,
+      "rewards/rejected": -4.952767372131348,
+      "sft_loss": 4.07918119430542,
+      "step": 5005
+    },
+    {
+      "epoch": 2.6813848469643755,
+      "grad_norm": 0.5812938263285328,
+      "learning_rate": 3.384335607654082e-08,
+      "logits/chosen": -0.5267273187637329,
+      "logits/rejected": -0.5056958198547363,
+      "logps/chosen": -4.424398899078369,
+      "logps/rejected": -5.259547233581543,
+      "loss": 0.0502,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.424398899078369,
+      "rewards/margins": 0.8351489305496216,
+      "rewards/rejected": -5.259547233581543,
+      "sft_loss": 4.068539619445801,
+      "step": 5010
+    },
+    {
+      "epoch": 2.684060879745777,
+      "grad_norm": 0.5304543852141715,
+      "learning_rate": 3.328238103371811e-08,
+      "logits/chosen": -0.5788456201553345,
+      "logits/rejected": -0.5353022217750549,
+      "logps/chosen": -4.434554576873779,
+      "logps/rejected": -5.322540760040283,
+      "loss": 0.0493,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.434554576873779,
+      "rewards/margins": 0.8879860639572144,
+      "rewards/rejected": -5.322540760040283,
+      "sft_loss": 4.054621696472168,
+      "step": 5015
+    },
+    {
+      "epoch": 2.6867369125271785,
+      "grad_norm": 0.6533298230537002,
+      "learning_rate": 3.272593408728169e-08,
+      "logits/chosen": -0.6691602468490601,
+      "logits/rejected": -0.43237876892089844,
+      "logps/chosen": -4.330901145935059,
+      "logps/rejected": -5.163939952850342,
+      "loss": 0.0499,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.330901145935059,
+      "rewards/margins": 0.8330386877059937,
+      "rewards/rejected": -5.163939952850342,
+      "sft_loss": 4.0684685707092285,
+      "step": 5020
+    },
+    {
+      "epoch": 2.6894129453085798,
+      "grad_norm": 0.459859314522557,
+      "learning_rate": 3.217402063588204e-08,
+      "logits/chosen": -0.6769046783447266,
+      "logits/rejected": -0.5177081823348999,
+      "logps/chosen": -4.403720855712891,
+      "logps/rejected": -5.134342670440674,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.403720855712891,
+      "rewards/margins": 0.7306212782859802,
+      "rewards/rejected": -5.134342670440674,
+      "sft_loss": 4.200765132904053,
+      "step": 5025
+    },
+    {
+      "epoch": 2.6920889780899815,
+      "grad_norm": 0.6249843436541217,
+      "learning_rate": 3.162664603418608e-08,
+      "logits/chosen": -0.5779106020927429,
+      "logits/rejected": -0.5267711281776428,
+      "logps/chosen": -4.2933197021484375,
+      "logps/rejected": -5.195624351501465,
+      "loss": 0.0499,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.2933197021484375,
+      "rewards/margins": 0.9023048281669617,
+      "rewards/rejected": -5.195624351501465,
+      "sft_loss": 4.026488304138184,
+      "step": 5030
+    },
+    {
+      "epoch": 2.694765010871383,
+      "grad_norm": 0.40991567740406004,
+      "learning_rate": 3.1083815592824416e-08,
+      "logits/chosen": -0.636762797832489,
+      "logits/rejected": -0.5826548337936401,
+      "logps/chosen": -4.438990592956543,
+      "logps/rejected": -5.237557411193848,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.438990592956543,
+      "rewards/margins": 0.7985668182373047,
+      "rewards/rejected": -5.237557411193848,
+      "sft_loss": 4.206995964050293,
+      "step": 5035
+    },
+    {
+      "epoch": 2.697441043652785,
+      "grad_norm": 0.5674535681916036,
+      "learning_rate": 3.054553457834053e-08,
+      "logits/chosen": -0.4564667344093323,
+      "logits/rejected": -0.5286347270011902,
+      "logps/chosen": -4.551898002624512,
+      "logps/rejected": -5.162118911743164,
+      "loss": 0.052,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.551898002624512,
+      "rewards/margins": 0.6102203130722046,
+      "rewards/rejected": -5.162118911743164,
+      "sft_loss": 4.228566646575928,
+      "step": 5040
+    },
+    {
+      "epoch": 2.700117076434186,
+      "grad_norm": 0.8191991141280918,
+      "learning_rate": 3.0011808213139036e-08,
+      "logits/chosen": -0.4883642792701721,
+      "logits/rejected": -0.5476293563842773,
+      "logps/chosen": -4.421045780181885,
+      "logps/rejected": -5.0714616775512695,
+      "loss": 0.0505,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.421045780181885,
+      "rewards/margins": 0.650415301322937,
+      "rewards/rejected": -5.0714616775512695,
+      "sft_loss": 4.127926826477051,
+      "step": 5045
+    },
+    {
+      "epoch": 2.702793109215588,
+      "grad_norm": 0.8811087819806706,
+      "learning_rate": 2.948264167543568e-08,
+      "logits/chosen": -0.6045140624046326,
+      "logits/rejected": -0.5384324789047241,
+      "logps/chosen": -4.425509452819824,
+      "logps/rejected": -5.067124366760254,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.425509452819824,
+      "rewards/margins": 0.6416141986846924,
+      "rewards/rejected": -5.067124366760254,
+      "sft_loss": 4.064424991607666,
+      "step": 5050
+    },
+    {
+      "epoch": 2.7054691419969896,
+      "grad_norm": 0.5335643620302218,
+      "learning_rate": 2.8958040099206216e-08,
+      "logits/chosen": -0.7181905508041382,
+      "logits/rejected": -0.6650737524032593,
+      "logps/chosen": -4.427859306335449,
+      "logps/rejected": -5.1806960105896,
+      "loss": 0.0503,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.427859306335449,
+      "rewards/margins": 0.7528366446495056,
+      "rewards/rejected": -5.1806960105896,
+      "sft_loss": 4.122951030731201,
+      "step": 5055
+    },
+    {
+      "epoch": 2.708145174778391,
+      "grad_norm": 0.6765859587891434,
+      "learning_rate": 2.843800857413775e-08,
+      "logits/chosen": -0.5532930493354797,
+      "logits/rejected": -0.5283568501472473,
+      "logps/chosen": -4.372910022735596,
+      "logps/rejected": -4.986554145812988,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.372910022735596,
+      "rewards/margins": 0.6136443614959717,
+      "rewards/rejected": -4.986554145812988,
+      "sft_loss": 4.0953049659729,
+      "step": 5060
+    },
+    {
+      "epoch": 2.7108212075597926,
+      "grad_norm": 0.6437133408390605,
+      "learning_rate": 2.7922552145578203e-08,
+      "logits/chosen": -0.5621504783630371,
+      "logits/rejected": -0.3205257058143616,
+      "logps/chosen": -4.362854957580566,
+      "logps/rejected": -5.120060920715332,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.362854957580566,
+      "rewards/margins": 0.7572060823440552,
+      "rewards/rejected": -5.120060920715332,
+      "sft_loss": 4.085061550140381,
+      "step": 5065
+    },
+    {
+      "epoch": 2.7134972403411943,
+      "grad_norm": 0.5586713434946048,
+      "learning_rate": 2.7411675814488277e-08,
+      "logits/chosen": -0.580437183380127,
+      "logits/rejected": -0.4562060832977295,
+      "logps/chosen": -4.351175785064697,
+      "logps/rejected": -5.0411834716796875,
+      "loss": 0.0501,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.351175785064697,
+      "rewards/margins": 0.690007209777832,
+      "rewards/rejected": -5.0411834716796875,
+      "sft_loss": 4.134765625,
+      "step": 5070
+    },
+    {
+      "epoch": 2.7161732731225956,
+      "grad_norm": 0.7816086223679309,
+      "learning_rate": 2.690538453739216e-08,
+      "logits/chosen": -0.5848197340965271,
+      "logits/rejected": -0.5601023435592651,
+      "logps/chosen": -4.249561309814453,
+      "logps/rejected": -4.906055927276611,
+      "loss": 0.0528,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.249561309814453,
+      "rewards/margins": 0.656494140625,
+      "rewards/rejected": -4.906055927276611,
+      "sft_loss": 4.066379070281982,
+      "step": 5075
+    },
+    {
+      "epoch": 2.7188493059039973,
+      "grad_norm": 0.4891229609407562,
+      "learning_rate": 2.6403683226330298e-08,
+      "logits/chosen": -0.6629522442817688,
+      "logits/rejected": -0.5027410387992859,
+      "logps/chosen": -4.347064018249512,
+      "logps/rejected": -5.038262367248535,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.347064018249512,
+      "rewards/margins": 0.6911985278129578,
+      "rewards/rejected": -5.038262367248535,
+      "sft_loss": 4.133894443511963,
+      "step": 5080
+    },
+    {
+      "epoch": 2.721525338685399,
+      "grad_norm": 0.5027988591263001,
+      "learning_rate": 2.5906576748810804e-08,
+      "logits/chosen": -0.6160419583320618,
+      "logits/rejected": -0.5071651339530945,
+      "logps/chosen": -4.447991371154785,
+      "logps/rejected": -5.372227668762207,
+      "loss": 0.0501,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -4.447991371154785,
+      "rewards/margins": 0.9242367744445801,
+      "rewards/rejected": -5.372227668762207,
+      "sft_loss": 4.256869316101074,
+      "step": 5085
+    },
+    {
+      "epoch": 2.7242013714668003,
+      "grad_norm": 0.5005779121654812,
+      "learning_rate": 2.5414069927763016e-08,
+      "logits/chosen": -0.7470074892044067,
+      "logits/rejected": -0.5773445963859558,
+      "logps/chosen": -4.272568702697754,
+      "logps/rejected": -5.214876174926758,
+      "loss": 0.049,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -4.272568702697754,
+      "rewards/margins": 0.9423072934150696,
+      "rewards/rejected": -5.214876174926758,
+      "sft_loss": 4.018656253814697,
+      "step": 5090
+    },
+    {
+      "epoch": 2.726877404248202,
+      "grad_norm": 0.4030114584917921,
+      "learning_rate": 2.4926167541490185e-08,
+      "logits/chosen": -0.7731162905693054,
+      "logits/rejected": -0.5590254068374634,
+      "logps/chosen": -4.330112934112549,
+      "logps/rejected": -5.2436041831970215,
+      "loss": 0.0505,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.330112934112549,
+      "rewards/margins": 0.9134915471076965,
+      "rewards/rejected": -5.2436041831970215,
+      "sft_loss": 4.119341850280762,
+      "step": 5095
+    },
+    {
+      "epoch": 2.7295534370296037,
+      "grad_norm": 0.5624609914644404,
+      "learning_rate": 2.4442874323623574e-08,
+      "logits/chosen": -0.5609525442123413,
+      "logits/rejected": -0.42731085419654846,
+      "logps/chosen": -4.410833835601807,
+      "logps/rejected": -5.249048709869385,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.410833835601807,
+      "rewards/margins": 0.8382146954536438,
+      "rewards/rejected": -5.249048709869385,
+      "sft_loss": 4.111285209655762,
+      "step": 5100
+    },
+    {
+      "epoch": 2.7322294698110055,
+      "grad_norm": 0.6188222511053089,
+      "learning_rate": 2.396419496307589e-08,
+      "logits/chosen": -0.6179810166358948,
+      "logits/rejected": -0.4628868103027344,
+      "logps/chosen": -4.559763431549072,
+      "logps/rejected": -5.188269138336182,
+      "loss": 0.0504,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.559763431549072,
+      "rewards/margins": 0.6285057067871094,
+      "rewards/rejected": -5.188269138336182,
+      "sft_loss": 4.189999580383301,
+      "step": 5105
+    },
+    {
+      "epoch": 2.7349055025924067,
+      "grad_norm": 0.45046317998459096,
+      "learning_rate": 2.349013410399653e-08,
+      "logits/chosen": -0.6038917303085327,
+      "logits/rejected": -0.5444539189338684,
+      "logps/chosen": -4.464918613433838,
+      "logps/rejected": -5.283292293548584,
+      "loss": 0.0501,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.464918613433838,
+      "rewards/margins": 0.81837397813797,
+      "rewards/rejected": -5.283292293548584,
+      "sft_loss": 4.040930271148682,
+      "step": 5110
+    },
+    {
+      "epoch": 2.7375815353738084,
+      "grad_norm": 0.4990731300671213,
+      "learning_rate": 2.3020696345725954e-08,
+      "logits/chosen": -0.6978880167007446,
+      "logits/rejected": -0.515345573425293,
+      "logps/chosen": -4.216785907745361,
+      "logps/rejected": -5.36154842376709,
+      "loss": 0.0484,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -4.216785907745361,
+      "rewards/margins": 1.144762635231018,
+      "rewards/rejected": -5.36154842376709,
+      "sft_loss": 3.9843826293945312,
+      "step": 5115
+    },
+    {
+      "epoch": 2.7402575681552097,
+      "grad_norm": 0.614831405515578,
+      "learning_rate": 2.2555886242751398e-08,
+      "logits/chosen": -0.624744713306427,
+      "logits/rejected": -0.5800564885139465,
+      "logps/chosen": -4.2822980880737305,
+      "logps/rejected": -5.1204705238342285,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.2822980880737305,
+      "rewards/margins": 0.8381722569465637,
+      "rewards/rejected": -5.1204705238342285,
+      "sft_loss": 4.099765300750732,
+      "step": 5120
+    },
+    {
+      "epoch": 2.7429336009366114,
+      "grad_norm": 0.6935432896593682,
+      "learning_rate": 2.2095708304662453e-08,
+      "logits/chosen": -0.7441667318344116,
+      "logits/rejected": -0.5041067600250244,
+      "logps/chosen": -4.457093238830566,
+      "logps/rejected": -5.133129596710205,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.457093238830566,
+      "rewards/margins": 0.6760369539260864,
+      "rewards/rejected": -5.133129596710205,
+      "sft_loss": 4.1581549644470215,
+      "step": 5125
+    },
+    {
+      "epoch": 2.745609633718013,
+      "grad_norm": 0.4785476752747039,
+      "learning_rate": 2.16401669961076e-08,
+      "logits/chosen": -0.7660086750984192,
+      "logits/rejected": -0.5496563911437988,
+      "logps/chosen": -4.31410551071167,
+      "logps/rejected": -5.084970951080322,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.31410551071167,
+      "rewards/margins": 0.7708655595779419,
+      "rewards/rejected": -5.084970951080322,
+      "sft_loss": 4.037846088409424,
+      "step": 5130
+    },
+    {
+      "epoch": 2.748285666499415,
+      "grad_norm": 0.5327286440548609,
+      "learning_rate": 2.1189266736750532e-08,
+      "logits/chosen": -0.5484969615936279,
+      "logits/rejected": -0.520624041557312,
+      "logps/chosen": -4.489085674285889,
+      "logps/rejected": -5.11185359954834,
+      "loss": 0.0523,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.489085674285889,
+      "rewards/margins": 0.622767448425293,
+      "rewards/rejected": -5.11185359954834,
+      "sft_loss": 4.165780067443848,
+      "step": 5135
+    },
+    {
+      "epoch": 2.750961699280816,
+      "grad_norm": 0.8101164199907651,
+      "learning_rate": 2.0743011901227623e-08,
+      "logits/chosen": -0.5449343919754028,
+      "logits/rejected": -0.4379898011684418,
+      "logps/chosen": -4.064428806304932,
+      "logps/rejected": -4.964702606201172,
+      "loss": 0.05,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.064428806304932,
+      "rewards/margins": 0.900274395942688,
+      "rewards/rejected": -4.964702606201172,
+      "sft_loss": 3.8239989280700684,
+      "step": 5140
+    },
+    {
+      "epoch": 2.753637732062218,
+      "grad_norm": 0.6566488278877074,
+      "learning_rate": 2.030140681910508e-08,
+      "logits/chosen": -0.5559419989585876,
+      "logits/rejected": -0.4396211504936218,
+      "logps/chosen": -4.3024468421936035,
+      "logps/rejected": -5.083981513977051,
+      "loss": 0.0501,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.3024468421936035,
+      "rewards/margins": 0.7815349698066711,
+      "rewards/rejected": -5.083981513977051,
+      "sft_loss": 4.013249397277832,
+      "step": 5145
+    },
+    {
+      "epoch": 2.756313764843619,
+      "grad_norm": 0.3999227713979821,
+      "learning_rate": 1.986445577483753e-08,
+      "logits/chosen": -0.6439892053604126,
+      "logits/rejected": -0.5110267400741577,
+      "logps/chosen": -4.2533369064331055,
+      "logps/rejected": -5.045928001403809,
+      "loss": 0.0503,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.2533369064331055,
+      "rewards/margins": 0.7925913333892822,
+      "rewards/rejected": -5.045928001403809,
+      "sft_loss": 4.006933689117432,
+      "step": 5150
+    },
+    {
+      "epoch": 2.758989797625021,
+      "grad_norm": 0.46329755465442035,
+      "learning_rate": 1.9432163007725765e-08,
+      "logits/chosen": -0.6546041369438171,
+      "logits/rejected": -0.5829042196273804,
+      "logps/chosen": -4.438974380493164,
+      "logps/rejected": -5.130999565124512,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.438974380493164,
+      "rewards/margins": 0.6920250654220581,
+      "rewards/rejected": -5.130999565124512,
+      "sft_loss": 4.231729507446289,
+      "step": 5155
+    },
+    {
+      "epoch": 2.7616658304064226,
+      "grad_norm": 0.41623003867322117,
+      "learning_rate": 1.9004532711876297e-08,
+      "logits/chosen": -0.5996496081352234,
+      "logits/rejected": -0.6131302714347839,
+      "logps/chosen": -4.398252487182617,
+      "logps/rejected": -5.066895961761475,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.398252487182617,
+      "rewards/margins": 0.6686434149742126,
+      "rewards/rejected": -5.066895961761475,
+      "sft_loss": 4.139983177185059,
+      "step": 5160
+    },
+    {
+      "epoch": 2.7643418631878243,
+      "grad_norm": 0.41866041657143405,
+      "learning_rate": 1.8581569036159928e-08,
+      "logits/chosen": -0.6335171461105347,
+      "logits/rejected": -0.4419315755367279,
+      "logps/chosen": -4.260961055755615,
+      "logps/rejected": -5.124476432800293,
+      "loss": 0.0503,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.260961055755615,
+      "rewards/margins": 0.8635152578353882,
+      "rewards/rejected": -5.124476432800293,
+      "sft_loss": 4.06007719039917,
+      "step": 5165
+    },
+    {
+      "epoch": 2.7670178959692255,
+      "grad_norm": 0.4901236585940214,
+      "learning_rate": 1.8163276084172285e-08,
+      "logits/chosen": -0.5955950021743774,
+      "logits/rejected": -0.5076081156730652,
+      "logps/chosen": -4.346221923828125,
+      "logps/rejected": -5.228376865386963,
+      "loss": 0.0491,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.346221923828125,
+      "rewards/margins": 0.8821552395820618,
+      "rewards/rejected": -5.228376865386963,
+      "sft_loss": 4.07955265045166,
+      "step": 5170
+    },
+    {
+      "epoch": 2.7696939287506273,
+      "grad_norm": 0.5751591218017986,
+      "learning_rate": 1.7749657914193194e-08,
+      "logits/chosen": -0.6155784726142883,
+      "logits/rejected": -0.5799046754837036,
+      "logps/chosen": -4.4383625984191895,
+      "logps/rejected": -5.276667594909668,
+      "loss": 0.0493,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.4383625984191895,
+      "rewards/margins": 0.8383051156997681,
+      "rewards/rejected": -5.276667594909668,
+      "sft_loss": 4.1238274574279785,
+      "step": 5175
+    },
+    {
+      "epoch": 2.7723699615320285,
+      "grad_norm": 0.5618583884966403,
+      "learning_rate": 1.7340718539148203e-08,
+      "logits/chosen": -0.5694500207901001,
+      "logits/rejected": -0.5331536531448364,
+      "logps/chosen": -4.376175880432129,
+      "logps/rejected": -5.064773082733154,
+      "loss": 0.052,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.376175880432129,
+      "rewards/margins": 0.6885972023010254,
+      "rewards/rejected": -5.064773082733154,
+      "sft_loss": 4.164933204650879,
+      "step": 5180
+    },
+    {
+      "epoch": 2.7750459943134302,
+      "grad_norm": 0.39402613135728615,
+      "learning_rate": 1.6936461926568724e-08,
+      "logits/chosen": -0.554807722568512,
+      "logits/rejected": -0.42144089937210083,
+      "logps/chosen": -4.39473819732666,
+      "logps/rejected": -5.201816558837891,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.39473819732666,
+      "rewards/margins": 0.8070778846740723,
+      "rewards/rejected": -5.201816558837891,
+      "sft_loss": 4.072976589202881,
+      "step": 5185
+    },
+    {
+      "epoch": 2.777722027094832,
+      "grad_norm": 0.5093891000184223,
+      "learning_rate": 1.6536891998554346e-08,
+      "logits/chosen": -0.6842007637023926,
+      "logits/rejected": -0.5274578332901001,
+      "logps/chosen": -4.328361511230469,
+      "logps/rejected": -5.09237003326416,
+      "loss": 0.0499,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -4.328361511230469,
+      "rewards/margins": 0.764008641242981,
+      "rewards/rejected": -5.09237003326416,
+      "sft_loss": 4.056042671203613,
+      "step": 5190
+    },
+    {
+      "epoch": 2.7803980598762337,
+      "grad_norm": 0.46814498565552387,
+      "learning_rate": 1.6142012631734093e-08,
+      "logits/chosen": -0.5290490388870239,
+      "logits/rejected": -0.4205712676048279,
+      "logps/chosen": -4.346985340118408,
+      "logps/rejected": -5.145593643188477,
+      "loss": 0.0491,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.346985340118408,
+      "rewards/margins": 0.7986089587211609,
+      "rewards/rejected": -5.145593643188477,
+      "sft_loss": 4.0815887451171875,
+      "step": 5195
+    },
+    {
+      "epoch": 2.783074092657635,
+      "grad_norm": 0.47857097670023907,
+      "learning_rate": 1.575182765722949e-08,
+      "logits/chosen": -0.7216336131095886,
+      "logits/rejected": -0.5649983882904053,
+      "logps/chosen": -4.365129470825195,
+      "logps/rejected": -5.156970977783203,
+      "loss": 0.0496,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.365129470825195,
+      "rewards/margins": 0.7918413877487183,
+      "rewards/rejected": -5.156970977783203,
+      "sft_loss": 3.9859061241149902,
+      "step": 5200
+    },
+    {
+      "epoch": 2.783074092657635,
+      "eval_logits/chosen": -0.14752337336540222,
+      "eval_logits/rejected": -0.06999073177576065,
+      "eval_logps/chosen": -4.349379062652588,
+      "eval_logps/rejected": -5.108426094055176,
+      "eval_loss": 0.05009545013308525,
+      "eval_rewards/accuracies": 0.6787833571434021,
+      "eval_rewards/chosen": -4.349379062652588,
+      "eval_rewards/margins": 0.7590476274490356,
+      "eval_rewards/rejected": -5.108426094055176,
+      "eval_runtime": 43.5685,
+      "eval_samples_per_second": 30.871,
+      "eval_sft_loss": 3.96047306060791,
+      "eval_steps_per_second": 7.735,
+      "step": 5200
+    },
+    {
+      "epoch": 2.7857501254390367,
+      "grad_norm": 0.7191144036543534,
+      "learning_rate": 1.536634086061672e-08,
+      "logits/chosen": -0.5561727285385132,
+      "logits/rejected": -0.565574049949646,
+      "logps/chosen": -4.195740699768066,
+      "logps/rejected": -5.171977996826172,
+      "loss": 0.0491,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.195740699768066,
+      "rewards/margins": 0.9762368202209473,
+      "rewards/rejected": -5.171977996826172,
+      "sft_loss": 3.954725742340088,
+      "step": 5205
+    },
+    {
+      "epoch": 2.788426158220438,
+      "grad_norm": 0.5988576367474885,
+      "learning_rate": 1.4985555981890495e-08,
+      "logits/chosen": -0.6057511568069458,
+      "logits/rejected": -0.5261834263801575,
+      "logps/chosen": -4.54041051864624,
+      "logps/rejected": -5.296923637390137,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.54041051864624,
+      "rewards/margins": 0.7565131187438965,
+      "rewards/rejected": -5.296923637390137,
+      "sft_loss": 4.197884559631348,
+      "step": 5210
+    },
+    {
+      "epoch": 2.7911021910018396,
+      "grad_norm": 0.4914619998876332,
+      "learning_rate": 1.4609476715427226e-08,
+      "logits/chosen": -0.5733352899551392,
+      "logits/rejected": -0.5202125906944275,
+      "logps/chosen": -4.2142791748046875,
+      "logps/rejected": -5.104830265045166,
+      "loss": 0.0494,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -4.2142791748046875,
+      "rewards/margins": 0.8905513882637024,
+      "rewards/rejected": -5.104830265045166,
+      "sft_loss": 4.010748863220215,
+      "step": 5215
+    },
+    {
+      "epoch": 2.7937782237832414,
+      "grad_norm": 0.5800766824514525,
+      "learning_rate": 1.4238106709949792e-08,
+      "logits/chosen": -0.5682021975517273,
+      "logits/rejected": -0.5216585397720337,
+      "logps/chosen": -4.326835632324219,
+      "logps/rejected": -5.297267436981201,
+      "loss": 0.049,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.326835632324219,
+      "rewards/margins": 0.9704321622848511,
+      "rewards/rejected": -5.297267436981201,
+      "sft_loss": 4.038890838623047,
+      "step": 5220
+    },
+    {
+      "epoch": 2.796454256564643,
+      "grad_norm": 0.6007376467369648,
+      "learning_rate": 1.3871449568491511e-08,
+      "logits/chosen": -0.5569092035293579,
+      "logits/rejected": -0.3917415738105774,
+      "logps/chosen": -4.43691349029541,
+      "logps/rejected": -5.082301616668701,
+      "loss": 0.0526,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.43691349029541,
+      "rewards/margins": 0.6453880071640015,
+      "rewards/rejected": -5.082301616668701,
+      "sft_loss": 4.11795711517334,
+      "step": 5225
+    },
+    {
+      "epoch": 2.7991302893460444,
+      "grad_norm": 0.5685464016697865,
+      "learning_rate": 1.3509508848361606e-08,
+      "logits/chosen": -0.6649554967880249,
+      "logits/rejected": -0.5384029150009155,
+      "logps/chosen": -4.373221397399902,
+      "logps/rejected": -5.240781784057617,
+      "loss": 0.0485,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -4.373221397399902,
+      "rewards/margins": 0.8675606846809387,
+      "rewards/rejected": -5.240781784057617,
+      "sft_loss": 3.96380615234375,
+      "step": 5230
+    },
+    {
+      "epoch": 2.801806322127446,
+      "grad_norm": 0.5009504983062434,
+      "learning_rate": 1.3152288061110517e-08,
+      "logits/chosen": -0.6154955625534058,
+      "logits/rejected": -0.48099485039711,
+      "logps/chosen": -4.260126113891602,
+      "logps/rejected": -5.21017599105835,
+      "loss": 0.0492,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.260126113891602,
+      "rewards/margins": 0.9500495791435242,
+      "rewards/rejected": -5.21017599105835,
+      "sft_loss": 3.9783453941345215,
+      "step": 5235
+    },
+    {
+      "epoch": 2.804482354908848,
+      "grad_norm": 0.5159901730284345,
+      "learning_rate": 1.2799790672495814e-08,
+      "logits/chosen": -0.662651002407074,
+      "logits/rejected": -0.4633462429046631,
+      "logps/chosen": -4.354351043701172,
+      "logps/rejected": -5.179112434387207,
+      "loss": 0.0498,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": -4.354351043701172,
+      "rewards/margins": 0.8247610926628113,
+      "rewards/rejected": -5.179112434387207,
+      "sft_loss": 4.13070821762085,
+      "step": 5240
+    },
+    {
+      "epoch": 2.807158387690249,
+      "grad_norm": 0.4095611145450703,
+      "learning_rate": 1.2452020102448835e-08,
+      "logits/chosen": -0.5761651396751404,
+      "logits/rejected": -0.5396815538406372,
+      "logps/chosen": -4.335955619812012,
+      "logps/rejected": -5.095559120178223,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.335955619812012,
+      "rewards/margins": 0.7596035599708557,
+      "rewards/rejected": -5.095559120178223,
+      "sft_loss": 4.165452480316162,
+      "step": 5245
+    },
+    {
+      "epoch": 2.8098344204716508,
+      "grad_norm": 0.6584993978510416,
+      "learning_rate": 1.2108979725041103e-08,
+      "logits/chosen": -0.7338468432426453,
+      "logits/rejected": -0.5876275897026062,
+      "logps/chosen": -4.284478664398193,
+      "logps/rejected": -5.134871482849121,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": -4.284478664398193,
+      "rewards/margins": 0.8503926396369934,
+      "rewards/rejected": -5.134871482849121,
+      "sft_loss": 4.059378623962402,
+      "step": 5250
+    },
+    {
+      "epoch": 2.8125104532530525,
+      "grad_norm": 0.5669141587129255,
+      "learning_rate": 1.1770672868451958e-08,
+      "logits/chosen": -0.6544702649116516,
+      "logits/rejected": -0.4560364782810211,
+      "logps/chosen": -4.261950492858887,
+      "logps/rejected": -5.159637451171875,
+      "loss": 0.0503,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.261950492858887,
+      "rewards/margins": 0.8976861834526062,
+      "rewards/rejected": -5.159637451171875,
+      "sft_loss": 4.083946228027344,
+      "step": 5255
+    },
+    {
+      "epoch": 2.8151864860344538,
+      "grad_norm": 0.45359611330774524,
+      "learning_rate": 1.1437102814935872e-08,
+      "logits/chosen": -0.5684564113616943,
+      "logits/rejected": -0.5300565361976624,
+      "logps/chosen": -4.360495090484619,
+      "logps/rejected": -5.112468719482422,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.360495090484619,
+      "rewards/margins": 0.7519737482070923,
+      "rewards/rejected": -5.112468719482422,
+      "sft_loss": 4.135025978088379,
+      "step": 5260
+    },
+    {
+      "epoch": 2.8178625188158555,
+      "grad_norm": 0.4332229391607074,
+      "learning_rate": 1.1108272800791018e-08,
+      "logits/chosen": -0.7341753840446472,
+      "logits/rejected": -0.5355566740036011,
+      "logps/chosen": -4.16178035736084,
+      "logps/rejected": -5.275792121887207,
+      "loss": 0.0479,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -4.16178035736084,
+      "rewards/margins": 1.1140128374099731,
+      "rewards/rejected": -5.275792121887207,
+      "sft_loss": 3.9122257232666016,
+      "step": 5265
+    },
+    {
+      "epoch": 2.820538551597257,
+      "grad_norm": 0.4923626008267502,
+      "learning_rate": 1.078418601632769e-08,
+      "logits/chosen": -0.5636851191520691,
+      "logits/rejected": -0.4564580023288727,
+      "logps/chosen": -4.271021842956543,
+      "logps/rejected": -5.0314130783081055,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -4.271021842956543,
+      "rewards/margins": 0.7603915929794312,
+      "rewards/rejected": -5.0314130783081055,
+      "sft_loss": 4.0438079833984375,
+      "step": 5270
+    },
+    {
+      "epoch": 2.8232145843786585,
+      "grad_norm": 0.6185752106230644,
+      "learning_rate": 1.0464845605837159e-08,
+      "logits/chosen": -0.6161797642707825,
+      "logits/rejected": -0.49774831533432007,
+      "logps/chosen": -4.328375339508057,
+      "logps/rejected": -5.104809761047363,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.328375339508057,
+      "rewards/margins": 0.7764348387718201,
+      "rewards/rejected": -5.104809761047363,
+      "sft_loss": 4.120706081390381,
+      "step": 5275
+    },
+    {
+      "epoch": 2.82589061716006,
+      "grad_norm": 0.4635994638293771,
+      "learning_rate": 1.0150254667561642e-08,
+      "logits/chosen": -0.6568474769592285,
+      "logits/rejected": -0.4739796221256256,
+      "logps/chosen": -4.326353549957275,
+      "logps/rejected": -5.257393836975098,
+      "loss": 0.0506,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -4.326353549957275,
+      "rewards/margins": 0.9310402870178223,
+      "rewards/rejected": -5.257393836975098,
+      "sft_loss": 3.9922804832458496,
+      "step": 5280
+    },
+    {
+      "epoch": 2.828566649941462,
+      "grad_norm": 0.46685510339292874,
+      "learning_rate": 9.840416253663719e-09,
+      "logits/chosen": -0.6774117350578308,
+      "logits/rejected": -0.5768987536430359,
+      "logps/chosen": -4.310311317443848,
+      "logps/rejected": -5.213183879852295,
+      "loss": 0.0505,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.310311317443848,
+      "rewards/margins": 0.9028725624084473,
+      "rewards/rejected": -5.213183879852295,
+      "sft_loss": 4.074436187744141,
+      "step": 5285
+    },
+    {
+      "epoch": 2.8312426827228636,
+      "grad_norm": 0.5414238630743532,
+      "learning_rate": 9.535333370197074e-09,
+      "logits/chosen": -0.6404070854187012,
+      "logits/rejected": -0.5115911364555359,
+      "logps/chosen": -4.246361255645752,
+      "logps/rejected": -5.055583477020264,
+      "loss": 0.0495,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.246361255645752,
+      "rewards/margins": 0.8092225193977356,
+      "rewards/rejected": -5.055583477020264,
+      "sft_loss": 3.9616026878356934,
+      "step": 5290
+    },
+    {
+      "epoch": 2.833918715504265,
+      "grad_norm": 0.54202729862122,
+      "learning_rate": 9.23500897707713e-09,
+      "logits/chosen": -0.6570879817008972,
+      "logits/rejected": -0.4803641438484192,
+      "logps/chosen": -4.478545188903809,
+      "logps/rejected": -5.1818037033081055,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.478545188903809,
+      "rewards/margins": 0.7032590508460999,
+      "rewards/rejected": -5.1818037033081055,
+      "sft_loss": 4.152520179748535,
+      "step": 5295
+    },
+    {
+      "epoch": 2.8365947482856666,
+      "grad_norm": 0.9681991761052725,
+      "learning_rate": 8.939445988052574e-09,
+      "logits/chosen": -0.5966919660568237,
+      "logits/rejected": -0.5533936023712158,
+      "logps/chosen": -4.413753509521484,
+      "logps/rejected": -5.19551420211792,
+      "loss": 0.0495,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.413753509521484,
+      "rewards/margins": 0.7817603349685669,
+      "rewards/rejected": -5.19551420211792,
+      "sft_loss": 4.069568634033203,
+      "step": 5300
+    },
+    {
+      "epoch": 2.839270781067068,
+      "grad_norm": 0.46309306509542547,
+      "learning_rate": 8.648647270676656e-09,
+      "logits/chosen": -0.5780660510063171,
+      "logits/rejected": -0.4947342872619629,
+      "logps/chosen": -4.218583583831787,
+      "logps/rejected": -5.101345539093018,
+      "loss": 0.0496,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.218583583831787,
+      "rewards/margins": 0.88276207447052,
+      "rewards/rejected": -5.101345539093018,
+      "sft_loss": 3.895846128463745,
+      "step": 5305
+    },
+    {
+      "epoch": 2.8419468138484696,
+      "grad_norm": 0.6254986509892569,
+      "learning_rate": 8.362615646279991e-09,
+      "logits/chosen": -0.7527583837509155,
+      "logits/rejected": -0.4959944784641266,
+      "logps/chosen": -4.352740287780762,
+      "logps/rejected": -5.29129695892334,
+      "loss": 0.0503,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.352740287780762,
+      "rewards/margins": 0.9385570287704468,
+      "rewards/rejected": -5.29129695892334,
+      "sft_loss": 4.110122203826904,
+      "step": 5310
+    },
+    {
+      "epoch": 2.8446228466298713,
+      "grad_norm": 0.562855496131138,
+      "learning_rate": 8.081353889942466e-09,
+      "logits/chosen": -0.5305291414260864,
+      "logits/rejected": -0.37018221616744995,
+      "logps/chosen": -4.3649582862854,
+      "logps/rejected": -5.108063697814941,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.3649582862854,
+      "rewards/margins": 0.7431057095527649,
+      "rewards/rejected": -5.108063697814941,
+      "sft_loss": 4.123278617858887,
+      "step": 5315
+    },
+    {
+      "epoch": 2.847298879411273,
+      "grad_norm": 0.48435276878243533,
+      "learning_rate": 7.804864730467042e-09,
+      "logits/chosen": -0.5514165759086609,
+      "logits/rejected": -0.5572249293327332,
+      "logps/chosen": -4.268599033355713,
+      "logps/rejected": -5.049825668334961,
+      "loss": 0.0499,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.268599033355713,
+      "rewards/margins": 0.7812260389328003,
+      "rewards/rejected": -5.049825668334961,
+      "sft_loss": 4.0047478675842285,
+      "step": 5320
+    },
+    {
+      "epoch": 2.8499749121926743,
+      "grad_norm": 0.5304477339600076,
+      "learning_rate": 7.533150850352665e-09,
+      "logits/chosen": -0.5211262106895447,
+      "logits/rejected": -0.35544973611831665,
+      "logps/chosen": -4.464164733886719,
+      "logps/rejected": -5.338298320770264,
+      "loss": 0.0493,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.464164733886719,
+      "rewards/margins": 0.8741337060928345,
+      "rewards/rejected": -5.338298320770264,
+      "sft_loss": 4.1555376052856445,
+      "step": 5325
+    },
+    {
+      "epoch": 2.852650944974076,
+      "grad_norm": 0.608494156805423,
+      "learning_rate": 7.2662148857686175e-09,
+      "logits/chosen": -0.5175211429595947,
+      "logits/rejected": -0.4477507472038269,
+      "logps/chosen": -4.422387599945068,
+      "logps/rejected": -5.1303510665893555,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.422387599945068,
+      "rewards/margins": 0.7079632878303528,
+      "rewards/rejected": -5.1303510665893555,
+      "sft_loss": 4.057928562164307,
+      "step": 5330
+    },
+    {
+      "epoch": 2.8553269777554773,
+      "grad_norm": 0.5363778155883481,
+      "learning_rate": 7.0040594265287635e-09,
+      "logits/chosen": -0.447273313999176,
+      "logits/rejected": -0.5438799858093262,
+      "logps/chosen": -4.418979644775391,
+      "logps/rejected": -4.9786481857299805,
+      "loss": 0.052,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -4.418979644775391,
+      "rewards/margins": 0.5596680641174316,
+      "rewards/rejected": -4.9786481857299805,
+      "sft_loss": 4.110053062438965,
+      "step": 5335
+    },
+    {
+      "epoch": 2.858003010536879,
+      "grad_norm": 0.5242168256380009,
+      "learning_rate": 6.746687016066566e-09,
+      "logits/chosen": -0.5645424723625183,
+      "logits/rejected": -0.5435028076171875,
+      "logps/chosen": -4.229454517364502,
+      "logps/rejected": -4.962915420532227,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.731249988079071,
+      "rewards/chosen": -4.229454517364502,
+      "rewards/margins": 0.733460545539856,
+      "rewards/rejected": -4.962915420532227,
+      "sft_loss": 4.003708839416504,
+      "step": 5340
+    },
+    {
+      "epoch": 2.8606790433182807,
+      "grad_norm": 1.1722025953068325,
+      "learning_rate": 6.494100151410276e-09,
+      "logits/chosen": -0.7569110989570618,
+      "logits/rejected": -0.6020675897598267,
+      "logps/chosen": -4.250548839569092,
+      "logps/rejected": -5.039128303527832,
+      "loss": 0.0506,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -4.250548839569092,
+      "rewards/margins": 0.7885786294937134,
+      "rewards/rejected": -5.039128303527832,
+      "sft_loss": 4.00954532623291,
+      "step": 5345
+    },
+    {
+      "epoch": 2.8633550760996824,
+      "grad_norm": 0.8999730124367986,
+      "learning_rate": 6.246301283158728e-09,
+      "logits/chosen": -0.5288999080657959,
+      "logits/rejected": -0.5689471364021301,
+      "logps/chosen": -4.336988925933838,
+      "logps/rejected": -5.015877723693848,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.336988925933838,
+      "rewards/margins": 0.6788896322250366,
+      "rewards/rejected": -5.015877723693848,
+      "sft_loss": 4.108680725097656,
+      "step": 5350
+    },
+    {
+      "epoch": 2.8660311088810837,
+      "grad_norm": 0.5531299179246978,
+      "learning_rate": 6.0032928154576944e-09,
+      "logits/chosen": -0.6178763508796692,
+      "logits/rejected": -0.5711098313331604,
+      "logps/chosen": -4.342726707458496,
+      "logps/rejected": -5.064497470855713,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.342726707458496,
+      "rewards/margins": 0.7217706441879272,
+      "rewards/rejected": -5.064497470855713,
+      "sft_loss": 4.154116630554199,
+      "step": 5355
+    },
+    {
+      "epoch": 2.8687071416624854,
+      "grad_norm": 0.8055111496097385,
+      "learning_rate": 5.76507710597629e-09,
+      "logits/chosen": -0.6592285633087158,
+      "logits/rejected": -0.4707743227481842,
+      "logps/chosen": -4.252751350402832,
+      "logps/rejected": -5.118173599243164,
+      "loss": 0.0503,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.252751350402832,
+      "rewards/margins": 0.8654224276542664,
+      "rewards/rejected": -5.118173599243164,
+      "sft_loss": 3.967607021331787,
+      "step": 5360
+    },
+    {
+      "epoch": 2.8713831744438867,
+      "grad_norm": 0.5251638072220567,
+      "learning_rate": 5.531656465884438e-09,
+      "logits/chosen": -0.6563533544540405,
+      "logits/rejected": -0.5304948091506958,
+      "logps/chosen": -4.314505577087402,
+      "logps/rejected": -5.1775736808776855,
+      "loss": 0.0491,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.314505577087402,
+      "rewards/margins": 0.8630677461624146,
+      "rewards/rejected": -5.1775736808776855,
+      "sft_loss": 4.001981258392334,
+      "step": 5365
+    },
+    {
+      "epoch": 2.8740592072252884,
+      "grad_norm": 0.9738682603396716,
+      "learning_rate": 5.303033159830217e-09,
+      "logits/chosen": -0.48909252882003784,
+      "logits/rejected": -0.4886673092842102,
+      "logps/chosen": -4.3931779861450195,
+      "logps/rejected": -4.965169906616211,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.3931779861450195,
+      "rewards/margins": 0.5719920992851257,
+      "rewards/rejected": -4.965169906616211,
+      "sft_loss": 4.030278205871582,
+      "step": 5370
+    },
+    {
+      "epoch": 2.87673524000669,
+      "grad_norm": 0.5118569792634934,
+      "learning_rate": 5.079209405917939e-09,
+      "logits/chosen": -0.5582677125930786,
+      "logits/rejected": -0.5140693187713623,
+      "logps/chosen": -4.305457592010498,
+      "logps/rejected": -5.290476322174072,
+      "loss": 0.0502,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -4.305457592010498,
+      "rewards/margins": 0.985019326210022,
+      "rewards/rejected": -5.290476322174072,
+      "sft_loss": 4.090975761413574,
+      "step": 5375
+    },
+    {
+      "epoch": 2.879411272788092,
+      "grad_norm": 0.49778330014464683,
+      "learning_rate": 4.860187375686664e-09,
+      "logits/chosen": -0.6722379922866821,
+      "logits/rejected": -0.42406749725341797,
+      "logps/chosen": -4.215766429901123,
+      "logps/rejected": -5.1766557693481445,
+      "loss": 0.0489,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -4.215766429901123,
+      "rewards/margins": 0.9608890414237976,
+      "rewards/rejected": -5.1766557693481445,
+      "sft_loss": 3.9776477813720703,
+      "step": 5380
+    },
+    {
+      "epoch": 2.882087305569493,
+      "grad_norm": 0.4995066285378187,
+      "learning_rate": 4.64596919408905e-09,
+      "logits/chosen": -0.4965580999851227,
+      "logits/rejected": -0.45552605390548706,
+      "logps/chosen": -4.254932403564453,
+      "logps/rejected": -4.939602851867676,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.254932403564453,
+      "rewards/margins": 0.6846706867218018,
+      "rewards/rejected": -4.939602851867676,
+      "sft_loss": 3.9634289741516113,
+      "step": 5385
+    },
+    {
+      "epoch": 2.884763338350895,
+      "grad_norm": 0.6083571045434075,
+      "learning_rate": 4.436556939470814e-09,
+      "logits/chosen": -0.6391115188598633,
+      "logits/rejected": -0.47817936539649963,
+      "logps/chosen": -4.613633632659912,
+      "logps/rejected": -5.136800765991211,
+      "loss": 0.0532,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.613633632659912,
+      "rewards/margins": 0.5231670141220093,
+      "rewards/rejected": -5.136800765991211,
+      "sft_loss": 4.38002347946167,
+      "step": 5390
+    },
+    {
+      "epoch": 2.887439371132296,
+      "grad_norm": 0.49564683122909814,
+      "learning_rate": 4.23195264355064e-09,
+      "logits/chosen": -0.7536950707435608,
+      "logits/rejected": -0.49495840072631836,
+      "logps/chosen": -4.2920918464660645,
+      "logps/rejected": -5.05672025680542,
+      "loss": 0.0507,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.2920918464660645,
+      "rewards/margins": 0.7646291851997375,
+      "rewards/rejected": -5.05672025680542,
+      "sft_loss": 4.073601722717285,
+      "step": 5395
+    },
+    {
+      "epoch": 2.890115403913698,
+      "grad_norm": 0.43853172793980605,
+      "learning_rate": 4.032158291400245e-09,
+      "logits/chosen": -0.6380362510681152,
+      "logits/rejected": -0.4227686822414398,
+      "logps/chosen": -4.206839561462402,
+      "logps/rejected": -5.371674060821533,
+      "loss": 0.0473,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -4.206839561462402,
+      "rewards/margins": 1.1648342609405518,
+      "rewards/rejected": -5.371674060821533,
+      "sft_loss": 3.8228912353515625,
+      "step": 5400
+    },
+    {
+      "epoch": 2.8927914366950995,
+      "grad_norm": 0.40022160682498903,
+      "learning_rate": 3.837175821425398e-09,
+      "logits/chosen": -0.5174564123153687,
+      "logits/rejected": -0.48399344086647034,
+      "logps/chosen": -4.494567394256592,
+      "logps/rejected": -5.1883416175842285,
+      "loss": 0.0518,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.494567394256592,
+      "rewards/margins": 0.693773627281189,
+      "rewards/rejected": -5.1883416175842285,
+      "sft_loss": 4.1761016845703125,
+      "step": 5405
+    },
+    {
+      "epoch": 2.8954674694765012,
+      "grad_norm": 0.5572661550149929,
+      "learning_rate": 3.6470071253467683e-09,
+      "logits/chosen": -0.6208726167678833,
+      "logits/rejected": -0.5416024923324585,
+      "logps/chosen": -4.29349422454834,
+      "logps/rejected": -5.193739891052246,
+      "loss": 0.0492,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.29349422454834,
+      "rewards/margins": 0.9002454876899719,
+      "rewards/rejected": -5.193739891052246,
+      "sft_loss": 3.985908031463623,
+      "step": 5410
+    },
+    {
+      "epoch": 2.8981435022579025,
+      "grad_norm": 0.4531036471053265,
+      "learning_rate": 3.461654048181939e-09,
+      "logits/chosen": -0.6595847010612488,
+      "logits/rejected": -0.4606492519378662,
+      "logps/chosen": -4.341272830963135,
+      "logps/rejected": -5.080430030822754,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.341272830963135,
+      "rewards/margins": 0.7391573786735535,
+      "rewards/rejected": -5.080430030822754,
+      "sft_loss": 4.150894641876221,
+      "step": 5415
+    },
+    {
+      "epoch": 2.9008195350393042,
+      "grad_norm": 0.49752924055659564,
+      "learning_rate": 3.281118388227255e-09,
+      "logits/chosen": -0.595501184463501,
+      "logits/rejected": -0.5304663777351379,
+      "logps/chosen": -4.371779441833496,
+      "logps/rejected": -4.99991512298584,
+      "loss": 0.0531,
+      "rewards/accuracies": 0.706250011920929,
+      "rewards/chosen": -4.371779441833496,
+      "rewards/margins": 0.628136157989502,
+      "rewards/rejected": -4.99991512298584,
+      "sft_loss": 4.091821193695068,
+      "step": 5420
+    },
+    {
+      "epoch": 2.903495567820706,
+      "grad_norm": 0.5344785084716002,
+      "learning_rate": 3.1054018970405048e-09,
+      "logits/chosen": -0.5903509855270386,
+      "logits/rejected": -0.44687825441360474,
+      "logps/chosen": -4.301074028015137,
+      "logps/rejected": -5.17934513092041,
+      "loss": 0.049,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -4.301074028015137,
+      "rewards/margins": 0.8782715797424316,
+      "rewards/rejected": -5.17934513092041,
+      "sft_loss": 3.9552109241485596,
+      "step": 5425
+    },
+    {
+      "epoch": 2.906171600602107,
+      "grad_norm": 0.5131492554199635,
+      "learning_rate": 2.9345062794238207e-09,
+      "logits/chosen": -0.6464307904243469,
+      "logits/rejected": -0.4529731869697571,
+      "logps/chosen": -4.344364166259766,
+      "logps/rejected": -5.18819522857666,
+      "loss": 0.0506,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -4.344364166259766,
+      "rewards/margins": 0.8438312411308289,
+      "rewards/rejected": -5.18819522857666,
+      "sft_loss": 4.132857322692871,
+      "step": 5430
+    },
+    {
+      "epoch": 2.908847633383509,
+      "grad_norm": 0.47960669183210924,
+      "learning_rate": 2.7684331934072492e-09,
+      "logits/chosen": -0.7158024311065674,
+      "logits/rejected": -0.6619073748588562,
+      "logps/chosen": -4.253783226013184,
+      "logps/rejected": -5.01469612121582,
+      "loss": 0.0514,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.253783226013184,
+      "rewards/margins": 0.7609127759933472,
+      "rewards/rejected": -5.01469612121582,
+      "sft_loss": 4.020027160644531,
+      "step": 5435
+    },
+    {
+      "epoch": 2.9115236661649107,
+      "grad_norm": 0.5277349713344069,
+      "learning_rate": 2.6071842502326526e-09,
+      "logits/chosen": -0.6982792615890503,
+      "logits/rejected": -0.5616760849952698,
+      "logps/chosen": -4.43766975402832,
+      "logps/rejected": -5.0740461349487305,
+      "loss": 0.0515,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.43766975402832,
+      "rewards/margins": 0.6363760828971863,
+      "rewards/rejected": -5.0740461349487305,
+      "sft_loss": 4.1986236572265625,
+      "step": 5440
+    },
+    {
+      "epoch": 2.9141996989463124,
+      "grad_norm": 0.6285782347051841,
+      "learning_rate": 2.450761014337888e-09,
+      "logits/chosen": -0.4401867985725403,
+      "logits/rejected": -0.4001937508583069,
+      "logps/chosen": -4.386246204376221,
+      "logps/rejected": -5.24971866607666,
+      "loss": 0.0524,
+      "rewards/accuracies": 0.6625000238418579,
+      "rewards/chosen": -4.386246204376221,
+      "rewards/margins": 0.8634727597236633,
+      "rewards/rejected": -5.24971866607666,
+      "sft_loss": 4.156008720397949,
+      "step": 5445
+    },
+    {
+      "epoch": 2.9168757317277136,
+      "grad_norm": 0.8007769761665051,
+      "learning_rate": 2.299165003341985e-09,
+      "logits/chosen": -0.4461418092250824,
+      "logits/rejected": -0.39837563037872314,
+      "logps/chosen": -4.265676021575928,
+      "logps/rejected": -5.094054698944092,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.265676021575928,
+      "rewards/margins": 0.8283787965774536,
+      "rewards/rejected": -5.094054698944092,
+      "sft_loss": 4.05593729019165,
+      "step": 5450
+    },
+    {
+      "epoch": 2.9195517645091154,
+      "grad_norm": 0.5820729781490392,
+      "learning_rate": 2.1523976880299945e-09,
+      "logits/chosen": -0.6315957307815552,
+      "logits/rejected": -0.4449182450771332,
+      "logps/chosen": -4.352351188659668,
+      "logps/rejected": -5.012256622314453,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.352351188659668,
+      "rewards/margins": 0.6599052548408508,
+      "rewards/rejected": -5.012256622314453,
+      "sft_loss": 4.167105674743652,
+      "step": 5455
+    },
+    {
+      "epoch": 2.9222277972905166,
+      "grad_norm": 0.636536197079115,
+      "learning_rate": 2.010460492339161e-09,
+      "logits/chosen": -0.5786579251289368,
+      "logits/rejected": -0.45049238204956055,
+      "logps/chosen": -4.320761203765869,
+      "logps/rejected": -5.0841498374938965,
+      "loss": 0.0493,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.320761203765869,
+      "rewards/margins": 0.7633887529373169,
+      "rewards/rejected": -5.0841498374938965,
+      "sft_loss": 3.9423680305480957,
+      "step": 5460
+    },
+    {
+      "epoch": 2.9249038300719183,
+      "grad_norm": 0.6277968475778644,
+      "learning_rate": 1.8733547933446614e-09,
+      "logits/chosen": -0.6489330530166626,
+      "logits/rejected": -0.40374070405960083,
+      "logps/chosen": -4.402930736541748,
+      "logps/rejected": -5.066678047180176,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.402930736541748,
+      "rewards/margins": 0.6637468934059143,
+      "rewards/rejected": -5.066678047180176,
+      "sft_loss": 4.137263298034668,
+      "step": 5465
+    },
+    {
+      "epoch": 2.92757986285332,
+      "grad_norm": 0.5994213815491749,
+      "learning_rate": 1.7410819212467231e-09,
+      "logits/chosen": -0.5600922703742981,
+      "logits/rejected": -0.5083585977554321,
+      "logps/chosen": -4.495439052581787,
+      "logps/rejected": -5.0145063400268555,
+      "loss": 0.0546,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -4.495439052581787,
+      "rewards/margins": 0.5190678238868713,
+      "rewards/rejected": -5.0145063400268555,
+      "sft_loss": 4.229347229003906,
+      "step": 5470
+    },
+    {
+      "epoch": 2.9302558956347218,
+      "grad_norm": 0.422530524920182,
+      "learning_rate": 1.613643159357192e-09,
+      "logits/chosen": -0.5021861791610718,
+      "logits/rejected": -0.6317010521888733,
+      "logps/chosen": -4.437952518463135,
+      "logps/rejected": -5.024625301361084,
+      "loss": 0.0516,
+      "rewards/accuracies": 0.6187499761581421,
+      "rewards/chosen": -4.437952518463135,
+      "rewards/margins": 0.5866727828979492,
+      "rewards/rejected": -5.024625301361084,
+      "sft_loss": 4.1623358726501465,
+      "step": 5475
+    },
+    {
+      "epoch": 2.932931928416123,
+      "grad_norm": 0.49640871995787955,
+      "learning_rate": 1.4910397440875967e-09,
+      "logits/chosen": -0.6009665131568909,
+      "logits/rejected": -0.5004478693008423,
+      "logps/chosen": -4.305395126342773,
+      "logps/rejected": -5.040763854980469,
+      "loss": 0.0506,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.305395126342773,
+      "rewards/margins": 0.7353684902191162,
+      "rewards/rejected": -5.040763854980469,
+      "sft_loss": 4.037181854248047,
+      "step": 5480
+    },
+    {
+      "epoch": 2.9356079611975248,
+      "grad_norm": 0.49228061861862676,
+      "learning_rate": 1.3732728649368253e-09,
+      "logits/chosen": -0.5226206183433533,
+      "logits/rejected": -0.3365953862667084,
+      "logps/chosen": -4.273961067199707,
+      "logps/rejected": -5.037269115447998,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.273961067199707,
+      "rewards/margins": 0.7633079290390015,
+      "rewards/rejected": -5.037269115447998,
+      "sft_loss": 4.002518653869629,
+      "step": 5485
+    },
+    {
+      "epoch": 2.938283993978926,
+      "grad_norm": 0.5793030543386408,
+      "learning_rate": 1.260343664479524e-09,
+      "logits/chosen": -0.5718457102775574,
+      "logits/rejected": -0.5662282109260559,
+      "logps/chosen": -4.469295024871826,
+      "logps/rejected": -5.060985565185547,
+      "loss": 0.0508,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -4.469295024871826,
+      "rewards/margins": 0.5916908979415894,
+      "rewards/rejected": -5.060985565185547,
+      "sft_loss": 4.07590389251709,
+      "step": 5490
+    },
+    {
+      "epoch": 2.9409600267603278,
+      "grad_norm": 0.5923705446849306,
+      "learning_rate": 1.1522532383554384e-09,
+      "logits/chosen": -0.6648327112197876,
+      "logits/rejected": -0.42489615082740784,
+      "logps/chosen": -4.377512454986572,
+      "logps/rejected": -5.144698619842529,
+      "loss": 0.0512,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.377512454986572,
+      "rewards/margins": 0.7671862840652466,
+      "rewards/rejected": -5.144698619842529,
+      "sft_loss": 4.193405628204346,
+      "step": 5495
+    },
+    {
+      "epoch": 2.9436360595417295,
+      "grad_norm": 0.38876523108866257,
+      "learning_rate": 1.049002635258256e-09,
+      "logits/chosen": -0.5083884000778198,
+      "logits/rejected": -0.4646090567111969,
+      "logps/chosen": -4.464354515075684,
+      "logps/rejected": -5.202513694763184,
+      "loss": 0.0504,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.464354515075684,
+      "rewards/margins": 0.7381576895713806,
+      "rewards/rejected": -5.202513694763184,
+      "sft_loss": 4.122217655181885,
+      "step": 5500
+    },
+    {
+      "epoch": 2.946312092323131,
+      "grad_norm": 0.5234987989786737,
+      "learning_rate": 9.505928569258358e-10,
+      "logits/chosen": -0.5698267817497253,
+      "logits/rejected": -0.6044186353683472,
+      "logps/chosen": -4.365851402282715,
+      "logps/rejected": -5.112934589385986,
+      "loss": 0.0509,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.365851402282715,
+      "rewards/margins": 0.7470830678939819,
+      "rewards/rejected": -5.112934589385986,
+      "sft_loss": 4.181767463684082,
+      "step": 5505
+    },
+    {
+      "epoch": 2.9489881251045325,
+      "grad_norm": 0.6937279836403201,
+      "learning_rate": 8.57024858130273e-10,
+      "logits/chosen": -0.5719789266586304,
+      "logits/rejected": -0.4153693616390228,
+      "logps/chosen": -4.365966320037842,
+      "logps/rejected": -5.392800807952881,
+      "loss": 0.0506,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -4.365966320037842,
+      "rewards/margins": 1.0268347263336182,
+      "rewards/rejected": -5.392800807952881,
+      "sft_loss": 4.19357967376709,
+      "step": 5510
+    },
+    {
+      "epoch": 2.951664157885934,
+      "grad_norm": 0.4074587556093628,
+      "learning_rate": 7.682995466686826e-10,
+      "logits/chosen": -0.6482599973678589,
+      "logits/rejected": -0.49886375665664673,
+      "logps/chosen": -4.476020812988281,
+      "logps/rejected": -5.329292297363281,
+      "loss": 0.0497,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.476020812988281,
+      "rewards/margins": 0.8532717823982239,
+      "rewards/rejected": -5.329292297363281,
+      "sft_loss": 4.033179759979248,
+      "step": 5515
+    },
+    {
+      "epoch": 2.9543401906673354,
+      "grad_norm": 0.637733982546982,
+      "learning_rate": 6.844177833543741e-10,
+      "logits/chosen": -0.5510967373847961,
+      "logits/rejected": -0.5156068801879883,
+      "logps/chosen": -4.321258544921875,
+      "logps/rejected": -5.055058002471924,
+      "loss": 0.0506,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": -4.321258544921875,
+      "rewards/margins": 0.7337992787361145,
+      "rewards/rejected": -5.055058002471924,
+      "sft_loss": 4.009609222412109,
+      "step": 5520
+    },
+    {
+      "epoch": 2.957016223448737,
+      "grad_norm": 0.564854251035776,
+      "learning_rate": 6.053803820087467e-10,
+      "logits/chosen": -0.5690053701400757,
+      "logits/rejected": -0.3823782801628113,
+      "logps/chosen": -4.465284824371338,
+      "logps/rejected": -5.242983818054199,
+      "loss": 0.0525,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": -4.465284824371338,
+      "rewards/margins": 0.7776988744735718,
+      "rewards/rejected": -5.242983818054199,
+      "sft_loss": 4.287803649902344,
+      "step": 5525
+    },
+    {
+      "epoch": 2.959692256230139,
+      "grad_norm": 0.7329322291633513,
+      "learning_rate": 5.311881094528514e-10,
+      "logits/chosen": -0.6794091463088989,
+      "logits/rejected": -0.4258442521095276,
+      "logps/chosen": -4.3857903480529785,
+      "logps/rejected": -5.122025966644287,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -4.3857903480529785,
+      "rewards/margins": 0.7362359762191772,
+      "rewards/rejected": -5.122025966644287,
+      "sft_loss": 4.057104587554932,
+      "step": 5530
+    },
+    {
+      "epoch": 2.9623682890115406,
+      "grad_norm": 0.9053958770939384,
+      "learning_rate": 4.6184168550050806e-10,
+      "logits/chosen": -0.5739349722862244,
+      "logits/rejected": -0.5423756837844849,
+      "logps/chosen": -4.460775852203369,
+      "logps/rejected": -5.1181230545043945,
+      "loss": 0.053,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.460775852203369,
+      "rewards/margins": 0.6573477387428284,
+      "rewards/rejected": -5.1181230545043945,
+      "sft_loss": 4.250723838806152,
+      "step": 5535
+    },
+    {
+      "epoch": 2.965044321792942,
+      "grad_norm": 0.5204857327137121,
+      "learning_rate": 3.973417829510328e-10,
+      "logits/chosen": -0.7034062147140503,
+      "logits/rejected": -0.5999553799629211,
+      "logps/chosen": -4.296984672546387,
+      "logps/rejected": -5.04316520690918,
+      "loss": 0.0505,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -4.296984672546387,
+      "rewards/margins": 0.7461800575256348,
+      "rewards/rejected": -5.04316520690918,
+      "sft_loss": 4.0506181716918945,
+      "step": 5540
+    },
+    {
+      "epoch": 2.9677203545743436,
+      "grad_norm": 0.7538807751340947,
+      "learning_rate": 3.3768902758274377e-10,
+      "logits/chosen": -0.5909171104431152,
+      "logits/rejected": -0.5124531388282776,
+      "logps/chosen": -4.239945411682129,
+      "logps/rejected": -5.1042375564575195,
+      "loss": 0.05,
+      "rewards/accuracies": 0.7562500238418579,
+      "rewards/chosen": -4.239945411682129,
+      "rewards/margins": 0.8642925024032593,
+      "rewards/rejected": -5.1042375564575195,
+      "sft_loss": 3.9758007526397705,
+      "step": 5545
+    },
+    {
+      "epoch": 2.970396387355745,
+      "grad_norm": 0.47749052206783726,
+      "learning_rate": 2.8288399814691e-10,
+      "logits/chosen": -0.472415030002594,
+      "logits/rejected": -0.45368775725364685,
+      "logps/chosen": -4.443525314331055,
+      "logps/rejected": -5.15672492980957,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.6812499761581421,
+      "rewards/chosen": -4.443525314331055,
+      "rewards/margins": 0.7131996154785156,
+      "rewards/rejected": -5.15672492980957,
+      "sft_loss": 4.1271562576293945,
+      "step": 5550
+    },
+    {
+      "epoch": 2.9730724201371466,
+      "grad_norm": 0.48131124563494715,
+      "learning_rate": 2.3292722636220066e-10,
+      "logits/chosen": -0.5993543267250061,
+      "logits/rejected": -0.4175679087638855,
+      "logps/chosen": -4.408750057220459,
+      "logps/rejected": -5.332371711730957,
+      "loss": 0.0494,
+      "rewards/accuracies": 0.6937500238418579,
+      "rewards/chosen": -4.408750057220459,
+      "rewards/margins": 0.9236211776733398,
+      "rewards/rejected": -5.332371711730957,
+      "sft_loss": 4.05026912689209,
+      "step": 5555
+    },
+    {
+      "epoch": 2.9757484529185483,
+      "grad_norm": 0.5310462303925957,
+      "learning_rate": 1.8781919690946668e-10,
+      "logits/chosen": -0.503473699092865,
+      "logits/rejected": -0.5251542925834656,
+      "logps/chosen": -4.388646602630615,
+      "logps/rejected": -5.023372650146484,
+      "loss": 0.0511,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.388646602630615,
+      "rewards/margins": 0.6347256898880005,
+      "rewards/rejected": -5.023372650146484,
+      "sft_loss": 4.1005964279174805,
+      "step": 5560
+    },
+    {
+      "epoch": 2.97842448569995,
+      "grad_norm": 0.5624959982604256,
+      "learning_rate": 1.4756034742696711e-10,
+      "logits/chosen": -0.6377474069595337,
+      "logits/rejected": -0.5445038676261902,
+      "logps/chosen": -4.326672077178955,
+      "logps/rejected": -5.114373683929443,
+      "loss": 0.0513,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -4.326672077178955,
+      "rewards/margins": 0.787702202796936,
+      "rewards/rejected": -5.114373683929443,
+      "sft_loss": 4.089654445648193,
+      "step": 5565
+    },
+    {
+      "epoch": 2.9811005184813513,
+      "grad_norm": 0.502159730033696,
+      "learning_rate": 1.12151068506261e-10,
+      "logits/chosen": -0.5310649871826172,
+      "logits/rejected": -0.43151599168777466,
+      "logps/chosen": -4.353323459625244,
+      "logps/rejected": -5.323481559753418,
+      "loss": 0.0481,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.353323459625244,
+      "rewards/margins": 0.9701583981513977,
+      "rewards/rejected": -5.323481559753418,
+      "sft_loss": 3.9440083503723145,
+      "step": 5570
+    },
+    {
+      "epoch": 2.983776551262753,
+      "grad_norm": 0.6644242224271146,
+      "learning_rate": 8.159170368826629e-11,
+      "logits/chosen": -0.5523242950439453,
+      "logits/rejected": -0.3813764452934265,
+      "logps/chosen": -4.306519508361816,
+      "logps/rejected": -5.125226974487305,
+      "loss": 0.0504,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -4.306519508361816,
+      "rewards/margins": 0.8187074661254883,
+      "rewards/rejected": -5.125226974487305,
+      "sft_loss": 3.987196445465088,
+      "step": 5575
+    },
+    {
+      "epoch": 2.9864525840441547,
+      "grad_norm": 0.5142404060584621,
+      "learning_rate": 5.588254946015114e-11,
+      "logits/chosen": -0.7060213088989258,
+      "logits/rejected": -0.4090491831302643,
+      "logps/chosen": -4.283689498901367,
+      "logps/rejected": -5.1850786209106445,
+      "loss": 0.0496,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -4.283689498901367,
+      "rewards/margins": 0.9013880491256714,
+      "rewards/rejected": -5.1850786209106445,
+      "sft_loss": 3.9925262928009033,
+      "step": 5580
+    },
+    {
+      "epoch": 2.989128616825556,
+      "grad_norm": 0.5090734891493847,
+      "learning_rate": 3.502385525216978e-11,
+      "logits/chosen": -0.6630311608314514,
+      "logits/rejected": -0.4777213931083679,
+      "logps/chosen": -4.301659107208252,
+      "logps/rejected": -5.119317054748535,
+      "loss": 0.051,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.301659107208252,
+      "rewards/margins": 0.8176582455635071,
+      "rewards/rejected": -5.119317054748535,
+      "sft_loss": 4.079021453857422,
+      "step": 5585
+    },
+    {
+      "epoch": 2.9918046496069577,
+      "grad_norm": 0.4086292888652854,
+      "learning_rate": 1.901582343555308e-11,
+      "logits/chosen": -0.5439545512199402,
+      "logits/rejected": -0.48325151205062866,
+      "logps/chosen": -4.553244590759277,
+      "logps/rejected": -5.258808612823486,
+      "loss": 0.0521,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -4.553244590759277,
+      "rewards/margins": 0.7055639028549194,
+      "rewards/rejected": -5.258808612823486,
+      "sft_loss": 4.205986022949219,
+      "step": 5590
+    },
+    {
+      "epoch": 2.9944806823883594,
+      "grad_norm": 0.5976393965645549,
+      "learning_rate": 7.858609320232634e-12,
+      "logits/chosen": -0.5710118412971497,
+      "logits/rejected": -0.3761889338493347,
+      "logps/chosen": -4.421603202819824,
+      "logps/rejected": -5.208468437194824,
+      "loss": 0.0505,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": -4.421603202819824,
+      "rewards/margins": 0.7868648767471313,
+      "rewards/rejected": -5.208468437194824,
+      "sft_loss": 4.181458950042725,
+      "step": 5595
+    },
+    {
+      "epoch": 2.9971567151697607,
+      "grad_norm": 0.5630905862516066,
+      "learning_rate": 1.5523211535639624e-12,
+      "logits/chosen": -0.577018678188324,
+      "logits/rejected": -0.4606627821922302,
+      "logps/chosen": -4.410850524902344,
+      "logps/rejected": -5.4495320320129395,
+      "loss": 0.0497,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": -4.410850524902344,
+      "rewards/margins": 1.0386813879013062,
+      "rewards/rejected": -5.4495320320129395,
+      "sft_loss": 4.128878593444824,
+      "step": 5600
+    },
+    {
+      "epoch": 2.9971567151697607,
+      "eval_logits/chosen": -0.1373719573020935,
+      "eval_logits/rejected": -0.0603991337120533,
+      "eval_logps/chosen": -4.343548774719238,
+      "eval_logps/rejected": -5.11141300201416,
+      "eval_loss": 0.05006532371044159,
+      "eval_rewards/accuracies": 0.6810088753700256,
+      "eval_rewards/chosen": -4.343548774719238,
+      "eval_rewards/margins": 0.7678641080856323,
+      "eval_rewards/rejected": -5.11141300201416,
+      "eval_runtime": 45.0553,
+      "eval_samples_per_second": 29.852,
+      "eval_sft_loss": 3.9427084922790527,
+      "eval_steps_per_second": 7.48,
+      "step": 5600
+    },
+    {
+      "epoch": 2.999297541394882,
+      "step": 5604,
+      "total_flos": 0.0,
+      "train_loss": 0.0595120464655691,
+      "train_runtime": 31911.443,
+      "train_samples_per_second": 5.621,
+      "train_steps_per_second": 0.176
+    }
+  ],
+  "logging_steps": 5,
+  "max_steps": 5604,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 1000000,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}