diff --git "a/trainer_state.json" "b/trainer_state.json"
--- "a/trainer_state.json"
+++ "b/trainer_state.json"
@@ -1,1436 +1,1436 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 0.9987943737441393,
+  "epoch": 0.9982631930527722,
   "eval_steps": 400,
-  "global_step": 466,
+  "global_step": 467,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.010716677829872739,
-      "grad_norm": 27.862872452731896,
-      "learning_rate": 5.3191489361702123e-08,
-      "logits/chosen": -2.2512857913970947,
-      "logits/rejected": -2.264533519744873,
-      "logps/chosen": -0.5397141575813293,
-      "logps/rejected": -0.540291428565979,
-      "loss": 0.8648,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -1.3492851257324219,
-      "rewards/margins": 0.0014432668685913086,
-      "rewards/rejected": -1.3507286310195923,
+      "epoch": 0.01068804275217101,
+      "grad_norm": 56.89954388226368,
+      "learning_rate": 1.0638297872340425e-07,
+      "logits/chosen": -1.0110526084899902,
+      "logits/rejected": -0.9819751977920532,
+      "logps/chosen": -0.27428290247917175,
+      "logps/rejected": -0.27152150869369507,
+      "loss": 3.074,
+      "rewards/accuracies": 0.4312500059604645,
+      "rewards/chosen": -2.7428293228149414,
+      "rewards/margins": -0.027614299207925797,
+      "rewards/rejected": -2.7152152061462402,
       "step": 5
     },
     {
-      "epoch": 0.021433355659745478,
-      "grad_norm": 35.31926398689821,
-      "learning_rate": 1.0638297872340425e-07,
-      "logits/chosen": -2.308763027191162,
-      "logits/rejected": -2.2997522354125977,
-      "logps/chosen": -0.5613104701042175,
-      "logps/rejected": -0.6120178699493408,
-      "loss": 0.8465,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -1.4032763242721558,
-      "rewards/margins": 0.1267683506011963,
-      "rewards/rejected": -1.5300445556640625,
+      "epoch": 0.02137608550434202,
+      "grad_norm": 41.09804027603237,
+      "learning_rate": 2.127659574468085e-07,
+      "logits/chosen": -1.0431499481201172,
+      "logits/rejected": -0.9762164354324341,
+      "logps/chosen": -0.29434844851493835,
+      "logps/rejected": -0.29961150884628296,
+      "loss": 3.0104,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -2.943484306335449,
+      "rewards/margins": 0.052630893886089325,
+      "rewards/rejected": -2.9961154460906982,
       "step": 10
     },
     {
-      "epoch": 0.032150033489618215,
-      "grad_norm": 22.32409182123902,
-      "learning_rate": 1.5957446808510638e-07,
-      "logits/chosen": -2.272329330444336,
-      "logits/rejected": -2.265493392944336,
-      "logps/chosen": -0.535502552986145,
-      "logps/rejected": -0.5462267994880676,
-      "loss": 0.8411,
-      "rewards/accuracies": 0.4937500059604645,
-      "rewards/chosen": -1.3387562036514282,
-      "rewards/margins": 0.026810690760612488,
-      "rewards/rejected": -1.3655668497085571,
+      "epoch": 0.03206412825651302,
+      "grad_norm": 54.12410975393932,
+      "learning_rate": 3.1914893617021275e-07,
+      "logits/chosen": -0.9705549478530884,
+      "logits/rejected": -0.9907904863357544,
+      "logps/chosen": -0.2644530236721039,
+      "logps/rejected": -0.300653874874115,
+      "loss": 2.9255,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -2.6445302963256836,
+      "rewards/margins": 0.36200839281082153,
+      "rewards/rejected": -3.0065386295318604,
       "step": 15
     },
     {
-      "epoch": 0.042866711319490956,
-      "grad_norm": 24.85973432029644,
-      "learning_rate": 2.127659574468085e-07,
-      "logits/chosen": -2.275291919708252,
-      "logits/rejected": -2.273228168487549,
-      "logps/chosen": -0.5528542399406433,
-      "logps/rejected": -0.5709605813026428,
-      "loss": 0.8452,
+      "epoch": 0.04275217100868404,
+      "grad_norm": 70.58531376681509,
+      "learning_rate": 4.25531914893617e-07,
+      "logits/chosen": -0.9636842608451843,
+      "logits/rejected": -0.9370673298835754,
+      "logps/chosen": -0.27763131260871887,
+      "logps/rejected": -0.29128938913345337,
+      "loss": 2.9498,
       "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -1.3821356296539307,
-      "rewards/margins": 0.04526572674512863,
-      "rewards/rejected": -1.4274013042449951,
+      "rewards/chosen": -2.776313304901123,
+      "rewards/margins": 0.13658040761947632,
+      "rewards/rejected": -2.912893772125244,
       "step": 20
     },
     {
-      "epoch": 0.0535833891493637,
-      "grad_norm": 66.37399964080045,
-      "learning_rate": 2.659574468085106e-07,
-      "logits/chosen": -2.227910041809082,
-      "logits/rejected": -2.2429375648498535,
-      "logps/chosen": -0.5119736790657043,
-      "logps/rejected": -0.5303855538368225,
-      "loss": 0.8369,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -1.279934287071228,
-      "rewards/margins": 0.046029649674892426,
-      "rewards/rejected": -1.3259638547897339,
+      "epoch": 0.053440213760855046,
+      "grad_norm": 56.29890733120682,
+      "learning_rate": 5.319148936170212e-07,
+      "logits/chosen": -1.021801233291626,
+      "logits/rejected": -0.9922389984130859,
+      "logps/chosen": -0.2716570198535919,
+      "logps/rejected": -0.27810558676719666,
+      "loss": 2.996,
+      "rewards/accuracies": 0.4937500059604645,
+      "rewards/chosen": -2.7165703773498535,
+      "rewards/margins": 0.06448549777269363,
+      "rewards/rejected": -2.7810559272766113,
       "step": 25
     },
     {
-      "epoch": 0.06430006697923643,
-      "grad_norm": 19.386202836307778,
-      "learning_rate": 3.1914893617021275e-07,
-      "logits/chosen": -2.3197097778320312,
-      "logits/rejected": -2.33433198928833,
-      "logps/chosen": -0.5134044885635376,
-      "logps/rejected": -0.5359271764755249,
-      "loss": 0.8261,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -1.2835114002227783,
-      "rewards/margins": 0.056306738406419754,
-      "rewards/rejected": -1.339818000793457,
+      "epoch": 0.06412825651302605,
+      "grad_norm": 43.82713664972921,
+      "learning_rate": 6.382978723404255e-07,
+      "logits/chosen": -0.9963313937187195,
+      "logits/rejected": -0.9516829252243042,
+      "logps/chosen": -0.2735390067100525,
+      "logps/rejected": -0.2788829803466797,
+      "loss": 2.9153,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -2.7353897094726562,
+      "rewards/margins": 0.0534403920173645,
+      "rewards/rejected": -2.788830041885376,
       "step": 30
     },
     {
-      "epoch": 0.07501674480910918,
-      "grad_norm": 20.601240257458635,
-      "learning_rate": 3.7234042553191484e-07,
-      "logits/chosen": -2.3346030712127686,
-      "logits/rejected": -2.3339004516601562,
-      "logps/chosen": -0.45776882767677307,
-      "logps/rejected": -0.4768803119659424,
-      "loss": 0.8405,
-      "rewards/accuracies": 0.518750011920929,
-      "rewards/chosen": -1.144421935081482,
-      "rewards/margins": 0.0477786585688591,
-      "rewards/rejected": -1.192200779914856,
+      "epoch": 0.07481629926519706,
+      "grad_norm": 70.24698865073822,
+      "learning_rate": 7.446808510638297e-07,
+      "logits/chosen": -1.052671194076538,
+      "logits/rejected": -0.9770715832710266,
+      "logps/chosen": -0.2941210865974426,
+      "logps/rejected": -0.32225456833839417,
+      "loss": 2.901,
+      "rewards/accuracies": 0.5062500238418579,
+      "rewards/chosen": -2.941210985183716,
+      "rewards/margins": 0.2813349962234497,
+      "rewards/rejected": -3.222545623779297,
       "step": 35
     },
     {
-      "epoch": 0.08573342263898191,
-      "grad_norm": 24.697322476213458,
-      "learning_rate": 4.25531914893617e-07,
-      "logits/chosen": -2.255758047103882,
-      "logits/rejected": -2.2820348739624023,
-      "logps/chosen": -0.5251036882400513,
-      "logps/rejected": -0.5213593244552612,
-      "loss": 0.8474,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": -1.312759280204773,
-      "rewards/margins": -0.009361037984490395,
-      "rewards/rejected": -1.3033982515335083,
+      "epoch": 0.08550434201736808,
+      "grad_norm": 55.30141312601411,
+      "learning_rate": 8.51063829787234e-07,
+      "logits/chosen": -1.0156581401824951,
+      "logits/rejected": -0.9718509912490845,
+      "logps/chosen": -0.28176331520080566,
+      "logps/rejected": -0.3227621614933014,
+      "loss": 2.9376,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -2.8176331520080566,
+      "rewards/margins": 0.40998831391334534,
+      "rewards/rejected": -3.227621555328369,
       "step": 40
     },
     {
-      "epoch": 0.09645010046885466,
-      "grad_norm": 19.817830752667202,
-      "learning_rate": 4.787234042553192e-07,
-      "logits/chosen": -2.3332431316375732,
-      "logits/rejected": -2.3259410858154297,
-      "logps/chosen": -0.4812852442264557,
-      "logps/rejected": -0.48691949248313904,
-      "loss": 0.8221,
-      "rewards/accuracies": 0.543749988079071,
-      "rewards/chosen": -1.2032132148742676,
-      "rewards/margins": 0.014085566624999046,
-      "rewards/rejected": -1.2172987461090088,
+      "epoch": 0.09619238476953908,
+      "grad_norm": 38.45710685921155,
+      "learning_rate": 9.574468085106384e-07,
+      "logits/chosen": -1.0545786619186401,
+      "logits/rejected": -1.0107605457305908,
+      "logps/chosen": -0.32836729288101196,
+      "logps/rejected": -0.37582582235336304,
+      "loss": 2.944,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -3.28367280960083,
+      "rewards/margins": 0.4745853841304779,
+      "rewards/rejected": -3.75825834274292,
       "step": 45
     },
     {
-      "epoch": 0.1071667782987274,
-      "grad_norm": 19.66835689612492,
-      "learning_rate": 4.999367579541646e-07,
-      "logits/chosen": -2.2813732624053955,
-      "logits/rejected": -2.271610975265503,
-      "logps/chosen": -0.45226341485977173,
-      "logps/rejected": -0.46558040380477905,
-      "loss": 0.8299,
-      "rewards/accuracies": 0.4937500059604645,
-      "rewards/chosen": -1.1306583881378174,
-      "rewards/margins": 0.03329245001077652,
-      "rewards/rejected": -1.1639509201049805,
+      "epoch": 0.10688042752171009,
+      "grad_norm": 127.64245160013748,
+      "learning_rate": 9.998741174712533e-07,
+      "logits/chosen": -1.040578842163086,
+      "logits/rejected": -0.9890502095222473,
+      "logps/chosen": -0.3216663599014282,
+      "logps/rejected": -0.3576571047306061,
+      "loss": 3.0077,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": -3.216662883758545,
+      "rewards/margins": 0.35990777611732483,
+      "rewards/rejected": -3.576570987701416,
       "step": 50
     },
     {
-      "epoch": 0.11788345612860013,
-      "grad_norm": 18.423253141965084,
-      "learning_rate": 4.99550394651012e-07,
-      "logits/chosen": -2.239288806915283,
-      "logits/rejected": -2.2441604137420654,
-      "logps/chosen": -0.4442782998085022,
-      "logps/rejected": -0.46781450510025024,
-      "loss": 0.8258,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -1.110695719718933,
-      "rewards/margins": 0.05884043127298355,
-      "rewards/rejected": -1.1695362329483032,
+      "epoch": 0.11756847027388109,
+      "grad_norm": 52.19635359720767,
+      "learning_rate": 9.991050648838675e-07,
+      "logits/chosen": -1.0747789144515991,
+      "logits/rejected": -1.0378679037094116,
+      "logps/chosen": -0.28550446033477783,
+      "logps/rejected": -0.3416834771633148,
+      "loss": 2.8502,
+      "rewards/accuracies": 0.5874999761581421,
+      "rewards/chosen": -2.8550446033477783,
+      "rewards/margins": 0.5617905855178833,
+      "rewards/rejected": -3.416834592819214,
       "step": 55
     },
     {
-      "epoch": 0.12860013395847286,
-      "grad_norm": 19.559581246861793,
-      "learning_rate": 4.988133448023071e-07,
-      "logits/chosen": -2.3177483081817627,
-      "logits/rejected": -2.3303709030151367,
-      "logps/chosen": -0.4191853106021881,
-      "logps/rejected": -0.41467294096946716,
-      "loss": 0.8268,
-      "rewards/accuracies": 0.4124999940395355,
-      "rewards/chosen": -1.0479633808135986,
-      "rewards/margins": -0.01128093060106039,
-      "rewards/rejected": -1.036682367324829,
+      "epoch": 0.1282565130260521,
+      "grad_norm": 45.58873626472617,
+      "learning_rate": 9.97637968732563e-07,
+      "logits/chosen": -1.0949262380599976,
+      "logits/rejected": -1.0604766607284546,
+      "logps/chosen": -0.3153451085090637,
+      "logps/rejected": -0.33443158864974976,
+      "loss": 2.8532,
+      "rewards/accuracies": 0.543749988079071,
+      "rewards/chosen": -3.1534507274627686,
+      "rewards/margins": 0.19086508452892303,
+      "rewards/rejected": -3.344316005706787,
       "step": 60
     },
     {
-      "epoch": 0.13931681178834562,
-      "grad_norm": 24.533483296865708,
-      "learning_rate": 4.977266441646379e-07,
-      "logits/chosen": -2.283264636993408,
-      "logits/rejected": -2.285848617553711,
-      "logps/chosen": -0.4495847225189209,
-      "logps/rejected": -0.46883049607276917,
-      "loss": 0.8155,
-      "rewards/accuracies": 0.4937500059604645,
-      "rewards/chosen": -1.1239618062973022,
-      "rewards/margins": 0.04811444133520126,
-      "rewards/rejected": -1.1720762252807617,
+      "epoch": 0.13894455577822312,
+      "grad_norm": 57.26986612669826,
+      "learning_rate": 9.954748808839674e-07,
+      "logits/chosen": -1.0162328481674194,
+      "logits/rejected": -0.9866952896118164,
+      "logps/chosen": -0.35168641805648804,
+      "logps/rejected": -0.39575284719467163,
+      "loss": 2.8058,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -3.516864061355591,
+      "rewards/margins": 0.4406643509864807,
+      "rewards/rejected": -3.9575283527374268,
       "step": 65
     },
     {
-      "epoch": 0.15003348961821836,
-      "grad_norm": 18.186755862107237,
-      "learning_rate": 4.962918198495654e-07,
-      "logits/chosen": -2.331956624984741,
-      "logits/rejected": -2.3319480419158936,
-      "logps/chosen": -0.46049419045448303,
-      "logps/rejected": -0.48135805130004883,
-      "loss": 0.8145,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -1.1512354612350464,
-      "rewards/margins": 0.05215970426797867,
-      "rewards/rejected": -1.203395128250122,
+      "epoch": 0.14963259853039412,
+      "grad_norm": 36.80891193149694,
+      "learning_rate": 9.926188266120295e-07,
+      "logits/chosen": -1.032679796218872,
+      "logits/rejected": -1.009883999824524,
+      "logps/chosen": -0.34069326519966125,
+      "logps/rejected": -0.4097404479980469,
+      "loss": 2.8492,
+      "rewards/accuracies": 0.550000011920929,
+      "rewards/chosen": -3.406932830810547,
+      "rewards/margins": 0.690471887588501,
+      "rewards/rejected": -4.097404479980469,
       "step": 70
     },
     {
-      "epoch": 0.1607501674480911,
-      "grad_norm": 41.3300960666502,
-      "learning_rate": 4.945108881776135e-07,
-      "logits/chosen": -2.3779914379119873,
-      "logits/rejected": -2.375589370727539,
-      "logps/chosen": -0.41506296396255493,
-      "logps/rejected": -0.435122013092041,
-      "loss": 0.8147,
+      "epoch": 0.16032064128256512,
+      "grad_norm": 52.89939132813969,
+      "learning_rate": 9.890738003669027e-07,
+      "logits/chosen": -0.9840335845947266,
+      "logits/rejected": -0.9154711961746216,
+      "logps/chosen": -0.3410232663154602,
+      "logps/rejected": -0.39265531301498413,
+      "loss": 2.8212,
       "rewards/accuracies": 0.543749988079071,
-      "rewards/chosen": -1.0376574993133545,
-      "rewards/margins": 0.050147540867328644,
-      "rewards/rejected": -1.087804913520813,
+      "rewards/chosen": -3.4102330207824707,
+      "rewards/margins": 0.5163205862045288,
+      "rewards/rejected": -3.926553249359131,
       "step": 75
     },
     {
-      "epoch": 0.17146684527796383,
-      "grad_norm": 19.713931611802217,
-      "learning_rate": 4.923863518447879e-07,
-      "logits/chosen": -2.3455538749694824,
-      "logits/rejected": -2.3334898948669434,
-      "logps/chosen": -0.44175705313682556,
-      "logps/rejected": -0.4695035517215729,
-      "loss": 0.8223,
-      "rewards/accuracies": 0.5562499761581421,
-      "rewards/chosen": -1.104392647743225,
-      "rewards/margins": 0.0693662017583847,
-      "rewards/rejected": -1.173758864402771,
+      "epoch": 0.17100868403473615,
+      "grad_norm": 43.121051189441694,
+      "learning_rate": 9.848447601883433e-07,
+      "logits/chosen": -0.9788318872451782,
+      "logits/rejected": -0.9655908346176147,
+      "logps/chosen": -0.3363918364048004,
+      "logps/rejected": -0.4268081784248352,
+      "loss": 2.7186,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": -3.3639183044433594,
+      "rewards/margins": 0.9041633605957031,
+      "rewards/rejected": -4.2680816650390625,
       "step": 80
     },
     {
-      "epoch": 0.18218352310783656,
-      "grad_norm": 20.212157179302782,
-      "learning_rate": 4.899211964056035e-07,
-      "logits/chosen": -2.296379566192627,
-      "logits/rejected": -2.317187786102295,
-      "logps/chosen": -0.41793909668922424,
-      "logps/rejected": -0.4339120388031006,
-      "loss": 0.8234,
-      "rewards/accuracies": 0.5562499761581421,
-      "rewards/chosen": -1.0448477268218994,
-      "rewards/margins": 0.03993237763643265,
-      "rewards/rejected": -1.084780216217041,
+      "epoch": 0.18169672678690715,
+      "grad_norm": 54.440571510963004,
+      "learning_rate": 9.799376207714444e-07,
+      "logits/chosen": -1.003329873085022,
+      "logits/rejected": -0.9817102551460266,
+      "logps/chosen": -0.3231724500656128,
+      "logps/rejected": -0.37237557768821716,
+      "loss": 2.7361,
+      "rewards/accuracies": 0.6312500238418579,
+      "rewards/chosen": -3.231724262237549,
+      "rewards/margins": 0.4920312464237213,
+      "rewards/rejected": -3.7237555980682373,
       "step": 85
     },
     {
-      "epoch": 0.19290020093770932,
-      "grad_norm": 19.946761983669234,
-      "learning_rate": 4.871188860775643e-07,
-      "logits/chosen": -2.217319965362549,
-      "logits/rejected": -2.2343027591705322,
-      "logps/chosen": -0.4441414475440979,
-      "logps/rejected": -0.463876336812973,
-      "loss": 0.8217,
-      "rewards/accuracies": 0.5562499761581421,
-      "rewards/chosen": -1.110353708267212,
-      "rewards/margins": 0.049337320029735565,
-      "rewards/rejected": -1.1596909761428833,
+      "epoch": 0.19238476953907815,
+      "grad_norm": 68.19418423227036,
+      "learning_rate": 9.743592451943998e-07,
+      "logits/chosen": -1.0398868322372437,
+      "logits/rejected": -1.0045406818389893,
+      "logps/chosen": -0.39259663224220276,
+      "logps/rejected": -0.46695399284362793,
+      "loss": 2.8464,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": -3.925966262817383,
+      "rewards/margins": 0.7435733675956726,
+      "rewards/rejected": -4.669539451599121,
       "step": 90
     },
     {
-      "epoch": 0.20361687876758205,
-      "grad_norm": 22.43054077775309,
-      "learning_rate": 4.839833588729897e-07,
-      "logits/chosen": -2.3150296211242676,
-      "logits/rejected": -2.305851936340332,
-      "logps/chosen": -0.4246097207069397,
-      "logps/rejected": -0.4527285695075989,
-      "loss": 0.8119,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -1.0615241527557373,
-      "rewards/margins": 0.07029719650745392,
-      "rewards/rejected": -1.1318213939666748,
+      "epoch": 0.20307281229124916,
+      "grad_norm": 48.310300313424634,
+      "learning_rate": 9.681174353198686e-07,
+      "logits/chosen": -1.1072306632995605,
+      "logits/rejected": -1.0259342193603516,
+      "logps/chosen": -0.4255724549293518,
+      "logps/rejected": -0.46118488907814026,
+      "loss": 2.7441,
+      "rewards/accuracies": 0.5562499761581421,
+      "rewards/chosen": -4.2557244300842285,
+      "rewards/margins": 0.3561245799064636,
+      "rewards/rejected": -4.611849308013916,
       "step": 95
     },
     {
-      "epoch": 0.2143335565974548,
-      "grad_norm": 22.6806216251904,
-      "learning_rate": 4.80519021065031e-07,
-      "logits/chosen": -2.2826828956604004,
-      "logits/rejected": -2.275033473968506,
-      "logps/chosen": -0.41437339782714844,
-      "logps/rejected": -0.4373084604740143,
-      "loss": 0.8065,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -1.0359336137771606,
-      "rewards/margins": 0.05733749270439148,
-      "rewards/rejected": -1.093271017074585,
+      "epoch": 0.21376085504342018,
+      "grad_norm": 65.13984774258292,
+      "learning_rate": 9.612209208833646e-07,
+      "logits/chosen": -1.0061118602752686,
+      "logits/rejected": -0.9820231199264526,
+      "logps/chosen": -0.4049428403377533,
+      "logps/rejected": -0.4770785868167877,
+      "loss": 2.7252,
+      "rewards/accuracies": 0.606249988079071,
+      "rewards/chosen": -4.049428462982178,
+      "rewards/margins": 0.7213573455810547,
+      "rewards/rejected": -4.770786285400391,
       "step": 100
     },
     {
-      "epoch": 0.22505023442732752,
-      "grad_norm": 19.45194393140105,
-      "learning_rate": 4.767307409956528e-07,
-      "logits/chosen": -2.2803359031677246,
-      "logits/rejected": -2.2837014198303223,
-      "logps/chosen": -0.4491514563560486,
-      "logps/rejected": -0.4561590254306793,
-      "loss": 0.8252,
-      "rewards/accuracies": 0.48124998807907104,
-      "rewards/chosen": -1.1228786706924438,
-      "rewards/margins": 0.01751893199980259,
-      "rewards/rejected": -1.1403974294662476,
+      "epoch": 0.22444889779559118,
+      "grad_norm": 67.14694271980007,
+      "learning_rate": 9.536793472839324e-07,
+      "logits/chosen": -1.0186512470245361,
+      "logits/rejected": -0.9668880701065063,
+      "logps/chosen": -0.40378451347351074,
+      "logps/rejected": -0.4971798360347748,
+      "loss": 2.7255,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -4.037845134735107,
+      "rewards/margins": 0.9339532852172852,
+      "rewards/rejected": -4.971798419952393,
       "step": 105
     },
     {
-      "epoch": 0.23576691225720026,
-      "grad_norm": 21.617685020101888,
-      "learning_rate": 4.726238422342805e-07,
-      "logits/chosen": -2.263587474822998,
-      "logits/rejected": -2.231074094772339,
-      "logps/chosen": -0.456240713596344,
-      "logps/rejected": -0.47300928831100464,
-      "loss": 0.8075,
-      "rewards/accuracies": 0.543749988079071,
-      "rewards/chosen": -1.1406018733978271,
-      "rewards/margins": 0.04192143306136131,
-      "rewards/rejected": -1.1825231313705444,
+      "epoch": 0.23513694054776219,
+      "grad_norm": 66.18323566645698,
+      "learning_rate": 9.455032620941839e-07,
+      "logits/chosen": -0.9757324457168579,
+      "logits/rejected": -0.9178048372268677,
+      "logps/chosen": -0.45775121450424194,
+      "logps/rejected": -0.5744206309318542,
+      "loss": 2.6384,
+      "rewards/accuracies": 0.668749988079071,
+      "rewards/chosen": -4.577511787414551,
+      "rewards/margins": 1.1666945219039917,
+      "rewards/rejected": -5.744206428527832,
       "step": 110
     },
     {
-      "epoch": 0.24648359008707302,
-      "grad_norm": 24.689073829083224,
-      "learning_rate": 4.6820409609673027e-07,
-      "logits/chosen": -2.3198399543762207,
-      "logits/rejected": -2.3183414936065674,
-      "logps/chosen": -0.4174441397190094,
-      "logps/rejected": -0.4455544054508209,
-      "loss": 0.8138,
-      "rewards/accuracies": 0.5062500238418579,
-      "rewards/chosen": -1.0436103343963623,
-      "rewards/margins": 0.07027561962604523,
-      "rewards/rejected": -1.1138859987258911,
+      "epoch": 0.2458249832999332,
+      "grad_norm": 59.4672474909725,
+      "learning_rate": 9.367041003085648e-07,
+      "logits/chosen": -1.0457837581634521,
+      "logits/rejected": -0.9855262041091919,
+      "logps/chosen": -0.5011107921600342,
+      "logps/rejected": -0.5669585466384888,
+      "loss": 2.5014,
+      "rewards/accuracies": 0.643750011920929,
+      "rewards/chosen": -5.011107444763184,
+      "rewards/margins": 0.6584769487380981,
+      "rewards/rejected": -5.66958475112915,
       "step": 115
     },
     {
-      "epoch": 0.2572002679169457,
-      "grad_norm": 21.210704415851712,
-      "learning_rate": 4.6347771353493173e-07,
-      "logits/chosen": -2.3105266094207764,
-      "logits/rejected": -2.3077518939971924,
-      "logps/chosen": -0.4340137839317322,
-      "logps/rejected": -0.4530583322048187,
-      "loss": 0.8106,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -1.0850344896316528,
-      "rewards/margins": 0.047611434012651443,
-      "rewards/rejected": -1.132645845413208,
+      "epoch": 0.2565130260521042,
+      "grad_norm": 66.03731752964929,
+      "learning_rate": 9.272941683504808e-07,
+      "logits/chosen": -0.9971206784248352,
+      "logits/rejected": -0.9091175198554993,
+      "logps/chosen": -0.5187912583351135,
+      "logps/rejected": -0.7068670392036438,
+      "loss": 2.3834,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -5.187912464141846,
+      "rewards/margins": 1.8807573318481445,
+      "rewards/rejected": -7.06866979598999,
       "step": 120
     },
     {
-      "epoch": 0.2679169457468185,
-      "grad_norm": 28.92966643864981,
-      "learning_rate": 4.5845133640884217e-07,
-      "logits/chosen": -2.2830965518951416,
-      "logits/rejected": -2.3132407665252686,
-      "logps/chosen": -0.45075589418411255,
-      "logps/rejected": -0.5100811719894409,
-      "loss": 0.7989,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -1.1268895864486694,
-      "rewards/margins": 0.14831334352493286,
-      "rewards/rejected": -1.275202989578247,
+      "epoch": 0.26720106880427524,
+      "grad_norm": 71.68229077695405,
+      "learning_rate": 9.172866268606513e-07,
+      "logits/chosen": -1.0721819400787354,
+      "logits/rejected": -1.030027151107788,
+      "logps/chosen": -0.5926575660705566,
+      "logps/rejected": -0.6795297861099243,
+      "loss": 2.3213,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -5.926576137542725,
+      "rewards/margins": 0.8687216639518738,
+      "rewards/rejected": -6.795297145843506,
       "step": 125
     },
     {
-      "epoch": 0.27863362357669125,
-      "grad_norm": 21.983830009995575,
-      "learning_rate": 4.531320281528169e-07,
-      "logits/chosen": -2.354926109313965,
-      "logits/rejected": -2.346691131591797,
-      "logps/chosen": -0.4212542474269867,
-      "logps/rejected": -0.46237748861312866,
-      "loss": 0.8063,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -1.053135633468628,
-      "rewards/margins": 0.10280808061361313,
-      "rewards/rejected": -1.1559436321258545,
+      "epoch": 0.27788911155644624,
+      "grad_norm": 64.96923591468992,
+      "learning_rate": 9.066954722907638e-07,
+      "logits/chosen": -1.1085387468338013,
+      "logits/rejected": -1.1034091711044312,
+      "logps/chosen": -0.5947206020355225,
+      "logps/rejected": -0.8544057607650757,
+      "loss": 2.1425,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -5.947205543518066,
+      "rewards/margins": 2.596850872039795,
+      "rewards/rejected": -8.54405689239502,
       "step": 130
     },
     {
-      "epoch": 0.289350301406564,
-      "grad_norm": 22.96804359323175,
-      "learning_rate": 4.4752726384955293e-07,
-      "logits/chosen": -2.366454601287842,
-      "logits/rejected": -2.3833343982696533,
-      "logps/chosen": -0.4526381492614746,
-      "logps/rejected": -0.5151073336601257,
-      "loss": 0.7816,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -1.1315953731536865,
-      "rewards/margins": 0.15617303550243378,
-      "rewards/rejected": -1.2877686023712158,
+      "epoch": 0.28857715430861725,
+      "grad_norm": 66.68661326155792,
+      "learning_rate": 8.955355173281707e-07,
+      "logits/chosen": -1.0726548433303833,
+      "logits/rejected": -1.0262948274612427,
+      "logps/chosen": -0.708818256855011,
+      "logps/rejected": -0.8760965466499329,
+      "loss": 2.0661,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -7.0881829261779785,
+      "rewards/margins": 1.67278254032135,
+      "rewards/rejected": -8.760965347290039,
       "step": 135
     },
     {
-      "epoch": 0.3000669792364367,
-      "grad_norm": 30.57012215821005,
-      "learning_rate": 4.4164491972555294e-07,
-      "logits/chosen": -2.4294421672821045,
-      "logits/rejected": -2.4487221240997314,
-      "logps/chosen": -0.44983214139938354,
-      "logps/rejected": -0.49694761633872986,
-      "loss": 0.8084,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -1.1245803833007812,
-      "rewards/margins": 0.11778867244720459,
-      "rewards/rejected": -1.2423690557479858,
+      "epoch": 0.29926519706078825,
+      "grad_norm": 88.39987976267922,
+      "learning_rate": 8.838223701790055e-07,
+      "logits/chosen": -1.144708275794983,
+      "logits/rejected": -1.1237266063690186,
+      "logps/chosen": -0.8580458760261536,
+      "logps/rejected": -1.0160259008407593,
+      "loss": 2.0786,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -8.580458641052246,
+      "rewards/margins": 1.5798008441925049,
+      "rewards/rejected": -10.160260200500488,
       "step": 140
     },
     {
-      "epoch": 0.31078365706630945,
-      "grad_norm": 21.180411511344364,
-      "learning_rate": 4.3549326208287393e-07,
-      "logits/chosen": -2.530730962753296,
-      "logits/rejected": -2.5269761085510254,
-      "logps/chosen": -0.42974090576171875,
-      "logps/rejected": -0.4626144766807556,
-      "loss": 0.7883,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -1.0743523836135864,
-      "rewards/margins": 0.08218394219875336,
-      "rewards/rejected": -1.1565362215042114,
+      "epoch": 0.30995323981295925,
+      "grad_norm": 90.83612810193658,
+      "learning_rate": 8.71572412738697e-07,
+      "logits/chosen": -1.0413812398910522,
+      "logits/rejected": -1.0159306526184082,
+      "logps/chosen": -0.9142902493476868,
+      "logps/rejected": -1.1587440967559814,
+      "loss": 2.0446,
+      "rewards/accuracies": 0.7437499761581421,
+      "rewards/chosen": -9.142904281616211,
+      "rewards/margins": 2.444538116455078,
+      "rewards/rejected": -11.587442398071289,
       "step": 145
     },
     {
-      "epoch": 0.3215003348961822,
-      "grad_norm": 29.400301381227408,
-      "learning_rate": 4.2908093568271187e-07,
-      "logits/chosen": -2.5069005489349365,
-      "logits/rejected": -2.4892706871032715,
-      "logps/chosen": -0.4535086750984192,
-      "logps/rejected": -0.4898821711540222,
-      "loss": 0.797,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -1.1337716579437256,
-      "rewards/margins": 0.09093374013900757,
-      "rewards/rejected": -1.224705457687378,
+      "epoch": 0.32064128256513025,
+      "grad_norm": 84.07324057563632,
+      "learning_rate": 8.588027776804058e-07,
+      "logits/chosen": -1.06027352809906,
+      "logits/rejected": -1.0439517498016357,
+      "logps/chosen": -1.0236120223999023,
+      "logps/rejected": -1.2801861763000488,
+      "loss": 1.9933,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -10.236120223999023,
+      "rewards/margins": 2.565741777420044,
+      "rewards/rejected": -12.801861763000488,
       "step": 150
     },
     {
-      "epoch": 0.3322170127260549,
-      "grad_norm": 28.140109365748334,
-      "learning_rate": 4.2241695159714843e-07,
-      "logits/chosen": -2.483699321746826,
-      "logits/rejected": -2.4972500801086426,
-      "logps/chosen": -0.4816765785217285,
-      "logps/rejected": -0.5146931409835815,
-      "loss": 0.8022,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -1.2041914463043213,
-      "rewards/margins": 0.08254122734069824,
-      "rewards/rejected": -1.2867326736450195,
+      "epoch": 0.33132932531730125,
+      "grad_norm": 83.2873729325751,
+      "learning_rate": 8.455313244934324e-07,
+      "logits/chosen": -1.0788214206695557,
+      "logits/rejected": -1.059463620185852,
+      "logps/chosen": -1.1024755239486694,
+      "logps/rejected": -1.424791932106018,
+      "loss": 2.0534,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -11.024755477905273,
+      "rewards/margins": 3.2231624126434326,
+      "rewards/rejected": -14.247919082641602,
       "step": 155
     },
     {
-      "epoch": 0.34293369055592765,
-      "grad_norm": 25.486526483358187,
-      "learning_rate": 4.1551067454613074e-07,
-      "logits/chosen": -2.505037784576416,
-      "logits/rejected": -2.5097804069519043,
-      "logps/chosen": -0.448524534702301,
-      "logps/rejected": -0.46093541383743286,
-      "loss": 0.8047,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -1.1213113069534302,
-      "rewards/margins": 0.031027212738990784,
-      "rewards/rejected": -1.1523386240005493,
+      "epoch": 0.3420173680694723,
+      "grad_norm": 91.00920515798947,
+      "learning_rate": 8.317766145051057e-07,
+      "logits/chosen": -1.115094780921936,
+      "logits/rejected": -1.0990221500396729,
+      "logps/chosen": -1.2077633142471313,
+      "logps/rejected": -1.5987141132354736,
+      "loss": 1.9827,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -12.077634811401367,
+      "rewards/margins": 3.909506320953369,
+      "rewards/rejected": -15.987141609191895,
       "step": 160
     },
     {
-      "epoch": 0.3536503683858004,
-      "grad_norm": 18.985578073968945,
-      "learning_rate": 4.0837180973747876e-07,
-      "logits/chosen": -2.515350103378296,
-      "logits/rejected": -2.528233528137207,
-      "logps/chosen": -0.43952083587646484,
-      "logps/rejected": -0.4735996127128601,
-      "loss": 0.7917,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -1.0988019704818726,
-      "rewards/margins": 0.08519701659679413,
-      "rewards/rejected": -1.1839991807937622,
+      "epoch": 0.3527054108216433,
+      "grad_norm": 65.16453388698586,
+      "learning_rate": 8.175578849210894e-07,
+      "logits/chosen": -1.1276443004608154,
+      "logits/rejected": -1.106072187423706,
+      "logps/chosen": -1.156090259552002,
+      "logps/rejected": -1.5160917043685913,
+      "loss": 1.8258,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -11.560903549194336,
+      "rewards/margins": 3.6000137329101562,
+      "rewards/rejected": -15.160917282104492,
       "step": 165
     },
     {
-      "epoch": 0.3643670462156731,
-      "grad_norm": 25.120865745392436,
-      "learning_rate": 4.0101038922841455e-07,
-      "logits/chosen": -2.576842784881592,
-      "logits/rejected": -2.5906457901000977,
-      "logps/chosen": -0.4653005599975586,
-      "logps/rejected": -0.4848781228065491,
-      "loss": 0.8051,
-      "rewards/accuracies": 0.53125,
-      "rewards/chosen": -1.163251280784607,
-      "rewards/margins": 0.0489438995718956,
-      "rewards/rejected": -1.2121951580047607,
+      "epoch": 0.3633934535738143,
+      "grad_norm": 89.17851329246218,
+      "learning_rate": 8.028950219204099e-07,
+      "logits/chosen": -1.133327841758728,
+      "logits/rejected": -1.1123793125152588,
+      "logps/chosen": -1.1282349824905396,
+      "logps/rejected": -1.5152461528778076,
+      "loss": 1.805,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -11.282350540161133,
+      "rewards/margins": 3.8701133728027344,
+      "rewards/rejected": -15.15246295928955,
       "step": 170
     },
     {
-      "epoch": 0.3750837240455459,
-      "grad_norm": 22.847481742575464,
-      "learning_rate": 3.934367578277783e-07,
-      "logits/chosen": -2.530890941619873,
-      "logits/rejected": -2.531864643096924,
-      "logps/chosen": -0.45446857810020447,
-      "logps/rejected": -0.5132210850715637,
-      "loss": 0.7911,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -1.1361713409423828,
-      "rewards/margins": 0.14688125252723694,
-      "rewards/rejected": -1.283052682876587,
+      "epoch": 0.3740814963259853,
+      "grad_norm": 90.91620530583262,
+      "learning_rate": 7.878085328428368e-07,
+      "logits/chosen": -1.1684499979019165,
+      "logits/rejected": -1.1230958700180054,
+      "logps/chosen": -1.199479341506958,
+      "logps/rejected": -1.480486273765564,
+      "loss": 1.7009,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -11.994794845581055,
+      "rewards/margins": 2.8100688457489014,
+      "rewards/rejected": -14.804861068725586,
       "step": 175
     },
     {
-      "epoch": 0.38580040187541864,
-      "grad_norm": 26.0995283209312,
-      "learning_rate": 3.856615585587434e-07,
-      "logits/chosen": -2.599412441253662,
-      "logits/rejected": -2.6105477809906006,
-      "logps/chosen": -0.4383458197116852,
-      "logps/rejected": -0.4895172715187073,
-      "loss": 0.7885,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": -1.0958645343780518,
-      "rewards/margins": 0.12792867422103882,
-      "rewards/rejected": -1.2237932682037354,
+      "epoch": 0.3847695390781563,
+      "grad_norm": 85.1133679017557,
+      "learning_rate": 7.723195175075135e-07,
+      "logits/chosen": -1.125327706336975,
+      "logits/rejected": -1.107810139656067,
+      "logps/chosen": -1.1546721458435059,
+      "logps/rejected": -1.5301127433776855,
+      "loss": 1.5941,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": -11.546720504760742,
+      "rewards/margins": 3.7544052600860596,
+      "rewards/rejected": -15.301126480102539,
       "step": 180
     },
     {
-      "epoch": 0.3965170797052914,
-      "grad_norm": 29.1301293940354,
-      "learning_rate": 3.776957177024581e-07,
-      "logits/chosen": -2.597064256668091,
-      "logits/rejected": -2.584998607635498,
-      "logps/chosen": -0.46641644835472107,
-      "logps/rejected": -0.5318672060966492,
-      "loss": 0.7907,
-      "rewards/accuracies": 0.5562499761581421,
-      "rewards/chosen": -1.1660410165786743,
-      "rewards/margins": 0.16362698376178741,
-      "rewards/rejected": -1.3296680450439453,
+      "epoch": 0.3954575818303273,
+      "grad_norm": 80.80074018938376,
+      "learning_rate": 7.564496387029531e-07,
+      "logits/chosen": -1.1530572175979614,
+      "logits/rejected": -1.1004054546356201,
+      "logps/chosen": -1.1977171897888184,
+      "logps/rejected": -1.6391410827636719,
+      "loss": 1.6323,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -11.977171897888184,
+      "rewards/margins": 4.414237976074219,
+      "rewards/rejected": -16.39141273498535,
       "step": 185
     },
     {
-      "epoch": 0.4072337575351641,
-      "grad_norm": 26.246700781484808,
-      "learning_rate": 3.69550429443633e-07,
-      "logits/chosen": -2.6202404499053955,
-      "logits/rejected": -2.6158204078674316,
-      "logps/chosen": -0.4914141595363617,
-      "logps/rejected": -0.4930477738380432,
-      "loss": 0.7991,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": -1.2285354137420654,
-      "rewards/margins": 0.004083993844687939,
-      "rewards/rejected": -1.2326194047927856,
+      "epoch": 0.4061456245824983,
+      "grad_norm": 87.8130211657885,
+      "learning_rate": 7.402210918896689e-07,
+      "logits/chosen": -1.1491271257400513,
+      "logits/rejected": -1.159403681755066,
+      "logps/chosen": -1.3192346096038818,
+      "logps/rejected": -1.8000110387802124,
+      "loss": 1.5253,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -13.192344665527344,
+      "rewards/margins": 4.807766437530518,
+      "rewards/rejected": -18.000110626220703,
       "step": 190
     },
     {
-      "epoch": 0.41795043536503684,
-      "grad_norm": 37.655825101171395,
-      "learning_rate": 3.612371401396502e-07,
-      "logits/chosen": -2.7039732933044434,
-      "logits/rejected": -2.6810288429260254,
-      "logps/chosen": -0.47953447699546814,
-      "logps/rejected": -0.5316765904426575,
-      "loss": 0.7925,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -1.198836088180542,
-      "rewards/margins": 0.13035525381565094,
-      "rewards/rejected": -1.3291913270950317,
+      "epoch": 0.4168336673346693,
+      "grad_norm": 84.55372178403601,
+      "learning_rate": 7.236565741578162e-07,
+      "logits/chosen": -1.1186103820800781,
+      "logits/rejected": -1.1020419597625732,
+      "logps/chosen": -1.367811918258667,
+      "logps/rejected": -1.7383407354354858,
+      "loss": 1.6656,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": -13.678120613098145,
+      "rewards/margins": 3.705289363861084,
+      "rewards/rejected": -17.38340950012207,
       "step": 195
     },
     {
-      "epoch": 0.4286671131949096,
-      "grad_norm": 22.671830849613123,
-      "learning_rate": 3.527675322353011e-07,
-      "logits/chosen": -2.711343288421631,
-      "logits/rejected": -2.724891185760498,
-      "logps/chosen": -0.4652802348136902,
-      "logps/rejected": -0.52509605884552,
-      "loss": 0.7948,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -1.1632006168365479,
-      "rewards/margins": 0.14953944087028503,
-      "rewards/rejected": -1.3127400875091553,
+      "epoch": 0.42752171008684037,
+      "grad_norm": 84.20111385217919,
+      "learning_rate": 7.067792524832603e-07,
+      "logits/chosen": -1.1123974323272705,
+      "logits/rejected": -1.1035972833633423,
+      "logps/chosen": -1.3991031646728516,
+      "logps/rejected": -1.8270728588104248,
+      "loss": 1.514,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": -13.991032600402832,
+      "rewards/margins": 4.279696464538574,
+      "rewards/rejected": -18.27073097229004,
       "step": 200
     },
     {
-      "epoch": 0.4393837910247823,
-      "grad_norm": 30.58821587730182,
-      "learning_rate": 3.4415350784575693e-07,
-      "logits/chosen": -2.7035841941833496,
-      "logits/rejected": -2.694777250289917,
-      "logps/chosen": -0.5165461897850037,
-      "logps/rejected": -0.573054850101471,
-      "loss": 0.7735,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": -1.291365623474121,
-      "rewards/margins": 0.1412714421749115,
-      "rewards/rejected": -1.432637095451355,
+      "epoch": 0.43820975283901137,
+      "grad_norm": 95.99111629240849,
+      "learning_rate": 6.896127313264642e-07,
+      "logits/chosen": -1.174342393875122,
+      "logits/rejected": -1.1257025003433228,
+      "logps/chosen": -1.463107705116272,
+      "logps/rejected": -1.938598871231079,
+      "loss": 1.6893,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -14.631075859069824,
+      "rewards/margins": 4.754912853240967,
+      "rewards/rejected": -19.385990142822266,
       "step": 205
     },
     {
-      "epoch": 0.45010046885465504,
-      "grad_norm": 31.58801707836199,
-      "learning_rate": 3.3540717203084224e-07,
-      "logits/chosen": -2.7756335735321045,
-      "logits/rejected": -2.751556873321533,
-      "logps/chosen": -0.5132556557655334,
-      "logps/rejected": -0.5941559076309204,
-      "loss": 0.786,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -1.2831389904022217,
-      "rewards/margins": 0.20225080847740173,
-      "rewards/rejected": -1.4853899478912354,
+      "epoch": 0.44889779559118237,
+      "grad_norm": 100.92269633403131,
+      "learning_rate": 6.721810196195174e-07,
+      "logits/chosen": -1.2214075326919556,
+      "logits/rejected": -1.2106014490127563,
+      "logps/chosen": -1.468076467514038,
+      "logps/rejected": -1.8905394077301025,
+      "loss": 1.573,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -14.680766105651855,
+      "rewards/margins": 4.224628925323486,
+      "rewards/rejected": -18.905391693115234,
       "step": 210
     },
     {
-      "epoch": 0.4608171466845278,
-      "grad_norm": 41.456925783555256,
-      "learning_rate": 3.265408157841153e-07,
-      "logits/chosen": -2.7932379245758057,
-      "logits/rejected": -2.7999000549316406,
-      "logps/chosen": -0.47902554273605347,
-      "logps/rejected": -0.5101593732833862,
-      "loss": 0.8028,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -1.197563886642456,
-      "rewards/margins": 0.07783453166484833,
-      "rewards/rejected": -1.2753984928131104,
+      "epoch": 0.45958583834335337,
+      "grad_norm": 98.80302089659659,
+      "learning_rate": 6.545084971874736e-07,
+      "logits/chosen": -1.1770942211151123,
+      "logits/rejected": -1.161029577255249,
+      "logps/chosen": -1.4916527271270752,
+      "logps/rejected": -1.9983189105987549,
+      "loss": 1.4988,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -14.916528701782227,
+      "rewards/margins": 5.0666608810424805,
+      "rewards/rejected": -19.98318862915039,
       "step": 215
     },
     {
-      "epoch": 0.4715338245144005,
-      "grad_norm": 31.036341061185052,
-      "learning_rate": 3.1756689876066187e-07,
-      "logits/chosen": -2.755951404571533,
-      "logits/rejected": -2.772839069366455,
-      "logps/chosen": -0.5013461112976074,
-      "logps/rejected": -0.5391478538513184,
-      "loss": 0.7933,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -1.2533652782440186,
-      "rewards/margins": 0.09450428932905197,
-      "rewards/rejected": -1.3478695154190063,
+      "epoch": 0.47027388109552437,
+      "grad_norm": 94.29735384618333,
+      "learning_rate": 6.3661988065096e-07,
+      "logits/chosen": -1.2350765466690063,
+      "logits/rejected": -1.2198480367660522,
+      "logps/chosen": -1.5811980962753296,
+      "logps/rejected": -2.1048731803894043,
+      "loss": 1.3966,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": -15.811983108520508,
+      "rewards/margins": 5.236747741699219,
+      "rewards/rejected": -21.048730850219727,
       "step": 220
     },
     {
-      "epoch": 0.4822505023442733,
-      "grad_norm": 26.21868089092331,
-      "learning_rate": 3.0849803176787184e-07,
-      "logits/chosen": -2.7325472831726074,
-      "logits/rejected": -2.7435898780822754,
-      "logps/chosen": -0.5336536765098572,
-      "logps/rejected": -0.5821245908737183,
-      "loss": 0.7835,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -1.3341342210769653,
-      "rewards/margins": 0.12117739021778107,
-      "rewards/rejected": -1.4553115367889404,
+      "epoch": 0.48096192384769537,
+      "grad_norm": 73.78349743811617,
+      "learning_rate": 6.185401888577487e-07,
+      "logits/chosen": -1.2252886295318604,
+      "logits/rejected": -1.195698618888855,
+      "logps/chosen": -1.6302112340927124,
+      "logps/rejected": -2.095083475112915,
+      "loss": 1.4949,
+      "rewards/accuracies": 0.768750011920929,
+      "rewards/chosen": -16.302112579345703,
+      "rewards/margins": 4.6487226486206055,
+      "rewards/rejected": -20.950834274291992,
       "step": 225
     },
     {
-      "epoch": 0.49296718017414604,
-      "grad_norm": 32.03132066446478,
-      "learning_rate": 2.993469590438065e-07,
-      "logits/chosen": -2.6822681427001953,
-      "logits/rejected": -2.7262749671936035,
-      "logps/chosen": -0.5299819111824036,
-      "logps/rejected": -0.5460516810417175,
-      "loss": 0.8012,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -1.3249547481536865,
-      "rewards/margins": 0.04017447307705879,
-      "rewards/rejected": -1.3651292324066162,
+      "epoch": 0.4916499665998664,
+      "grad_norm": 72.86824044049841,
+      "learning_rate": 6.002947078916364e-07,
+      "logits/chosen": -1.2797940969467163,
+      "logits/rejected": -1.2363423109054565,
+      "logps/chosen": -1.5519064664840698,
+      "logps/rejected": -2.0577468872070312,
+      "loss": 1.3568,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -15.519063949584961,
+      "rewards/margins": 5.05840539932251,
+      "rewards/rejected": -20.577470779418945,
       "step": 230
     },
     {
-      "epoch": 0.5036838580040187,
-      "grad_norm": 29.43042758117792,
-      "learning_rate": 2.901265403480597e-07,
-      "logits/chosen": -2.7072153091430664,
-      "logits/rejected": -2.7308104038238525,
-      "logps/chosen": -0.5455118417739868,
-      "logps/rejected": -0.5925866961479187,
-      "loss": 0.7764,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -1.3637795448303223,
-      "rewards/margins": 0.11768718808889389,
-      "rewards/rejected": -1.4814666509628296,
+      "epoch": 0.5023380093520374,
+      "grad_norm": 87.56760475487422,
+      "learning_rate": 5.819089557075688e-07,
+      "logits/chosen": -1.3070341348648071,
+      "logits/rejected": -1.2829915285110474,
+      "logps/chosen": -1.5769996643066406,
+      "logps/rejected": -2.101868152618408,
+      "loss": 1.4121,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": -15.769996643066406,
+      "rewards/margins": 5.248685359954834,
+      "rewards/rejected": -21.0186824798584,
       "step": 235
     },
     {
-      "epoch": 0.5144005358338914,
-      "grad_norm": 29.703263320129235,
-      "learning_rate": 2.808497328902782e-07,
-      "logits/chosen": -2.6819677352905273,
-      "logits/rejected": -2.6889100074768066,
-      "logps/chosen": -0.5209198594093323,
-      "logps/rejected": -0.5574553608894348,
-      "loss": 0.8044,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -1.3022996187210083,
-      "rewards/margins": 0.09133879840373993,
-      "rewards/rejected": -1.3936383724212646,
+      "epoch": 0.5130260521042084,
+      "grad_norm": 97.95206336790469,
+      "learning_rate": 5.634086464424742e-07,
+      "logits/chosen": -1.2722411155700684,
+      "logits/rejected": -1.2728557586669922,
+      "logps/chosen": -1.5129470825195312,
+      "logps/rejected": -2.0472776889801025,
+      "loss": 1.4543,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -15.12946891784668,
+      "rewards/margins": 5.343307018280029,
+      "rewards/rejected": -20.4727783203125,
       "step": 240
     },
     {
-      "epoch": 0.5251172136637642,
-      "grad_norm": 37.052180806405566,
-      "learning_rate": 2.715295731217396e-07,
-      "logits/chosen": -2.7002854347229004,
-      "logits/rejected": -2.70177960395813,
-      "logps/chosen": -0.5708497762680054,
-      "logps/rejected": -0.5927925109863281,
-      "loss": 0.8003,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -1.4271246194839478,
-      "rewards/margins": 0.05485675856471062,
-      "rewards/rejected": -1.4819812774658203,
+      "epoch": 0.5237140948563794,
+      "grad_norm": 91.68979328640371,
+      "learning_rate": 5.448196544517167e-07,
+      "logits/chosen": -1.3419743776321411,
+      "logits/rejected": -1.2915674448013306,
+      "logps/chosen": -1.5888983011245728,
+      "logps/rejected": -2.2375972270965576,
+      "loss": 1.3296,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -15.888982772827148,
+      "rewards/margins": 6.486991882324219,
+      "rewards/rejected": -22.375974655151367,
       "step": 245
     },
     {
-      "epoch": 0.535833891493637,
-      "grad_norm": 32.762361065320945,
-      "learning_rate": 2.621791584155725e-07,
-      "logits/chosen": -2.6251308917999268,
-      "logits/rejected": -2.614744186401367,
-      "logps/chosen": -0.5160342454910278,
-      "logps/rejected": -0.6052494645118713,
-      "loss": 0.7709,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -1.2900855541229248,
-      "rewards/margins": 0.2230379283428192,
-      "rewards/rejected": -1.5131235122680664,
+      "epoch": 0.5344021376085505,
+      "grad_norm": 90.04751296677803,
+      "learning_rate": 5.26167978121472e-07,
+      "logits/chosen": -1.3019921779632568,
+      "logits/rejected": -1.287535309791565,
+      "logps/chosen": -1.632799744606018,
+      "logps/rejected": -2.2480623722076416,
+      "loss": 1.3388,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -16.3279972076416,
+      "rewards/margins": 6.1526265144348145,
+      "rewards/rejected": -22.48062515258789,
       "step": 250
     },
     {
-      "epoch": 0.5465505693235098,
-      "grad_norm": 31.109425612718024,
-      "learning_rate": 2.52811628661367e-07,
-      "logits/chosen": -2.7665226459503174,
-      "logits/rejected": -2.7633566856384277,
-      "logps/chosen": -0.5172406435012817,
-      "logps/rejected": -0.55857914686203,
-      "loss": 0.7997,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -1.2931016683578491,
-      "rewards/margins": 0.10334619134664536,
-      "rewards/rejected": -1.3964478969573975,
+      "epoch": 0.5450901803607214,
+      "grad_norm": 72.22778279348458,
+      "learning_rate": 5.074797035076318e-07,
+      "logits/chosen": -1.3449289798736572,
+      "logits/rejected": -1.321571707725525,
+      "logps/chosen": -1.6949834823608398,
+      "logps/rejected": -2.1791787147521973,
+      "loss": 1.4266,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -16.949832916259766,
+      "rewards/margins": 4.84195613861084,
+      "rewards/rejected": -21.791790008544922,
       "step": 255
     },
     {
-      "epoch": 0.5572672471533825,
-      "grad_norm": 34.78606161386547,
-      "learning_rate": 2.434401478000353e-07,
-      "logits/chosen": -2.760376214981079,
-      "logits/rejected": -2.7761454582214355,
-      "logps/chosen": -0.5151620507240295,
-      "logps/rejected": -0.5613347887992859,
-      "loss": 0.778,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": -1.287904977798462,
-      "rewards/margins": 0.11543188244104385,
-      "rewards/rejected": -1.4033368825912476,
+      "epoch": 0.5557782231128925,
+      "grad_norm": 82.32130834680915,
+      "learning_rate": 4.887809678520975e-07,
+      "logits/chosen": -1.3022774457931519,
+      "logits/rejected": -1.2787460088729858,
+      "logps/chosen": -1.6087223291397095,
+      "logps/rejected": -2.112210750579834,
+      "loss": 1.4418,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -16.087223052978516,
+      "rewards/margins": 5.034885883331299,
+      "rewards/rejected": -21.122108459472656,
       "step": 260
     },
     {
-      "epoch": 0.5679839249832552,
-      "grad_norm": 26.46498554148562,
-      "learning_rate": 2.340778853248759e-07,
-      "logits/chosen": -2.7599408626556396,
-      "logits/rejected": -2.7651569843292236,
-      "logps/chosen": -0.5734432935714722,
-      "logps/rejected": -0.6467745900154114,
-      "loss": 0.7688,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -1.4336082935333252,
-      "rewards/margins": 0.18332824110984802,
-      "rewards/rejected": -1.616936445236206,
+      "epoch": 0.5664662658650634,
+      "grad_norm": 82.10988858763459,
+      "learning_rate": 4.700979230274829e-07,
+      "logits/chosen": -1.263526201248169,
+      "logits/rejected": -1.2527527809143066,
+      "logps/chosen": -1.7088844776153564,
+      "logps/rejected": -2.2491986751556396,
+      "loss": 1.2866,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -17.08884620666504,
+      "rewards/margins": 5.40314245223999,
+      "rewards/rejected": -22.491985321044922,
       "step": 265
     },
     {
-      "epoch": 0.578700602813128,
-      "grad_norm": 34.04025832229061,
-      "learning_rate": 2.247379977748339e-07,
-      "logits/chosen": -2.7183938026428223,
-      "logits/rejected": -2.75644588470459,
-      "logps/chosen": -0.5665550231933594,
-      "logps/rejected": -0.6176514625549316,
-      "loss": 0.7952,
-      "rewards/accuracies": 0.518750011920929,
-      "rewards/chosen": -1.4163875579833984,
-      "rewards/margins": 0.12774114310741425,
-      "rewards/rejected": -1.544128656387329,
+      "epoch": 0.5771543086172345,
+      "grad_norm": 74.22531378642971,
+      "learning_rate": 4.514566989613559e-07,
+      "logits/chosen": -1.276107907295227,
+      "logits/rejected": -1.2517645359039307,
+      "logps/chosen": -1.5033190250396729,
+      "logps/rejected": -2.076201915740967,
+      "loss": 1.2308,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -15.033190727233887,
+      "rewards/margins": 5.728825569152832,
+      "rewards/rejected": -20.762014389038086,
       "step": 270
     },
     {
-      "epoch": 0.5894172806430007,
-      "grad_norm": 31.258853663890047,
-      "learning_rate": 2.1543361024596573e-07,
-      "logits/chosen": -2.7444400787353516,
-      "logits/rejected": -2.745250940322876,
-      "logps/chosen": -0.5861095190048218,
-      "logps/rejected": -0.6748340725898743,
-      "loss": 0.758,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -1.4652737379074097,
-      "rewards/margins": 0.22181133925914764,
-      "rewards/rejected": -1.6870851516723633,
+      "epoch": 0.5878423513694054,
+      "grad_norm": 66.56842436857252,
+      "learning_rate": 4.328833670911724e-07,
+      "logits/chosen": -1.2568957805633545,
+      "logits/rejected": -1.2262030839920044,
+      "logps/chosen": -1.562673568725586,
+      "logps/rejected": -2.0573482513427734,
+      "loss": 1.4699,
+      "rewards/accuracies": 0.8062499761581421,
+      "rewards/chosen": -15.626736640930176,
+      "rewards/margins": 4.946745872497559,
+      "rewards/rejected": -20.573482513427734,
       "step": 275
     },
     {
-      "epoch": 0.6001339584728734,
-      "grad_norm": 46.09656975376248,
-      "learning_rate": 2.0617779794709082e-07,
-      "logits/chosen": -2.724043607711792,
-      "logits/rejected": -2.74163818359375,
-      "logps/chosen": -0.5856515169143677,
-      "logps/rejected": -0.643699049949646,
-      "loss": 0.7812,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -1.464128851890564,
-      "rewards/margins": 0.14511898159980774,
-      "rewards/rejected": -1.6092478036880493,
+      "epoch": 0.5985303941215765,
+      "grad_norm": 71.08794378986906,
+      "learning_rate": 4.144039039010124e-07,
+      "logits/chosen": -1.3348051309585571,
+      "logits/rejected": -1.3145146369934082,
+      "logps/chosen": -1.6333341598510742,
+      "logps/rejected": -2.22784686088562,
+      "loss": 1.3131,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -16.33333969116211,
+      "rewards/margins": 5.945128440856934,
+      "rewards/rejected": -22.27846908569336,
       "step": 280
     },
     {
-      "epoch": 0.6108506363027462,
-      "grad_norm": 43.94493014255228,
-      "learning_rate": 1.9698356782554687e-07,
-      "logits/chosen": -2.6912693977355957,
-      "logits/rejected": -2.707552194595337,
-      "logps/chosen": -0.5880975723266602,
-      "logps/rejected": -0.6368181705474854,
-      "loss": 0.7642,
-      "rewards/accuracies": 0.59375,
-      "rewards/chosen": -1.4702438116073608,
-      "rewards/margins": 0.12180168926715851,
-      "rewards/rejected": -1.592045545578003,
+      "epoch": 0.6092184368737475,
+      "grad_norm": 93.72278509593903,
+      "learning_rate": 3.960441545911204e-07,
+      "logits/chosen": -1.297483205795288,
+      "logits/rejected": -1.2733924388885498,
+      "logps/chosen": -1.650708794593811,
+      "logps/rejected": -2.298219680786133,
+      "loss": 1.0059,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -16.50708770751953,
+      "rewards/margins": 6.4751081466674805,
+      "rewards/rejected": -22.982196807861328,
       "step": 285
     },
     {
-      "epoch": 0.6215673141326189,
-      "grad_norm": 49.663136616239086,
-      "learning_rate": 1.878638402888727e-07,
-      "logits/chosen": -2.7193799018859863,
-      "logits/rejected": -2.722045421600342,
-      "logps/chosen": -0.6236746311187744,
-      "logps/rejected": -0.6843193173408508,
-      "loss": 0.785,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -1.559186577796936,
-      "rewards/margins": 0.15161168575286865,
-      "rewards/rejected": -1.7107985019683838,
+      "epoch": 0.6199064796259185,
+      "grad_norm": 76.68999166591557,
+      "learning_rate": 3.778297969310529e-07,
+      "logits/chosen": -1.32032310962677,
+      "logits/rejected": -1.2831968069076538,
+      "logps/chosen": -1.706647276878357,
+      "logps/rejected": -2.2490804195404053,
+      "loss": 1.3171,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -17.066471099853516,
+      "rewards/margins": 5.424333572387695,
+      "rewards/rejected": -22.490802764892578,
       "step": 290
     },
     {
-      "epoch": 0.6322839919624916,
-      "grad_norm": 34.23744852441083,
-      "learning_rate": 1.7883143104810223e-07,
-      "logits/chosen": -2.6754257678985596,
-      "logits/rejected": -2.673750877380371,
-      "logps/chosen": -0.6068626046180725,
-      "logps/rejected": -0.7437673807144165,
-      "loss": 0.754,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -1.5171564817428589,
-      "rewards/margins": 0.3422619104385376,
-      "rewards/rejected": -1.859418511390686,
+      "epoch": 0.6305945223780896,
+      "grad_norm": 87.09453084365835,
+      "learning_rate": 3.5978630534699865e-07,
+      "logits/chosen": -1.2636520862579346,
+      "logits/rejected": -1.2527822256088257,
+      "logps/chosen": -1.6968777179718018,
+      "logps/rejected": -2.257763624191284,
+      "loss": 1.1554,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -16.96877670288086,
+      "rewards/margins": 5.608861446380615,
+      "rewards/rejected": -22.57763671875,
       "step": 295
     },
     {
-      "epoch": 0.6430006697923644,
-      "grad_norm": 33.8142047010383,
-      "learning_rate": 1.6989903310818548e-07,
-      "logits/chosen": -2.714789867401123,
-      "logits/rejected": -2.710334062576294,
-      "logps/chosen": -0.6445562243461609,
-      "logps/rejected": -0.7172508239746094,
-      "loss": 0.7768,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": -1.611390471458435,
-      "rewards/margins": 0.1817365437746048,
-      "rewards/rejected": -1.7931270599365234,
+      "epoch": 0.6412825651302605,
+      "grad_norm": 82.76223672337171,
+      "learning_rate": 3.4193891529348795e-07,
+      "logits/chosen": -1.206903100013733,
+      "logits/rejected": -1.1841256618499756,
+      "logps/chosen": -1.7743552923202515,
+      "logps/rejected": -2.2721633911132812,
+      "loss": 1.5577,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": -17.74355125427246,
+      "rewards/margins": 4.978079795837402,
+      "rewards/rejected": -22.72163200378418,
       "step": 300
     },
     {
-      "epoch": 0.6537173476222371,
-      "grad_norm": 89.8750741255919,
-      "learning_rate": 1.610791989308466e-07,
-      "logits/chosen": -2.693514347076416,
-      "logits/rejected": -2.6736016273498535,
-      "logps/chosen": -0.6287732720375061,
-      "logps/rejected": -0.701014518737793,
-      "loss": 0.7907,
-      "rewards/accuracies": 0.5874999761581421,
-      "rewards/chosen": -1.5719332695007324,
-      "rewards/margins": 0.1806029975414276,
-      "rewards/rejected": -1.7525361776351929,
+      "epoch": 0.6519706078824316,
+      "grad_norm": 78.16246422530884,
+      "learning_rate": 3.243125879593286e-07,
+      "logits/chosen": -1.3001482486724854,
+      "logits/rejected": -1.2663486003875732,
+      "logps/chosen": -1.6906330585479736,
+      "logps/rejected": -2.1780858039855957,
+      "loss": 1.3471,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": -16.906330108642578,
+      "rewards/margins": 4.874526023864746,
+      "rewards/rejected": -21.78085708618164,
       "step": 305
     },
     {
-      "epoch": 0.6644340254521098,
-      "grad_norm": 49.27896755369507,
-      "learning_rate": 1.523843227949418e-07,
-      "logits/chosen": -2.791170597076416,
-      "logits/rejected": -2.7779464721679688,
-      "logps/chosen": -0.6191990971565247,
-      "logps/rejected": -0.690567672252655,
-      "loss": 0.7726,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -1.5479977130889893,
-      "rewards/margins": 0.17842140793800354,
-      "rewards/rejected": -1.72641921043396,
+      "epoch": 0.6626586506346025,
+      "grad_norm": 91.76909016032246,
+      "learning_rate": 3.069319753571269e-07,
+      "logits/chosen": -1.3407663106918335,
+      "logits/rejected": -1.3254361152648926,
+      "logps/chosen": -1.6954444646835327,
+      "logps/rejected": -2.250342845916748,
+      "loss": 1.3722,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -16.95444679260254,
+      "rewards/margins": 5.548983573913574,
+      "rewards/rejected": -22.503429412841797,
       "step": 310
     },
     {
-      "epoch": 0.6751507032819826,
-      "grad_norm": 52.532294308547925,
-      "learning_rate": 1.4382662337910867e-07,
-      "logits/chosen": -2.7822985649108887,
-      "logits/rejected": -2.8041725158691406,
-      "logps/chosen": -0.546491801738739,
-      "logps/rejected": -0.6088187098503113,
-      "loss": 0.7633,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -1.3662294149398804,
-      "rewards/margins": 0.15581734478473663,
-      "rewards/rejected": -1.5220468044281006,
+      "epoch": 0.6733466933867736,
+      "grad_norm": 83.8485405296481,
+      "learning_rate": 2.898213858452173e-07,
+      "logits/chosen": -1.3287131786346436,
+      "logits/rejected": -1.2830489873886108,
+      "logps/chosen": -1.7108901739120483,
+      "logps/rejected": -2.263986587524414,
+      "loss": 1.2997,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -17.108901977539062,
+      "rewards/margins": 5.530962944030762,
+      "rewards/rejected": -22.63986587524414,
       "step": 315
     },
     {
-      "epoch": 0.6858673811118553,
-      "grad_norm": 31.304061633836678,
-      "learning_rate": 1.3541812659118104e-07,
-      "logits/chosen": -2.784820556640625,
-      "logits/rejected": -2.790811777114868,
-      "logps/chosen": -0.6415971517562866,
-      "logps/rejected": -0.6807955503463745,
-      "loss": 0.7832,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": -1.6039928197860718,
-      "rewards/margins": 0.09799607843160629,
-      "rewards/rejected": -1.701988935470581,
+      "epoch": 0.6840347361389446,
+      "grad_norm": 110.4735398683659,
+      "learning_rate": 2.730047501302266e-07,
+      "logits/chosen": -1.3247960805892944,
+      "logits/rejected": -1.3241022825241089,
+      "logps/chosen": -1.7461349964141846,
+      "logps/rejected": -2.435873508453369,
+      "loss": 1.1823,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -17.461349487304688,
+      "rewards/margins": 6.8973846435546875,
+      "rewards/rejected": -24.358732223510742,
       "step": 320
     },
     {
-      "epoch": 0.696584058941728,
-      "grad_norm": 42.827372642417416,
-      "learning_rate": 1.2717064866850047e-07,
-      "logits/chosen": -2.8045849800109863,
-      "logits/rejected": -2.7954859733581543,
-      "logps/chosen": -0.6314974427223206,
-      "logps/rejected": -0.7085036635398865,
-      "loss": 0.7874,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": -1.5787436962127686,
-      "rewards/margins": 0.19251549243927002,
-      "rewards/rejected": -1.771259069442749,
+      "epoch": 0.6947227788911156,
+      "grad_norm": 83.494848255409,
+      "learning_rate": 2.5650558779781635e-07,
+      "logits/chosen": -1.332975149154663,
+      "logits/rejected": -1.2938920259475708,
+      "logps/chosen": -1.795236349105835,
+      "logps/rejected": -2.5173287391662598,
+      "loss": 1.2571,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": -17.952363967895508,
+      "rewards/margins": 7.220925331115723,
+      "rewards/rejected": -25.173290252685547,
       "step": 325
     },
     {
-      "epoch": 0.7073007367716008,
-      "grad_norm": 35.26979827058928,
-      "learning_rate": 1.1909577957287117e-07,
-      "logits/chosen": -2.8660318851470947,
-      "logits/rejected": -2.870734691619873,
-      "logps/chosen": -0.5516513586044312,
-      "logps/rejected": -0.605838418006897,
-      "loss": 0.7555,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -1.379128336906433,
-      "rewards/margins": 0.13546761870384216,
-      "rewards/rejected": -1.5145959854125977,
+      "epoch": 0.7054108216432866,
+      "grad_norm": 67.71202935932357,
+      "learning_rate": 2.403469744184154e-07,
+      "logits/chosen": -1.2552909851074219,
+      "logits/rejected": -1.2208917140960693,
+      "logps/chosen": -1.8006776571273804,
+      "logps/rejected": -2.3320443630218506,
+      "loss": 1.3218,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -18.006778717041016,
+      "rewards/margins": 5.3136701583862305,
+      "rewards/rejected": -23.32044792175293,
       "step": 330
     },
     {
-      "epoch": 0.7180174146014735,
-      "grad_norm": 33.88925580800264,
-      "learning_rate": 1.1120486670349441e-07,
-      "logits/chosen": -2.82328724861145,
-      "logits/rejected": -2.847529649734497,
-      "logps/chosen": -0.6149914860725403,
-      "logps/rejected": -0.6911052465438843,
-      "loss": 0.7548,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -1.5374788045883179,
-      "rewards/margins": 0.19028423726558685,
-      "rewards/rejected": -1.7277631759643555,
+      "epoch": 0.7160988643954576,
+      "grad_norm": 100.05067628008052,
+      "learning_rate": 2.2455150927394878e-07,
+      "logits/chosen": -1.2909272909164429,
+      "logits/rejected": -1.279365062713623,
+      "logps/chosen": -1.7781559228897095,
+      "logps/rejected": -2.3740553855895996,
+      "loss": 1.2001,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": -17.781558990478516,
+      "rewards/margins": 5.9589948654174805,
+      "rewards/rejected": -23.740550994873047,
       "step": 335
     },
     {
-      "epoch": 0.7287340924313462,
-      "grad_norm": 31.71698090485735,
-      "learning_rate": 1.0350899895076948e-07,
-      "logits/chosen": -2.790008068084717,
-      "logits/rejected": -2.7927775382995605,
-      "logps/chosen": -0.6008199453353882,
-      "logps/rejected": -0.6924527287483215,
-      "loss": 0.7686,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -1.5020498037338257,
-      "rewards/margins": 0.2290818691253662,
-      "rewards/rejected": -1.7311317920684814,
+      "epoch": 0.7267869071476286,
+      "grad_norm": 97.4036448779786,
+      "learning_rate": 2.0914128375069722e-07,
+      "logits/chosen": -1.2993793487548828,
+      "logits/rejected": -1.2727620601654053,
+      "logps/chosen": -1.6889365911483765,
+      "logps/rejected": -2.2718539237976074,
+      "loss": 1.3272,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -16.889366149902344,
+      "rewards/margins": 5.829171180725098,
+      "rewards/rejected": -22.718538284301758,
       "step": 340
     },
     {
-      "epoch": 0.739450770261219,
-      "grad_norm": 36.716335914223706,
-      "learning_rate": 9.601899111337069e-08,
-      "logits/chosen": -2.885432481765747,
-      "logits/rejected": -2.8697662353515625,
-      "logps/chosen": -0.6102738976478577,
-      "logps/rejected": -0.6805952191352844,
-      "loss": 0.7819,
-      "rewards/accuracies": 0.65625,
-      "rewards/chosen": -1.5256847143173218,
-      "rewards/margins": 0.17580333352088928,
-      "rewards/rejected": -1.7014878988265991,
+      "epoch": 0.7374749498997996,
+      "grad_norm": 80.17854676377132,
+      "learning_rate": 1.9413785044249676e-07,
+      "logits/chosen": -1.3267228603363037,
+      "logits/rejected": -1.307962417602539,
+      "logps/chosen": -1.7601877450942993,
+      "logps/rejected": -2.4487035274505615,
+      "loss": 1.268,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -17.601877212524414,
+      "rewards/margins": 6.885159492492676,
+      "rewards/rejected": -24.487037658691406,
       "step": 345
     },
     {
-      "epoch": 0.7501674480910918,
-      "grad_norm": 33.17344514912859,
-      "learning_rate": 8.874536870049765e-08,
-      "logits/chosen": -2.865382432937622,
-      "logits/rejected": -2.88032865524292,
-      "logps/chosen": -0.6345846056938171,
-      "logps/rejected": -0.7329587936401367,
-      "loss": 0.7599,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -1.5864615440368652,
-      "rewards/margins": 0.24593544006347656,
-      "rewards/rejected": -1.8323971033096313,
+      "epoch": 0.7481629926519706,
+      "grad_norm": 93.74425146775721,
+      "learning_rate": 1.7956219300748792e-07,
+      "logits/chosen": -1.301977515220642,
+      "logits/rejected": -1.3074698448181152,
+      "logps/chosen": -1.6333377361297607,
+      "logps/rejected": -2.2102160453796387,
+      "loss": 1.3055,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -16.333377838134766,
+      "rewards/margins": 5.768781661987305,
+      "rewards/rejected": -22.102161407470703,
       "step": 350
     },
     {
-      "epoch": 0.7608841259209645,
-      "grad_norm": 35.10962301829223,
-      "learning_rate": 8.169835314065663e-08,
-      "logits/chosen": -2.787919521331787,
-      "logits/rejected": -2.7910847663879395,
-      "logps/chosen": -0.6197460889816284,
-      "logps/rejected": -0.6998850107192993,
-      "loss": 0.7762,
-      "rewards/accuracies": 0.6187499761581421,
-      "rewards/chosen": -1.5493652820587158,
-      "rewards/margins": 0.20034721493721008,
-      "rewards/rejected": -1.749712586402893,
+      "epoch": 0.7588510354041417,
+      "grad_norm": 81.82295078846326,
+      "learning_rate": 1.6543469682057104e-07,
+      "logits/chosen": -1.231096625328064,
+      "logits/rejected": -1.248560905456543,
+      "logps/chosen": -1.6608985662460327,
+      "logps/rejected": -2.275078535079956,
+      "loss": 1.1623,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -16.608983993530273,
+      "rewards/margins": 6.141798496246338,
+      "rewards/rejected": -22.750782012939453,
       "step": 355
     },
     {
-      "epoch": 0.7716008037508373,
-      "grad_norm": 34.02497610424062,
-      "learning_rate": 7.488784741775816e-08,
-      "logits/chosen": -2.8549702167510986,
-      "logits/rejected": -2.8631627559661865,
-      "logps/chosen": -0.6374293565750122,
-      "logps/rejected": -0.7279438972473145,
-      "loss": 0.7685,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -1.5935734510421753,
-      "rewards/margins": 0.2262863665819168,
-      "rewards/rejected": -1.8198598623275757,
+      "epoch": 0.7695390781563126,
+      "grad_norm": 83.05095937518779,
+      "learning_rate": 1.5177512046261666e-07,
+      "logits/chosen": -1.2908953428268433,
+      "logits/rejected": -1.2959532737731934,
+      "logps/chosen": -1.669966459274292,
+      "logps/rejected": -2.3578410148620605,
+      "loss": 1.2917,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -16.69966697692871,
+      "rewards/margins": 6.878741264343262,
+      "rewards/rejected": -23.578407287597656,
       "step": 360
     },
     {
-      "epoch": 0.78231748158071,
-      "grad_norm": 34.499708849490524,
-      "learning_rate": 6.832342215471715e-08,
-      "logits/chosen": -2.90082049369812,
-      "logits/rejected": -2.8912737369537354,
-      "logps/chosen": -0.6256529688835144,
-      "logps/rejected": -0.6695230603218079,
-      "loss": 0.765,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -1.5641324520111084,
-      "rewards/margins": 0.10967500507831573,
-      "rewards/rejected": -1.6738075017929077,
+      "epoch": 0.7802271209084837,
+      "grad_norm": 74.89454351181128,
+      "learning_rate": 1.3860256808630427e-07,
+      "logits/chosen": -1.3286449909210205,
+      "logits/rejected": -1.2749364376068115,
+      "logps/chosen": -1.7136112451553345,
+      "logps/rejected": -2.3888180255889893,
+      "loss": 1.2062,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -17.136112213134766,
+      "rewards/margins": 6.752066135406494,
+      "rewards/rejected": -23.888179779052734,
       "step": 365
     },
     {
-      "epoch": 0.7930341594105828,
-      "grad_norm": 40.533056688484706,
-      "learning_rate": 6.201430216410985e-08,
-      "logits/chosen": -2.840193510055542,
-      "logits/rejected": -2.845524549484253,
-      "logps/chosen": -0.638153076171875,
-      "logps/rejected": -0.6873365044593811,
-      "loss": 0.7559,
-      "rewards/accuracies": 0.543749988079071,
-      "rewards/chosen": -1.5953826904296875,
-      "rewards/margins": 0.12295844405889511,
-      "rewards/rejected": -1.7183412313461304,
+      "epoch": 0.7909151636606546,
+      "grad_norm": 86.53704054424027,
+      "learning_rate": 1.2593546269723647e-07,
+      "logits/chosen": -1.251564860343933,
+      "logits/rejected": -1.2445452213287354,
+      "logps/chosen": -1.689417839050293,
+      "logps/rejected": -2.242030620574951,
+      "loss": 1.2397,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -16.89417839050293,
+      "rewards/margins": 5.526127338409424,
+      "rewards/rejected": -22.420307159423828,
       "step": 370
     },
     {
-      "epoch": 0.8037508372404555,
-      "grad_norm": 61.33843154278605,
-      "learning_rate": 5.596935348479029e-08,
-      "logits/chosen": -2.8891007900238037,
-      "logits/rejected": -2.9143340587615967,
-      "logps/chosen": -0.6350052952766418,
-      "logps/rejected": -0.680306613445282,
-      "loss": 0.7693,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -1.5875132083892822,
-      "rewards/margins": 0.11325322091579437,
-      "rewards/rejected": -1.7007665634155273,
+      "epoch": 0.8016032064128257,
+      "grad_norm": 67.08154175834267,
+      "learning_rate": 1.1379152038770029e-07,
+      "logits/chosen": -1.2969611883163452,
+      "logits/rejected": -1.3048449754714966,
+      "logps/chosen": -1.8122236728668213,
+      "logps/rejected": -2.4301235675811768,
+      "loss": 1.1934,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -18.122234344482422,
+      "rewards/margins": 6.178996562957764,
+      "rewards/rejected": -24.301233291625977,
       "step": 375
     },
     {
-      "epoch": 0.8144675150703282,
-      "grad_norm": 50.258970661764174,
-      "learning_rate": 5.01970709226808e-08,
-      "logits/chosen": -2.8725762367248535,
-      "logits/rejected": -2.8825366497039795,
-      "logps/chosen": -0.6515634059906006,
-      "logps/rejected": -0.7237032651901245,
-      "loss": 0.7826,
-      "rewards/accuracies": 0.606249988079071,
-      "rewards/chosen": -1.6289085149765015,
-      "rewards/margins": 0.1803499013185501,
-      "rewards/rejected": -1.809258222579956,
+      "epoch": 0.8122912491649966,
+      "grad_norm": 117.54208827116818,
+      "learning_rate": 1.0218772555910954e-07,
+      "logits/chosen": -1.2999813556671143,
+      "logits/rejected": -1.2861607074737549,
+      "logps/chosen": -1.7071046829223633,
+      "logps/rejected": -2.316257953643799,
+      "loss": 1.3229,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -17.071046829223633,
+      "rewards/margins": 6.091533184051514,
+      "rewards/rejected": -23.162578582763672,
       "step": 380
     },
     {
-      "epoch": 0.825184192900201,
-      "grad_norm": 42.96591025859426,
-      "learning_rate": 4.4705566113246635e-08,
-      "logits/chosen": -2.857102870941162,
-      "logits/rejected": -2.857473850250244,
-      "logps/chosen": -0.641956090927124,
-      "logps/rejected": -0.7416104078292847,
-      "loss": 0.7604,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -1.6048901081085205,
-      "rewards/margins": 0.2491360902786255,
-      "rewards/rejected": -1.854026198387146,
+      "epoch": 0.8229792919171677,
+      "grad_norm": 81.44656810755579,
+      "learning_rate": 9.114030716778432e-08,
+      "logits/chosen": -1.2965797185897827,
+      "logits/rejected": -1.2802777290344238,
+      "logps/chosen": -1.7108449935913086,
+      "logps/rejected": -2.4812264442443848,
+      "loss": 1.0893,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -17.108448028564453,
+      "rewards/margins": 7.703813076019287,
+      "rewards/rejected": -24.8122615814209,
       "step": 385
     },
     {
-      "epoch": 0.8359008707300737,
-      "grad_norm": 33.97745224085272,
-      "learning_rate": 3.950255612243042e-08,
-      "logits/chosen": -2.8420250415802,
-      "logits/rejected": -2.840378522872925,
-      "logps/chosen": -0.6417650580406189,
-      "logps/rejected": -0.7058727145195007,
-      "loss": 0.7771,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": -1.6044127941131592,
-      "rewards/margins": 0.16026917099952698,
-      "rewards/rejected": -1.7646818161010742,
+      "epoch": 0.8336673346693386,
+      "grad_norm": 77.70445335852864,
+      "learning_rate": 8.066471602728803e-08,
+      "logits/chosen": -1.3073532581329346,
+      "logits/rejected": -1.298214316368103,
+      "logps/chosen": -1.7747161388397217,
+      "logps/rejected": -2.438636302947998,
+      "loss": 1.243,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": -17.747161865234375,
+      "rewards/margins": 6.639204502105713,
+      "rewards/rejected": -24.38636589050293,
       "step": 390
     },
     {
-      "epoch": 0.8466175485599464,
-      "grad_norm": 41.64546946228987,
-      "learning_rate": 3.459535260206384e-08,
-      "logits/chosen": -2.836892604827881,
-      "logits/rejected": -2.8546109199523926,
-      "logps/chosen": -0.7084933519363403,
-      "logps/rejected": -0.8302263021469116,
-      "loss": 0.7662,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": -1.771233320236206,
-      "rewards/margins": 0.3043324947357178,
-      "rewards/rejected": -2.075565814971924,
+      "epoch": 0.8443553774215097,
+      "grad_norm": 83.85991252299944,
+      "learning_rate": 7.077560319906694e-08,
+      "logits/chosen": -1.3133912086486816,
+      "logits/rejected": -1.2960683107376099,
+      "logps/chosen": -1.6985034942626953,
+      "logps/rejected": -2.307638645172119,
+      "loss": 1.1851,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -16.985034942626953,
+      "rewards/margins": 6.091352462768555,
+      "rewards/rejected": -23.07638931274414,
       "step": 395
     },
     {
-      "epoch": 0.8573342263898192,
-      "grad_norm": 38.05882539644461,
-      "learning_rate": 2.999085151499764e-08,
-      "logits/chosen": -2.8660452365875244,
-      "logits/rejected": -2.886165142059326,
-      "logps/chosen": -0.6565605401992798,
-      "logps/rejected": -0.758074164390564,
-      "loss": 0.7494,
-      "rewards/accuracies": 0.643750011920929,
-      "rewards/chosen": -1.6414012908935547,
-      "rewards/margins": 0.25378426909446716,
-      "rewards/rejected": -1.8951854705810547,
+      "epoch": 0.8550434201736807,
+      "grad_norm": 71.13795001226813,
+      "learning_rate": 6.148679950161672e-08,
+      "logits/chosen": -1.3109194040298462,
+      "logits/rejected": -1.2997634410858154,
+      "logps/chosen": -1.7445151805877686,
+      "logps/rejected": -2.31221604347229,
+      "loss": 1.1882,
+      "rewards/accuracies": 0.831250011920929,
+      "rewards/chosen": -17.445152282714844,
+      "rewards/margins": 5.677010536193848,
+      "rewards/rejected": -23.122159957885742,
       "step": 400
     },
     {
-      "epoch": 0.8573342263898192,
-      "eval_logits/chosen": -2.7592597007751465,
-      "eval_logits/rejected": -2.7572638988494873,
-      "eval_logps/chosen": -0.676339328289032,
-      "eval_logps/rejected": -0.7378701567649841,
-      "eval_loss": 0.7909048795700073,
-      "eval_rewards/accuracies": 0.602393627166748,
-      "eval_rewards/chosen": -1.690848469734192,
-      "eval_rewards/margins": 0.15382711589336395,
-      "eval_rewards/rejected": -1.8446754217147827,
-      "eval_runtime": 102.5,
-      "eval_samples_per_second": 29.21,
-      "eval_steps_per_second": 1.834,
+      "epoch": 0.8550434201736807,
+      "eval_logits/chosen": -1.4831929206848145,
+      "eval_logits/rejected": -1.4942151308059692,
+      "eval_logps/chosen": -1.7345198392868042,
+      "eval_logps/rejected": -2.3351681232452393,
+      "eval_loss": 1.17947518825531,
+      "eval_rewards/accuracies": 0.8434959053993225,
+      "eval_rewards/chosen": -17.345199584960938,
+      "eval_rewards/margins": 6.006482124328613,
+      "eval_rewards/rejected": -23.351680755615234,
+      "eval_runtime": 66.8816,
+      "eval_samples_per_second": 29.32,
+      "eval_steps_per_second": 1.839,
       "step": 400
     },
     {
-      "epoch": 0.8680509042196919,
-      "grad_norm": 36.86643037307591,
-      "learning_rate": 2.5695523444387747e-08,
-      "logits/chosen": -2.9111428260803223,
-      "logits/rejected": -2.910399913787842,
-      "logps/chosen": -0.6198450326919556,
-      "logps/rejected": -0.7135489583015442,
-      "loss": 0.7679,
-      "rewards/accuracies": 0.706250011920929,
-      "rewards/chosen": -1.5496125221252441,
-      "rewards/margins": 0.23425981402397156,
-      "rewards/rejected": -1.7838722467422485,
+      "epoch": 0.8657314629258517,
+      "grad_norm": 93.04526178302538,
+      "learning_rate": 5.2811296166831666e-08,
+      "logits/chosen": -1.2737401723861694,
+      "logits/rejected": -1.2966753244400024,
+      "logps/chosen": -1.8380101919174194,
+      "logps/rejected": -2.4519333839416504,
+      "loss": 1.1965,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -18.380102157592773,
+      "rewards/margins": 6.139228820800781,
+      "rewards/rejected": -24.51933479309082,
       "step": 405
     },
     {
-      "epoch": 0.8787675820495646,
-      "grad_norm": 42.73305785852333,
-      "learning_rate": 2.1715404500756875e-08,
-      "logits/chosen": -2.8384289741516113,
-      "logits/rejected": -2.8592209815979004,
-      "logps/chosen": -0.648736834526062,
-      "logps/rejected": -0.7135356664657593,
-      "loss": 0.7489,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": -1.6218420267105103,
-      "rewards/margins": 0.16199705004692078,
-      "rewards/rejected": -1.7838389873504639,
+      "epoch": 0.8764195056780227,
+      "grad_norm": 146.600711895034,
+      "learning_rate": 4.4761226670592066e-08,
+      "logits/chosen": -1.29901921749115,
+      "logits/rejected": -1.287117838859558,
+      "logps/chosen": -1.744970679283142,
+      "logps/rejected": -2.3512680530548096,
+      "loss": 1.2833,
+      "rewards/accuracies": 0.8500000238418579,
+      "rewards/chosen": -17.44970703125,
+      "rewards/margins": 6.062973499298096,
+      "rewards/rejected": -23.512680053710938,
       "step": 410
     },
     {
-      "epoch": 0.8894842598794374,
-      "grad_norm": 40.990552105099376,
-      "learning_rate": 1.8056087839608546e-08,
-      "logits/chosen": -2.8605241775512695,
-      "logits/rejected": -2.8745930194854736,
-      "logps/chosen": -0.6644041538238525,
-      "logps/rejected": -0.7614117860794067,
-      "loss": 0.7642,
-      "rewards/accuracies": 0.668749988079071,
-      "rewards/chosen": -1.661010503768921,
-      "rewards/margins": 0.2425190508365631,
-      "rewards/rejected": -1.9035295248031616,
+      "epoch": 0.8871075484301937,
+      "grad_norm": 75.88048221665215,
+      "learning_rate": 3.734784976300165e-08,
+      "logits/chosen": -1.2868596315383911,
+      "logits/rejected": -1.2446833848953247,
+      "logps/chosen": -1.6970937252044678,
+      "logps/rejected": -2.4172720909118652,
+      "loss": 1.2743,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -16.970937728881836,
+      "rewards/margins": 7.201780796051025,
+      "rewards/rejected": -24.172718048095703,
       "step": 415
     },
     {
-      "epoch": 0.9002009377093101,
-      "grad_norm": 43.0964450321845,
-      "learning_rate": 1.4722715801513935e-08,
-      "logits/chosen": -2.884080171585083,
-      "logits/rejected": -2.8876559734344482,
-      "logps/chosen": -0.6492477655410767,
-      "logps/rejected": -0.7468755841255188,
-      "loss": 0.757,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -1.6231193542480469,
-      "rewards/margins": 0.2440694123506546,
-      "rewards/rejected": -1.8671890497207642,
+      "epoch": 0.8977955911823647,
+      "grad_norm": 89.93720385918785,
+      "learning_rate": 3.058153372200695e-08,
+      "logits/chosen": -1.322579026222229,
+      "logits/rejected": -1.2816030979156494,
+      "logps/chosen": -1.6371269226074219,
+      "logps/rejected": -2.3092269897460938,
+      "loss": 1.1839,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": -16.371267318725586,
+      "rewards/margins": 6.720999240875244,
+      "rewards/rejected": -23.092267990112305,
       "step": 420
     },
     {
-      "epoch": 0.9109176155391828,
-      "grad_norm": 35.62063577969192,
-      "learning_rate": 1.1719972685717395e-08,
-      "logits/chosen": -2.870063304901123,
-      "logits/rejected": -2.8697421550750732,
-      "logps/chosen": -0.6383547186851501,
-      "logps/rejected": -0.7067494988441467,
-      "loss": 0.7706,
-      "rewards/accuracies": 0.581250011920929,
-      "rewards/chosen": -1.5958869457244873,
-      "rewards/margins": 0.1709868460893631,
-      "rewards/rejected": -1.7668737173080444,
+      "epoch": 0.9084836339345357,
+      "grad_norm": 87.9502722305557,
+      "learning_rate": 2.4471741852423233e-08,
+      "logits/chosen": -1.320749044418335,
+      "logits/rejected": -1.3164869546890259,
+      "logps/chosen": -1.8138576745986938,
+      "logps/rejected": -2.4155681133270264,
+      "loss": 1.3872,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -18.13857650756836,
+      "rewards/margins": 6.0171027183532715,
+      "rewards/rejected": -24.15567970275879,
       "step": 425
     },
     {
-      "epoch": 0.9216342933690556,
-      "grad_norm": 43.124891991653534,
-      "learning_rate": 9.05207816741485e-09,
-      "logits/chosen": -2.836172580718994,
-      "logits/rejected": -2.8423163890838623,
-      "logps/chosen": -0.6527460813522339,
-      "logps/rejected": -0.730507493019104,
-      "loss": 0.7819,
-      "rewards/accuracies": 0.6312500238418579,
-      "rewards/chosen": -1.63186514377594,
-      "rewards/margins": 0.19440340995788574,
-      "rewards/rejected": -1.8262684345245361,
+      "epoch": 0.9191716766867067,
+      "grad_norm": 86.56225821481218,
+      "learning_rate": 1.9027019250647036e-08,
+      "logits/chosen": -1.2986432313919067,
+      "logits/rejected": -1.2903183698654175,
+      "logps/chosen": -1.7905200719833374,
+      "logps/rejected": -2.436361789703369,
+      "loss": 1.2511,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": -17.905202865600586,
+      "rewards/margins": 6.458415985107422,
+      "rewards/rejected": -24.363616943359375,
       "step": 430
     },
     {
-      "epoch": 0.9323509711989283,
-      "grad_norm": 36.86465436137469,
-      "learning_rate": 6.722781367956481e-09,
-      "logits/chosen": -2.782316207885742,
-      "logits/rejected": -2.7888705730438232,
-      "logps/chosen": -0.6667356491088867,
-      "logps/rejected": -0.7845855355262756,
-      "loss": 0.7675,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -1.6668390035629272,
-      "rewards/margins": 0.2946247458457947,
-      "rewards/rejected": -1.9614636898040771,
+      "epoch": 0.9298597194388778,
+      "grad_norm": 72.5185778330974,
+      "learning_rate": 1.4254980853566246e-08,
+      "logits/chosen": -1.2492395639419556,
+      "logits/rejected": -1.2206861972808838,
+      "logps/chosen": -1.657965064048767,
+      "logps/rejected": -2.2997374534606934,
+      "loss": 1.1762,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -16.57965087890625,
+      "rewards/margins": 6.417726039886475,
+      "rewards/rejected": -22.99737548828125,
       "step": 435
     },
     {
-      "epoch": 0.943067649028801,
-      "grad_norm": 47.527494016819155,
-      "learning_rate": 4.735355586305861e-09,
-      "logits/chosen": -2.836718797683716,
-      "logits/rejected": -2.8620688915252686,
-      "logps/chosen": -0.6587570309638977,
-      "logps/rejected": -0.7252873778343201,
-      "loss": 0.7738,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -1.6468925476074219,
-      "rewards/margins": 0.16632597148418427,
-      "rewards/rejected": -1.8132184743881226,
+      "epoch": 0.9405477621910487,
+      "grad_norm": 93.87912624876707,
+      "learning_rate": 1.016230078838226e-08,
+      "logits/chosen": -1.2905051708221436,
+      "logits/rejected": -1.2409080266952515,
+      "logps/chosen": -1.7906429767608643,
+      "logps/rejected": -2.375685453414917,
+      "loss": 1.2152,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": -17.906429290771484,
+      "rewards/margins": 5.850423812866211,
+      "rewards/rejected": -23.756855010986328,
       "step": 440
     },
     {
-      "epoch": 0.9537843268586738,
-      "grad_norm": 40.42551390475137,
-      "learning_rate": 3.0925936991598454e-09,
-      "logits/chosen": -2.839256763458252,
-      "logits/rejected": -2.8521199226379395,
-      "logps/chosen": -0.6958639025688171,
-      "logps/rejected": -0.7931674718856812,
-      "loss": 0.7647,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": -1.7396596670150757,
-      "rewards/margins": 0.243258997797966,
-      "rewards/rejected": -1.9829187393188477,
+      "epoch": 0.9512358049432198,
+      "grad_norm": 68.35754825310542,
+      "learning_rate": 6.754703038239329e-09,
+      "logits/chosen": -1.238844633102417,
+      "logits/rejected": -1.2277412414550781,
+      "logps/chosen": -1.769052267074585,
+      "logps/rejected": -2.5054378509521484,
+      "loss": 1.1183,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -17.690523147583008,
+      "rewards/margins": 7.363855838775635,
+      "rewards/rejected": -25.054378509521484,
       "step": 445
     },
     {
-      "epoch": 0.9645010046885466,
-      "grad_norm": 42.24998694242168,
-      "learning_rate": 1.7968042361930225e-09,
-      "logits/chosen": -2.821035623550415,
-      "logits/rejected": -2.843494415283203,
-      "logps/chosen": -0.6622462868690491,
-      "logps/rejected": -0.7363108396530151,
-      "loss": 0.7558,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": -1.6556155681610107,
-      "rewards/margins": 0.1851612776517868,
-      "rewards/rejected": -1.8407771587371826,
+      "epoch": 0.9619238476953907,
+      "grad_norm": 83.54543382327525,
+      "learning_rate": 4.036953436716895e-09,
+      "logits/chosen": -1.346346139907837,
+      "logits/rejected": -1.3324317932128906,
+      "logps/chosen": -1.7270358800888062,
+      "logps/rejected": -2.3151395320892334,
+      "loss": 1.242,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -17.27035903930664,
+      "rewards/margins": 5.881035804748535,
+      "rewards/rejected": -23.15139389038086,
       "step": 450
     },
     {
-      "epoch": 0.9752176825184193,
-      "grad_norm": 58.74520439596834,
-      "learning_rate": 8.498081359423759e-10,
-      "logits/chosen": -2.8467049598693848,
-      "logits/rejected": -2.8465356826782227,
-      "logps/chosen": -0.6843534708023071,
-      "logps/rejected": -0.7486766576766968,
-      "loss": 0.7873,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": -1.7108838558197021,
-      "rewards/margins": 0.1608080118894577,
-      "rewards/rejected": -1.8716917037963867,
+      "epoch": 0.9726118904475618,
+      "grad_norm": 96.76907591863952,
+      "learning_rate": 2.0128530023804656e-09,
+      "logits/chosen": -1.3069255352020264,
+      "logits/rejected": -1.277972936630249,
+      "logps/chosen": -1.7244638204574585,
+      "logps/rejected": -2.4305450916290283,
+      "loss": 1.1012,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": -17.244640350341797,
+      "rewards/margins": 7.060812473297119,
+      "rewards/rejected": -24.305450439453125,
       "step": 455
     },
     {
-      "epoch": 0.9859343603482921,
-      "grad_norm": 41.029699707171545,
-      "learning_rate": 2.5293618689056774e-10,
-      "logits/chosen": -2.8341171741485596,
-      "logits/rejected": -2.8409810066223145,
-      "logps/chosen": -0.6728862524032593,
-      "logps/rejected": -0.7803421020507812,
-      "loss": 0.7617,
-      "rewards/accuracies": 0.6625000238418579,
-      "rewards/chosen": -1.6822154521942139,
-      "rewards/margins": 0.2686397135257721,
-      "rewards/rejected": -1.9508554935455322,
+      "epoch": 0.9832999331997327,
+      "grad_norm": 95.7750256826231,
+      "learning_rate": 6.852326227130833e-10,
+      "logits/chosen": -1.3161975145339966,
+      "logits/rejected": -1.3086355924606323,
+      "logps/chosen": -1.768693208694458,
+      "logps/rejected": -2.425410747528076,
+      "loss": 1.1545,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": -17.686931610107422,
+      "rewards/margins": 6.567179203033447,
+      "rewards/rejected": -24.25411033630371,
       "step": 460
     },
     {
-      "epoch": 0.9966510381781648,
-      "grad_norm": 36.64296263080341,
-      "learning_rate": 7.02715734424153e-12,
-      "logits/chosen": -2.8702194690704346,
-      "logits/rejected": -2.8706626892089844,
-      "logps/chosen": -0.6983669996261597,
-      "logps/rejected": -0.7460958361625671,
-      "loss": 0.7864,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -1.745917558670044,
-      "rewards/margins": 0.11932210624217987,
-      "rewards/rejected": -1.8652395009994507,
+      "epoch": 0.9939879759519038,
+      "grad_norm": 90.4800906374845,
+      "learning_rate": 5.594909486328348e-11,
+      "logits/chosen": -1.2925775051116943,
+      "logits/rejected": -1.3013789653778076,
+      "logps/chosen": -1.8294912576675415,
+      "logps/rejected": -2.505312204360962,
+      "loss": 1.2602,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": -18.294912338256836,
+      "rewards/margins": 6.758206844329834,
+      "rewards/rejected": -25.053119659423828,
       "step": 465
     },
     {
-      "epoch": 0.9987943737441393,
-      "step": 466,
+      "epoch": 0.9982631930527722,
+      "step": 467,
       "total_flos": 0.0,
-      "train_loss": 0.7914171927476645,
-      "train_runtime": 8402.7408,
-      "train_samples_per_second": 7.107,
-      "train_steps_per_second": 0.055
+      "train_loss": 1.7882541670789045,
+      "train_runtime": 8154.3088,
+      "train_samples_per_second": 7.343,
+      "train_steps_per_second": 0.057
     }
   ],
   "logging_steps": 5,
-  "max_steps": 466,
+  "max_steps": 467,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 1,
   "save_steps": 1000000,