{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9987943737441393,
  "eval_steps": 400,
  "global_step": 466,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.010716677829872739,
      "grad_norm": 8.776682238147446,
      "learning_rate": 5.3191489361702123e-08,
      "logits/chosen": -2.8905415534973145,
      "logits/rejected": -2.8877615928649902,
      "logps/chosen": -0.9742914438247681,
      "logps/rejected": -0.9831219911575317,
      "loss": 0.0001,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.9742914438247681,
      "rewards/margins": 0.008830604143440723,
      "rewards/rejected": -0.9831219911575317,
      "step": 5
    },
    {
      "epoch": 0.021433355659745478,
      "grad_norm": 9.810248541108757,
      "learning_rate": 1.0638297872340425e-07,
      "logits/chosen": -2.9070327281951904,
      "logits/rejected": -2.8929152488708496,
      "logps/chosen": -0.978168785572052,
      "logps/rejected": -1.001371145248413,
      "loss": -0.0,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.978168785572052,
      "rewards/margins": 0.023202406242489815,
      "rewards/rejected": -1.001371145248413,
      "step": 10
    },
    {
      "epoch": 0.032150033489618215,
      "grad_norm": 7.0995253496741135,
      "learning_rate": 1.5957446808510638e-07,
      "logits/chosen": -2.895216703414917,
      "logits/rejected": -2.9180421829223633,
      "logps/chosen": -0.9596306085586548,
      "logps/rejected": -0.9847423434257507,
      "loss": -0.0001,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.9596306085586548,
      "rewards/margins": 0.02511175535619259,
      "rewards/rejected": -0.9847423434257507,
      "step": 15
    },
    {
      "epoch": 0.042866711319490956,
      "grad_norm": 7.785343608601427,
      "learning_rate": 2.127659574468085e-07,
      "logits/chosen": -2.902580976486206,
      "logits/rejected": -2.9009933471679688,
      "logps/chosen": -0.9715825319290161,
      "logps/rejected": -0.9759886860847473,
      "loss": -0.0,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.9715825319290161,
      "rewards/margins": 0.00440616998821497,
      "rewards/rejected": -0.9759886860847473,
      "step": 20
    },
    {
      "epoch": 0.0535833891493637,
      "grad_norm": 6.677386701450202,
      "learning_rate": 2.659574468085106e-07,
      "logits/chosen": -2.8314852714538574,
      "logits/rejected": -2.8363277912139893,
      "logps/chosen": -0.9519669413566589,
      "logps/rejected": -1.0002533197402954,
      "loss": -0.0003,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.9519669413566589,
      "rewards/margins": 0.048286281526088715,
      "rewards/rejected": -1.0002533197402954,
      "step": 25
    },
    {
      "epoch": 0.06430006697923643,
      "grad_norm": 8.4552870442767,
      "learning_rate": 3.1914893617021275e-07,
      "logits/chosen": -2.895077705383301,
      "logits/rejected": -2.9221832752227783,
      "logps/chosen": -0.983117938041687,
      "logps/rejected": -0.9829673767089844,
      "loss": -0.0002,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.983117938041687,
      "rewards/margins": -0.00015044640167616308,
      "rewards/rejected": -0.9829673767089844,
      "step": 30
    },
    {
      "epoch": 0.07501674480910918,
      "grad_norm": 7.60248558460589,
      "learning_rate": 3.7234042553191484e-07,
      "logits/chosen": -2.9559242725372314,
      "logits/rejected": -2.9249794483184814,
      "logps/chosen": -0.95380699634552,
      "logps/rejected": -0.9984177350997925,
      "loss": -0.0004,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.95380699634552,
      "rewards/margins": 0.04461073502898216,
      "rewards/rejected": -0.9984177350997925,
      "step": 35
    },
    {
      "epoch": 0.08573342263898191,
      "grad_norm": 8.131750300550957,
      "learning_rate": 4.25531914893617e-07,
      "logits/chosen": -2.8956754207611084,
      "logits/rejected": -2.8939049243927,
      "logps/chosen": -0.9987373352050781,
      "logps/rejected": -0.9930660128593445,
      "loss": -0.0015,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.9987373352050781,
      "rewards/margins": -0.005671312101185322,
      "rewards/rejected": -0.9930660128593445,
      "step": 40
    },
    {
      "epoch": 0.09645010046885466,
      "grad_norm": 7.488747662199227,
      "learning_rate": 4.787234042553192e-07,
      "logits/chosen": -2.8674330711364746,
      "logits/rejected": -2.856853485107422,
      "logps/chosen": -0.9764472246170044,
      "logps/rejected": -0.9902997016906738,
      "loss": -0.001,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.9764472246170044,
      "rewards/margins": 0.013852563686668873,
      "rewards/rejected": -0.9902997016906738,
      "step": 45
    },
    {
      "epoch": 0.1071667782987274,
      "grad_norm": 7.447612263721868,
      "learning_rate": 4.999367579541646e-07,
      "logits/chosen": -2.875795364379883,
      "logits/rejected": -2.8871512413024902,
      "logps/chosen": -0.9369581937789917,
      "logps/rejected": -0.9167054891586304,
      "loss": -0.0027,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.9369581937789917,
      "rewards/margins": -0.020252685993909836,
      "rewards/rejected": -0.9167054891586304,
      "step": 50
    },
    {
      "epoch": 0.11788345612860013,
      "grad_norm": 8.432358893020524,
      "learning_rate": 4.99550394651012e-07,
      "logits/chosen": -2.915107011795044,
      "logits/rejected": -2.925875186920166,
      "logps/chosen": -0.9737237095832825,
      "logps/rejected": -1.0020536184310913,
      "loss": -0.0022,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.9737237095832825,
      "rewards/margins": 0.028329964727163315,
      "rewards/rejected": -1.0020536184310913,
      "step": 55
    },
    {
      "epoch": 0.12860013395847286,
      "grad_norm": 9.97321343058875,
      "learning_rate": 4.988133448023071e-07,
      "logits/chosen": -2.8224921226501465,
      "logits/rejected": -2.8447964191436768,
      "logps/chosen": -0.9149552583694458,
      "logps/rejected": -0.9350795745849609,
      "loss": -0.0041,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9149552583694458,
      "rewards/margins": 0.02012430503964424,
      "rewards/rejected": -0.9350795745849609,
      "step": 60
    },
    {
      "epoch": 0.13931681178834562,
      "grad_norm": 8.412802955896868,
      "learning_rate": 4.977266441646379e-07,
      "logits/chosen": -2.900336503982544,
      "logits/rejected": -2.9115452766418457,
      "logps/chosen": -0.8882263898849487,
      "logps/rejected": -0.9012020230293274,
      "loss": -0.0057,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.8882263898849487,
      "rewards/margins": 0.012975621037185192,
      "rewards/rejected": -0.9012020230293274,
      "step": 65
    },
    {
      "epoch": 0.15003348961821836,
      "grad_norm": 8.21023142941744,
      "learning_rate": 4.962918198495654e-07,
      "logits/chosen": -2.816331386566162,
      "logits/rejected": -2.839690685272217,
      "logps/chosen": -0.9164741635322571,
      "logps/rejected": -0.9583846926689148,
      "loss": -0.007,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.9164741635322571,
      "rewards/margins": 0.04191048443317413,
      "rewards/rejected": -0.9583846926689148,
      "step": 70
    },
    {
      "epoch": 0.1607501674480911,
      "grad_norm": 8.354728288045035,
      "learning_rate": 4.945108881776135e-07,
      "logits/chosen": -2.8184382915496826,
      "logits/rejected": -2.81858229637146,
      "logps/chosen": -0.8914631009101868,
      "logps/rejected": -0.9294856190681458,
      "loss": -0.0066,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.8914631009101868,
      "rewards/margins": 0.038022562861442566,
      "rewards/rejected": -0.9294856190681458,
      "step": 75
    },
    {
      "epoch": 0.17146684527796383,
      "grad_norm": 9.546162491226012,
      "learning_rate": 4.923863518447879e-07,
      "logits/chosen": -2.840766191482544,
      "logits/rejected": -2.8635830879211426,
      "logps/chosen": -0.887871265411377,
      "logps/rejected": -0.958912193775177,
      "loss": -0.006,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.887871265411377,
      "rewards/margins": 0.07104082405567169,
      "rewards/rejected": -0.958912193775177,
      "step": 80
    },
    {
      "epoch": 0.18218352310783656,
      "grad_norm": 9.354318155678731,
      "learning_rate": 4.899211964056035e-07,
      "logits/chosen": -2.854689359664917,
      "logits/rejected": -2.866008758544922,
      "logps/chosen": -0.9082525968551636,
      "logps/rejected": -0.9574602842330933,
      "loss": -0.0132,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.9082525968551636,
      "rewards/margins": 0.04920773208141327,
      "rewards/rejected": -0.9574602842330933,
      "step": 85
    },
    {
      "epoch": 0.19290020093770932,
      "grad_norm": 9.38237658589797,
      "learning_rate": 4.871188860775643e-07,
      "logits/chosen": -2.830294132232666,
      "logits/rejected": -2.8332040309906006,
      "logps/chosen": -0.9233630895614624,
      "logps/rejected": -0.9633795022964478,
      "loss": -0.0142,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.9233630895614624,
      "rewards/margins": 0.040016382932662964,
      "rewards/rejected": -0.9633795022964478,
      "step": 90
    },
    {
      "epoch": 0.20361687876758205,
      "grad_norm": 8.979685614931006,
      "learning_rate": 4.839833588729897e-07,
      "logits/chosen": -2.8387644290924072,
      "logits/rejected": -2.8495821952819824,
      "logps/chosen": -0.9509484171867371,
      "logps/rejected": -0.974162220954895,
      "loss": -0.0085,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.9509484171867371,
      "rewards/margins": 0.02321377769112587,
      "rewards/rejected": -0.974162220954895,
      "step": 95
    },
    {
      "epoch": 0.2143335565974548,
      "grad_norm": 9.427728619757332,
      "learning_rate": 4.80519021065031e-07,
      "logits/chosen": -2.860654354095459,
      "logits/rejected": -2.8680408000946045,
      "logps/chosen": -0.9869228601455688,
      "logps/rejected": -1.0385963916778564,
      "loss": -0.015,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.9869228601455688,
      "rewards/margins": 0.05167357996106148,
      "rewards/rejected": -1.0385963916778564,
      "step": 100
    },
    {
      "epoch": 0.22505023442732752,
      "grad_norm": 8.191485445947501,
      "learning_rate": 4.767307409956528e-07,
      "logits/chosen": -2.859048843383789,
      "logits/rejected": -2.8625106811523438,
      "logps/chosen": -0.8909996151924133,
      "logps/rejected": -0.9282458424568176,
      "loss": -0.0154,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.8909996151924133,
      "rewards/margins": 0.03724620118737221,
      "rewards/rejected": -0.9282458424568176,
      "step": 105
    },
    {
      "epoch": 0.23576691225720026,
      "grad_norm": 9.657244741544229,
      "learning_rate": 4.726238422342805e-07,
      "logits/chosen": -2.8523192405700684,
      "logits/rejected": -2.84688138961792,
      "logps/chosen": -0.9256642460823059,
      "logps/rejected": -0.9944097399711609,
      "loss": -0.0136,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.9256642460823059,
      "rewards/margins": 0.06874561309814453,
      "rewards/rejected": -0.9944097399711609,
      "step": 110
    },
    {
      "epoch": 0.24648359008707302,
      "grad_norm": 11.45037121550009,
      "learning_rate": 4.6820409609673027e-07,
      "logits/chosen": -2.895726203918457,
      "logits/rejected": -2.900205612182617,
      "logps/chosen": -0.9234002828598022,
      "logps/rejected": -0.938769519329071,
      "loss": -0.0216,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.9234002828598022,
      "rewards/margins": 0.015369323082268238,
      "rewards/rejected": -0.938769519329071,
      "step": 115
    },
    {
      "epoch": 0.2572002679169457,
      "grad_norm": 9.77719736840442,
      "learning_rate": 4.6347771353493173e-07,
      "logits/chosen": -2.9136910438537598,
      "logits/rejected": -2.930593252182007,
      "logps/chosen": -0.9225813150405884,
      "logps/rejected": -0.917222797870636,
      "loss": -0.0168,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9225813150405884,
      "rewards/margins": -0.005358507391065359,
      "rewards/rejected": -0.917222797870636,
      "step": 120
    },
    {
      "epoch": 0.2679169457468185,
      "grad_norm": 10.417139695198527,
      "learning_rate": 4.5845133640884217e-07,
      "logits/chosen": -2.898656129837036,
      "logits/rejected": -2.8996481895446777,
      "logps/chosen": -0.9208585619926453,
      "logps/rejected": -0.9479470252990723,
      "loss": -0.0193,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.9208585619926453,
      "rewards/margins": 0.0270883496850729,
      "rewards/rejected": -0.9479470252990723,
      "step": 125
    },
    {
      "epoch": 0.27863362357669125,
      "grad_norm": 11.697763554027677,
      "learning_rate": 4.531320281528169e-07,
      "logits/chosen": -2.9303739070892334,
      "logits/rejected": -2.9241251945495605,
      "logps/chosen": -0.9110285639762878,
      "logps/rejected": -0.9726667404174805,
      "loss": -0.0227,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.9110285639762878,
      "rewards/margins": 0.06163806468248367,
      "rewards/rejected": -0.9726667404174805,
      "step": 130
    },
    {
      "epoch": 0.289350301406564,
      "grad_norm": 11.144902490144164,
      "learning_rate": 4.4752726384955293e-07,
      "logits/chosen": -2.9404830932617188,
      "logits/rejected": -2.947343111038208,
      "logps/chosen": -0.9363792538642883,
      "logps/rejected": -0.9744508862495422,
      "loss": -0.0195,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.9363792538642883,
      "rewards/margins": 0.03807152435183525,
      "rewards/rejected": -0.9744508862495422,
      "step": 135
    },
    {
      "epoch": 0.3000669792364367,
      "grad_norm": 11.921950508886521,
      "learning_rate": 4.4164491972555294e-07,
      "logits/chosen": -2.8976681232452393,
      "logits/rejected": -2.898315906524658,
      "logps/chosen": -0.9219269752502441,
      "logps/rejected": -0.9640507698059082,
      "loss": -0.0259,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.9219269752502441,
      "rewards/margins": 0.04212382063269615,
      "rewards/rejected": -0.9640507698059082,
      "step": 140
    },
    {
      "epoch": 0.31078365706630945,
      "grad_norm": 13.471098814591679,
      "learning_rate": 4.3549326208287393e-07,
      "logits/chosen": -2.9619617462158203,
      "logits/rejected": -2.9274888038635254,
      "logps/chosen": -0.9526675343513489,
      "logps/rejected": -1.007359266281128,
      "loss": -0.0355,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.9526675343513489,
      "rewards/margins": 0.0546916201710701,
      "rewards/rejected": -1.007359266281128,
      "step": 145
    },
    {
      "epoch": 0.3215003348961822,
      "grad_norm": 12.766870563789471,
      "learning_rate": 4.2908093568271187e-07,
      "logits/chosen": -2.9606540203094482,
      "logits/rejected": -2.9726486206054688,
      "logps/chosen": -0.9194633364677429,
      "logps/rejected": -0.9709534645080566,
      "loss": -0.0371,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.9194633364677429,
      "rewards/margins": 0.05149019509553909,
      "rewards/rejected": -0.9709534645080566,
      "step": 150
    },
    {
      "epoch": 0.3322170127260549,
      "grad_norm": 12.28868938106036,
      "learning_rate": 4.2241695159714843e-07,
      "logits/chosen": -3.043233633041382,
      "logits/rejected": -3.0254199504852295,
      "logps/chosen": -0.9320791959762573,
      "logps/rejected": -0.9708281755447388,
      "loss": -0.0274,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.9320791959762573,
      "rewards/margins": 0.038748957216739655,
      "rewards/rejected": -0.9708281755447388,
      "step": 155
    },
    {
      "epoch": 0.34293369055592765,
      "grad_norm": 16.212616603609877,
      "learning_rate": 4.1551067454613074e-07,
      "logits/chosen": -2.943720579147339,
      "logits/rejected": -2.928537607192993,
      "logps/chosen": -0.921017050743103,
      "logps/rejected": -0.9882138967514038,
      "loss": -0.0359,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.921017050743103,
      "rewards/margins": 0.06719694286584854,
      "rewards/rejected": -0.9882138967514038,
      "step": 160
    },
    {
      "epoch": 0.3536503683858004,
      "grad_norm": 15.792664568932544,
      "learning_rate": 4.0837180973747876e-07,
      "logits/chosen": -3.0298409461975098,
      "logits/rejected": -3.0069289207458496,
      "logps/chosen": -0.9174503087997437,
      "logps/rejected": -0.9726656079292297,
      "loss": -0.0342,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.9174503087997437,
      "rewards/margins": 0.05521535873413086,
      "rewards/rejected": -0.9726656079292297,
      "step": 165
    },
    {
      "epoch": 0.3643670462156731,
      "grad_norm": 10.787104513853434,
      "learning_rate": 4.0101038922841455e-07,
      "logits/chosen": -2.968522548675537,
      "logits/rejected": -2.9739623069763184,
      "logps/chosen": -0.9771224856376648,
      "logps/rejected": -1.0373847484588623,
      "loss": -0.0466,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.9771224856376648,
      "rewards/margins": 0.06026231497526169,
      "rewards/rejected": -1.0373847484588623,
      "step": 170
    },
    {
      "epoch": 0.3750837240455459,
      "grad_norm": 13.17830711446757,
      "learning_rate": 3.934367578277783e-07,
      "logits/chosen": -2.9837470054626465,
      "logits/rejected": -2.9880218505859375,
      "logps/chosen": -1.000985026359558,
      "logps/rejected": -1.0419981479644775,
      "loss": -0.0381,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.000985026359558,
      "rewards/margins": 0.041013117879629135,
      "rewards/rejected": -1.0419981479644775,
      "step": 175
    },
    {
      "epoch": 0.38580040187541864,
      "grad_norm": 14.72612824066128,
      "learning_rate": 3.856615585587434e-07,
      "logits/chosen": -2.9933667182922363,
      "logits/rejected": -3.0047719478607178,
      "logps/chosen": -1.0653940439224243,
      "logps/rejected": -1.1087322235107422,
      "loss": -0.0319,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -1.0653940439224243,
      "rewards/margins": 0.043338216841220856,
      "rewards/rejected": -1.1087322235107422,
      "step": 180
    },
    {
      "epoch": 0.3965170797052914,
      "grad_norm": 13.397319728202142,
      "learning_rate": 3.776957177024581e-07,
      "logits/chosen": -2.9812378883361816,
      "logits/rejected": -2.9986259937286377,
      "logps/chosen": -1.0316323041915894,
      "logps/rejected": -1.0616919994354248,
      "loss": -0.033,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.0316323041915894,
      "rewards/margins": 0.030059661716222763,
      "rewards/rejected": -1.0616919994354248,
      "step": 185
    },
    {
      "epoch": 0.4072337575351641,
      "grad_norm": 14.059564629221503,
      "learning_rate": 3.69550429443633e-07,
      "logits/chosen": -2.8960227966308594,
      "logits/rejected": -2.913637161254883,
      "logps/chosen": -1.012019157409668,
      "logps/rejected": -1.215237021446228,
      "loss": -0.0447,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.012019157409668,
      "rewards/margins": 0.20321793854236603,
      "rewards/rejected": -1.215237021446228,
      "step": 190
    },
    {
      "epoch": 0.41795043536503684,
      "grad_norm": 14.952805148520074,
      "learning_rate": 3.612371401396502e-07,
      "logits/chosen": -3.007540225982666,
      "logits/rejected": -2.9880564212799072,
      "logps/chosen": -1.0366661548614502,
      "logps/rejected": -1.0943790674209595,
      "loss": -0.0368,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.0366661548614502,
      "rewards/margins": 0.05771297961473465,
      "rewards/rejected": -1.0943790674209595,
      "step": 195
    },
    {
      "epoch": 0.4286671131949096,
      "grad_norm": 14.170450035952996,
      "learning_rate": 3.527675322353011e-07,
      "logits/chosen": -2.922602415084839,
      "logits/rejected": -2.9457390308380127,
      "logps/chosen": -0.970561146736145,
      "logps/rejected": -1.0033313035964966,
      "loss": -0.0473,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.970561146736145,
      "rewards/margins": 0.032770149409770966,
      "rewards/rejected": -1.0033313035964966,
      "step": 200
    },
    {
      "epoch": 0.4393837910247823,
      "grad_norm": 15.488305590600266,
      "learning_rate": 3.4415350784575693e-07,
      "logits/chosen": -3.004404067993164,
      "logits/rejected": -2.9882898330688477,
      "logps/chosen": -0.9613070487976074,
      "logps/rejected": -1.0151447057724,
      "loss": -0.044,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.9613070487976074,
      "rewards/margins": 0.053837649524211884,
      "rewards/rejected": -1.0151447057724,
      "step": 205
    },
    {
      "epoch": 0.45010046885465504,
      "grad_norm": 13.543647630514888,
      "learning_rate": 3.3540717203084224e-07,
      "logits/chosen": -2.956808090209961,
      "logits/rejected": -2.9622817039489746,
      "logps/chosen": -0.9170505404472351,
      "logps/rejected": -0.9884069561958313,
      "loss": -0.0514,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.9170505404472351,
      "rewards/margins": 0.07135634124279022,
      "rewards/rejected": -0.9884069561958313,
      "step": 210
    },
    {
      "epoch": 0.4608171466845278,
      "grad_norm": 14.183913951293327,
      "learning_rate": 3.265408157841153e-07,
      "logits/chosen": -2.9880080223083496,
      "logits/rejected": -2.986445903778076,
      "logps/chosen": -0.9918737411499023,
      "logps/rejected": -1.0286133289337158,
      "loss": -0.0446,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.9918737411499023,
      "rewards/margins": 0.036739740520715714,
      "rewards/rejected": -1.0286133289337158,
      "step": 215
    },
    {
      "epoch": 0.4715338245144005,
      "grad_norm": 16.687524624276737,
      "learning_rate": 3.1756689876066187e-07,
      "logits/chosen": -2.940664291381836,
      "logits/rejected": -2.940556049346924,
      "logps/chosen": -0.9756967425346375,
      "logps/rejected": -1.0494619607925415,
      "loss": -0.0496,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.9756967425346375,
      "rewards/margins": 0.07376515865325928,
      "rewards/rejected": -1.0494619607925415,
      "step": 220
    },
    {
      "epoch": 0.4822505023442733,
      "grad_norm": 15.106449736541158,
      "learning_rate": 3.0849803176787184e-07,
      "logits/chosen": -2.978764057159424,
      "logits/rejected": -3.02057147026062,
      "logps/chosen": -0.9675830602645874,
      "logps/rejected": -1.0782067775726318,
      "loss": -0.0528,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.9675830602645874,
      "rewards/margins": 0.11062371730804443,
      "rewards/rejected": -1.0782067775726318,
      "step": 225
    },
    {
      "epoch": 0.49296718017414604,
      "grad_norm": 19.732642999044504,
      "learning_rate": 2.993469590438065e-07,
      "logits/chosen": -2.91894793510437,
      "logits/rejected": -2.92021107673645,
      "logps/chosen": -0.9777423143386841,
      "logps/rejected": -1.0478038787841797,
      "loss": -0.0487,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.9777423143386841,
      "rewards/margins": 0.07006163895130157,
      "rewards/rejected": -1.0478038787841797,
      "step": 230
    },
    {
      "epoch": 0.5036838580040187,
      "grad_norm": 15.470655040763342,
      "learning_rate": 2.901265403480597e-07,
      "logits/chosen": -2.868373394012451,
      "logits/rejected": -2.8822197914123535,
      "logps/chosen": -0.9746626615524292,
      "logps/rejected": -1.109392523765564,
      "loss": -0.0532,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.9746626615524292,
      "rewards/margins": 0.13472989201545715,
      "rewards/rejected": -1.109392523765564,
      "step": 235
    },
    {
      "epoch": 0.5144005358338914,
      "grad_norm": 18.117444556863273,
      "learning_rate": 2.808497328902782e-07,
      "logits/chosen": -2.829807758331299,
      "logits/rejected": -2.837395191192627,
      "logps/chosen": -1.0404951572418213,
      "logps/rejected": -1.1007030010223389,
      "loss": -0.0453,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.0404951572418213,
      "rewards/margins": 0.06020789220929146,
      "rewards/rejected": -1.1007030010223389,
      "step": 240
    },
    {
      "epoch": 0.5251172136637642,
      "grad_norm": 18.628678925066545,
      "learning_rate": 2.715295731217396e-07,
      "logits/chosen": -2.81119966506958,
      "logits/rejected": -2.815497636795044,
      "logps/chosen": -1.0369569063186646,
      "logps/rejected": -1.112683892250061,
      "loss": -0.0596,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -1.0369569063186646,
      "rewards/margins": 0.07572712004184723,
      "rewards/rejected": -1.112683892250061,
      "step": 245
    },
    {
      "epoch": 0.535833891493637,
      "grad_norm": 17.866763466807367,
      "learning_rate": 2.621791584155725e-07,
      "logits/chosen": -2.83900785446167,
      "logits/rejected": -2.859978199005127,
      "logps/chosen": -1.1023390293121338,
      "logps/rejected": -1.1515963077545166,
      "loss": -0.0617,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.1023390293121338,
      "rewards/margins": 0.04925721138715744,
      "rewards/rejected": -1.1515963077545166,
      "step": 250
    },
    {
      "epoch": 0.5465505693235098,
      "grad_norm": 17.478329638335467,
      "learning_rate": 2.52811628661367e-07,
      "logits/chosen": -2.8983101844787598,
      "logits/rejected": -2.908754348754883,
      "logps/chosen": -1.0288032293319702,
      "logps/rejected": -1.134108066558838,
      "loss": -0.0631,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -1.0288032293319702,
      "rewards/margins": 0.10530481487512589,
      "rewards/rejected": -1.134108066558838,
      "step": 255
    },
    {
      "epoch": 0.5572672471533825,
      "grad_norm": 15.049601923886751,
      "learning_rate": 2.434401478000353e-07,
      "logits/chosen": -2.8707144260406494,
      "logits/rejected": -2.876131057739258,
      "logps/chosen": -1.0516908168792725,
      "logps/rejected": -1.1409704685211182,
      "loss": -0.0405,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.0516908168792725,
      "rewards/margins": 0.08927960693836212,
      "rewards/rejected": -1.1409704685211182,
      "step": 260
    },
    {
      "epoch": 0.5679839249832552,
      "grad_norm": 14.714810625194913,
      "learning_rate": 2.340778853248759e-07,
      "logits/chosen": -2.86578369140625,
      "logits/rejected": -2.8567006587982178,
      "logps/chosen": -1.0564695596694946,
      "logps/rejected": -1.199512004852295,
      "loss": -0.0695,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.0564695596694946,
      "rewards/margins": 0.143042653799057,
      "rewards/rejected": -1.199512004852295,
      "step": 265
    },
    {
      "epoch": 0.578700602813128,
      "grad_norm": 18.053080062016626,
      "learning_rate": 2.247379977748339e-07,
      "logits/chosen": -2.9022586345672607,
      "logits/rejected": -2.9201228618621826,
      "logps/chosen": -1.0403499603271484,
      "logps/rejected": -1.1265124082565308,
      "loss": -0.0673,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.0403499603271484,
      "rewards/margins": 0.08616244792938232,
      "rewards/rejected": -1.1265124082565308,
      "step": 270
    },
    {
      "epoch": 0.5894172806430007,
      "grad_norm": 14.15566071309189,
      "learning_rate": 2.1543361024596573e-07,
      "logits/chosen": -2.9802463054656982,
      "logits/rejected": -2.988959550857544,
      "logps/chosen": -0.9800962209701538,
      "logps/rejected": -1.0803120136260986,
      "loss": -0.0525,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.9800962209701538,
      "rewards/margins": 0.1002158373594284,
      "rewards/rejected": -1.0803120136260986,
      "step": 275
    },
    {
      "epoch": 0.6001339584728734,
      "grad_norm": 17.56629547418189,
      "learning_rate": 2.0617779794709082e-07,
      "logits/chosen": -2.8620638847351074,
      "logits/rejected": -2.8629186153411865,
      "logps/chosen": -1.103073000907898,
      "logps/rejected": -1.196608304977417,
      "loss": -0.0632,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.103073000907898,
      "rewards/margins": 0.09353531897068024,
      "rewards/rejected": -1.196608304977417,
      "step": 280
    },
    {
      "epoch": 0.6108506363027462,
      "grad_norm": 15.46761530330997,
      "learning_rate": 1.9698356782554687e-07,
      "logits/chosen": -2.9556548595428467,
      "logits/rejected": -2.9529995918273926,
      "logps/chosen": -1.109263300895691,
      "logps/rejected": -1.1731650829315186,
      "loss": -0.0564,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.109263300895691,
      "rewards/margins": 0.06390184909105301,
      "rewards/rejected": -1.1731650829315186,
      "step": 285
    },
    {
      "epoch": 0.6215673141326189,
      "grad_norm": 16.8037567850014,
      "learning_rate": 1.878638402888727e-07,
      "logits/chosen": -2.9940178394317627,
      "logits/rejected": -2.9970099925994873,
      "logps/chosen": -0.9795197248458862,
      "logps/rejected": -1.0418453216552734,
      "loss": -0.0668,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.9795197248458862,
      "rewards/margins": 0.06232555955648422,
      "rewards/rejected": -1.0418453216552734,
      "step": 290
    },
    {
      "epoch": 0.6322839919624916,
      "grad_norm": 16.60747046570722,
      "learning_rate": 1.7883143104810223e-07,
      "logits/chosen": -2.9415085315704346,
      "logits/rejected": -2.9544739723205566,
      "logps/chosen": -1.069349765777588,
      "logps/rejected": -1.2494323253631592,
      "loss": -0.0739,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.069349765777588,
      "rewards/margins": 0.1800827533006668,
      "rewards/rejected": -1.2494323253631592,
      "step": 295
    },
    {
      "epoch": 0.6430006697923644,
      "grad_norm": 15.422934564351497,
      "learning_rate": 1.6989903310818548e-07,
      "logits/chosen": -3.0317275524139404,
      "logits/rejected": -3.049278736114502,
      "logps/chosen": -1.0510863065719604,
      "logps/rejected": -1.1395753622055054,
      "loss": -0.0517,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.0510863065719604,
      "rewards/margins": 0.08848897367715836,
      "rewards/rejected": -1.1395753622055054,
      "step": 300
    },
    {
      "epoch": 0.6537173476222371,
      "grad_norm": 17.56071364411128,
      "learning_rate": 1.610791989308466e-07,
      "logits/chosen": -2.9625232219696045,
      "logits/rejected": -2.9480719566345215,
      "logps/chosen": -1.0810691118240356,
      "logps/rejected": -1.1474703550338745,
      "loss": -0.0711,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.0810691118240356,
      "rewards/margins": 0.06640125066041946,
      "rewards/rejected": -1.1474703550338745,
      "step": 305
    },
    {
      "epoch": 0.6644340254521098,
      "grad_norm": 19.400175535223255,
      "learning_rate": 1.523843227949418e-07,
      "logits/chosen": -3.0158638954162598,
      "logits/rejected": -3.014029026031494,
      "logps/chosen": -1.054434061050415,
      "logps/rejected": -1.1792142391204834,
      "loss": -0.0711,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.054434061050415,
      "rewards/margins": 0.12478007376194,
      "rewards/rejected": -1.1792142391204834,
      "step": 310
    },
    {
      "epoch": 0.6751507032819826,
      "grad_norm": 18.063066961906372,
      "learning_rate": 1.4382662337910867e-07,
      "logits/chosen": -2.9173102378845215,
      "logits/rejected": -2.9165103435516357,
      "logps/chosen": -1.1207153797149658,
      "logps/rejected": -1.2241191864013672,
      "loss": -0.0582,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.1207153797149658,
      "rewards/margins": 0.10340368747711182,
      "rewards/rejected": -1.2241191864013672,
      "step": 315
    },
    {
      "epoch": 0.6858673811118553,
      "grad_norm": 17.656998709658712,
      "learning_rate": 1.3541812659118104e-07,
      "logits/chosen": -2.924173593521118,
      "logits/rejected": -2.901674747467041,
      "logps/chosen": -1.1284511089324951,
      "logps/rejected": -1.2347378730773926,
      "loss": -0.0605,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.1284511089324951,
      "rewards/margins": 0.10628668963909149,
      "rewards/rejected": -1.2347378730773926,
      "step": 320
    },
    {
      "epoch": 0.696584058941728,
      "grad_norm": 17.674353062779463,
      "learning_rate": 1.2717064866850047e-07,
      "logits/chosen": -2.9069221019744873,
      "logits/rejected": -2.9266703128814697,
      "logps/chosen": -1.0635308027267456,
      "logps/rejected": -1.2443645000457764,
      "loss": -0.076,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.0635308027267456,
      "rewards/margins": 0.1808338761329651,
      "rewards/rejected": -1.2443645000457764,
      "step": 325
    },
    {
      "epoch": 0.7073007367716008,
      "grad_norm": 16.692003983984833,
      "learning_rate": 1.1909577957287117e-07,
      "logits/chosen": -2.9482550621032715,
      "logits/rejected": -2.9383444786071777,
      "logps/chosen": -1.1115998029708862,
      "logps/rejected": -1.269691824913025,
      "loss": -0.0875,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.1115998029708862,
      "rewards/margins": 0.15809203684329987,
      "rewards/rejected": -1.269691824913025,
      "step": 330
    },
    {
      "epoch": 0.7180174146014735,
      "grad_norm": 16.832132437998734,
      "learning_rate": 1.1120486670349441e-07,
      "logits/chosen": -3.0022974014282227,
      "logits/rejected": -3.0084285736083984,
      "logps/chosen": -1.1076433658599854,
      "logps/rejected": -1.2158708572387695,
      "loss": -0.0631,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.1076433658599854,
      "rewards/margins": 0.10822737216949463,
      "rewards/rejected": -1.2158708572387695,
      "step": 335
    },
    {
      "epoch": 0.7287340924313462,
      "grad_norm": 16.746834458412387,
      "learning_rate": 1.0350899895076948e-07,
      "logits/chosen": -2.960242509841919,
      "logits/rejected": -2.9590137004852295,
      "logps/chosen": -1.1618969440460205,
      "logps/rejected": -1.3054715394973755,
      "loss": -0.063,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.1618969440460205,
      "rewards/margins": 0.14357443153858185,
      "rewards/rejected": -1.3054715394973755,
      "step": 340
    },
    {
      "epoch": 0.739450770261219,
      "grad_norm": 16.472011449839883,
      "learning_rate": 9.601899111337069e-08,
      "logits/chosen": -2.9834091663360596,
      "logits/rejected": -2.9975457191467285,
      "logps/chosen": -1.126527190208435,
      "logps/rejected": -1.3144327402114868,
      "loss": -0.0777,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -1.126527190208435,
      "rewards/margins": 0.18790560960769653,
      "rewards/rejected": -1.3144327402114868,
      "step": 345
    },
    {
      "epoch": 0.7501674480910918,
      "grad_norm": 17.215572295903478,
      "learning_rate": 8.874536870049765e-08,
      "logits/chosen": -2.969423294067383,
      "logits/rejected": -2.969230890274048,
      "logps/chosen": -1.127826452255249,
      "logps/rejected": -1.271470069885254,
      "loss": -0.0752,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.127826452255249,
      "rewards/margins": 0.1436435580253601,
      "rewards/rejected": -1.271470069885254,
      "step": 350
    },
    {
      "epoch": 0.7608841259209645,
      "grad_norm": 19.033752258043222,
      "learning_rate": 8.169835314065663e-08,
      "logits/chosen": -2.9632205963134766,
      "logits/rejected": -2.9548847675323486,
      "logps/chosen": -1.1820392608642578,
      "logps/rejected": -1.2698676586151123,
      "loss": -0.0668,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -1.1820392608642578,
      "rewards/margins": 0.08782847970724106,
      "rewards/rejected": -1.2698676586151123,
      "step": 355
    },
    {
      "epoch": 0.7716008037508373,
      "grad_norm": 17.064795493163807,
      "learning_rate": 7.488784741775816e-08,
      "logits/chosen": -2.976466178894043,
      "logits/rejected": -2.9713473320007324,
      "logps/chosen": -1.054131269454956,
      "logps/rejected": -1.1566834449768066,
      "loss": -0.0663,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.054131269454956,
      "rewards/margins": 0.10255217552185059,
      "rewards/rejected": -1.1566834449768066,
      "step": 360
    },
    {
      "epoch": 0.78231748158071,
      "grad_norm": 17.868156711335107,
      "learning_rate": 6.832342215471715e-08,
      "logits/chosen": -2.98659086227417,
      "logits/rejected": -2.986912488937378,
      "logps/chosen": -1.1025737524032593,
      "logps/rejected": -1.242219090461731,
      "loss": -0.0823,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.1025737524032593,
      "rewards/margins": 0.13964524865150452,
      "rewards/rejected": -1.242219090461731,
      "step": 365
    },
    {
      "epoch": 0.7930341594105828,
      "grad_norm": 24.714287844140323,
      "learning_rate": 6.201430216410985e-08,
      "logits/chosen": -3.0034446716308594,
      "logits/rejected": -3.001354217529297,
      "logps/chosen": -1.1147871017456055,
      "logps/rejected": -1.200984239578247,
      "loss": -0.081,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.1147871017456055,
      "rewards/margins": 0.08619704097509384,
      "rewards/rejected": -1.200984239578247,
      "step": 370
    },
    {
      "epoch": 0.8037508372404555,
      "grad_norm": 20.89166421346418,
      "learning_rate": 5.596935348479029e-08,
      "logits/chosen": -2.9388105869293213,
      "logits/rejected": -2.948521137237549,
      "logps/chosen": -1.2442572116851807,
      "logps/rejected": -1.393157958984375,
      "loss": -0.0861,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.2442572116851807,
      "rewards/margins": 0.1489005833864212,
      "rewards/rejected": -1.393157958984375,
      "step": 375
    },
    {
      "epoch": 0.8144675150703282,
      "grad_norm": 22.406198534031674,
      "learning_rate": 5.01970709226808e-08,
      "logits/chosen": -3.028179883956909,
      "logits/rejected": -3.009589672088623,
      "logps/chosen": -1.0915172100067139,
      "logps/rejected": -1.243607997894287,
      "loss": -0.0837,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.0915172100067139,
      "rewards/margins": 0.1520906686782837,
      "rewards/rejected": -1.243607997894287,
      "step": 380
    },
    {
      "epoch": 0.825184192900201,
      "grad_norm": 24.898962036749563,
      "learning_rate": 4.4705566113246635e-08,
      "logits/chosen": -2.9715943336486816,
      "logits/rejected": -2.949638605117798,
      "logps/chosen": -1.228461503982544,
      "logps/rejected": -1.3865272998809814,
      "loss": -0.0897,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.228461503982544,
      "rewards/margins": 0.15806564688682556,
      "rewards/rejected": -1.3865272998809814,
      "step": 385
    },
    {
      "epoch": 0.8359008707300737,
      "grad_norm": 19.449984589836117,
      "learning_rate": 3.950255612243042e-08,
      "logits/chosen": -2.951904058456421,
      "logits/rejected": -2.928208827972412,
      "logps/chosen": -1.2001017332077026,
      "logps/rejected": -1.3440535068511963,
      "loss": -0.0724,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -1.2001017332077026,
      "rewards/margins": 0.1439516693353653,
      "rewards/rejected": -1.3440535068511963,
      "step": 390
    },
    {
      "epoch": 0.8466175485599464,
      "grad_norm": 19.857222600639766,
      "learning_rate": 3.459535260206384e-08,
      "logits/chosen": -2.9977169036865234,
      "logits/rejected": -3.020946979522705,
      "logps/chosen": -1.096103549003601,
      "logps/rejected": -1.21526300907135,
      "loss": -0.0898,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.096103549003601,
      "rewards/margins": 0.11915931850671768,
      "rewards/rejected": -1.21526300907135,
      "step": 395
    },
    {
      "epoch": 0.8573342263898192,
      "grad_norm": 21.73532568075866,
      "learning_rate": 2.999085151499764e-08,
      "logits/chosen": -2.9534425735473633,
      "logits/rejected": -2.9359939098358154,
      "logps/chosen": -1.0908204317092896,
      "logps/rejected": -1.356081485748291,
      "loss": -0.0905,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -1.0908204317092896,
      "rewards/margins": 0.2652612626552582,
      "rewards/rejected": -1.356081485748291,
      "step": 400
    },
    {
      "epoch": 0.8573342263898192,
      "eval_logits/chosen": -2.8999884128570557,
      "eval_logits/rejected": -2.892754316329956,
      "eval_logps/chosen": -1.1346372365951538,
      "eval_logps/rejected": -1.2466005086898804,
      "eval_loss": -0.07080772519111633,
      "eval_rewards/accuracies": 0.5757978558540344,
      "eval_rewards/chosen": -1.1346372365951538,
      "eval_rewards/margins": 0.11196334660053253,
      "eval_rewards/rejected": -1.2466005086898804,
      "eval_runtime": 432.4273,
      "eval_samples_per_second": 6.924,
      "eval_steps_per_second": 0.435,
      "step": 400
    },
    {
      "epoch": 0.8680509042196919,
      "grad_norm": 19.440021908588918,
      "learning_rate": 2.5695523444387747e-08,
      "logits/chosen": -2.9388387203216553,
      "logits/rejected": -2.9628615379333496,
      "logps/chosen": -1.111882209777832,
      "logps/rejected": -1.2351758480072021,
      "loss": -0.09,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.111882209777832,
      "rewards/margins": 0.12329357862472534,
      "rewards/rejected": -1.2351758480072021,
      "step": 405
    },
    {
      "epoch": 0.8787675820495646,
      "grad_norm": 19.454852119565878,
      "learning_rate": 2.1715404500756875e-08,
      "logits/chosen": -2.98136305809021,
      "logits/rejected": -3.0059518814086914,
      "logps/chosen": -1.090047836303711,
      "logps/rejected": -1.203460693359375,
      "loss": -0.0823,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.090047836303711,
      "rewards/margins": 0.11341279745101929,
      "rewards/rejected": -1.203460693359375,
      "step": 410
    },
    {
      "epoch": 0.8894842598794374,
      "grad_norm": 27.119138813457198,
      "learning_rate": 1.8056087839608546e-08,
      "logits/chosen": -2.9424166679382324,
      "logits/rejected": -2.9254748821258545,
      "logps/chosen": -1.1260377168655396,
      "logps/rejected": -1.3184921741485596,
      "loss": -0.1086,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.1260377168655396,
      "rewards/margins": 0.19245454668998718,
      "rewards/rejected": -1.3184921741485596,
      "step": 415
    },
    {
      "epoch": 0.9002009377093101,
      "grad_norm": 19.3060602023721,
      "learning_rate": 1.4722715801513935e-08,
      "logits/chosen": -2.9605276584625244,
      "logits/rejected": -2.967475414276123,
      "logps/chosen": -1.0344493389129639,
      "logps/rejected": -1.0916310548782349,
      "loss": -0.0802,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.0344493389129639,
      "rewards/margins": 0.057181768119335175,
      "rewards/rejected": -1.0916310548782349,
      "step": 420
    },
    {
      "epoch": 0.9109176155391828,
      "grad_norm": 20.048601077954377,
      "learning_rate": 1.1719972685717395e-08,
      "logits/chosen": -2.970942258834839,
      "logits/rejected": -2.9757351875305176,
      "logps/chosen": -1.0653189420700073,
      "logps/rejected": -1.3184258937835693,
      "loss": -0.1043,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.0653189420700073,
      "rewards/margins": 0.2531070411205292,
      "rewards/rejected": -1.3184258937835693,
      "step": 425
    },
    {
      "epoch": 0.9216342933690556,
      "grad_norm": 21.66418470338373,
      "learning_rate": 9.05207816741485e-09,
      "logits/chosen": -2.939943313598633,
      "logits/rejected": -2.9273343086242676,
      "logps/chosen": -1.2503459453582764,
      "logps/rejected": -1.4009029865264893,
      "loss": -0.1013,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.2503459453582764,
      "rewards/margins": 0.15055687725543976,
      "rewards/rejected": -1.4009029865264893,
      "step": 430
    },
    {
      "epoch": 0.9323509711989283,
      "grad_norm": 21.774320932145653,
      "learning_rate": 6.722781367956481e-09,
      "logits/chosen": -3.0610954761505127,
      "logits/rejected": -3.0484843254089355,
      "logps/chosen": -1.0789310932159424,
      "logps/rejected": -1.2742266654968262,
      "loss": -0.1009,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.0789310932159424,
      "rewards/margins": 0.19529545307159424,
      "rewards/rejected": -1.2742266654968262,
      "step": 435
    },
    {
      "epoch": 0.943067649028801,
      "grad_norm": 19.319233646112725,
      "learning_rate": 4.735355586305861e-09,
      "logits/chosen": -2.94116473197937,
      "logits/rejected": -2.949489116668701,
      "logps/chosen": -1.1311179399490356,
      "logps/rejected": -1.3173315525054932,
      "loss": -0.0993,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.1311179399490356,
      "rewards/margins": 0.1862136423587799,
      "rewards/rejected": -1.3173315525054932,
      "step": 440
    },
    {
      "epoch": 0.9537843268586738,
      "grad_norm": 19.556600822562356,
      "learning_rate": 3.0925936991598454e-09,
      "logits/chosen": -2.9415977001190186,
      "logits/rejected": -2.944352388381958,
      "logps/chosen": -1.1981102228164673,
      "logps/rejected": -1.329925775527954,
      "loss": -0.0746,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.1981102228164673,
      "rewards/margins": 0.1318153738975525,
      "rewards/rejected": -1.329925775527954,
      "step": 445
    },
    {
      "epoch": 0.9645010046885466,
      "grad_norm": 24.43813858171124,
      "learning_rate": 1.7968042361930225e-09,
      "logits/chosen": -2.958519697189331,
      "logits/rejected": -2.9468445777893066,
      "logps/chosen": -1.1451342105865479,
      "logps/rejected": -1.265005111694336,
      "loss": -0.0894,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -1.1451342105865479,
      "rewards/margins": 0.11987074464559555,
      "rewards/rejected": -1.265005111694336,
      "step": 450
    },
    {
      "epoch": 0.9752176825184193,
      "grad_norm": 23.495311684580507,
      "learning_rate": 8.498081359423759e-10,
      "logits/chosen": -2.9030869007110596,
      "logits/rejected": -2.932225465774536,
      "logps/chosen": -1.1186686754226685,
      "logps/rejected": -1.335756540298462,
      "loss": -0.1008,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.1186686754226685,
      "rewards/margins": 0.21708789467811584,
      "rewards/rejected": -1.335756540298462,
      "step": 455
    },
    {
      "epoch": 0.9859343603482921,
      "grad_norm": 20.358552883416746,
      "learning_rate": 2.5293618689056774e-10,
      "logits/chosen": -2.9370243549346924,
      "logits/rejected": -2.953622341156006,
      "logps/chosen": -1.1718332767486572,
      "logps/rejected": -1.4025189876556396,
      "loss": -0.0805,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.1718332767486572,
      "rewards/margins": 0.230685755610466,
      "rewards/rejected": -1.4025189876556396,
      "step": 460
    },
    {
      "epoch": 0.9966510381781648,
      "grad_norm": 27.657325299971138,
      "learning_rate": 7.02715734424153e-12,
      "logits/chosen": -2.9429261684417725,
      "logits/rejected": -2.951002359390259,
      "logps/chosen": -1.2534146308898926,
      "logps/rejected": -1.3702634572982788,
      "loss": -0.0688,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.2534146308898926,
      "rewards/margins": 0.11684894561767578,
      "rewards/rejected": -1.3702634572982788,
      "step": 465
    },
    {
      "epoch": 0.9987943737441393,
      "step": 466,
      "total_flos": 0.0,
      "train_loss": 0.0,
      "train_runtime": 4.3106,
      "train_samples_per_second": 13854.197,
      "train_steps_per_second": 108.105
    }
  ],
  "logging_steps": 5,
  "max_steps": 466,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 1000000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}