{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.998691442030882,
  "eval_steps": 500,
  "global_step": 477,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.010468463752944255,
      "grad_norm": 15.141688725114767,
      "learning_rate": 5.208333333333333e-08,
      "logits/chosen": -2.770766496658325,
      "logits/rejected": -2.728191375732422,
      "logps/chosen": -1.027971625328064,
      "logps/rejected": -1.1730705499649048,
      "loss": 2.0284,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.027971625328064,
      "rewards/margins": 0.14509883522987366,
      "rewards/rejected": -1.1730705499649048,
      "step": 5
    },
    {
      "epoch": 0.02093692750588851,
      "grad_norm": 13.969517964651633,
      "learning_rate": 1.0416666666666667e-07,
      "logits/chosen": -2.7511062622070312,
      "logits/rejected": -2.7253737449645996,
      "logps/chosen": -0.9954013824462891,
      "logps/rejected": -1.0719196796417236,
      "loss": 2.002,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.9954013824462891,
      "rewards/margins": 0.07651824504137039,
      "rewards/rejected": -1.0719196796417236,
      "step": 10
    },
    {
      "epoch": 0.031405391258832765,
      "grad_norm": 13.915856984059875,
      "learning_rate": 1.5624999999999999e-07,
      "logits/chosen": -2.7306742668151855,
      "logits/rejected": -2.697096347808838,
      "logps/chosen": -0.9895243644714355,
      "logps/rejected": -1.147679328918457,
      "loss": 1.9874,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.9895243644714355,
      "rewards/margins": 0.15815496444702148,
      "rewards/rejected": -1.147679328918457,
      "step": 15
    },
    {
      "epoch": 0.04187385501177702,
      "grad_norm": 13.588156017248505,
      "learning_rate": 2.0833333333333333e-07,
      "logits/chosen": -2.8030834197998047,
      "logits/rejected": -2.709643602371216,
      "logps/chosen": -1.0228091478347778,
      "logps/rejected": -1.1238880157470703,
      "loss": 2.0265,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -1.0228091478347778,
      "rewards/margins": 0.10107902437448502,
      "rewards/rejected": -1.1238880157470703,
      "step": 20
    },
    {
      "epoch": 0.05234231876472128,
      "grad_norm": 14.042807702810773,
      "learning_rate": 2.604166666666667e-07,
      "logits/chosen": -2.7991013526916504,
      "logits/rejected": -2.7758243083953857,
      "logps/chosen": -1.0302048921585083,
      "logps/rejected": -1.086944818496704,
      "loss": 2.0091,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -1.0302048921585083,
      "rewards/margins": 0.05673985555768013,
      "rewards/rejected": -1.086944818496704,
      "step": 25
    },
    {
      "epoch": 0.06281078251766553,
      "grad_norm": 15.069602245174066,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": -2.7733917236328125,
      "logits/rejected": -2.7317938804626465,
      "logps/chosen": -0.9855572581291199,
      "logps/rejected": -1.0908451080322266,
      "loss": 2.0118,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.9855572581291199,
      "rewards/margins": 0.1052878275513649,
      "rewards/rejected": -1.0908451080322266,
      "step": 30
    },
    {
      "epoch": 0.07327924627060979,
      "grad_norm": 14.14588992747414,
      "learning_rate": 3.645833333333333e-07,
      "logits/chosen": -2.7773802280426025,
      "logits/rejected": -2.6910452842712402,
      "logps/chosen": -1.0310332775115967,
      "logps/rejected": -1.1711452007293701,
      "loss": 1.9903,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.0310332775115967,
      "rewards/margins": 0.1401120126247406,
      "rewards/rejected": -1.1711452007293701,
      "step": 35
    },
    {
      "epoch": 0.08374771002355404,
      "grad_norm": 20.106972236529245,
      "learning_rate": 4.1666666666666667e-07,
      "logits/chosen": -2.7931580543518066,
      "logits/rejected": -2.6956787109375,
      "logps/chosen": -1.011854887008667,
      "logps/rejected": -1.2990570068359375,
      "loss": 1.9463,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.011854887008667,
      "rewards/margins": 0.28720206022262573,
      "rewards/rejected": -1.2990570068359375,
      "step": 40
    },
    {
      "epoch": 0.0942161737764983,
      "grad_norm": 15.19327904046142,
      "learning_rate": 4.6874999999999996e-07,
      "logits/chosen": -2.8017444610595703,
      "logits/rejected": -2.7550606727600098,
      "logps/chosen": -0.9639647603034973,
      "logps/rejected": -1.142878770828247,
      "loss": 1.9512,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.9639647603034973,
      "rewards/margins": 0.17891402542591095,
      "rewards/rejected": -1.142878770828247,
      "step": 45
    },
    {
      "epoch": 0.10468463752944256,
      "grad_norm": 20.91684390311358,
      "learning_rate": 4.999731868769026e-07,
      "logits/chosen": -2.7584781646728516,
      "logits/rejected": -2.6904306411743164,
      "logps/chosen": -1.0117145776748657,
      "logps/rejected": -1.30343496799469,
      "loss": 1.9653,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.0117145776748657,
      "rewards/margins": 0.2917203903198242,
      "rewards/rejected": -1.30343496799469,
      "step": 50
    },
    {
      "epoch": 0.11515310128238682,
      "grad_norm": 16.247858117319236,
      "learning_rate": 4.996716052911017e-07,
      "logits/chosen": -2.754804849624634,
      "logits/rejected": -2.71933650970459,
      "logps/chosen": -1.0166981220245361,
      "logps/rejected": -1.268345832824707,
      "loss": 1.9611,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -1.0166981220245361,
      "rewards/margins": 0.25164780020713806,
      "rewards/rejected": -1.268345832824707,
      "step": 55
    },
    {
      "epoch": 0.12562156503533106,
      "grad_norm": 16.39781017532518,
      "learning_rate": 4.990353313429303e-07,
      "logits/chosen": -2.8165841102600098,
      "logits/rejected": -2.7710437774658203,
      "logps/chosen": -1.049938440322876,
      "logps/rejected": -1.2433346509933472,
      "loss": 1.9164,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.049938440322876,
      "rewards/margins": 0.19339624047279358,
      "rewards/rejected": -1.2433346509933472,
      "step": 60
    },
    {
      "epoch": 0.1360900287882753,
      "grad_norm": 15.239254500208718,
      "learning_rate": 4.980652179769217e-07,
      "logits/chosen": -2.8142337799072266,
      "logits/rejected": -2.6995322704315186,
      "logps/chosen": -1.0191227197647095,
      "logps/rejected": -1.544886827468872,
      "loss": 1.9025,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.0191227197647095,
      "rewards/margins": 0.5257640480995178,
      "rewards/rejected": -1.544886827468872,
      "step": 65
    },
    {
      "epoch": 0.14655849254121958,
      "grad_norm": 21.0566472158153,
      "learning_rate": 4.967625656594781e-07,
      "logits/chosen": -2.7174973487854004,
      "logits/rejected": -2.6667487621307373,
      "logps/chosen": -1.0492310523986816,
      "logps/rejected": -1.3482974767684937,
      "loss": 1.9091,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -1.0492310523986816,
      "rewards/margins": 0.2990665137767792,
      "rewards/rejected": -1.3482974767684937,
      "step": 70
    },
    {
      "epoch": 0.15702695629416383,
      "grad_norm": 30.761967865275885,
      "learning_rate": 4.951291206355559e-07,
      "logits/chosen": -2.773061752319336,
      "logits/rejected": -2.698946952819824,
      "logps/chosen": -0.9925507307052612,
      "logps/rejected": -1.3563807010650635,
      "loss": 1.8891,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9925507307052612,
      "rewards/margins": 0.36382991075515747,
      "rewards/rejected": -1.3563807010650635,
      "step": 75
    },
    {
      "epoch": 0.16749542004710807,
      "grad_norm": 31.297744502118277,
      "learning_rate": 4.93167072587771e-07,
      "logits/chosen": -2.738257884979248,
      "logits/rejected": -2.6784729957580566,
      "logps/chosen": -1.0087690353393555,
      "logps/rejected": -1.4246900081634521,
      "loss": 1.9382,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.0087690353393555,
      "rewards/margins": 0.4159211218357086,
      "rewards/rejected": -1.4246900081634521,
      "step": 80
    },
    {
      "epoch": 0.17796388380005235,
      "grad_norm": 24.407477818447635,
      "learning_rate": 4.908790517010636e-07,
      "logits/chosen": -2.747776508331299,
      "logits/rejected": -2.7003962993621826,
      "logps/chosen": -0.9973554611206055,
      "logps/rejected": -1.3728947639465332,
      "loss": 1.9613,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.9973554611206055,
      "rewards/margins": 0.3755393624305725,
      "rewards/rejected": -1.3728947639465332,
      "step": 85
    },
    {
      "epoch": 0.1884323475529966,
      "grad_norm": 31.20847987328103,
      "learning_rate": 4.882681251368548e-07,
      "logits/chosen": -2.705894708633423,
      "logits/rejected": -2.6819891929626465,
      "logps/chosen": -1.0237383842468262,
      "logps/rejected": -1.404168725013733,
      "loss": 1.88,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.0237383842468262,
      "rewards/margins": 0.3804302513599396,
      "rewards/rejected": -1.404168725013733,
      "step": 90
    },
    {
      "epoch": 0.19890081130594087,
      "grad_norm": 30.65914852598083,
      "learning_rate": 4.853377929214243e-07,
      "logits/chosen": -2.776717185974121,
      "logits/rejected": -2.6825289726257324,
      "logps/chosen": -1.038529872894287,
      "logps/rejected": -1.3572471141815186,
      "loss": 1.9062,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.038529872894287,
      "rewards/margins": 0.31871724128723145,
      "rewards/rejected": -1.3572471141815186,
      "step": 95
    },
    {
      "epoch": 0.2093692750588851,
      "grad_norm": 34.08856475096774,
      "learning_rate": 4.820919832540181e-07,
      "logits/chosen": -2.74165678024292,
      "logits/rejected": -2.6503825187683105,
      "logps/chosen": -1.051953673362732,
      "logps/rejected": -1.4617860317230225,
      "loss": 1.9147,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -1.051953673362732,
      "rewards/margins": 0.4098321497440338,
      "rewards/rejected": -1.4617860317230225,
      "step": 100
    },
    {
      "epoch": 0.21983773881182936,
      "grad_norm": 28.24345375738832,
      "learning_rate": 4.785350472409791e-07,
      "logits/chosen": -2.743473529815674,
      "logits/rejected": -2.7197020053863525,
      "logps/chosen": -0.931172251701355,
      "logps/rejected": -1.453914999961853,
      "loss": 1.8781,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.931172251701355,
      "rewards/margins": 0.522742748260498,
      "rewards/rejected": -1.453914999961853,
      "step": 105
    },
    {
      "epoch": 0.23030620256477363,
      "grad_norm": 35.845899664243596,
      "learning_rate": 4.7467175306295647e-07,
      "logits/chosen": -2.817007064819336,
      "logits/rejected": -2.762336254119873,
      "logps/chosen": -1.0500761270523071,
      "logps/rejected": -1.4565006494522095,
      "loss": 1.88,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.0500761270523071,
      "rewards/margins": 0.40642452239990234,
      "rewards/rejected": -1.4565006494522095,
      "step": 110
    },
    {
      "epoch": 0.24077466631771788,
      "grad_norm": 23.382933706781618,
      "learning_rate": 4.70507279583015e-07,
      "logits/chosen": -2.7697396278381348,
      "logits/rejected": -2.710322856903076,
      "logps/chosen": -1.0177266597747803,
      "logps/rejected": -1.5666338205337524,
      "loss": 1.868,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.0177266597747803,
      "rewards/margins": 0.5489073991775513,
      "rewards/rejected": -1.5666338205337524,
      "step": 115
    },
    {
      "epoch": 0.2512431300706621,
      "grad_norm": 29.59771539393517,
      "learning_rate": 4.6604720940421207e-07,
      "logits/chosen": -2.7786221504211426,
      "logits/rejected": -2.737684965133667,
      "logps/chosen": -1.0584137439727783,
      "logps/rejected": -1.4742605686187744,
      "loss": 1.9062,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.0584137439727783,
      "rewards/margins": 0.41584667563438416,
      "rewards/rejected": -1.4742605686187744,
      "step": 120
    },
    {
      "epoch": 0.26171159382360637,
      "grad_norm": 24.555574989451102,
      "learning_rate": 4.612975213859487e-07,
      "logits/chosen": -2.8200080394744873,
      "logits/rejected": -2.7997546195983887,
      "logps/chosen": -1.058563470840454,
      "logps/rejected": -1.3519017696380615,
      "loss": 1.9069,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -1.058563470840454,
      "rewards/margins": 0.29333844780921936,
      "rewards/rejected": -1.3519017696380615,
      "step": 125
    },
    {
      "epoch": 0.2721800575765506,
      "grad_norm": 26.634009671559898,
      "learning_rate": 4.5626458262912735e-07,
      "logits/chosen": -2.788752794265747,
      "logits/rejected": -2.754631519317627,
      "logps/chosen": -1.0312286615371704,
      "logps/rejected": -1.2794463634490967,
      "loss": 1.8977,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -1.0312286615371704,
      "rewards/margins": 0.24821774661540985,
      "rewards/rejected": -1.2794463634490967,
      "step": 130
    },
    {
      "epoch": 0.2826485213294949,
      "grad_norm": 32.240888174368,
      "learning_rate": 4.5095513994085974e-07,
      "logits/chosen": -2.7877840995788574,
      "logits/rejected": -2.7001712322235107,
      "logps/chosen": -1.0880383253097534,
      "logps/rejected": -1.7845348119735718,
      "loss": 1.8912,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.0880383253097534,
      "rewards/margins": 0.696496307849884,
      "rewards/rejected": -1.7845348119735718,
      "step": 135
    },
    {
      "epoch": 0.29311698508243916,
      "grad_norm": 25.76204815720163,
      "learning_rate": 4.453763107901675e-07,
      "logits/chosen": -2.8313488960266113,
      "logits/rejected": -2.780134677886963,
      "logps/chosen": -0.9982128143310547,
      "logps/rejected": -1.4504693746566772,
      "loss": 1.8155,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.9982128143310547,
      "rewards/margins": 0.4522566795349121,
      "rewards/rejected": -1.4504693746566772,
      "step": 140
    },
    {
      "epoch": 0.3035854488353834,
      "grad_norm": 21.7794898263259,
      "learning_rate": 4.395355737667985e-07,
      "logits/chosen": -2.809255361557007,
      "logits/rejected": -2.761308193206787,
      "logps/chosen": -1.0206363201141357,
      "logps/rejected": -1.419768214225769,
      "loss": 1.8605,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.0206363201141357,
      "rewards/margins": 0.39913198351860046,
      "rewards/rejected": -1.419768214225769,
      "step": 145
    },
    {
      "epoch": 0.31405391258832765,
      "grad_norm": 26.507909597761785,
      "learning_rate": 4.3344075855595097e-07,
      "logits/chosen": -2.849714517593384,
      "logits/rejected": -2.8313772678375244,
      "logps/chosen": -0.9701302647590637,
      "logps/rejected": -1.3971799612045288,
      "loss": 1.8345,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.9701302647590637,
      "rewards/margins": 0.42704978585243225,
      "rewards/rejected": -1.3971799612045288,
      "step": 150
    },
    {
      "epoch": 0.3245223763412719,
      "grad_norm": 28.367719379820283,
      "learning_rate": 4.271000354423425e-07,
      "logits/chosen": -2.7890210151672363,
      "logits/rejected": -2.757753849029541,
      "logps/chosen": -1.0669995546340942,
      "logps/rejected": -1.4297670125961304,
      "loss": 1.894,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.0669995546340942,
      "rewards/margins": 0.36276736855506897,
      "rewards/rejected": -1.4297670125961304,
      "step": 155
    },
    {
      "epoch": 0.33499084009421615,
      "grad_norm": 32.029509535293904,
      "learning_rate": 4.2052190435769554e-07,
      "logits/chosen": -2.7710726261138916,
      "logits/rejected": -2.7451701164245605,
      "logps/chosen": -1.0376036167144775,
      "logps/rejected": -1.4104793071746826,
      "loss": 1.8838,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.0376036167144775,
      "rewards/margins": 0.37287575006484985,
      "rewards/rejected": -1.4104793071746826,
      "step": 160
    },
    {
      "epoch": 0.34545930384716045,
      "grad_norm": 25.20687040858121,
      "learning_rate": 4.137151834863213e-07,
      "logits/chosen": -2.7923319339752197,
      "logits/rejected": -2.707219362258911,
      "logps/chosen": -0.9815086126327515,
      "logps/rejected": -1.4057986736297607,
      "loss": 1.8435,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.9815086126327515,
      "rewards/margins": 0.4242900311946869,
      "rewards/rejected": -1.4057986736297607,
      "step": 165
    },
    {
      "epoch": 0.3559277676001047,
      "grad_norm": 28.006785803959723,
      "learning_rate": 4.0668899744407567e-07,
      "logits/chosen": -2.8175511360168457,
      "logits/rejected": -2.752450466156006,
      "logps/chosen": -1.0611494779586792,
      "logps/rejected": -1.5021220445632935,
      "loss": 1.8575,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.0611494779586792,
      "rewards/margins": 0.4409725069999695,
      "rewards/rejected": -1.5021220445632935,
      "step": 170
    },
    {
      "epoch": 0.36639623135304894,
      "grad_norm": 25.596200434230976,
      "learning_rate": 3.994527650465352e-07,
      "logits/chosen": -2.7964577674865723,
      "logits/rejected": -2.709230661392212,
      "logps/chosen": -1.1404192447662354,
      "logps/rejected": -1.6063992977142334,
      "loss": 1.9076,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.1404192447662354,
      "rewards/margins": 0.4659802317619324,
      "rewards/rejected": -1.6063992977142334,
      "step": 175
    },
    {
      "epoch": 0.3768646951059932,
      "grad_norm": 25.892094106029592,
      "learning_rate": 3.920161866827889e-07,
      "logits/chosen": -2.7817273139953613,
      "logits/rejected": -2.7617640495300293,
      "logps/chosen": -1.0391079187393188,
      "logps/rejected": -1.5876353979110718,
      "loss": 1.8544,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.0391079187393188,
      "rewards/margins": 0.5485275387763977,
      "rewards/rejected": -1.5876353979110718,
      "step": 180
    },
    {
      "epoch": 0.38733315885893743,
      "grad_norm": 29.42463620034665,
      "learning_rate": 3.8438923131177237e-07,
      "logits/chosen": -2.7785096168518066,
      "logits/rejected": -2.6860692501068115,
      "logps/chosen": -1.0492454767227173,
      "logps/rejected": -1.6226046085357666,
      "loss": 1.8247,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.0492454767227173,
      "rewards/margins": 0.573358952999115,
      "rewards/rejected": -1.6226046085357666,
      "step": 185
    },
    {
      "epoch": 0.39780162261188173,
      "grad_norm": 27.921360264789193,
      "learning_rate": 3.765821230985757e-07,
      "logits/chosen": -2.7853875160217285,
      "logits/rejected": -2.7454705238342285,
      "logps/chosen": -1.0294712781906128,
      "logps/rejected": -1.4654343128204346,
      "loss": 1.9244,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0294712781906128,
      "rewards/margins": 0.4359629154205322,
      "rewards/rejected": -1.4654343128204346,
      "step": 190
    },
    {
      "epoch": 0.408270086364826,
      "grad_norm": 29.87078620394206,
      "learning_rate": 3.6860532770864005e-07,
      "logits/chosen": -2.701105833053589,
      "logits/rejected": -2.6695313453674316,
      "logps/chosen": -1.098097324371338,
      "logps/rejected": -1.4831833839416504,
      "loss": 1.899,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.098097324371338,
      "rewards/margins": 0.3850858509540558,
      "rewards/rejected": -1.4831833839416504,
      "step": 195
    },
    {
      "epoch": 0.4187385501177702,
      "grad_norm": 28.048650935368237,
      "learning_rate": 3.604695382782159e-07,
      "logits/chosen": -2.738492250442505,
      "logits/rejected": -2.7187023162841797,
      "logps/chosen": -1.0665818452835083,
      "logps/rejected": -1.46578049659729,
      "loss": 1.8737,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.0665818452835083,
      "rewards/margins": 0.39919885993003845,
      "rewards/rejected": -1.46578049659729,
      "step": 200
    },
    {
      "epoch": 0.42920701387071447,
      "grad_norm": 27.850087928191183,
      "learning_rate": 3.5218566107988867e-07,
      "logits/chosen": -2.6582818031311035,
      "logits/rejected": -2.5809996128082275,
      "logps/chosen": -0.9890678524971008,
      "logps/rejected": -1.4678758382797241,
      "loss": 1.8403,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.9890678524971008,
      "rewards/margins": 0.47880783677101135,
      "rewards/rejected": -1.4678758382797241,
      "step": 205
    },
    {
      "epoch": 0.4396754776236587,
      "grad_norm": 29.109484926295075,
      "learning_rate": 3.4376480090239047e-07,
      "logits/chosen": -2.6842644214630127,
      "logits/rejected": -2.627836227416992,
      "logps/chosen": -1.075455665588379,
      "logps/rejected": -1.4202773571014404,
      "loss": 1.8863,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.075455665588379,
      "rewards/margins": 0.34482160210609436,
      "rewards/rejected": -1.4202773571014404,
      "step": 210
    },
    {
      "epoch": 0.45014394137660296,
      "grad_norm": 23.99752736324705,
      "learning_rate": 3.3521824616429284e-07,
      "logits/chosen": -2.716024875640869,
      "logits/rejected": -2.6916396617889404,
      "logps/chosen": -1.0032670497894287,
      "logps/rejected": -1.5087072849273682,
      "loss": 1.8264,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -1.0032670497894287,
      "rewards/margins": 0.5054404139518738,
      "rewards/rejected": -1.5087072849273682,
      "step": 215
    },
    {
      "epoch": 0.46061240512954726,
      "grad_norm": 31.21916182230841,
      "learning_rate": 3.265574537815398e-07,
      "logits/chosen": -2.742882013320923,
      "logits/rejected": -2.658285617828369,
      "logps/chosen": -1.033542275428772,
      "logps/rejected": -1.5640771389007568,
      "loss": 1.8307,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.033542275428772,
      "rewards/margins": 0.5305347442626953,
      "rewards/rejected": -1.5640771389007568,
      "step": 220
    },
    {
      "epoch": 0.4710808688824915,
      "grad_norm": 26.98263419792695,
      "learning_rate": 3.1779403380910425e-07,
      "logits/chosen": -2.7341504096984863,
      "logits/rejected": -2.6276047229766846,
      "logps/chosen": -0.988808810710907,
      "logps/rejected": -1.5687158107757568,
      "loss": 1.8334,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.988808810710907,
      "rewards/margins": 0.5799070596694946,
      "rewards/rejected": -1.5687158107757568,
      "step": 225
    },
    {
      "epoch": 0.48154933263543576,
      "grad_norm": 29.311922113138767,
      "learning_rate": 3.0893973387735683e-07,
      "logits/chosen": -2.74092698097229,
      "logits/rejected": -2.6369271278381348,
      "logps/chosen": -1.009143352508545,
      "logps/rejected": -1.665408730506897,
      "loss": 1.8283,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.009143352508545,
      "rewards/margins": 0.6562654376029968,
      "rewards/rejected": -1.665408730506897,
      "step": 230
    },
    {
      "epoch": 0.49201779638838,
      "grad_norm": 32.62916846952035,
      "learning_rate": 3.000064234440111e-07,
      "logits/chosen": -2.729424476623535,
      "logits/rejected": -2.672799587249756,
      "logps/chosen": -1.0022896528244019,
      "logps/rejected": -1.5322816371917725,
      "loss": 1.86,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -1.0022896528244019,
      "rewards/margins": 0.5299919247627258,
      "rewards/rejected": -1.5322816371917725,
      "step": 235
    },
    {
      "epoch": 0.5024862601413242,
      "grad_norm": 25.60849652215575,
      "learning_rate": 2.910060778827554e-07,
      "logits/chosen": -2.666738986968994,
      "logits/rejected": -2.6147377490997314,
      "logps/chosen": -1.0380260944366455,
      "logps/rejected": -1.5499128103256226,
      "loss": 1.8806,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.0380260944366455,
      "rewards/margins": 0.5118868947029114,
      "rewards/rejected": -1.5499128103256226,
      "step": 240
    },
    {
      "epoch": 0.5129547238942685,
      "grad_norm": 23.307769710548996,
      "learning_rate": 2.8195076242990116e-07,
      "logits/chosen": -2.7083492279052734,
      "logits/rejected": -2.608227014541626,
      "logps/chosen": -1.1102759838104248,
      "logps/rejected": -1.5368731021881104,
      "loss": 1.8599,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.1102759838104248,
      "rewards/margins": 0.42659682035446167,
      "rewards/rejected": -1.5368731021881104,
      "step": 245
    },
    {
      "epoch": 0.5234231876472127,
      "grad_norm": 30.713855522791444,
      "learning_rate": 2.7285261601056697e-07,
      "logits/chosen": -2.621891498565674,
      "logits/rejected": -2.572918653488159,
      "logps/chosen": -1.0056602954864502,
      "logps/rejected": -1.5289640426635742,
      "loss": 1.8611,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.0056602954864502,
      "rewards/margins": 0.5233036875724792,
      "rewards/rejected": -1.5289640426635742,
      "step": 250
    },
    {
      "epoch": 0.533891651400157,
      "grad_norm": 21.027582284131125,
      "learning_rate": 2.6372383496608186e-07,
      "logits/chosen": -2.679741859436035,
      "logits/rejected": -2.588026762008667,
      "logps/chosen": -1.0767912864685059,
      "logps/rejected": -1.6802423000335693,
      "loss": 1.7962,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -1.0767912864685059,
      "rewards/margins": 0.603451132774353,
      "rewards/rejected": -1.6802423000335693,
      "step": 255
    },
    {
      "epoch": 0.5443601151531012,
      "grad_norm": 21.083992579626138,
      "learning_rate": 2.5457665670441937e-07,
      "logits/chosen": -2.639758825302124,
      "logits/rejected": -2.539482831954956,
      "logps/chosen": -0.9617997407913208,
      "logps/rejected": -1.643094778060913,
      "loss": 1.8469,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.9617997407913208,
      "rewards/margins": 0.6812952160835266,
      "rewards/rejected": -1.643094778060913,
      "step": 260
    },
    {
      "epoch": 0.5548285789060455,
      "grad_norm": 34.336932322095805,
      "learning_rate": 2.454233432955807e-07,
      "logits/chosen": -2.6934094429016113,
      "logits/rejected": -2.635524034500122,
      "logps/chosen": -1.0399844646453857,
      "logps/rejected": -1.3895108699798584,
      "loss": 1.8185,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -1.0399844646453857,
      "rewards/margins": 0.3495265543460846,
      "rewards/rejected": -1.3895108699798584,
      "step": 265
    },
    {
      "epoch": 0.5652970426589898,
      "grad_norm": 35.56183499916552,
      "learning_rate": 2.3627616503391812e-07,
      "logits/chosen": -2.678424596786499,
      "logits/rejected": -2.6192078590393066,
      "logps/chosen": -1.1347005367279053,
      "logps/rejected": -1.5019694566726685,
      "loss": 1.8876,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.1347005367279053,
      "rewards/margins": 0.36726871132850647,
      "rewards/rejected": -1.5019694566726685,
      "step": 270
    },
    {
      "epoch": 0.575765506411934,
      "grad_norm": 21.13133035084058,
      "learning_rate": 2.2714738398943308e-07,
      "logits/chosen": -2.749488592147827,
      "logits/rejected": -2.629812717437744,
      "logps/chosen": -1.0328668355941772,
      "logps/rejected": -1.6207387447357178,
      "loss": 1.833,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.0328668355941772,
      "rewards/margins": 0.5878719091415405,
      "rewards/rejected": -1.6207387447357178,
      "step": 275
    },
    {
      "epoch": 0.5862339701648783,
      "grad_norm": 27.44411354972676,
      "learning_rate": 2.1804923757009882e-07,
      "logits/chosen": -2.7630438804626465,
      "logits/rejected": -2.692211866378784,
      "logps/chosen": -0.968722939491272,
      "logps/rejected": -1.3731743097305298,
      "loss": 1.8453,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.968722939491272,
      "rewards/margins": 0.40445131063461304,
      "rewards/rejected": -1.3731743097305298,
      "step": 280
    },
    {
      "epoch": 0.5967024339178225,
      "grad_norm": 32.499361452307625,
      "learning_rate": 2.089939221172446e-07,
      "logits/chosen": -2.682573080062866,
      "logits/rejected": -2.678396224975586,
      "logps/chosen": -1.0675628185272217,
      "logps/rejected": -1.5514373779296875,
      "loss": 1.8267,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.0675628185272217,
      "rewards/margins": 0.4838746190071106,
      "rewards/rejected": -1.5514373779296875,
      "step": 285
    },
    {
      "epoch": 0.6071708976707668,
      "grad_norm": 34.51002212540645,
      "learning_rate": 1.9999357655598891e-07,
      "logits/chosen": -2.7195992469787598,
      "logits/rejected": -2.6691336631774902,
      "logps/chosen": -1.0737675428390503,
      "logps/rejected": -1.6511163711547852,
      "loss": 1.8055,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.0737675428390503,
      "rewards/margins": 0.5773489475250244,
      "rewards/rejected": -1.6511163711547852,
      "step": 290
    },
    {
      "epoch": 0.6176393614237111,
      "grad_norm": 34.81875700322843,
      "learning_rate": 1.9106026612264315e-07,
      "logits/chosen": -2.7475931644439697,
      "logits/rejected": -2.661341667175293,
      "logps/chosen": -1.0815327167510986,
      "logps/rejected": -1.6415588855743408,
      "loss": 1.8548,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.0815327167510986,
      "rewards/margins": 0.5600259900093079,
      "rewards/rejected": -1.6415588855743408,
      "step": 295
    },
    {
      "epoch": 0.6281078251766553,
      "grad_norm": 39.656088577829394,
      "learning_rate": 1.8220596619089573e-07,
      "logits/chosen": -2.650444507598877,
      "logits/rejected": -2.601776123046875,
      "logps/chosen": -1.0726263523101807,
      "logps/rejected": -1.5858640670776367,
      "loss": 1.8352,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.0726263523101807,
      "rewards/margins": 0.5132376551628113,
      "rewards/rejected": -1.5858640670776367,
      "step": 300
    },
    {
      "epoch": 0.6385762889295996,
      "grad_norm": 34.266407421018876,
      "learning_rate": 1.7344254621846017e-07,
      "logits/chosen": -2.6358659267425537,
      "logits/rejected": -2.589916944503784,
      "logps/chosen": -1.0825355052947998,
      "logps/rejected": -1.695123314857483,
      "loss": 1.8122,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.0825355052947998,
      "rewards/margins": 0.6125878691673279,
      "rewards/rejected": -1.695123314857483,
      "step": 305
    },
    {
      "epoch": 0.6490447526825438,
      "grad_norm": 29.29392494715919,
      "learning_rate": 1.647817538357072e-07,
      "logits/chosen": -2.6732287406921387,
      "logits/rejected": -2.6671674251556396,
      "logps/chosen": -0.9606215357780457,
      "logps/rejected": -1.507188081741333,
      "loss": 1.8154,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.9606215357780457,
      "rewards/margins": 0.5465664863586426,
      "rewards/rejected": -1.507188081741333,
      "step": 310
    },
    {
      "epoch": 0.6595132164354881,
      "grad_norm": 30.886215191002822,
      "learning_rate": 1.562351990976095e-07,
      "logits/chosen": -2.6945061683654785,
      "logits/rejected": -2.6447794437408447,
      "logps/chosen": -0.9400749206542969,
      "logps/rejected": -1.5918124914169312,
      "loss": 1.8296,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.9400749206542969,
      "rewards/margins": 0.651737630367279,
      "rewards/rejected": -1.5918124914169312,
      "step": 315
    },
    {
      "epoch": 0.6699816801884323,
      "grad_norm": 40.31455490523793,
      "learning_rate": 1.478143389201113e-07,
      "logits/chosen": -2.6877899169921875,
      "logits/rejected": -2.6543657779693604,
      "logps/chosen": -1.0309157371520996,
      "logps/rejected": -1.5069587230682373,
      "loss": 1.7854,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.0309157371520996,
      "rewards/margins": 0.47604283690452576,
      "rewards/rejected": -1.5069587230682373,
      "step": 320
    },
    {
      "epoch": 0.6804501439413766,
      "grad_norm": 33.29916453849858,
      "learning_rate": 1.3953046172178413e-07,
      "logits/chosen": -2.712068557739258,
      "logits/rejected": -2.6253581047058105,
      "logps/chosen": -1.0362852811813354,
      "logps/rejected": -1.6363003253936768,
      "loss": 1.7979,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.0362852811813354,
      "rewards/margins": 0.6000149846076965,
      "rewards/rejected": -1.6363003253936768,
      "step": 325
    },
    {
      "epoch": 0.6909186076943209,
      "grad_norm": 28.615351737244207,
      "learning_rate": 1.3139467229135998e-07,
      "logits/chosen": -2.672591209411621,
      "logits/rejected": -2.624178886413574,
      "logps/chosen": -1.0226625204086304,
      "logps/rejected": -1.4031275510787964,
      "loss": 1.8499,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0226625204086304,
      "rewards/margins": 0.38046500086784363,
      "rewards/rejected": -1.4031275510787964,
      "step": 330
    },
    {
      "epoch": 0.7013870714472651,
      "grad_norm": 31.872437043858163,
      "learning_rate": 1.2341787690142435e-07,
      "logits/chosen": -2.783500909805298,
      "logits/rejected": -2.7123568058013916,
      "logps/chosen": -0.9628676176071167,
      "logps/rejected": -1.5226846933364868,
      "loss": 1.8281,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.9628676176071167,
      "rewards/margins": 0.5598170161247253,
      "rewards/rejected": -1.5226846933364868,
      "step": 335
    },
    {
      "epoch": 0.7118555352002094,
      "grad_norm": 27.878376041354297,
      "learning_rate": 1.1561076868822755e-07,
      "logits/chosen": -2.7173564434051514,
      "logits/rejected": -2.6413800716400146,
      "logps/chosen": -1.1935253143310547,
      "logps/rejected": -1.5975086688995361,
      "loss": 1.8896,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.1935253143310547,
      "rewards/margins": 0.40398329496383667,
      "rewards/rejected": -1.5975086688995361,
      "step": 340
    },
    {
      "epoch": 0.7223239989531536,
      "grad_norm": 36.976534101742224,
      "learning_rate": 1.0798381331721107e-07,
      "logits/chosen": -2.7495174407958984,
      "logits/rejected": -2.700282573699951,
      "logps/chosen": -1.0389212369918823,
      "logps/rejected": -1.5958582162857056,
      "loss": 1.8433,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -1.0389212369918823,
      "rewards/margins": 0.5569368600845337,
      "rewards/rejected": -1.5958582162857056,
      "step": 345
    },
    {
      "epoch": 0.7327924627060979,
      "grad_norm": 30.288124940758514,
      "learning_rate": 1.0054723495346482e-07,
      "logits/chosen": -2.7106757164001465,
      "logits/rejected": -2.687342882156372,
      "logps/chosen": -1.0641244649887085,
      "logps/rejected": -1.5430529117584229,
      "loss": 1.8735,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.0641244649887085,
      "rewards/margins": 0.47892847657203674,
      "rewards/rejected": -1.5430529117584229,
      "step": 350
    },
    {
      "epoch": 0.7432609264590422,
      "grad_norm": 30.72874948597524,
      "learning_rate": 9.331100255592436e-08,
      "logits/chosen": -2.7169158458709717,
      "logits/rejected": -2.66363787651062,
      "logps/chosen": -0.9323965311050415,
      "logps/rejected": -1.5993990898132324,
      "loss": 1.7736,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9323965311050415,
      "rewards/margins": 0.6670026779174805,
      "rewards/rejected": -1.5993990898132324,
      "step": 355
    },
    {
      "epoch": 0.7537293902119864,
      "grad_norm": 22.0079277817506,
      "learning_rate": 8.628481651367875e-08,
      "logits/chosen": -2.746021270751953,
      "logits/rejected": -2.6950621604919434,
      "logps/chosen": -0.9703041315078735,
      "logps/rejected": -1.4327679872512817,
      "loss": 1.783,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.9703041315078735,
      "rewards/margins": 0.4624637961387634,
      "rewards/rejected": -1.4327679872512817,
      "step": 360
    },
    {
      "epoch": 0.7641978539649307,
      "grad_norm": 33.139790420388316,
      "learning_rate": 7.947809564230445e-08,
      "logits/chosen": -2.6830432415008545,
      "logits/rejected": -2.5992660522460938,
      "logps/chosen": -1.0758744478225708,
      "logps/rejected": -1.6359386444091797,
      "loss": 1.8553,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -1.0758744478225708,
      "rewards/margins": 0.5600640773773193,
      "rewards/rejected": -1.6359386444091797,
      "step": 365
    },
    {
      "epoch": 0.7746663177178749,
      "grad_norm": 42.978648090018346,
      "learning_rate": 7.289996455765748e-08,
      "logits/chosen": -2.7361717224121094,
      "logits/rejected": -2.6721107959747314,
      "logps/chosen": -1.0527479648590088,
      "logps/rejected": -1.7067893743515015,
      "loss": 1.8183,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.0527479648590088,
      "rewards/margins": 0.6540414094924927,
      "rewards/rejected": -1.7067893743515015,
      "step": 370
    },
    {
      "epoch": 0.7851347814708192,
      "grad_norm": 24.61410992203062,
      "learning_rate": 6.655924144404906e-08,
      "logits/chosen": -2.715893268585205,
      "logits/rejected": -2.7327394485473633,
      "logps/chosen": -1.0623342990875244,
      "logps/rejected": -1.5285212993621826,
      "loss": 1.8189,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.0623342990875244,
      "rewards/margins": 0.4661870002746582,
      "rewards/rejected": -1.5285212993621826,
      "step": 375
    },
    {
      "epoch": 0.7956032452237635,
      "grad_norm": 29.084948650681554,
      "learning_rate": 6.046442623320145e-08,
      "logits/chosen": -2.7230355739593506,
      "logits/rejected": -2.665907621383667,
      "logps/chosen": -1.0696966648101807,
      "logps/rejected": -1.5225203037261963,
      "loss": 1.7794,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -1.0696966648101807,
      "rewards/margins": 0.4528239369392395,
      "rewards/rejected": -1.5225203037261963,
      "step": 380
    },
    {
      "epoch": 0.8060717089767077,
      "grad_norm": 25.930022257563827,
      "learning_rate": 5.4623689209832484e-08,
      "logits/chosen": -2.7370214462280273,
      "logits/rejected": -2.6596484184265137,
      "logps/chosen": -1.0190281867980957,
      "logps/rejected": -1.4870203733444214,
      "loss": 1.8012,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -1.0190281867980957,
      "rewards/margins": 0.46799221634864807,
      "rewards/rejected": -1.4870203733444214,
      "step": 385
    },
    {
      "epoch": 0.816540172729652,
      "grad_norm": 30.16253644602296,
      "learning_rate": 4.904486005914027e-08,
      "logits/chosen": -2.755403518676758,
      "logits/rejected": -2.6998536586761475,
      "logps/chosen": -1.0058339834213257,
      "logps/rejected": -1.6741714477539062,
      "loss": 1.7816,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.0058339834213257,
      "rewards/margins": 0.6683375239372253,
      "rewards/rejected": -1.6741714477539062,
      "step": 390
    },
    {
      "epoch": 0.8270086364825961,
      "grad_norm": 32.100503895364014,
      "learning_rate": 4.373541737087263e-08,
      "logits/chosen": -2.6867947578430176,
      "logits/rejected": -2.6116786003112793,
      "logps/chosen": -0.9958473443984985,
      "logps/rejected": -1.5259888172149658,
      "loss": 1.8098,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.9958473443984985,
      "rewards/margins": 0.5301414132118225,
      "rewards/rejected": -1.5259888172149658,
      "step": 395
    },
    {
      "epoch": 0.8374771002355405,
      "grad_norm": 39.50777886532678,
      "learning_rate": 3.8702478614051345e-08,
      "logits/chosen": -2.6500821113586426,
      "logits/rejected": -2.587491273880005,
      "logps/chosen": -0.9868978261947632,
      "logps/rejected": -1.5105135440826416,
      "loss": 1.81,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.9868978261947632,
      "rewards/margins": 0.523615837097168,
      "rewards/rejected": -1.5105135440826416,
      "step": 400
    },
    {
      "epoch": 0.8479455639884846,
      "grad_norm": 39.172472564486576,
      "learning_rate": 3.3952790595787986e-08,
      "logits/chosen": -2.7077910900115967,
      "logits/rejected": -2.671496868133545,
      "logps/chosen": -1.0807896852493286,
      "logps/rejected": -1.6041944026947021,
      "loss": 1.8475,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.0807896852493286,
      "rewards/margins": 0.5234047174453735,
      "rewards/rejected": -1.6041944026947021,
      "step": 405
    },
    {
      "epoch": 0.8584140277414289,
      "grad_norm": 34.16755927550673,
      "learning_rate": 2.9492720416985e-08,
      "logits/chosen": -2.729513645172119,
      "logits/rejected": -2.68589448928833,
      "logps/chosen": -0.9558256268501282,
      "logps/rejected": -1.4403427839279175,
      "loss": 1.8083,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.9558256268501282,
      "rewards/margins": 0.4845171868801117,
      "rewards/rejected": -1.4403427839279175,
      "step": 410
    },
    {
      "epoch": 0.8688824914943732,
      "grad_norm": 27.600146859315487,
      "learning_rate": 2.5328246937043525e-08,
      "logits/chosen": -2.758931875228882,
      "logits/rejected": -2.693338394165039,
      "logps/chosen": -1.004372239112854,
      "logps/rejected": -1.6196565628051758,
      "loss": 1.7709,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -1.004372239112854,
      "rewards/margins": 0.615284264087677,
      "rewards/rejected": -1.6196565628051758,
      "step": 415
    },
    {
      "epoch": 0.8793509552473174,
      "grad_norm": 26.767731856804396,
      "learning_rate": 2.1464952759020856e-08,
      "logits/chosen": -2.7129735946655273,
      "logits/rejected": -2.6572775840759277,
      "logps/chosen": -0.9905831217765808,
      "logps/rejected": -1.5554085969924927,
      "loss": 1.8411,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.9905831217765808,
      "rewards/margins": 0.5648255944252014,
      "rewards/rejected": -1.5554085969924927,
      "step": 420
    },
    {
      "epoch": 0.8898194190002617,
      "grad_norm": 32.1190541072846,
      "learning_rate": 1.7908016745981856e-08,
      "logits/chosen": -2.656250476837158,
      "logits/rejected": -2.645900249481201,
      "logps/chosen": -1.1461454629898071,
      "logps/rejected": -1.604714035987854,
      "loss": 1.8218,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -1.1461454629898071,
      "rewards/margins": 0.45856842398643494,
      "rewards/rejected": -1.604714035987854,
      "step": 425
    },
    {
      "epoch": 0.9002878827532059,
      "grad_norm": 38.87548467523877,
      "learning_rate": 1.4662207078575684e-08,
      "logits/chosen": -2.661616325378418,
      "logits/rejected": -2.617453098297119,
      "logps/chosen": -1.1022528409957886,
      "logps/rejected": -1.628826379776001,
      "loss": 1.8088,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -1.1022528409957886,
      "rewards/margins": 0.5265735387802124,
      "rewards/rejected": -1.628826379776001,
      "step": 430
    },
    {
      "epoch": 0.9107563465061502,
      "grad_norm": 34.292720836263,
      "learning_rate": 1.1731874863145142e-08,
      "logits/chosen": -2.7260875701904297,
      "logits/rejected": -2.685364246368408,
      "logps/chosen": -1.0480232238769531,
      "logps/rejected": -1.6929914951324463,
      "loss": 1.8315,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.0480232238769531,
      "rewards/margins": 0.6449683308601379,
      "rewards/rejected": -1.6929914951324463,
      "step": 435
    },
    {
      "epoch": 0.9212248102590945,
      "grad_norm": 36.60770449520838,
      "learning_rate": 9.12094829893642e-09,
      "logits/chosen": -2.6789753437042236,
      "logits/rejected": -2.6536145210266113,
      "logps/chosen": -1.0216642618179321,
      "logps/rejected": -1.547012448310852,
      "loss": 1.8478,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -1.0216642618179321,
      "rewards/margins": 0.5253482460975647,
      "rewards/rejected": -1.547012448310852,
      "step": 440
    },
    {
      "epoch": 0.9316932740120387,
      "grad_norm": 34.43665423362751,
      "learning_rate": 6.832927412229017e-09,
      "logits/chosen": -2.721832752227783,
      "logits/rejected": -2.660407781600952,
      "logps/chosen": -1.0646559000015259,
      "logps/rejected": -1.554300308227539,
      "loss": 1.8533,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -1.0646559000015259,
      "rewards/margins": 0.4896443486213684,
      "rewards/rejected": -1.554300308227539,
      "step": 445
    },
    {
      "epoch": 0.942161737764983,
      "grad_norm": 30.770535415455328,
      "learning_rate": 4.8708793644441086e-09,
      "logits/chosen": -2.6228880882263184,
      "logits/rejected": -2.5837976932525635,
      "logps/chosen": -1.0409653186798096,
      "logps/rejected": -1.537603735923767,
      "loss": 1.7905,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.0409653186798096,
      "rewards/margins": 0.4966384768486023,
      "rewards/rejected": -1.537603735923767,
      "step": 450
    },
    {
      "epoch": 0.9526302015179272,
      "grad_norm": 27.915122014592697,
      "learning_rate": 3.2374343405217884e-09,
      "logits/chosen": -2.6800639629364014,
      "logits/rejected": -2.6128344535827637,
      "logps/chosen": -1.132541298866272,
      "logps/rejected": -1.5116033554077148,
      "loss": 1.825,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -1.132541298866272,
      "rewards/margins": 0.3790620267391205,
      "rewards/rejected": -1.5116033554077148,
      "step": 455
    },
    {
      "epoch": 0.9630986652708715,
      "grad_norm": 31.401214396279837,
      "learning_rate": 1.9347820230782295e-09,
      "logits/chosen": -2.6828560829162598,
      "logits/rejected": -2.6430163383483887,
      "logps/chosen": -0.9926983118057251,
      "logps/rejected": -1.5756382942199707,
      "loss": 1.7905,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.9926983118057251,
      "rewards/margins": 0.5829401612281799,
      "rewards/rejected": -1.5756382942199707,
      "step": 460
    },
    {
      "epoch": 0.9735671290238157,
      "grad_norm": 38.51782864695839,
      "learning_rate": 9.64668657069706e-10,
      "logits/chosen": -2.7383384704589844,
      "logits/rejected": -2.6524176597595215,
      "logps/chosen": -1.0901256799697876,
      "logps/rejected": -1.620060682296753,
      "loss": 1.8476,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -1.0901256799697876,
      "rewards/margins": 0.5299350023269653,
      "rewards/rejected": -1.620060682296753,
      "step": 465
    },
    {
      "epoch": 0.98403559277676,
      "grad_norm": 27.923910450585048,
      "learning_rate": 3.2839470889836627e-10,
      "logits/chosen": -2.7122910022735596,
      "logits/rejected": -2.6457693576812744,
      "logps/chosen": -0.9990774393081665,
      "logps/rejected": -1.4476706981658936,
      "loss": 1.8714,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.9990774393081665,
      "rewards/margins": 0.4485931396484375,
      "rewards/rejected": -1.4476706981658936,
      "step": 470
    },
    {
      "epoch": 0.9945040565297043,
      "grad_norm": 33.978087151320835,
      "learning_rate": 2.6813123097352287e-11,
      "logits/chosen": -2.7624480724334717,
      "logits/rejected": -2.6619315147399902,
      "logps/chosen": -1.042180061340332,
      "logps/rejected": -1.5233901739120483,
      "loss": 1.8233,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -1.042180061340332,
      "rewards/margins": 0.4812101721763611,
      "rewards/rejected": -1.5233901739120483,
      "step": 475
    },
    {
      "epoch": 0.998691442030882,
      "step": 477,
      "total_flos": 0.0,
      "train_loss": 1.8644854217705238,
      "train_runtime": 7378.4782,
      "train_samples_per_second": 8.286,
      "train_steps_per_second": 0.065
    }
  ],
  "logging_steps": 5,
  "max_steps": 477,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 1000000,
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}