{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.998691442030882,
  "eval_steps": 500,
  "global_step": 477,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.010468463752944255,
      "grad_norm": 522503.4218003003,
      "learning_rate": 5.208333333333333e-08,
      "logits/chosen": -2.7708499431610107,
      "logits/rejected": -2.7286171913146973,
      "logps/chosen": -253.5421142578125,
      "logps/rejected": -244.00076293945312,
      "loss": 125000.2375,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -253.5421142578125,
      "rewards/margins": -9.541345596313477,
      "rewards/rejected": -244.00076293945312,
      "step": 5
    },
    {
      "epoch": 0.02093692750588851,
      "grad_norm": 409413.4458768586,
      "learning_rate": 1.0416666666666667e-07,
      "logits/chosen": -2.7510223388671875,
      "logits/rejected": -2.7252488136291504,
      "logps/chosen": -268.2208557128906,
      "logps/rejected": -243.4846649169922,
      "loss": 124997.8,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -268.2208557128906,
      "rewards/margins": -24.73613929748535,
      "rewards/rejected": -243.4846649169922,
      "step": 10
    },
    {
      "epoch": 0.031405391258832765,
      "grad_norm": 414950.26842560945,
      "learning_rate": 1.5624999999999999e-07,
      "logits/chosen": -2.730611562728882,
      "logits/rejected": -2.6970362663269043,
      "logps/chosen": -236.9029541015625,
      "logps/rejected": -273.0437316894531,
      "loss": 124982.75,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -236.9029541015625,
      "rewards/margins": 36.140785217285156,
      "rewards/rejected": -273.0437316894531,
      "step": 15
    },
    {
      "epoch": 0.04187385501177702,
      "grad_norm": 718250.9758832247,
      "learning_rate": 2.0833333333333333e-07,
      "logits/chosen": -2.802532434463501,
      "logits/rejected": -2.7090563774108887,
      "logps/chosen": -301.2122802734375,
      "logps/rejected": -269.02008056640625,
      "loss": 124929.4,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -301.2122802734375,
      "rewards/margins": -32.19221496582031,
      "rewards/rejected": -269.02008056640625,
      "step": 20
    },
    {
      "epoch": 0.05234231876472128,
      "grad_norm": 444864.8327529458,
      "learning_rate": 2.604166666666667e-07,
      "logits/chosen": -2.796382188796997,
      "logits/rejected": -2.7728114128112793,
      "logps/chosen": -274.8233947753906,
      "logps/rejected": -246.075927734375,
      "loss": 124820.5,
      "rewards/accuracies": 0.4437499940395355,
      "rewards/chosen": -274.8233947753906,
      "rewards/margins": -28.747509002685547,
      "rewards/rejected": -246.075927734375,
      "step": 25
    },
    {
      "epoch": 0.06281078251766553,
      "grad_norm": 458498.66919273353,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": -2.7689995765686035,
      "logits/rejected": -2.7268056869506836,
      "logps/chosen": -240.1889190673828,
      "logps/rejected": -226.0777587890625,
      "loss": 124675.575,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -240.1889190673828,
      "rewards/margins": -14.111166000366211,
      "rewards/rejected": -226.0777587890625,
      "step": 30
    },
    {
      "epoch": 0.07327924627060979,
      "grad_norm": 450902.00202128256,
      "learning_rate": 3.645833333333333e-07,
      "logits/chosen": -2.7725298404693604,
      "logits/rejected": -2.6864283084869385,
      "logps/chosen": -274.01287841796875,
      "logps/rejected": -278.74652099609375,
      "loss": 124375.275,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -274.01287841796875,
      "rewards/margins": 4.733607292175293,
      "rewards/rejected": -278.74652099609375,
      "step": 35
    },
    {
      "epoch": 0.08374771002355404,
      "grad_norm": 417057.7715157791,
      "learning_rate": 4.1666666666666667e-07,
      "logits/chosen": -2.7862133979797363,
      "logits/rejected": -2.6889536380767822,
      "logps/chosen": -294.39178466796875,
      "logps/rejected": -261.9639892578125,
      "loss": 123992.45,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -294.39178466796875,
      "rewards/margins": -32.427791595458984,
      "rewards/rejected": -261.9639892578125,
      "step": 40
    },
    {
      "epoch": 0.0942161737764983,
      "grad_norm": 466508.70788908063,
      "learning_rate": 4.6874999999999996e-07,
      "logits/chosen": -2.7797024250030518,
      "logits/rejected": -2.731426477432251,
      "logps/chosen": -278.8966979980469,
      "logps/rejected": -264.8343200683594,
      "loss": 123537.05,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -278.8966979980469,
      "rewards/margins": -14.062413215637207,
      "rewards/rejected": -264.8343200683594,
      "step": 45
    },
    {
      "epoch": 0.10468463752944256,
      "grad_norm": 559270.5129260804,
      "learning_rate": 4.999731868769026e-07,
      "logits/chosen": -2.708888053894043,
      "logits/rejected": -2.639033079147339,
      "logps/chosen": -292.9398498535156,
      "logps/rejected": -284.3505554199219,
      "loss": 122464.5875,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -292.9398498535156,
      "rewards/margins": -8.58930492401123,
      "rewards/rejected": -284.3505554199219,
      "step": 50
    },
    {
      "epoch": 0.11515310128238682,
      "grad_norm": 521515.212841857,
      "learning_rate": 4.996716052911017e-07,
      "logits/chosen": -2.685326099395752,
      "logits/rejected": -2.643601894378662,
      "logps/chosen": -283.38983154296875,
      "logps/rejected": -281.60528564453125,
      "loss": 122310.025,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -283.38983154296875,
      "rewards/margins": -1.784542441368103,
      "rewards/rejected": -281.60528564453125,
      "step": 55
    },
    {
      "epoch": 0.12562156503533106,
      "grad_norm": 729100.4219221757,
      "learning_rate": 4.990353313429303e-07,
      "logits/chosen": -2.724234104156494,
      "logits/rejected": -2.681450605392456,
      "logps/chosen": -270.0491027832031,
      "logps/rejected": -281.71820068359375,
      "loss": 121167.4375,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -270.0491027832031,
      "rewards/margins": 11.669103622436523,
      "rewards/rejected": -281.71820068359375,
      "step": 60
    },
    {
      "epoch": 0.1360900287882753,
      "grad_norm": 961680.3751689971,
      "learning_rate": 4.980652179769217e-07,
      "logits/chosen": -2.718280792236328,
      "logits/rejected": -2.597512722015381,
      "logps/chosen": -310.9617004394531,
      "logps/rejected": -293.2528381347656,
      "loss": 119799.15,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -310.9617004394531,
      "rewards/margins": -17.708873748779297,
      "rewards/rejected": -293.2528381347656,
      "step": 65
    },
    {
      "epoch": 0.14655849254121958,
      "grad_norm": 783593.0959446996,
      "learning_rate": 4.967625656594781e-07,
      "logits/chosen": -2.6035261154174805,
      "logits/rejected": -2.5565543174743652,
      "logps/chosen": -290.8232421875,
      "logps/rejected": -287.1130065917969,
      "loss": 119810.2,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -290.8232421875,
      "rewards/margins": -3.710245132446289,
      "rewards/rejected": -287.1130065917969,
      "step": 70
    },
    {
      "epoch": 0.15702695629416383,
      "grad_norm": 745801.2137469078,
      "learning_rate": 4.951291206355559e-07,
      "logits/chosen": -2.7136483192443848,
      "logits/rejected": -2.6402573585510254,
      "logps/chosen": -291.23773193359375,
      "logps/rejected": -318.0777587890625,
      "loss": 119278.1375,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -291.23773193359375,
      "rewards/margins": 26.84000015258789,
      "rewards/rejected": -318.0777587890625,
      "step": 75
    },
    {
      "epoch": 0.16749542004710807,
      "grad_norm": 907046.1955290881,
      "learning_rate": 4.93167072587771e-07,
      "logits/chosen": -2.718100070953369,
      "logits/rejected": -2.6574056148529053,
      "logps/chosen": -292.93060302734375,
      "logps/rejected": -307.5615234375,
      "loss": 119745.725,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -292.93060302734375,
      "rewards/margins": 14.630932807922363,
      "rewards/rejected": -307.5615234375,
      "step": 80
    },
    {
      "epoch": 0.17796388380005235,
      "grad_norm": 860624.0897837883,
      "learning_rate": 4.908790517010636e-07,
      "logits/chosen": -2.757253408432007,
      "logits/rejected": -2.69834566116333,
      "logps/chosen": -295.21539306640625,
      "logps/rejected": -327.061767578125,
      "loss": 118244.975,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -295.21539306640625,
      "rewards/margins": 31.846384048461914,
      "rewards/rejected": -327.061767578125,
      "step": 85
    },
    {
      "epoch": 0.1884323475529966,
      "grad_norm": 1180769.397532202,
      "learning_rate": 4.882681251368548e-07,
      "logits/chosen": -2.7516987323760986,
      "logits/rejected": -2.7340774536132812,
      "logps/chosen": -311.9580993652344,
      "logps/rejected": -325.9947814941406,
      "loss": 118380.25,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -311.9580993652344,
      "rewards/margins": 14.036661148071289,
      "rewards/rejected": -325.9947814941406,
      "step": 90
    },
    {
      "epoch": 0.19890081130594087,
      "grad_norm": 1435209.2727647137,
      "learning_rate": 4.853377929214243e-07,
      "logits/chosen": -2.7984097003936768,
      "logits/rejected": -2.713646411895752,
      "logps/chosen": -308.3005676269531,
      "logps/rejected": -312.81201171875,
      "loss": 116816.6,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -308.3005676269531,
      "rewards/margins": 4.5114593505859375,
      "rewards/rejected": -312.81201171875,
      "step": 95
    },
    {
      "epoch": 0.2093692750588851,
      "grad_norm": 1728193.4126168345,
      "learning_rate": 4.820919832540181e-07,
      "logits/chosen": -2.70627498626709,
      "logits/rejected": -2.6122021675109863,
      "logps/chosen": -283.68487548828125,
      "logps/rejected": -284.87298583984375,
      "loss": 118125.7625,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -283.68487548828125,
      "rewards/margins": 1.188114881515503,
      "rewards/rejected": -284.87298583984375,
      "step": 100
    },
    {
      "epoch": 0.21983773881182936,
      "grad_norm": 1249095.5568791889,
      "learning_rate": 4.785350472409791e-07,
      "logits/chosen": -2.555546283721924,
      "logits/rejected": -2.499777317047119,
      "logps/chosen": -262.59674072265625,
      "logps/rejected": -337.804931640625,
      "loss": 116391.2,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -262.59674072265625,
      "rewards/margins": 75.20821380615234,
      "rewards/rejected": -337.804931640625,
      "step": 105
    },
    {
      "epoch": 0.23030620256477363,
      "grad_norm": 1324636.8678255256,
      "learning_rate": 4.7467175306295647e-07,
      "logits/chosen": -2.2604401111602783,
      "logits/rejected": -2.107639789581299,
      "logps/chosen": -314.15606689453125,
      "logps/rejected": -354.67578125,
      "loss": 115764.325,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -314.15606689453125,
      "rewards/margins": 40.519737243652344,
      "rewards/rejected": -354.67578125,
      "step": 110
    },
    {
      "epoch": 0.24077466631771788,
      "grad_norm": 2088676.398368818,
      "learning_rate": 4.70507279583015e-07,
      "logits/chosen": -2.022336959838867,
      "logits/rejected": -1.7513967752456665,
      "logps/chosen": -331.6026611328125,
      "logps/rejected": -372.3267517089844,
      "loss": 115322.7375,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -331.6026611328125,
      "rewards/margins": 40.724082946777344,
      "rewards/rejected": -372.3267517089844,
      "step": 115
    },
    {
      "epoch": 0.2512431300706621,
      "grad_norm": 1807091.2744565222,
      "learning_rate": 4.6604720940421207e-07,
      "logits/chosen": -1.804964303970337,
      "logits/rejected": -1.5805766582489014,
      "logps/chosen": -315.6778869628906,
      "logps/rejected": -387.36260986328125,
      "loss": 115043.6,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -315.6778869628906,
      "rewards/margins": 71.68473052978516,
      "rewards/rejected": -387.36260986328125,
      "step": 120
    },
    {
      "epoch": 0.26171159382360637,
      "grad_norm": 1582536.6614139078,
      "learning_rate": 4.612975213859487e-07,
      "logits/chosen": -2.094388961791992,
      "logits/rejected": -1.9840008020401,
      "logps/chosen": -285.61419677734375,
      "logps/rejected": -304.2431335449219,
      "loss": 116704.675,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -285.61419677734375,
      "rewards/margins": 18.628963470458984,
      "rewards/rejected": -304.2431335449219,
      "step": 125
    },
    {
      "epoch": 0.2721800575765506,
      "grad_norm": 1405438.3906735398,
      "learning_rate": 4.5626458262912735e-07,
      "logits/chosen": -2.012509822845459,
      "logits/rejected": -1.8286924362182617,
      "logps/chosen": -295.90277099609375,
      "logps/rejected": -341.51812744140625,
      "loss": 116832.5,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -295.90277099609375,
      "rewards/margins": 45.61534881591797,
      "rewards/rejected": -341.51812744140625,
      "step": 130
    },
    {
      "epoch": 0.2826485213294949,
      "grad_norm": 2166459.423278389,
      "learning_rate": 4.5095513994085974e-07,
      "logits/chosen": -1.7868998050689697,
      "logits/rejected": -1.1601707935333252,
      "logps/chosen": -296.5223693847656,
      "logps/rejected": -369.28326416015625,
      "loss": 115705.0,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -296.5223693847656,
      "rewards/margins": 72.76090240478516,
      "rewards/rejected": -369.28326416015625,
      "step": 135
    },
    {
      "epoch": 0.29311698508243916,
      "grad_norm": 1788707.2451489503,
      "learning_rate": 4.453763107901675e-07,
      "logits/chosen": -1.2310272455215454,
      "logits/rejected": -0.7306966781616211,
      "logps/chosen": -284.04351806640625,
      "logps/rejected": -365.6307678222656,
      "loss": 112320.4,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -284.04351806640625,
      "rewards/margins": 81.58724975585938,
      "rewards/rejected": -365.6307678222656,
      "step": 140
    },
    {
      "epoch": 0.3035854488353834,
      "grad_norm": 2512886.744124689,
      "learning_rate": 4.395355737667985e-07,
      "logits/chosen": -1.370542049407959,
      "logits/rejected": -0.847873330116272,
      "logps/chosen": -295.51165771484375,
      "logps/rejected": -344.68634033203125,
      "loss": 116938.0875,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -295.51165771484375,
      "rewards/margins": 49.174705505371094,
      "rewards/rejected": -344.68634033203125,
      "step": 145
    },
    {
      "epoch": 0.31405391258832765,
      "grad_norm": 2459660.1241130554,
      "learning_rate": 4.3344075855595097e-07,
      "logits/chosen": -1.8194392919540405,
      "logits/rejected": -1.4853712320327759,
      "logps/chosen": -293.786865234375,
      "logps/rejected": -332.8251953125,
      "loss": 112010.725,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -293.786865234375,
      "rewards/margins": 39.03831100463867,
      "rewards/rejected": -332.8251953125,
      "step": 150
    },
    {
      "epoch": 0.3245223763412719,
      "grad_norm": 4807009.727933751,
      "learning_rate": 4.271000354423425e-07,
      "logits/chosen": -1.3470885753631592,
      "logits/rejected": -0.8607180714607239,
      "logps/chosen": -337.14300537109375,
      "logps/rejected": -391.9334716796875,
      "loss": 115519.375,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -337.14300537109375,
      "rewards/margins": 54.79045867919922,
      "rewards/rejected": -391.9334716796875,
      "step": 155
    },
    {
      "epoch": 0.33499084009421615,
      "grad_norm": 2699245.2043745625,
      "learning_rate": 4.2052190435769554e-07,
      "logits/chosen": -1.7011409997940063,
      "logits/rejected": -1.335893988609314,
      "logps/chosen": -274.394287109375,
      "logps/rejected": -330.22100830078125,
      "loss": 114255.0875,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -274.394287109375,
      "rewards/margins": 55.82673263549805,
      "rewards/rejected": -330.22100830078125,
      "step": 160
    },
    {
      "epoch": 0.34545930384716045,
      "grad_norm": 840943.395175269,
      "learning_rate": 4.137151834863213e-07,
      "logits/chosen": -2.163696527481079,
      "logits/rejected": -1.7751731872558594,
      "logps/chosen": -273.88092041015625,
      "logps/rejected": -331.56085205078125,
      "loss": 116034.3125,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -273.88092041015625,
      "rewards/margins": 57.679969787597656,
      "rewards/rejected": -331.56085205078125,
      "step": 165
    },
    {
      "epoch": 0.3559277676001047,
      "grad_norm": 1277262.6579787417,
      "learning_rate": 4.0668899744407567e-07,
      "logits/chosen": -1.8461663722991943,
      "logits/rejected": -1.260231852531433,
      "logps/chosen": -303.04083251953125,
      "logps/rejected": -325.74371337890625,
      "loss": 114209.3125,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -303.04083251953125,
      "rewards/margins": 22.702842712402344,
      "rewards/rejected": -325.74371337890625,
      "step": 170
    },
    {
      "epoch": 0.36639623135304894,
      "grad_norm": 1135243.1010669412,
      "learning_rate": 3.994527650465352e-07,
      "logits/chosen": -1.029491901397705,
      "logits/rejected": -0.0737704187631607,
      "logps/chosen": -286.8111877441406,
      "logps/rejected": -299.5812072753906,
      "loss": 115237.225,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -286.8111877441406,
      "rewards/margins": 12.770014762878418,
      "rewards/rejected": -299.5812072753906,
      "step": 175
    },
    {
      "epoch": 0.3768646951059932,
      "grad_norm": 1925685.8821038436,
      "learning_rate": 3.920161866827889e-07,
      "logits/chosen": -0.454359769821167,
      "logits/rejected": -0.02063882350921631,
      "logps/chosen": -297.41461181640625,
      "logps/rejected": -369.3190002441406,
      "loss": 113950.1,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -297.41461181640625,
      "rewards/margins": 71.90437316894531,
      "rewards/rejected": -369.3190002441406,
      "step": 180
    },
    {
      "epoch": 0.38733315885893743,
      "grad_norm": 2698235.0873295115,
      "learning_rate": 3.8438923131177237e-07,
      "logits/chosen": -0.5734198689460754,
      "logits/rejected": 0.27511700987815857,
      "logps/chosen": -316.64971923828125,
      "logps/rejected": -350.3505554199219,
      "loss": 113131.425,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -316.64971923828125,
      "rewards/margins": 33.700801849365234,
      "rewards/rejected": -350.3505554199219,
      "step": 185
    },
    {
      "epoch": 0.39780162261188173,
      "grad_norm": 2263250.8365393346,
      "learning_rate": 3.765821230985757e-07,
      "logits/chosen": -0.4672353267669678,
      "logits/rejected": -0.028242725878953934,
      "logps/chosen": -283.13873291015625,
      "logps/rejected": -305.73968505859375,
      "loss": 115869.625,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -283.13873291015625,
      "rewards/margins": 22.600967407226562,
      "rewards/rejected": -305.73968505859375,
      "step": 190
    },
    {
      "epoch": 0.408270086364826,
      "grad_norm": 2336859.8229430458,
      "learning_rate": 3.6860532770864005e-07,
      "logits/chosen": -0.3446078598499298,
      "logits/rejected": 0.17580614984035492,
      "logps/chosen": -345.1635437011719,
      "logps/rejected": -373.7042236328125,
      "loss": 115610.2375,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -345.1635437011719,
      "rewards/margins": 28.540664672851562,
      "rewards/rejected": -373.7042236328125,
      "step": 195
    },
    {
      "epoch": 0.4187385501177702,
      "grad_norm": 2875002.890975931,
      "learning_rate": 3.604695382782159e-07,
      "logits/chosen": -0.3627869188785553,
      "logits/rejected": -0.14643223583698273,
      "logps/chosen": -305.51885986328125,
      "logps/rejected": -352.6678466796875,
      "loss": 116295.75,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -305.51885986328125,
      "rewards/margins": 47.14897537231445,
      "rewards/rejected": -352.6678466796875,
      "step": 200
    },
    {
      "epoch": 0.42920701387071447,
      "grad_norm": 1400861.091712678,
      "learning_rate": 3.5218566107988867e-07,
      "logits/chosen": -0.8571730852127075,
      "logits/rejected": -0.25482162833213806,
      "logps/chosen": -296.7934875488281,
      "logps/rejected": -329.7360534667969,
      "loss": 114386.15,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -296.7934875488281,
      "rewards/margins": 32.94261932373047,
      "rewards/rejected": -329.7360534667969,
      "step": 205
    },
    {
      "epoch": 0.4396754776236587,
      "grad_norm": 2247391.450727784,
      "learning_rate": 3.4376480090239047e-07,
      "logits/chosen": -0.7361810803413391,
      "logits/rejected": -0.26914888620376587,
      "logps/chosen": -324.780517578125,
      "logps/rejected": -346.3909606933594,
      "loss": 115300.375,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -324.780517578125,
      "rewards/margins": 21.61046028137207,
      "rewards/rejected": -346.3909606933594,
      "step": 210
    },
    {
      "epoch": 0.45014394137660296,
      "grad_norm": 1942071.0568026109,
      "learning_rate": 3.3521824616429284e-07,
      "logits/chosen": -0.7115877270698547,
      "logits/rejected": -0.16935975849628448,
      "logps/chosen": -306.1327819824219,
      "logps/rejected": -375.35125732421875,
      "loss": 113532.725,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -306.1327819824219,
      "rewards/margins": 69.21846008300781,
      "rewards/rejected": -375.35125732421875,
      "step": 215
    },
    {
      "epoch": 0.46061240512954726,
      "grad_norm": 1880893.803266421,
      "learning_rate": 3.265574537815398e-07,
      "logits/chosen": -0.7726610898971558,
      "logits/rejected": 0.11326490342617035,
      "logps/chosen": -315.4923095703125,
      "logps/rejected": -381.5898132324219,
      "loss": 113337.5125,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -315.4923095703125,
      "rewards/margins": 66.09752655029297,
      "rewards/rejected": -381.5898132324219,
      "step": 220
    },
    {
      "epoch": 0.4710808688824915,
      "grad_norm": 1945629.7722817352,
      "learning_rate": 3.1779403380910425e-07,
      "logits/chosen": -0.5554154515266418,
      "logits/rejected": 0.3164888024330139,
      "logps/chosen": -310.39068603515625,
      "logps/rejected": -347.70709228515625,
      "loss": 112926.525,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -310.39068603515625,
      "rewards/margins": 37.316429138183594,
      "rewards/rejected": -347.70709228515625,
      "step": 225
    },
    {
      "epoch": 0.48154933263543576,
      "grad_norm": 1616156.0747634866,
      "learning_rate": 3.0893973387735683e-07,
      "logits/chosen": -0.8544542193412781,
      "logits/rejected": 0.16107910871505737,
      "logps/chosen": -327.91864013671875,
      "logps/rejected": -367.1671447753906,
      "loss": 112795.225,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -327.91864013671875,
      "rewards/margins": 39.24848175048828,
      "rewards/rejected": -367.1671447753906,
      "step": 230
    },
    {
      "epoch": 0.49201779638838,
      "grad_norm": 1973066.2630310976,
      "learning_rate": 3.000064234440111e-07,
      "logits/chosen": -0.6506497263908386,
      "logits/rejected": 0.023212980479002,
      "logps/chosen": -294.9061279296875,
      "logps/rejected": -359.06591796875,
      "loss": 113001.925,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -294.9061279296875,
      "rewards/margins": 64.1597671508789,
      "rewards/rejected": -359.06591796875,
      "step": 235
    },
    {
      "epoch": 0.5024862601413242,
      "grad_norm": 2122255.375505896,
      "learning_rate": 2.910060778827554e-07,
      "logits/chosen": -0.884017288684845,
      "logits/rejected": -0.20214924216270447,
      "logps/chosen": -304.61309814453125,
      "logps/rejected": -344.1042175292969,
      "loss": 112705.575,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -304.61309814453125,
      "rewards/margins": 39.491153717041016,
      "rewards/rejected": -344.1042175292969,
      "step": 240
    },
    {
      "epoch": 0.5129547238942685,
      "grad_norm": 1750645.3017169987,
      "learning_rate": 2.8195076242990116e-07,
      "logits/chosen": -0.5472243428230286,
      "logits/rejected": 0.13178296387195587,
      "logps/chosen": -284.5224609375,
      "logps/rejected": -338.53375244140625,
      "loss": 114159.5,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -284.5224609375,
      "rewards/margins": 54.01129150390625,
      "rewards/rejected": -338.53375244140625,
      "step": 245
    },
    {
      "epoch": 0.5234231876472127,
      "grad_norm": 2085184.1734165123,
      "learning_rate": 2.7285261601056697e-07,
      "logits/chosen": -0.4131326675415039,
      "logits/rejected": 0.3880499005317688,
      "logps/chosen": -274.2272033691406,
      "logps/rejected": -340.6448669433594,
      "loss": 114531.3375,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -274.2272033691406,
      "rewards/margins": 66.4177017211914,
      "rewards/rejected": -340.6448669433594,
      "step": 250
    },
    {
      "epoch": 0.533891651400157,
      "grad_norm": 2304054.790659312,
      "learning_rate": 2.6372383496608186e-07,
      "logits/chosen": -0.17990997433662415,
      "logits/rejected": 0.9603084325790405,
      "logps/chosen": -331.6031188964844,
      "logps/rejected": -412.93817138671875,
      "loss": 111151.35,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -331.6031188964844,
      "rewards/margins": 81.3350830078125,
      "rewards/rejected": -412.93817138671875,
      "step": 255
    },
    {
      "epoch": 0.5443601151531012,
      "grad_norm": 1940084.13729787,
      "learning_rate": 2.5457665670441937e-07,
      "logits/chosen": -0.2644157409667969,
      "logits/rejected": 0.6259841918945312,
      "logps/chosen": -258.68670654296875,
      "logps/rejected": -366.4393615722656,
      "loss": 112844.35,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -258.68670654296875,
      "rewards/margins": 107.7526626586914,
      "rewards/rejected": -366.4393615722656,
      "step": 260
    },
    {
      "epoch": 0.5548285789060455,
      "grad_norm": 1521044.9308265387,
      "learning_rate": 2.454233432955807e-07,
      "logits/chosen": -0.47308096289634705,
      "logits/rejected": 0.20548152923583984,
      "logps/chosen": -286.80633544921875,
      "logps/rejected": -340.56756591796875,
      "loss": 112564.5375,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -286.80633544921875,
      "rewards/margins": 53.76123809814453,
      "rewards/rejected": -340.56756591796875,
      "step": 265
    },
    {
      "epoch": 0.5652970426589898,
      "grad_norm": 1873803.7134507762,
      "learning_rate": 2.3627616503391812e-07,
      "logits/chosen": -0.5675733685493469,
      "logits/rejected": 0.14759287238121033,
      "logps/chosen": -301.10040283203125,
      "logps/rejected": -370.1958312988281,
      "loss": 114260.7375,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -301.10040283203125,
      "rewards/margins": 69.09541320800781,
      "rewards/rejected": -370.1958312988281,
      "step": 270
    },
    {
      "epoch": 0.575765506411934,
      "grad_norm": 2362564.8339613965,
      "learning_rate": 2.2714738398943308e-07,
      "logits/chosen": -0.6608392000198364,
      "logits/rejected": 0.4885464608669281,
      "logps/chosen": -315.5157775878906,
      "logps/rejected": -383.5878601074219,
      "loss": 111772.275,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -315.5157775878906,
      "rewards/margins": 68.07210540771484,
      "rewards/rejected": -383.5878601074219,
      "step": 275
    },
    {
      "epoch": 0.5862339701648783,
      "grad_norm": 1807476.8899846808,
      "learning_rate": 2.1804923757009882e-07,
      "logits/chosen": -0.5532770752906799,
      "logits/rejected": 0.43744096159935,
      "logps/chosen": -315.10870361328125,
      "logps/rejected": -358.2113952636719,
      "loss": 112631.6,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -315.10870361328125,
      "rewards/margins": 43.10268020629883,
      "rewards/rejected": -358.2113952636719,
      "step": 280
    },
    {
      "epoch": 0.5967024339178225,
      "grad_norm": 2237959.642261439,
      "learning_rate": 2.089939221172446e-07,
      "logits/chosen": -0.46047574281692505,
      "logits/rejected": 0.012615549378097057,
      "logps/chosen": -293.84136962890625,
      "logps/rejected": -370.7669372558594,
      "loss": 112695.3,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -293.84136962890625,
      "rewards/margins": 76.92558288574219,
      "rewards/rejected": -370.7669372558594,
      "step": 285
    },
    {
      "epoch": 0.6071708976707668,
      "grad_norm": 1554217.5345301214,
      "learning_rate": 1.9999357655598891e-07,
      "logits/chosen": -0.5754300355911255,
      "logits/rejected": 0.33466261625289917,
      "logps/chosen": -303.59539794921875,
      "logps/rejected": -346.04412841796875,
      "loss": 112625.2125,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -303.59539794921875,
      "rewards/margins": 42.44871520996094,
      "rewards/rejected": -346.04412841796875,
      "step": 290
    },
    {
      "epoch": 0.6176393614237111,
      "grad_norm": 2521456.999656128,
      "learning_rate": 1.9106026612264315e-07,
      "logits/chosen": -0.687907338142395,
      "logits/rejected": 0.15663442015647888,
      "logps/chosen": -279.992919921875,
      "logps/rejected": -332.9551696777344,
      "loss": 114791.675,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -279.992919921875,
      "rewards/margins": 52.962249755859375,
      "rewards/rejected": -332.9551696777344,
      "step": 295
    },
    {
      "epoch": 0.6281078251766553,
      "grad_norm": 3193163.1405664305,
      "learning_rate": 1.8220596619089573e-07,
      "logits/chosen": -0.7139306664466858,
      "logits/rejected": -0.06598733365535736,
      "logps/chosen": -264.813232421875,
      "logps/rejected": -318.2864685058594,
      "loss": 112895.5875,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -264.813232421875,
      "rewards/margins": 53.47322463989258,
      "rewards/rejected": -318.2864685058594,
      "step": 300
    },
    {
      "epoch": 0.6385762889295996,
      "grad_norm": 2896442.2441057838,
      "learning_rate": 1.7344254621846017e-07,
      "logits/chosen": -0.5134263634681702,
      "logits/rejected": 0.272099107503891,
      "logps/chosen": -293.1112060546875,
      "logps/rejected": -366.1026306152344,
      "loss": 111599.1625,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -293.1112060546875,
      "rewards/margins": 72.99146270751953,
      "rewards/rejected": -366.1026306152344,
      "step": 305
    },
    {
      "epoch": 0.6490447526825438,
      "grad_norm": 1410328.2640552768,
      "learning_rate": 1.647817538357072e-07,
      "logits/chosen": -0.5094229578971863,
      "logits/rejected": -0.08179249614477158,
      "logps/chosen": -262.71868896484375,
      "logps/rejected": -373.6757507324219,
      "loss": 110514.8625,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -262.71868896484375,
      "rewards/margins": 110.95704650878906,
      "rewards/rejected": -373.6757507324219,
      "step": 310
    },
    {
      "epoch": 0.6595132164354881,
      "grad_norm": 4370337.480639941,
      "learning_rate": 1.562351990976095e-07,
      "logits/chosen": -0.5516433715820312,
      "logits/rejected": 0.3842604160308838,
      "logps/chosen": -299.07208251953125,
      "logps/rejected": -386.3035583496094,
      "loss": 111146.3375,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -299.07208251953125,
      "rewards/margins": 87.2314682006836,
      "rewards/rejected": -386.3035583496094,
      "step": 315
    },
    {
      "epoch": 0.6699816801884323,
      "grad_norm": 8860713.857687093,
      "learning_rate": 1.478143389201113e-07,
      "logits/chosen": -0.3542352616786957,
      "logits/rejected": 0.15903237462043762,
      "logps/chosen": -277.9214782714844,
      "logps/rejected": -365.1288146972656,
      "loss": 112815.525,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -277.9214782714844,
      "rewards/margins": 87.20732879638672,
      "rewards/rejected": -365.1288146972656,
      "step": 320
    },
    {
      "epoch": 0.6804501439413766,
      "grad_norm": 1908239.1278667776,
      "learning_rate": 1.3953046172178413e-07,
      "logits/chosen": -0.5815322399139404,
      "logits/rejected": 0.323032021522522,
      "logps/chosen": -323.28436279296875,
      "logps/rejected": -390.740234375,
      "loss": 112318.4,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -323.28436279296875,
      "rewards/margins": 67.45586395263672,
      "rewards/rejected": -390.740234375,
      "step": 325
    },
    {
      "epoch": 0.6909186076943209,
      "grad_norm": 2119739.073105214,
      "learning_rate": 1.3139467229135998e-07,
      "logits/chosen": -0.668155312538147,
      "logits/rejected": 0.13567589223384857,
      "logps/chosen": -295.8731384277344,
      "logps/rejected": -347.6330261230469,
      "loss": 111457.7,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -295.8731384277344,
      "rewards/margins": 51.7598762512207,
      "rewards/rejected": -347.6330261230469,
      "step": 330
    },
    {
      "epoch": 0.7013870714472651,
      "grad_norm": 1927999.1666317754,
      "learning_rate": 1.2341787690142435e-07,
      "logits/chosen": -0.8385793566703796,
      "logits/rejected": -0.16031041741371155,
      "logps/chosen": -299.18060302734375,
      "logps/rejected": -360.9104919433594,
      "loss": 112842.25,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -299.18060302734375,
      "rewards/margins": 61.72993087768555,
      "rewards/rejected": -360.9104919433594,
      "step": 335
    },
    {
      "epoch": 0.7118555352002094,
      "grad_norm": 2176227.517015282,
      "learning_rate": 1.1561076868822755e-07,
      "logits/chosen": -0.7431761622428894,
      "logits/rejected": 0.11613886058330536,
      "logps/chosen": -331.2103271484375,
      "logps/rejected": -360.3314208984375,
      "loss": 111618.6375,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -331.2103271484375,
      "rewards/margins": 29.121089935302734,
      "rewards/rejected": -360.3314208984375,
      "step": 340
    },
    {
      "epoch": 0.7223239989531536,
      "grad_norm": 2699423.898748738,
      "learning_rate": 1.0798381331721107e-07,
      "logits/chosen": -0.7944961786270142,
      "logits/rejected": 0.2726106643676758,
      "logps/chosen": -324.1297302246094,
      "logps/rejected": -362.2281188964844,
      "loss": 111899.3375,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -324.1297302246094,
      "rewards/margins": 38.09839630126953,
      "rewards/rejected": -362.2281188964844,
      "step": 345
    },
    {
      "epoch": 0.7327924627060979,
      "grad_norm": 2320601.542393335,
      "learning_rate": 1.0054723495346482e-07,
      "logits/chosen": -0.2713714838027954,
      "logits/rejected": 0.22024962306022644,
      "logps/chosen": -288.8630676269531,
      "logps/rejected": -365.87994384765625,
      "loss": 113524.075,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -288.8630676269531,
      "rewards/margins": 77.01687622070312,
      "rewards/rejected": -365.87994384765625,
      "step": 350
    },
    {
      "epoch": 0.7432609264590422,
      "grad_norm": 1815809.2243324781,
      "learning_rate": 9.331100255592436e-08,
      "logits/chosen": -0.2964836657047272,
      "logits/rejected": 0.5565654039382935,
      "logps/chosen": -277.563232421875,
      "logps/rejected": -366.7838134765625,
      "loss": 111167.625,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -277.563232421875,
      "rewards/margins": 89.22059631347656,
      "rewards/rejected": -366.7838134765625,
      "step": 355
    },
    {
      "epoch": 0.7537293902119864,
      "grad_norm": 1971255.6245351008,
      "learning_rate": 8.628481651367875e-08,
      "logits/chosen": -0.7251734137535095,
      "logits/rejected": 0.05562416836619377,
      "logps/chosen": -304.60357666015625,
      "logps/rejected": -355.7652587890625,
      "loss": 109653.975,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -304.60357666015625,
      "rewards/margins": 51.16161346435547,
      "rewards/rejected": -355.7652587890625,
      "step": 360
    },
    {
      "epoch": 0.7641978539649307,
      "grad_norm": 1754900.7076158663,
      "learning_rate": 7.947809564230445e-08,
      "logits/chosen": -0.45269888639450073,
      "logits/rejected": 0.3153532147407532,
      "logps/chosen": -322.9766845703125,
      "logps/rejected": -375.75787353515625,
      "loss": 114208.8,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -322.9766845703125,
      "rewards/margins": 52.78118896484375,
      "rewards/rejected": -375.75787353515625,
      "step": 365
    },
    {
      "epoch": 0.7746663177178749,
      "grad_norm": 2147071.573360362,
      "learning_rate": 7.289996455765748e-08,
      "logits/chosen": -0.43487948179244995,
      "logits/rejected": 0.42789965867996216,
      "logps/chosen": -311.0076599121094,
      "logps/rejected": -392.62152099609375,
      "loss": 112070.675,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -311.0076599121094,
      "rewards/margins": 81.61390686035156,
      "rewards/rejected": -392.62152099609375,
      "step": 370
    },
    {
      "epoch": 0.7851347814708192,
      "grad_norm": 1709073.4926486812,
      "learning_rate": 6.655924144404906e-08,
      "logits/chosen": -0.30552688241004944,
      "logits/rejected": -0.23342788219451904,
      "logps/chosen": -273.34320068359375,
      "logps/rejected": -361.71856689453125,
      "loss": 110961.7625,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -273.34320068359375,
      "rewards/margins": 88.37535095214844,
      "rewards/rejected": -361.71856689453125,
      "step": 375
    },
    {
      "epoch": 0.7956032452237635,
      "grad_norm": 2722896.7250441867,
      "learning_rate": 6.046442623320145e-08,
      "logits/chosen": -0.41763028502464294,
      "logits/rejected": 0.005231809802353382,
      "logps/chosen": -307.29693603515625,
      "logps/rejected": -393.7823181152344,
      "loss": 111852.7,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -307.29693603515625,
      "rewards/margins": 86.4853744506836,
      "rewards/rejected": -393.7823181152344,
      "step": 380
    },
    {
      "epoch": 0.8060717089767077,
      "grad_norm": 1745300.231202083,
      "learning_rate": 5.4623689209832484e-08,
      "logits/chosen": -0.718239426612854,
      "logits/rejected": -0.026650678366422653,
      "logps/chosen": -292.75494384765625,
      "logps/rejected": -368.7347717285156,
      "loss": 112271.8625,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -292.75494384765625,
      "rewards/margins": 75.97982788085938,
      "rewards/rejected": -368.7347717285156,
      "step": 385
    },
    {
      "epoch": 0.816540172729652,
      "grad_norm": 2201936.896729985,
      "learning_rate": 4.904486005914027e-08,
      "logits/chosen": -0.9130474328994751,
      "logits/rejected": 0.17849242687225342,
      "logps/chosen": -299.4166259765625,
      "logps/rejected": -363.7040100097656,
      "loss": 110293.55,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -299.4166259765625,
      "rewards/margins": 64.2873764038086,
      "rewards/rejected": -363.7040100097656,
      "step": 390
    },
    {
      "epoch": 0.8270086364825961,
      "grad_norm": 1965017.8644740335,
      "learning_rate": 4.373541737087263e-08,
      "logits/chosen": -0.7144075036048889,
      "logits/rejected": 0.521481454372406,
      "logps/chosen": -278.52801513671875,
      "logps/rejected": -330.89202880859375,
      "loss": 109924.925,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -278.52801513671875,
      "rewards/margins": 52.3640251159668,
      "rewards/rejected": -330.89202880859375,
      "step": 395
    },
    {
      "epoch": 0.8374771002355405,
      "grad_norm": 2507424.209111181,
      "learning_rate": 3.8702478614051345e-08,
      "logits/chosen": -0.6051787734031677,
      "logits/rejected": 0.003916704561561346,
      "logps/chosen": -281.8033447265625,
      "logps/rejected": -354.2074279785156,
      "loss": 113172.8375,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -281.8033447265625,
      "rewards/margins": 72.404052734375,
      "rewards/rejected": -354.2074279785156,
      "step": 400
    },
    {
      "epoch": 0.8479455639884846,
      "grad_norm": 2422379.6896536425,
      "learning_rate": 3.3952790595787986e-08,
      "logits/chosen": -0.6138535737991333,
      "logits/rejected": -0.022265207022428513,
      "logps/chosen": -283.72808837890625,
      "logps/rejected": -363.5780944824219,
      "loss": 112426.925,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -283.72808837890625,
      "rewards/margins": 79.85002899169922,
      "rewards/rejected": -363.5780944824219,
      "step": 405
    },
    {
      "epoch": 0.8584140277414289,
      "grad_norm": 2043445.172936004,
      "learning_rate": 2.9492720416985e-08,
      "logits/chosen": -0.7124518752098083,
      "logits/rejected": -0.10028399527072906,
      "logps/chosen": -277.91021728515625,
      "logps/rejected": -341.11846923828125,
      "loss": 112355.5875,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -277.91021728515625,
      "rewards/margins": 63.208221435546875,
      "rewards/rejected": -341.11846923828125,
      "step": 410
    },
    {
      "epoch": 0.8688824914943732,
      "grad_norm": 1929711.202147663,
      "learning_rate": 2.5328246937043525e-08,
      "logits/chosen": -0.5992367267608643,
      "logits/rejected": 0.41105660796165466,
      "logps/chosen": -325.5799560546875,
      "logps/rejected": -357.8695983886719,
      "loss": 110899.8875,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -325.5799560546875,
      "rewards/margins": 32.289649963378906,
      "rewards/rejected": -357.8695983886719,
      "step": 415
    },
    {
      "epoch": 0.8793509552473174,
      "grad_norm": 1596356.9629490953,
      "learning_rate": 2.1464952759020856e-08,
      "logits/chosen": -0.666754961013794,
      "logits/rejected": -0.003174859331920743,
      "logps/chosen": -269.9102478027344,
      "logps/rejected": -335.4573974609375,
      "loss": 111847.95,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -269.9102478027344,
      "rewards/margins": 65.54713439941406,
      "rewards/rejected": -335.4573974609375,
      "step": 420
    },
    {
      "epoch": 0.8898194190002617,
      "grad_norm": 1777896.953612317,
      "learning_rate": 1.7908016745981856e-08,
      "logits/chosen": -0.5347632765769958,
      "logits/rejected": 0.15892794728279114,
      "logps/chosen": -334.6045227050781,
      "logps/rejected": -374.3296813964844,
      "loss": 111778.7375,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -334.6045227050781,
      "rewards/margins": 39.72520065307617,
      "rewards/rejected": -374.3296813964844,
      "step": 425
    },
    {
      "epoch": 0.9002878827532059,
      "grad_norm": 1803597.2077325762,
      "learning_rate": 1.4662207078575684e-08,
      "logits/chosen": -0.5875300765037537,
      "logits/rejected": 0.1222870722413063,
      "logps/chosen": -282.852783203125,
      "logps/rejected": -356.6959228515625,
      "loss": 111337.7875,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -282.852783203125,
      "rewards/margins": 73.84312438964844,
      "rewards/rejected": -356.6959228515625,
      "step": 430
    },
    {
      "epoch": 0.9107563465061502,
      "grad_norm": 2178153.4810825037,
      "learning_rate": 1.1731874863145142e-08,
      "logits/chosen": -0.4485841691493988,
      "logits/rejected": 0.1706048995256424,
      "logps/chosen": -246.25521850585938,
      "logps/rejected": -336.5995178222656,
      "loss": 112746.675,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -246.25521850585938,
      "rewards/margins": 90.34431457519531,
      "rewards/rejected": -336.5995178222656,
      "step": 435
    },
    {
      "epoch": 0.9212248102590945,
      "grad_norm": 2249926.728268449,
      "learning_rate": 9.12094829893642e-09,
      "logits/chosen": -0.23983442783355713,
      "logits/rejected": 0.3341619372367859,
      "logps/chosen": -297.7192077636719,
      "logps/rejected": -405.0928649902344,
      "loss": 112266.5125,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -297.7192077636719,
      "rewards/margins": 107.37367248535156,
      "rewards/rejected": -405.0928649902344,
      "step": 440
    },
    {
      "epoch": 0.9316932740120387,
      "grad_norm": 2093448.7772529407,
      "learning_rate": 6.832927412229017e-09,
      "logits/chosen": -0.5585839152336121,
      "logits/rejected": 0.15140660107135773,
      "logps/chosen": -316.87335205078125,
      "logps/rejected": -385.70098876953125,
      "loss": 112199.175,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -316.87335205078125,
      "rewards/margins": 68.82764434814453,
      "rewards/rejected": -385.70098876953125,
      "step": 445
    },
    {
      "epoch": 0.942161737764983,
      "grad_norm": 2219595.1116894637,
      "learning_rate": 4.8708793644441086e-09,
      "logits/chosen": -0.3891911506652832,
      "logits/rejected": 0.024116378277540207,
      "logps/chosen": -270.6603088378906,
      "logps/rejected": -366.87481689453125,
      "loss": 110792.1375,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -270.6603088378906,
      "rewards/margins": 96.21451568603516,
      "rewards/rejected": -366.87481689453125,
      "step": 450
    },
    {
      "epoch": 0.9526302015179272,
      "grad_norm": 2056046.2252654296,
      "learning_rate": 3.2374343405217884e-09,
      "logits/chosen": -0.3999771475791931,
      "logits/rejected": 0.3102295994758606,
      "logps/chosen": -297.7739562988281,
      "logps/rejected": -349.3359069824219,
      "loss": 111786.575,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -297.7739562988281,
      "rewards/margins": 51.561973571777344,
      "rewards/rejected": -349.3359069824219,
      "step": 455
    },
    {
      "epoch": 0.9630986652708715,
      "grad_norm": 1596322.4991768566,
      "learning_rate": 1.9347820230782295e-09,
      "logits/chosen": -0.33234018087387085,
      "logits/rejected": 0.4286741614341736,
      "logps/chosen": -302.89599609375,
      "logps/rejected": -369.64752197265625,
      "loss": 110058.4875,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -302.89599609375,
      "rewards/margins": 66.75153350830078,
      "rewards/rejected": -369.64752197265625,
      "step": 460
    },
    {
      "epoch": 0.9735671290238157,
      "grad_norm": 2030583.2053086923,
      "learning_rate": 9.64668657069706e-10,
      "logits/chosen": -0.7010616064071655,
      "logits/rejected": 0.4352653920650482,
      "logps/chosen": -332.9463195800781,
      "logps/rejected": -413.53594970703125,
      "loss": 109834.325,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -332.9463195800781,
      "rewards/margins": 80.58956146240234,
      "rewards/rejected": -413.53594970703125,
      "step": 465
    },
    {
      "epoch": 0.98403559277676,
      "grad_norm": 1771708.6547032476,
      "learning_rate": 3.2839470889836627e-10,
      "logits/chosen": -0.673961877822876,
      "logits/rejected": 0.0036758959759026766,
      "logps/chosen": -312.06292724609375,
      "logps/rejected": -398.3697814941406,
      "loss": 112512.225,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -312.06292724609375,
      "rewards/margins": 86.30685424804688,
      "rewards/rejected": -398.3697814941406,
      "step": 470
    },
    {
      "epoch": 0.9945040565297043,
      "grad_norm": 1840231.2095069257,
      "learning_rate": 2.6813123097352287e-11,
      "logits/chosen": -0.7123033404350281,
      "logits/rejected": 0.20311255753040314,
      "logps/chosen": -341.65423583984375,
      "logps/rejected": -375.7768859863281,
      "loss": 111344.2125,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -341.65423583984375,
      "rewards/margins": 34.122650146484375,
      "rewards/rejected": -375.7768859863281,
      "step": 475
    },
    {
      "epoch": 0.998691442030882,
      "step": 477,
      "total_flos": 0.0,
      "train_loss": 114854.6911687631,
      "train_runtime": 7479.76,
      "train_samples_per_second": 8.173,
      "train_steps_per_second": 0.064
    }
  ],
  "logging_steps": 5,
  "max_steps": 477,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 1000000,
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}