{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.0,
  "eval_steps": 100,
  "global_step": 478,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0041841004184100415,
      "grad_norm": 7.099414444882459,
      "learning_rate": 1.0416666666666666e-08,
      "logits/chosen": -2.6615781784057617,
      "logits/rejected": -2.691082239151001,
      "logps/chosen": -298.690673828125,
      "logps/rejected": -323.7967834472656,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.04184100418410042,
      "grad_norm": 6.23352657034775,
      "learning_rate": 1.0416666666666667e-07,
      "logits/chosen": -2.628962278366089,
      "logits/rejected": -2.581556797027588,
      "logps/chosen": -270.5624694824219,
      "logps/rejected": -274.7192077636719,
      "loss": 0.6931,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.00030235512531362474,
      "rewards/margins": 0.00018380279652774334,
      "rewards/rejected": 0.00011855230695800856,
      "step": 10
    },
    {
      "epoch": 0.08368200836820083,
      "grad_norm": 6.403910934005757,
      "learning_rate": 2.0833333333333333e-07,
      "logits/chosen": -2.6337690353393555,
      "logits/rejected": -2.5815837383270264,
      "logps/chosen": -283.6520080566406,
      "logps/rejected": -254.223876953125,
      "loss": 0.692,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": 0.0022390992380678654,
      "rewards/margins": 0.002958392258733511,
      "rewards/rejected": -0.000719292729627341,
      "step": 20
    },
    {
      "epoch": 0.12552301255230125,
      "grad_norm": 6.382530025546531,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": -2.596775531768799,
      "logits/rejected": -2.5497207641601562,
      "logps/chosen": -273.9563293457031,
      "logps/rejected": -294.90093994140625,
      "loss": 0.6863,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": 0.01745353825390339,
      "rewards/margins": 0.0119100296869874,
      "rewards/rejected": 0.005543507635593414,
      "step": 30
    },
    {
      "epoch": 0.16736401673640167,
      "grad_norm": 7.236510898066947,
      "learning_rate": 4.1666666666666667e-07,
      "logits/chosen": -2.595787525177002,
      "logits/rejected": -2.56512188911438,
      "logps/chosen": -289.8186340332031,
      "logps/rejected": -260.59027099609375,
      "loss": 0.6711,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.044044919312000275,
      "rewards/margins": 0.045689694583415985,
      "rewards/rejected": -0.0016447717789560556,
      "step": 40
    },
    {
      "epoch": 0.20920502092050208,
      "grad_norm": 8.532614175897798,
      "learning_rate": 4.999733114418725e-07,
      "logits/chosen": -2.5427632331848145,
      "logits/rejected": -2.5035042762756348,
      "logps/chosen": -262.0881042480469,
      "logps/rejected": -270.3690185546875,
      "loss": 0.6456,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.023291397839784622,
      "rewards/margins": 0.13293734192848206,
      "rewards/rejected": -0.15622875094413757,
      "step": 50
    },
    {
      "epoch": 0.2510460251046025,
      "grad_norm": 10.272678361666578,
      "learning_rate": 4.990398100856366e-07,
      "logits/chosen": -2.531106472015381,
      "logits/rejected": -2.470402240753174,
      "logps/chosen": -302.3480224609375,
      "logps/rejected": -291.5530700683594,
      "loss": 0.6203,
      "rewards/accuracies": 0.7093750238418579,
      "rewards/chosen": -0.09494151920080185,
      "rewards/margins": 0.23186612129211426,
      "rewards/rejected": -0.3268076777458191,
      "step": 60
    },
    {
      "epoch": 0.2928870292887029,
      "grad_norm": 14.725483063714895,
      "learning_rate": 4.967775735898179e-07,
      "logits/chosen": -2.4730429649353027,
      "logits/rejected": -2.4408392906188965,
      "logps/chosen": -316.3986511230469,
      "logps/rejected": -333.5642395019531,
      "loss": 0.5917,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.1742347776889801,
      "rewards/margins": 0.31590718030929565,
      "rewards/rejected": -0.49014195799827576,
      "step": 70
    },
    {
      "epoch": 0.33472803347280333,
      "grad_norm": 10.498750585183611,
      "learning_rate": 4.931986719649298e-07,
      "logits/chosen": -2.521245002746582,
      "logits/rejected": -2.4482836723327637,
      "logps/chosen": -308.7649841308594,
      "logps/rejected": -326.46234130859375,
      "loss": 0.6031,
      "rewards/accuracies": 0.6968749761581421,
      "rewards/chosen": -0.30625176429748535,
      "rewards/margins": 0.29028043150901794,
      "rewards/rejected": -0.5965322256088257,
      "step": 80
    },
    {
      "epoch": 0.37656903765690375,
      "grad_norm": 12.712503521629706,
      "learning_rate": 4.883222001996351e-07,
      "logits/chosen": -2.414280414581299,
      "logits/rejected": -2.352023124694824,
      "logps/chosen": -291.20147705078125,
      "logps/rejected": -300.1272888183594,
      "loss": 0.5797,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.383798211812973,
      "rewards/margins": 0.3239237368106842,
      "rewards/rejected": -0.7077219486236572,
      "step": 90
    },
    {
      "epoch": 0.41841004184100417,
      "grad_norm": 23.30871369910813,
      "learning_rate": 4.821741763807186e-07,
      "logits/chosen": -1.6760666370391846,
      "logits/rejected": -1.5858938694000244,
      "logps/chosen": -309.859375,
      "logps/rejected": -349.28802490234375,
      "loss": 0.5692,
      "rewards/accuracies": 0.6968749761581421,
      "rewards/chosen": -0.44263941049575806,
      "rewards/margins": 0.4911220669746399,
      "rewards/rejected": -0.9337614178657532,
      "step": 100
    },
    {
      "epoch": 0.41841004184100417,
      "eval_logits/chosen": -1.3128046989440918,
      "eval_logits/rejected": -1.1362591981887817,
      "eval_logps/chosen": -299.5389099121094,
      "eval_logps/rejected": -355.61285400390625,
      "eval_loss": 0.5671091079711914,
      "eval_rewards/accuracies": 0.734375,
      "eval_rewards/chosen": -0.3690897822380066,
      "eval_rewards/margins": 0.560417652130127,
      "eval_rewards/rejected": -0.9295073747634888,
      "eval_runtime": 93.5851,
      "eval_samples_per_second": 21.371,
      "eval_steps_per_second": 0.342,
      "step": 100
    },
    {
      "epoch": 0.4602510460251046,
      "grad_norm": 18.679015114468058,
      "learning_rate": 4.747874028753375e-07,
      "logits/chosen": -0.9447735548019409,
      "logits/rejected": -0.7094529867172241,
      "logps/chosen": -332.88055419921875,
      "logps/rejected": -359.24169921875,
      "loss": 0.5669,
      "rewards/accuracies": 0.7281249761581421,
      "rewards/chosen": -0.5535979270935059,
      "rewards/margins": 0.5540369749069214,
      "rewards/rejected": -1.1076349020004272,
      "step": 110
    },
    {
      "epoch": 0.502092050209205,
      "grad_norm": 16.44393963859546,
      "learning_rate": 4.662012913161997e-07,
      "logits/chosen": -0.5595887303352356,
      "logits/rejected": -0.3426797091960907,
      "logps/chosen": -330.4803161621094,
      "logps/rejected": -391.5724792480469,
      "loss": 0.5393,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.7413566708564758,
      "rewards/margins": 0.6902385950088501,
      "rewards/rejected": -1.4315952062606812,
      "step": 120
    },
    {
      "epoch": 0.5439330543933054,
      "grad_norm": 19.802781403351748,
      "learning_rate": 4.5646165232345103e-07,
      "logits/chosen": -0.4343790113925934,
      "logits/rejected": -0.07114946097135544,
      "logps/chosen": -362.0838928222656,
      "logps/rejected": -396.4689025878906,
      "loss": 0.5338,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.6583347916603088,
      "rewards/margins": 0.7389912009239197,
      "rewards/rejected": -1.397326111793518,
      "step": 130
    },
    {
      "epoch": 0.5857740585774058,
      "grad_norm": 21.162587940908836,
      "learning_rate": 4.456204510851956e-07,
      "logits/chosen": -0.3260354697704315,
      "logits/rejected": 0.02392721176147461,
      "logps/chosen": -345.8163757324219,
      "logps/rejected": -386.79254150390625,
      "loss": 0.5444,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.7646363377571106,
      "rewards/margins": 0.6039775013923645,
      "rewards/rejected": -1.368613839149475,
      "step": 140
    },
    {
      "epoch": 0.6276150627615062,
      "grad_norm": 21.73672532038991,
      "learning_rate": 4.337355301007335e-07,
      "logits/chosen": -0.20636191964149475,
      "logits/rejected": 0.28736037015914917,
      "logps/chosen": -364.95477294921875,
      "logps/rejected": -408.7835998535156,
      "loss": 0.5282,
      "rewards/accuracies": 0.715624988079071,
      "rewards/chosen": -0.8889427185058594,
      "rewards/margins": 0.6887326240539551,
      "rewards/rejected": -1.577675223350525,
      "step": 150
    },
    {
      "epoch": 0.6694560669456067,
      "grad_norm": 21.022249543596576,
      "learning_rate": 4.2087030056579986e-07,
      "logits/chosen": 0.2184087336063385,
      "logits/rejected": 0.6583582162857056,
      "logps/chosen": -384.6106872558594,
      "logps/rejected": -409.9974060058594,
      "loss": 0.5064,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.9010273218154907,
      "rewards/margins": 0.6821587681770325,
      "rewards/rejected": -1.583186149597168,
      "step": 160
    },
    {
      "epoch": 0.7112970711297071,
      "grad_norm": 19.31934084124254,
      "learning_rate": 4.070934040463998e-07,
      "logits/chosen": 0.3104431927204132,
      "logits/rejected": 0.8817472457885742,
      "logps/chosen": -350.32891845703125,
      "logps/rejected": -423.04266357421875,
      "loss": 0.5153,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.741706132888794,
      "rewards/margins": 0.7905541062355042,
      "rewards/rejected": -1.5322601795196533,
      "step": 170
    },
    {
      "epoch": 0.7531380753138075,
      "grad_norm": 17.210834759479862,
      "learning_rate": 3.9247834624635404e-07,
      "logits/chosen": 1.03719961643219,
      "logits/rejected": 1.7225978374481201,
      "logps/chosen": -386.9837646484375,
      "logps/rejected": -437.9752502441406,
      "loss": 0.5063,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0653451681137085,
      "rewards/margins": 0.8046163320541382,
      "rewards/rejected": -1.8699613809585571,
      "step": 180
    },
    {
      "epoch": 0.7949790794979079,
      "grad_norm": 18.78911912939503,
      "learning_rate": 3.7710310482256523e-07,
      "logits/chosen": 1.121408224105835,
      "logits/rejected": 1.6289663314819336,
      "logps/chosen": -369.95733642578125,
      "logps/rejected": -439.37548828125,
      "loss": 0.5163,
      "rewards/accuracies": 0.746874988079071,
      "rewards/chosen": -1.1254816055297852,
      "rewards/margins": 0.7835719585418701,
      "rewards/rejected": -1.9090534448623657,
      "step": 190
    },
    {
      "epoch": 0.8368200836820083,
      "grad_norm": 18.234200389798175,
      "learning_rate": 3.610497133404795e-07,
      "logits/chosen": 1.011256217956543,
      "logits/rejected": 1.5305925607681274,
      "logps/chosen": -390.50341796875,
      "logps/rejected": -459.53875732421875,
      "loss": 0.5056,
      "rewards/accuracies": 0.7593749761581421,
      "rewards/chosen": -1.0241918563842773,
      "rewards/margins": 0.81651771068573,
      "rewards/rejected": -1.8407094478607178,
      "step": 200
    },
    {
      "epoch": 0.8368200836820083,
      "eval_logits/chosen": 0.9604065418243408,
      "eval_logits/rejected": 1.4953534603118896,
      "eval_logps/chosen": -358.1262512207031,
      "eval_logps/rejected": -447.7286376953125,
      "eval_loss": 0.5104337334632874,
      "eval_rewards/accuracies": 0.78515625,
      "eval_rewards/chosen": -0.9549632668495178,
      "eval_rewards/margins": 0.8957023620605469,
      "eval_rewards/rejected": -1.8506656885147095,
      "eval_runtime": 93.4351,
      "eval_samples_per_second": 21.405,
      "eval_steps_per_second": 0.342,
      "step": 200
    },
    {
      "epoch": 0.8786610878661087,
      "grad_norm": 19.215200164648973,
      "learning_rate": 3.4440382358952115e-07,
      "logits/chosen": 1.0344123840332031,
      "logits/rejected": 1.8179776668548584,
      "logps/chosen": -402.72003173828125,
      "logps/rejected": -460.1229553222656,
      "loss": 0.5066,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0471458435058594,
      "rewards/margins": 0.8377550840377808,
      "rewards/rejected": -1.884900689125061,
      "step": 210
    },
    {
      "epoch": 0.9205020920502092,
      "grad_norm": 22.998405474723544,
      "learning_rate": 3.272542485937368e-07,
      "logits/chosen": 0.9624277353286743,
      "logits/rejected": 1.7323452234268188,
      "logps/chosen": -374.59521484375,
      "logps/rejected": -442.35650634765625,
      "loss": 0.4894,
      "rewards/accuracies": 0.784375011920929,
      "rewards/chosen": -0.9290931820869446,
      "rewards/margins": 0.8601258397102356,
      "rewards/rejected": -1.7892189025878906,
      "step": 220
    },
    {
      "epoch": 0.9623430962343096,
      "grad_norm": 17.30285752760936,
      "learning_rate": 3.096924887558854e-07,
      "logits/chosen": 0.9907492399215698,
      "logits/rejected": 1.7148042917251587,
      "logps/chosen": -386.2179260253906,
      "logps/rejected": -461.5531311035156,
      "loss": 0.5016,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -1.0336068868637085,
      "rewards/margins": 0.8331363797187805,
      "rewards/rejected": -1.8667433261871338,
      "step": 230
    },
    {
      "epoch": 1.00418410041841,
      "grad_norm": 17.247084086602182,
      "learning_rate": 2.9181224366319943e-07,
      "logits/chosen": 0.6638993620872498,
      "logits/rejected": 1.3943182229995728,
      "logps/chosen": -376.9907531738281,
      "logps/rejected": -443.1883850097656,
      "loss": 0.4867,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": -0.9322942495346069,
      "rewards/margins": 0.7798601388931274,
      "rewards/rejected": -1.7121543884277344,
      "step": 240
    },
    {
      "epoch": 1.0460251046025104,
      "grad_norm": 27.106663640280367,
      "learning_rate": 2.7370891215954565e-07,
      "logits/chosen": 1.1816644668579102,
      "logits/rejected": 2.409132480621338,
      "logps/chosen": -390.9169006347656,
      "logps/rejected": -476.27032470703125,
      "loss": 0.3951,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -1.0209319591522217,
      "rewards/margins": 1.2181396484375,
      "rewards/rejected": -2.2390713691711426,
      "step": 250
    },
    {
      "epoch": 1.0878661087866108,
      "grad_norm": 26.07881941521921,
      "learning_rate": 2.55479083351317e-07,
      "logits/chosen": 1.634642243385315,
      "logits/rejected": 2.725491762161255,
      "logps/chosen": -382.5552673339844,
      "logps/rejected": -490.21063232421875,
      "loss": 0.3905,
      "rewards/accuracies": 0.815625011920929,
      "rewards/chosen": -1.040998935699463,
      "rewards/margins": 1.2747693061828613,
      "rewards/rejected": -2.3157684803009033,
      "step": 260
    },
    {
      "epoch": 1.1297071129707112,
      "grad_norm": 25.1217443727487,
      "learning_rate": 2.3722002126275822e-07,
      "logits/chosen": 1.4133918285369873,
      "logits/rejected": 2.5140790939331055,
      "logps/chosen": -383.543212890625,
      "logps/rejected": -506.23443603515625,
      "loss": 0.3874,
      "rewards/accuracies": 0.8343750238418579,
      "rewards/chosen": -1.2637007236480713,
      "rewards/margins": 1.3295375108718872,
      "rewards/rejected": -2.593238353729248,
      "step": 270
    },
    {
      "epoch": 1.1715481171548117,
      "grad_norm": 25.52463427219616,
      "learning_rate": 2.19029145890313e-07,
      "logits/chosen": 1.2717490196228027,
      "logits/rejected": 2.5123631954193115,
      "logps/chosen": -398.04620361328125,
      "logps/rejected": -503.71759033203125,
      "loss": 0.3954,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -1.2178623676300049,
      "rewards/margins": 1.3320947885513306,
      "rewards/rejected": -2.549957275390625,
      "step": 280
    },
    {
      "epoch": 1.213389121338912,
      "grad_norm": 19.330696144971178,
      "learning_rate": 2.0100351342479216e-07,
      "logits/chosen": 1.1333314180374146,
      "logits/rejected": 2.094285011291504,
      "logps/chosen": -396.33905029296875,
      "logps/rejected": -513.3264770507812,
      "loss": 0.381,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.1736336946487427,
      "rewards/margins": 1.3145819902420044,
      "rewards/rejected": -2.488215684890747,
      "step": 290
    },
    {
      "epoch": 1.2552301255230125,
      "grad_norm": 25.82255359759937,
      "learning_rate": 1.8323929841460178e-07,
      "logits/chosen": 0.8372614979743958,
      "logits/rejected": 1.8510090112686157,
      "logps/chosen": -397.23175048828125,
      "logps/rejected": -493.86578369140625,
      "loss": 0.3744,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -1.2290160655975342,
      "rewards/margins": 1.2341445684432983,
      "rewards/rejected": -2.463160753250122,
      "step": 300
    },
    {
      "epoch": 1.2552301255230125,
      "eval_logits/chosen": 0.9870985150337219,
      "eval_logits/rejected": 1.8526870012283325,
      "eval_logps/chosen": -397.17913818359375,
      "eval_logps/rejected": -518.840576171875,
      "eval_loss": 0.5120064616203308,
      "eval_rewards/accuracies": 0.76171875,
      "eval_rewards/chosen": -1.3454917669296265,
      "eval_rewards/margins": 1.2162930965423584,
      "eval_rewards/rejected": -2.5617849826812744,
      "eval_runtime": 93.0364,
      "eval_samples_per_second": 21.497,
      "eval_steps_per_second": 0.344,
      "step": 300
    },
    {
      "epoch": 1.297071129707113,
      "grad_norm": 40.35910050212018,
      "learning_rate": 1.6583128063291573e-07,
      "logits/chosen": 0.8788517713546753,
      "logits/rejected": 2.132133722305298,
      "logps/chosen": -439.266845703125,
      "logps/rejected": -540.0487060546875,
      "loss": 0.3858,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.3913695812225342,
      "rewards/margins": 1.3974393606185913,
      "rewards/rejected": -2.788808822631836,
      "step": 310
    },
    {
      "epoch": 1.3389121338912133,
      "grad_norm": 20.432466646869965,
      "learning_rate": 1.488723393865766e-07,
      "logits/chosen": 0.7624516487121582,
      "logits/rejected": 1.9204070568084717,
      "logps/chosen": -394.59002685546875,
      "logps/rejected": -506.1437072753906,
      "loss": 0.3764,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -1.1876527070999146,
      "rewards/margins": 1.364680528640747,
      "rewards/rejected": -2.552333354949951,
      "step": 320
    },
    {
      "epoch": 1.3807531380753137,
      "grad_norm": 24.117995597732122,
      "learning_rate": 1.3245295796480788e-07,
      "logits/chosen": 1.2566739320755005,
      "logits/rejected": 2.4089713096618652,
      "logps/chosen": -403.59539794921875,
      "logps/rejected": -535.1661987304688,
      "loss": 0.3785,
      "rewards/accuracies": 0.8656250238418579,
      "rewards/chosen": -1.3696684837341309,
      "rewards/margins": 1.466200828552246,
      "rewards/rejected": -2.835869312286377,
      "step": 330
    },
    {
      "epoch": 1.4225941422594142,
      "grad_norm": 26.170896618734517,
      "learning_rate": 1.1666074087171627e-07,
      "logits/chosen": 1.6827380657196045,
      "logits/rejected": 2.633002519607544,
      "logps/chosen": -409.3313903808594,
      "logps/rejected": -560.8604736328125,
      "loss": 0.3673,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.4851205348968506,
      "rewards/margins": 1.4507629871368408,
      "rewards/rejected": -2.9358837604522705,
      "step": 340
    },
    {
      "epoch": 1.4644351464435146,
      "grad_norm": 20.814302126974475,
      "learning_rate": 1.0157994641835734e-07,
      "logits/chosen": 1.3171794414520264,
      "logits/rejected": 2.3436312675476074,
      "logps/chosen": -415.7732849121094,
      "logps/rejected": -539.7247924804688,
      "loss": 0.3782,
      "rewards/accuracies": 0.8218749761581421,
      "rewards/chosen": -1.373610496520996,
      "rewards/margins": 1.4075970649719238,
      "rewards/rejected": -2.78120756149292,
      "step": 350
    },
    {
      "epoch": 1.506276150627615,
      "grad_norm": 21.900774053482067,
      "learning_rate": 8.729103716819111e-08,
      "logits/chosen": 1.4213756322860718,
      "logits/rejected": 2.515977382659912,
      "logps/chosen": -440.63311767578125,
      "logps/rejected": -566.4862060546875,
      "loss": 0.3606,
      "rewards/accuracies": 0.846875011920929,
      "rewards/chosen": -1.460523009300232,
      "rewards/margins": 1.5627386569976807,
      "rewards/rejected": -3.023261308670044,
      "step": 360
    },
    {
      "epoch": 1.5481171548117154,
      "grad_norm": 24.209127764578607,
      "learning_rate": 7.387025063449081e-08,
      "logits/chosen": 1.1703312397003174,
      "logits/rejected": 2.1595053672790527,
      "logps/chosen": -427.52734375,
      "logps/rejected": -536.962646484375,
      "loss": 0.3769,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.4095176458358765,
      "rewards/margins": 1.3942286968231201,
      "rewards/rejected": -2.803746461868286,
      "step": 370
    },
    {
      "epoch": 1.5899581589958158,
      "grad_norm": 25.431449474931732,
      "learning_rate": 6.138919252022435e-08,
      "logits/chosen": 1.452207326889038,
      "logits/rejected": 2.379581928253174,
      "logps/chosen": -434.0857849121094,
      "logps/rejected": -585.3619384765625,
      "loss": 0.366,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.3957841396331787,
      "rewards/margins": 1.5939209461212158,
      "rewards/rejected": -2.9897048473358154,
      "step": 380
    },
    {
      "epoch": 1.6317991631799162,
      "grad_norm": 22.87966472658563,
      "learning_rate": 4.991445467064689e-08,
      "logits/chosen": 1.3533505201339722,
      "logits/rejected": 2.646073579788208,
      "logps/chosen": -424.43267822265625,
      "logps/rejected": -543.8640747070312,
      "loss": 0.3723,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -1.4647841453552246,
      "rewards/margins": 1.4622496366500854,
      "rewards/rejected": -2.9270341396331787,
      "step": 390
    },
    {
      "epoch": 1.6736401673640167,
      "grad_norm": 21.87896636487824,
      "learning_rate": 3.9507259776993954e-08,
      "logits/chosen": 1.3778632879257202,
      "logits/rejected": 2.785320281982422,
      "logps/chosen": -429.8521423339844,
      "logps/rejected": -541.7579345703125,
      "loss": 0.351,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -1.3404476642608643,
      "rewards/margins": 1.5822999477386475,
      "rewards/rejected": -2.922747850418091,
      "step": 400
    },
    {
      "epoch": 1.6736401673640167,
      "eval_logits/chosen": 1.4005358219146729,
      "eval_logits/rejected": 2.178215265274048,
      "eval_logps/chosen": -414.581298828125,
      "eval_logps/rejected": -536.5388793945312,
      "eval_loss": 0.5018947124481201,
      "eval_rewards/accuracies": 0.76953125,
      "eval_rewards/chosen": -1.519513487815857,
      "eval_rewards/margins": 1.219254732131958,
      "eval_rewards/rejected": -2.7387683391571045,
      "eval_runtime": 92.8387,
      "eval_samples_per_second": 21.543,
      "eval_steps_per_second": 0.345,
      "step": 400
    },
    {
      "epoch": 1.715481171548117,
      "grad_norm": 25.54848296222211,
      "learning_rate": 3.022313472693447e-08,
      "logits/chosen": 1.3364863395690918,
      "logits/rejected": 2.4736454486846924,
      "logps/chosen": -421.3280334472656,
      "logps/rejected": -570.7842407226562,
      "loss": 0.375,
      "rewards/accuracies": 0.846875011920929,
      "rewards/chosen": -1.4803876876831055,
      "rewards/margins": 1.5594526529312134,
      "rewards/rejected": -3.0398402214050293,
      "step": 410
    },
    {
      "epoch": 1.7573221757322175,
      "grad_norm": 30.706537474329608,
      "learning_rate": 2.2111614344599684e-08,
      "logits/chosen": 1.4454032182693481,
      "logits/rejected": 2.485903263092041,
      "logps/chosen": -436.48260498046875,
      "logps/rejected": -557.8076171875,
      "loss": 0.361,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.5660855770111084,
      "rewards/margins": 1.5376800298690796,
      "rewards/rejected": -3.1037657260894775,
      "step": 420
    },
    {
      "epoch": 1.799163179916318,
      "grad_norm": 23.906131529756504,
      "learning_rate": 1.521597710086439e-08,
      "logits/chosen": 1.5828850269317627,
      "logits/rejected": 2.5288591384887695,
      "logps/chosen": -428.4192810058594,
      "logps/rejected": -542.4400634765625,
      "loss": 0.369,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.4776898622512817,
      "rewards/margins": 1.4163352251052856,
      "rewards/rejected": -2.8940250873565674,
      "step": 430
    },
    {
      "epoch": 1.8410041841004183,
      "grad_norm": 28.22708684609244,
      "learning_rate": 9.57301420397924e-09,
      "logits/chosen": 1.6976782083511353,
      "logits/rejected": 2.871784210205078,
      "logps/chosen": -424.5814514160156,
      "logps/rejected": -530.4151000976562,
      "loss": 0.3655,
      "rewards/accuracies": 0.8531249761581421,
      "rewards/chosen": -1.5554591417312622,
      "rewards/margins": 1.416295051574707,
      "rewards/rejected": -2.971754550933838,
      "step": 440
    },
    {
      "epoch": 1.8828451882845187,
      "grad_norm": 21.46648253004346,
      "learning_rate": 5.212833302556258e-09,
      "logits/chosen": 1.5777668952941895,
      "logits/rejected": 2.6181626319885254,
      "logps/chosen": -431.70086669921875,
      "logps/rejected": -552.4868774414062,
      "loss": 0.3454,
      "rewards/accuracies": 0.871874988079071,
      "rewards/chosen": -1.5065062046051025,
      "rewards/margins": 1.5084437131881714,
      "rewards/rejected": -3.0149497985839844,
      "step": 450
    },
    {
      "epoch": 1.9246861924686192,
      "grad_norm": 24.44249151761102,
      "learning_rate": 2.158697848236607e-09,
      "logits/chosen": 1.7532329559326172,
      "logits/rejected": 2.691193103790283,
      "logps/chosen": -401.5673828125,
      "logps/rejected": -556.6257934570312,
      "loss": 0.3685,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.477992057800293,
      "rewards/margins": 1.5126928091049194,
      "rewards/rejected": -2.990684986114502,
      "step": 460
    },
    {
      "epoch": 1.9665271966527196,
      "grad_norm": 25.733721301575518,
      "learning_rate": 4.269029751107489e-10,
      "logits/chosen": 1.2774773836135864,
      "logits/rejected": 2.5834484100341797,
      "logps/chosen": -431.2181701660156,
      "logps/rejected": -525.5928955078125,
      "loss": 0.3641,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.4548835754394531,
      "rewards/margins": 1.3838748931884766,
      "rewards/rejected": -2.838758707046509,
      "step": 470
    },
    {
      "epoch": 2.0,
      "step": 478,
      "total_flos": 0.0,
      "train_loss": 0.47010813497599196,
      "train_runtime": 14181.5338,
      "train_samples_per_second": 8.622,
      "train_steps_per_second": 0.034
    }
  ],
  "logging_steps": 10,
  "max_steps": 478,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}