{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.0,
  "eval_steps": 100,
  "global_step": 574,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.003484320557491289,
      "grad_norm": 497.3575626103625,
      "learning_rate": 1.5517241379310344e-08,
      "logits/chosen": -2.5345611572265625,
      "logits/rejected": -2.581700563430786,
      "logps/chosen": -60.002105712890625,
      "logps/rejected": -99.98374938964844,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.03484320557491289,
      "grad_norm": 462.4792154122565,
      "learning_rate": 1.5517241379310344e-07,
      "logits/chosen": -2.563889503479004,
      "logits/rejected": -2.562492609024048,
      "logps/chosen": -59.64662551879883,
      "logps/rejected": -73.37532043457031,
      "loss": 0.6974,
      "rewards/accuracies": 0.2083333283662796,
      "rewards/chosen": 0.011204251088202,
      "rewards/margins": 0.00639230664819479,
      "rewards/rejected": 0.004811946302652359,
      "step": 10
    },
    {
      "epoch": 0.06968641114982578,
      "grad_norm": 537.1826669031543,
      "learning_rate": 3.103448275862069e-07,
      "logits/chosen": -2.6071362495422363,
      "logits/rejected": -2.565964698791504,
      "logps/chosen": -103.9787826538086,
      "logps/rejected": -94.84336853027344,
      "loss": 0.6775,
      "rewards/accuracies": 0.3375000059604645,
      "rewards/chosen": 0.12751302123069763,
      "rewards/margins": 0.08386950194835663,
      "rewards/rejected": 0.0436435230076313,
      "step": 20
    },
    {
      "epoch": 0.10452961672473868,
      "grad_norm": 586.0340590697643,
      "learning_rate": 4.6551724137931035e-07,
      "logits/chosen": -2.5952959060668945,
      "logits/rejected": -2.5755515098571777,
      "logps/chosen": -81.9345932006836,
      "logps/rejected": -91.35636901855469,
      "loss": 0.6673,
      "rewards/accuracies": 0.33125001192092896,
      "rewards/chosen": 0.48329487442970276,
      "rewards/margins": 0.3252241015434265,
      "rewards/rejected": 0.15807083249092102,
      "step": 30
    },
    {
      "epoch": 0.13937282229965156,
      "grad_norm": 508.2061569040689,
      "learning_rate": 6.206896551724138e-07,
      "logits/chosen": -2.495903491973877,
      "logits/rejected": -2.4941375255584717,
      "logps/chosen": -78.31864929199219,
      "logps/rejected": -74.04932403564453,
      "loss": 0.6704,
      "rewards/accuracies": 0.3187499940395355,
      "rewards/chosen": -0.3442053496837616,
      "rewards/margins": 0.5570718050003052,
      "rewards/rejected": -0.9012771844863892,
      "step": 40
    },
    {
      "epoch": 0.17421602787456447,
      "grad_norm": 371.6317110957719,
      "learning_rate": 7.758620689655172e-07,
      "logits/chosen": -2.522507667541504,
      "logits/rejected": -2.5270426273345947,
      "logps/chosen": -63.30371856689453,
      "logps/rejected": -75.92526245117188,
      "loss": 0.7487,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": 0.5464689135551453,
      "rewards/margins": 0.3932306468486786,
      "rewards/rejected": 0.15323826670646667,
      "step": 50
    },
    {
      "epoch": 0.20905923344947736,
      "grad_norm": 345.88352506142644,
      "learning_rate": 8.965116279069767e-07,
      "logits/chosen": -2.493328094482422,
      "logits/rejected": -2.4874939918518066,
      "logps/chosen": -70.6145248413086,
      "logps/rejected": -66.24369049072266,
      "loss": 0.7394,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 2.2291111946105957,
      "rewards/margins": 0.3673287332057953,
      "rewards/rejected": 1.8617826700210571,
      "step": 60
    },
    {
      "epoch": 0.24390243902439024,
      "grad_norm": 461.9022420517148,
      "learning_rate": 8.790697674418603e-07,
      "logits/chosen": -2.5233232975006104,
      "logits/rejected": -2.518942356109619,
      "logps/chosen": -61.38080978393555,
      "logps/rejected": -66.21412658691406,
      "loss": 0.8029,
      "rewards/accuracies": 0.2874999940395355,
      "rewards/chosen": 2.4556429386138916,
      "rewards/margins": 0.5528993606567383,
      "rewards/rejected": 1.9027433395385742,
      "step": 70
    },
    {
      "epoch": 0.2787456445993031,
      "grad_norm": 445.0623855333619,
      "learning_rate": 8.616279069767441e-07,
      "logits/chosen": -2.464928388595581,
      "logits/rejected": -2.4558563232421875,
      "logps/chosen": -72.87749481201172,
      "logps/rejected": -75.74545288085938,
      "loss": 0.8699,
      "rewards/accuracies": 0.3062500059604645,
      "rewards/chosen": 2.397139310836792,
      "rewards/margins": 0.8753674626350403,
      "rewards/rejected": 1.521772027015686,
      "step": 80
    },
    {
      "epoch": 0.313588850174216,
      "grad_norm": 523.0685685145581,
      "learning_rate": 8.441860465116279e-07,
      "logits/chosen": -2.514057159423828,
      "logits/rejected": -2.529362201690674,
      "logps/chosen": -63.073158264160156,
      "logps/rejected": -67.64947509765625,
      "loss": 0.9667,
      "rewards/accuracies": 0.2874999940395355,
      "rewards/chosen": 2.5018441677093506,
      "rewards/margins": 0.38231611251831055,
      "rewards/rejected": 2.11952805519104,
      "step": 90
    },
    {
      "epoch": 0.34843205574912894,
      "grad_norm": 538.3596051073214,
      "learning_rate": 8.267441860465116e-07,
      "logits/chosen": -2.5051705837249756,
      "logits/rejected": -2.5038981437683105,
      "logps/chosen": -72.73633575439453,
      "logps/rejected": -79.80201721191406,
      "loss": 0.9734,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 2.179957389831543,
      "rewards/margins": 1.1052770614624023,
      "rewards/rejected": 1.0746803283691406,
      "step": 100
    },
    {
      "epoch": 0.34843205574912894,
      "eval_logits/chosen": -2.5853304862976074,
      "eval_logits/rejected": -2.569460868835449,
      "eval_logps/chosen": -72.08978271484375,
      "eval_logps/rejected": -79.57595825195312,
      "eval_loss": 0.9202933311462402,
      "eval_rewards/accuracies": 0.329365074634552,
      "eval_rewards/chosen": 2.065248489379883,
      "eval_rewards/margins": 0.730187714099884,
      "eval_rewards/rejected": 1.3350608348846436,
      "eval_runtime": 113.407,
      "eval_samples_per_second": 17.636,
      "eval_steps_per_second": 0.556,
      "step": 100
    },
    {
      "epoch": 0.3832752613240418,
      "grad_norm": 565.1538822918781,
      "learning_rate": 8.093023255813954e-07,
      "logits/chosen": -2.517366409301758,
      "logits/rejected": -2.486677646636963,
      "logps/chosen": -71.63627624511719,
      "logps/rejected": -61.792930603027344,
      "loss": 0.9796,
      "rewards/accuracies": 0.2562499940395355,
      "rewards/chosen": 1.3190182447433472,
      "rewards/margins": 0.27183371782302856,
      "rewards/rejected": 1.0471845865249634,
      "step": 110
    },
    {
      "epoch": 0.4181184668989547,
      "grad_norm": 415.3172615536407,
      "learning_rate": 7.91860465116279e-07,
      "logits/chosen": -2.5475993156433105,
      "logits/rejected": -2.518019437789917,
      "logps/chosen": -75.4908676147461,
      "logps/rejected": -65.90876770019531,
      "loss": 0.9023,
      "rewards/accuracies": 0.30000001192092896,
      "rewards/chosen": 1.6948095560073853,
      "rewards/margins": 1.0387893915176392,
      "rewards/rejected": 0.6560201644897461,
      "step": 120
    },
    {
      "epoch": 0.4529616724738676,
      "grad_norm": 654.650382698047,
      "learning_rate": 7.744186046511627e-07,
      "logits/chosen": -2.57270884513855,
      "logits/rejected": -2.5545814037323,
      "logps/chosen": -82.57270812988281,
      "logps/rejected": -87.82828521728516,
      "loss": 1.0166,
      "rewards/accuracies": 0.34375,
      "rewards/chosen": 1.1249371767044067,
      "rewards/margins": 1.3999314308166504,
      "rewards/rejected": -0.2749941945075989,
      "step": 130
    },
    {
      "epoch": 0.4878048780487805,
      "grad_norm": 392.83640221764773,
      "learning_rate": 7.569767441860465e-07,
      "logits/chosen": -2.4829936027526855,
      "logits/rejected": -2.474769353866577,
      "logps/chosen": -78.97613525390625,
      "logps/rejected": -70.12017059326172,
      "loss": 0.9314,
      "rewards/accuracies": 0.32499998807907104,
      "rewards/chosen": 1.8655914068222046,
      "rewards/margins": 1.3116776943206787,
      "rewards/rejected": 0.5539135932922363,
      "step": 140
    },
    {
      "epoch": 0.5226480836236934,
      "grad_norm": 496.0616334612732,
      "learning_rate": 7.395348837209303e-07,
      "logits/chosen": -2.5784010887145996,
      "logits/rejected": -2.5391345024108887,
      "logps/chosen": -77.13214111328125,
      "logps/rejected": -78.45569610595703,
      "loss": 1.0681,
      "rewards/accuracies": 0.28125,
      "rewards/chosen": 2.022892713546753,
      "rewards/margins": 1.0811107158660889,
      "rewards/rejected": 0.9417816400527954,
      "step": 150
    },
    {
      "epoch": 0.5574912891986062,
      "grad_norm": 572.3657252794802,
      "learning_rate": 7.220930232558139e-07,
      "logits/chosen": -2.579862117767334,
      "logits/rejected": -2.5984551906585693,
      "logps/chosen": -62.170875549316406,
      "logps/rejected": -70.7127914428711,
      "loss": 0.996,
      "rewards/accuracies": 0.28125,
      "rewards/chosen": 2.024418354034424,
      "rewards/margins": 0.9407089948654175,
      "rewards/rejected": 1.0837091207504272,
      "step": 160
    },
    {
      "epoch": 0.5923344947735192,
      "grad_norm": 568.8903494255211,
      "learning_rate": 7.046511627906976e-07,
      "logits/chosen": -2.5988354682922363,
      "logits/rejected": -2.5848004817962646,
      "logps/chosen": -66.76710510253906,
      "logps/rejected": -75.21239471435547,
      "loss": 0.9987,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 1.8922548294067383,
      "rewards/margins": 0.944841206073761,
      "rewards/rejected": 0.9474137425422668,
      "step": 170
    },
    {
      "epoch": 0.627177700348432,
      "grad_norm": 521.2606597836349,
      "learning_rate": 6.872093023255814e-07,
      "logits/chosen": -2.6132636070251465,
      "logits/rejected": -2.602483034133911,
      "logps/chosen": -89.0395736694336,
      "logps/rejected": -84.7410659790039,
      "loss": 1.0952,
      "rewards/accuracies": 0.3375000059604645,
      "rewards/chosen": 2.4231231212615967,
      "rewards/margins": 0.9014472961425781,
      "rewards/rejected": 1.5216760635375977,
      "step": 180
    },
    {
      "epoch": 0.662020905923345,
      "grad_norm": 416.32272877425527,
      "learning_rate": 6.697674418604651e-07,
      "logits/chosen": -2.6020960807800293,
      "logits/rejected": -2.594320297241211,
      "logps/chosen": -68.85299682617188,
      "logps/rejected": -79.95862579345703,
      "loss": 1.0472,
      "rewards/accuracies": 0.2874999940395355,
      "rewards/chosen": 1.8906943798065186,
      "rewards/margins": 0.5408729314804077,
      "rewards/rejected": 1.3498214483261108,
      "step": 190
    },
    {
      "epoch": 0.6968641114982579,
      "grad_norm": 736.2428171588583,
      "learning_rate": 6.523255813953487e-07,
      "logits/chosen": -2.6261801719665527,
      "logits/rejected": -2.6321043968200684,
      "logps/chosen": -86.841064453125,
      "logps/rejected": -90.11552429199219,
      "loss": 0.9883,
      "rewards/accuracies": 0.36250001192092896,
      "rewards/chosen": 3.2811522483825684,
      "rewards/margins": 1.6169086694717407,
      "rewards/rejected": 1.6642436981201172,
      "step": 200
    },
    {
      "epoch": 0.6968641114982579,
      "eval_logits/chosen": -2.5874733924865723,
      "eval_logits/rejected": -2.5707614421844482,
      "eval_logps/chosen": -71.6690673828125,
      "eval_logps/rejected": -79.74640655517578,
      "eval_loss": 1.0967082977294922,
      "eval_rewards/accuracies": 0.3373015820980072,
      "eval_rewards/chosen": 2.4270544052124023,
      "eval_rewards/margins": 1.2385817766189575,
      "eval_rewards/rejected": 1.1884726285934448,
      "eval_runtime": 113.2711,
      "eval_samples_per_second": 17.657,
      "eval_steps_per_second": 0.556,
      "step": 200
    },
    {
      "epoch": 0.7317073170731707,
      "grad_norm": 760.3151343554975,
      "learning_rate": 6.348837209302325e-07,
      "logits/chosen": -2.594675064086914,
      "logits/rejected": -2.570430278778076,
      "logps/chosen": -66.980224609375,
      "logps/rejected": -62.78594207763672,
      "loss": 1.0072,
      "rewards/accuracies": 0.3375000059604645,
      "rewards/chosen": 2.011408805847168,
      "rewards/margins": 1.4795125722885132,
      "rewards/rejected": 0.5318960547447205,
      "step": 210
    },
    {
      "epoch": 0.7665505226480837,
      "grad_norm": 481.77954463638264,
      "learning_rate": 6.174418604651163e-07,
      "logits/chosen": -2.645392894744873,
      "logits/rejected": -2.6270740032196045,
      "logps/chosen": -70.76057434082031,
      "logps/rejected": -69.80204772949219,
      "loss": 1.1057,
      "rewards/accuracies": 0.24375000596046448,
      "rewards/chosen": 2.319939136505127,
      "rewards/margins": 0.7193705439567566,
      "rewards/rejected": 1.6005685329437256,
      "step": 220
    },
    {
      "epoch": 0.8013937282229965,
      "grad_norm": 765.046306381631,
      "learning_rate": 6e-07,
      "logits/chosen": -2.6732935905456543,
      "logits/rejected": -2.65371036529541,
      "logps/chosen": -86.65088653564453,
      "logps/rejected": -87.35969543457031,
      "loss": 1.1788,
      "rewards/accuracies": 0.3375000059604645,
      "rewards/chosen": 2.8807787895202637,
      "rewards/margins": 2.20210599899292,
      "rewards/rejected": 0.6786726713180542,
      "step": 230
    },
    {
      "epoch": 0.8362369337979094,
      "grad_norm": 548.8932956260087,
      "learning_rate": 5.825581395348838e-07,
      "logits/chosen": -2.6739866733551025,
      "logits/rejected": -2.6417675018310547,
      "logps/chosen": -83.05066680908203,
      "logps/rejected": -77.82533264160156,
      "loss": 1.1372,
      "rewards/accuracies": 0.34375,
      "rewards/chosen": 3.1742072105407715,
      "rewards/margins": 1.1869542598724365,
      "rewards/rejected": 1.9872528314590454,
      "step": 240
    },
    {
      "epoch": 0.8710801393728222,
      "grad_norm": 616.7927885159357,
      "learning_rate": 5.651162790697674e-07,
      "logits/chosen": -2.671238899230957,
      "logits/rejected": -2.637321949005127,
      "logps/chosen": -92.645263671875,
      "logps/rejected": -88.57808685302734,
      "loss": 0.8737,
      "rewards/accuracies": 0.3499999940395355,
      "rewards/chosen": 2.4114432334899902,
      "rewards/margins": 1.2348735332489014,
      "rewards/rejected": 1.1765693426132202,
      "step": 250
    },
    {
      "epoch": 0.9059233449477352,
      "grad_norm": 478.21518650366113,
      "learning_rate": 5.476744186046511e-07,
      "logits/chosen": -2.555266857147217,
      "logits/rejected": -2.5690598487854004,
      "logps/chosen": -57.567100524902344,
      "logps/rejected": -65.19673919677734,
      "loss": 1.1295,
      "rewards/accuracies": 0.3062500059604645,
      "rewards/chosen": 1.2747440338134766,
      "rewards/margins": 0.9036104083061218,
      "rewards/rejected": 0.37113362550735474,
      "step": 260
    },
    {
      "epoch": 0.9407665505226481,
      "grad_norm": 730.4342654590606,
      "learning_rate": 5.302325581395349e-07,
      "logits/chosen": -2.6346664428710938,
      "logits/rejected": -2.633521556854248,
      "logps/chosen": -67.13279724121094,
      "logps/rejected": -82.27578735351562,
      "loss": 1.0712,
      "rewards/accuracies": 0.3062500059604645,
      "rewards/chosen": 1.5027388334274292,
      "rewards/margins": 1.282596468925476,
      "rewards/rejected": 0.2201424092054367,
      "step": 270
    },
    {
      "epoch": 0.975609756097561,
      "grad_norm": 595.5490958688126,
      "learning_rate": 5.127906976744186e-07,
      "logits/chosen": -2.5305819511413574,
      "logits/rejected": -2.510344982147217,
      "logps/chosen": -65.64894104003906,
      "logps/rejected": -70.33460998535156,
      "loss": 0.9263,
      "rewards/accuracies": 0.32499998807907104,
      "rewards/chosen": 1.8794810771942139,
      "rewards/margins": 1.4496572017669678,
      "rewards/rejected": 0.42982417345046997,
      "step": 280
    },
    {
      "epoch": 1.0104529616724738,
      "grad_norm": 23.979212075374758,
      "learning_rate": 4.953488372093023e-07,
      "logits/chosen": -2.573585033416748,
      "logits/rejected": -2.5460705757141113,
      "logps/chosen": -66.83503723144531,
      "logps/rejected": -65.80335998535156,
      "loss": 0.7697,
      "rewards/accuracies": 0.38749998807907104,
      "rewards/chosen": 4.500079154968262,
      "rewards/margins": 5.2650299072265625,
      "rewards/rejected": -0.7649505734443665,
      "step": 290
    },
    {
      "epoch": 1.0452961672473868,
      "grad_norm": 44.13210545620442,
      "learning_rate": 4.779069767441861e-07,
      "logits/chosen": -2.6247591972351074,
      "logits/rejected": -2.610288381576538,
      "logps/chosen": -56.94911575317383,
      "logps/rejected": -78.56959533691406,
      "loss": 0.4215,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": 8.255823135375977,
      "rewards/margins": 16.339366912841797,
      "rewards/rejected": -8.08354377746582,
      "step": 300
    },
    {
      "epoch": 1.0452961672473868,
      "eval_logits/chosen": -2.656022548675537,
      "eval_logits/rejected": -2.6403489112854004,
      "eval_logps/chosen": -70.90100860595703,
      "eval_logps/rejected": -79.04632568359375,
      "eval_loss": 1.1233958005905151,
      "eval_rewards/accuracies": 0.3313491940498352,
      "eval_rewards/chosen": 3.0875895023345947,
      "eval_rewards/margins": 1.2970443964004517,
      "eval_rewards/rejected": 1.790545105934143,
      "eval_runtime": 113.4658,
      "eval_samples_per_second": 17.626,
      "eval_steps_per_second": 0.555,
      "step": 300
    },
    {
      "epoch": 1.0801393728222997,
      "grad_norm": 40.17101348318783,
      "learning_rate": 4.604651162790698e-07,
      "logits/chosen": -2.611358165740967,
      "logits/rejected": -2.6116092205047607,
      "logps/chosen": -58.17817306518555,
      "logps/rejected": -85.24242401123047,
      "loss": 0.4163,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 8.134294509887695,
      "rewards/margins": 16.730432510375977,
      "rewards/rejected": -8.596138000488281,
      "step": 310
    },
    {
      "epoch": 1.1149825783972125,
      "grad_norm": 153.73635089362784,
      "learning_rate": 4.4302325581395346e-07,
      "logits/chosen": -2.647315740585327,
      "logits/rejected": -2.6344220638275146,
      "logps/chosen": -64.29966735839844,
      "logps/rejected": -88.07272338867188,
      "loss": 0.3886,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 8.3077974319458,
      "rewards/margins": 16.785139083862305,
      "rewards/rejected": -8.477343559265137,
      "step": 320
    },
    {
      "epoch": 1.1498257839721253,
      "grad_norm": 196.76703226808354,
      "learning_rate": 4.255813953488372e-07,
      "logits/chosen": -2.6384284496307373,
      "logits/rejected": -2.6142385005950928,
      "logps/chosen": -73.04707336425781,
      "logps/rejected": -88.73326873779297,
      "loss": 0.4236,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": 9.5239896774292,
      "rewards/margins": 18.215595245361328,
      "rewards/rejected": -8.691606521606445,
      "step": 330
    },
    {
      "epoch": 1.1846689895470384,
      "grad_norm": 16.30501354867415,
      "learning_rate": 4.081395348837209e-07,
      "logits/chosen": -2.6152892112731934,
      "logits/rejected": -2.617434024810791,
      "logps/chosen": -72.81196594238281,
      "logps/rejected": -107.9176254272461,
      "loss": 0.4009,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 9.404792785644531,
      "rewards/margins": 20.270700454711914,
      "rewards/rejected": -10.8659086227417,
      "step": 340
    },
    {
      "epoch": 1.2195121951219512,
      "grad_norm": 253.70590340563848,
      "learning_rate": 3.9069767441860464e-07,
      "logits/chosen": -2.627079486846924,
      "logits/rejected": -2.594177007675171,
      "logps/chosen": -57.3041877746582,
      "logps/rejected": -77.43646240234375,
      "loss": 0.4007,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 9.71183967590332,
      "rewards/margins": 19.823144912719727,
      "rewards/rejected": -10.111306190490723,
      "step": 350
    },
    {
      "epoch": 1.254355400696864,
      "grad_norm": 137.93033306808474,
      "learning_rate": 3.7325581395348837e-07,
      "logits/chosen": -2.603165626525879,
      "logits/rejected": -2.573925495147705,
      "logps/chosen": -62.46228790283203,
      "logps/rejected": -73.45440673828125,
      "loss": 0.374,
      "rewards/accuracies": 0.4437499940395355,
      "rewards/chosen": 8.073513984680176,
      "rewards/margins": 14.77312183380127,
      "rewards/rejected": -6.699607849121094,
      "step": 360
    },
    {
      "epoch": 1.289198606271777,
      "grad_norm": 2.3357390678870833,
      "learning_rate": 3.5581395348837204e-07,
      "logits/chosen": -2.562800407409668,
      "logits/rejected": -2.5790553092956543,
      "logps/chosen": -61.26555633544922,
      "logps/rejected": -86.61184692382812,
      "loss": 0.4449,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 8.428425788879395,
      "rewards/margins": 18.987430572509766,
      "rewards/rejected": -10.559003829956055,
      "step": 370
    },
    {
      "epoch": 1.32404181184669,
      "grad_norm": 240.07418332327254,
      "learning_rate": 3.383720930232558e-07,
      "logits/chosen": -2.55873966217041,
      "logits/rejected": -2.556434392929077,
      "logps/chosen": -77.422607421875,
      "logps/rejected": -101.52474212646484,
      "loss": 0.4144,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 8.128644943237305,
      "rewards/margins": 21.078929901123047,
      "rewards/rejected": -12.950288772583008,
      "step": 380
    },
    {
      "epoch": 1.3588850174216027,
      "grad_norm": 26.553648537118264,
      "learning_rate": 3.2093023255813955e-07,
      "logits/chosen": -2.6296944618225098,
      "logits/rejected": -2.6129908561706543,
      "logps/chosen": -56.90552520751953,
      "logps/rejected": -84.36430358886719,
      "loss": 0.378,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 8.677267074584961,
      "rewards/margins": 19.946393966674805,
      "rewards/rejected": -11.269128799438477,
      "step": 390
    },
    {
      "epoch": 1.3937282229965158,
      "grad_norm": 11.915044642787674,
      "learning_rate": 3.034883720930232e-07,
      "logits/chosen": -2.6192545890808105,
      "logits/rejected": -2.591175079345703,
      "logps/chosen": -77.75074768066406,
      "logps/rejected": -112.25565338134766,
      "loss": 0.393,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": 7.220816135406494,
      "rewards/margins": 19.133440017700195,
      "rewards/rejected": -11.912622451782227,
      "step": 400
    },
    {
      "epoch": 1.3937282229965158,
      "eval_logits/chosen": -2.625042676925659,
      "eval_logits/rejected": -2.609307050704956,
      "eval_logps/chosen": -74.64740753173828,
      "eval_logps/rejected": -83.32994842529297,
      "eval_loss": 1.223362684249878,
      "eval_rewards/accuracies": 0.32341268658638,
      "eval_rewards/chosen": -0.1343114674091339,
      "eval_rewards/margins": 1.759063720703125,
      "eval_rewards/rejected": -1.893375277519226,
      "eval_runtime": 113.3813,
      "eval_samples_per_second": 17.64,
      "eval_steps_per_second": 0.556,
      "step": 400
    },
    {
      "epoch": 1.4285714285714286,
      "grad_norm": 265.4416418102539,
      "learning_rate": 2.8604651162790695e-07,
      "logits/chosen": -2.6114742755889893,
      "logits/rejected": -2.601598024368286,
      "logps/chosen": -73.08380889892578,
      "logps/rejected": -94.53117370605469,
      "loss": 0.6924,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": 8.195378303527832,
      "rewards/margins": 20.368560791015625,
      "rewards/rejected": -12.17318344116211,
      "step": 410
    },
    {
      "epoch": 1.4634146341463414,
      "grad_norm": 110.44055155377453,
      "learning_rate": 2.6860465116279067e-07,
      "logits/chosen": -2.651491641998291,
      "logits/rejected": -2.65342378616333,
      "logps/chosen": -69.68745422363281,
      "logps/rejected": -97.7309341430664,
      "loss": 0.4257,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 5.508909702301025,
      "rewards/margins": 16.567991256713867,
      "rewards/rejected": -11.059081077575684,
      "step": 420
    },
    {
      "epoch": 1.4982578397212545,
      "grad_norm": 0.6603991710359594,
      "learning_rate": 2.511627906976744e-07,
      "logits/chosen": -2.606182336807251,
      "logits/rejected": -2.5925304889678955,
      "logps/chosen": -62.015174865722656,
      "logps/rejected": -85.03865814208984,
      "loss": 0.3861,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": 5.7165632247924805,
      "rewards/margins": 16.509204864501953,
      "rewards/rejected": -10.792640686035156,
      "step": 430
    },
    {
      "epoch": 1.533101045296167,
      "grad_norm": 171.49719572579374,
      "learning_rate": 2.3372093023255815e-07,
      "logits/chosen": -2.5812370777130127,
      "logits/rejected": -2.5880465507507324,
      "logps/chosen": -62.13433074951172,
      "logps/rejected": -84.38856506347656,
      "loss": 0.4158,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": 6.746127128601074,
      "rewards/margins": 16.110437393188477,
      "rewards/rejected": -9.364312171936035,
      "step": 440
    },
    {
      "epoch": 1.5679442508710801,
      "grad_norm": 28.499265540380463,
      "learning_rate": 2.1627906976744185e-07,
      "logits/chosen": -2.641430139541626,
      "logits/rejected": -2.6030945777893066,
      "logps/chosen": -79.95354461669922,
      "logps/rejected": -103.62571716308594,
      "loss": 0.4177,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": 9.364060401916504,
      "rewards/margins": 24.554065704345703,
      "rewards/rejected": -15.1900053024292,
      "step": 450
    },
    {
      "epoch": 1.6027874564459932,
      "grad_norm": 208.9071332592839,
      "learning_rate": 1.9883720930232558e-07,
      "logits/chosen": -2.6206653118133545,
      "logits/rejected": -2.597672939300537,
      "logps/chosen": -66.5615463256836,
      "logps/rejected": -93.40229797363281,
      "loss": 0.4043,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 8.271448135375977,
      "rewards/margins": 22.373382568359375,
      "rewards/rejected": -14.101933479309082,
      "step": 460
    },
    {
      "epoch": 1.6376306620209058,
      "grad_norm": 0.3844536104459883,
      "learning_rate": 1.8139534883720928e-07,
      "logits/chosen": -2.5973286628723145,
      "logits/rejected": -2.5908195972442627,
      "logps/chosen": -54.78232955932617,
      "logps/rejected": -81.273681640625,
      "loss": 0.4316,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 5.607256889343262,
      "rewards/margins": 13.865686416625977,
      "rewards/rejected": -8.258430480957031,
      "step": 470
    },
    {
      "epoch": 1.6724738675958188,
      "grad_norm": 150.7317542423224,
      "learning_rate": 1.63953488372093e-07,
      "logits/chosen": -2.599632501602173,
      "logits/rejected": -2.584341049194336,
      "logps/chosen": -49.672950744628906,
      "logps/rejected": -57.88109588623047,
      "loss": 0.4093,
      "rewards/accuracies": 0.35624998807907104,
      "rewards/chosen": 3.7741661071777344,
      "rewards/margins": 8.84811782836914,
      "rewards/rejected": -5.073951721191406,
      "step": 480
    },
    {
      "epoch": 1.7073170731707317,
      "grad_norm": 51.065123641891695,
      "learning_rate": 1.4651162790697676e-07,
      "logits/chosen": -2.571542978286743,
      "logits/rejected": -2.561433792114258,
      "logps/chosen": -66.06110382080078,
      "logps/rejected": -81.25625610351562,
      "loss": 0.5041,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": 6.158200740814209,
      "rewards/margins": 15.785115242004395,
      "rewards/rejected": -9.626913070678711,
      "step": 490
    },
    {
      "epoch": 1.7421602787456445,
      "grad_norm": 9.459943139553493,
      "learning_rate": 1.2906976744186046e-07,
      "logits/chosen": -2.496741771697998,
      "logits/rejected": -2.4906914234161377,
      "logps/chosen": -68.98286437988281,
      "logps/rejected": -96.44841003417969,
      "loss": 0.3986,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 6.282340049743652,
      "rewards/margins": 17.225933074951172,
      "rewards/rejected": -10.943593978881836,
      "step": 500
    },
    {
      "epoch": 1.7421602787456445,
      "eval_logits/chosen": -2.607004404067993,
      "eval_logits/rejected": -2.5908732414245605,
      "eval_logps/chosen": -74.26602935791016,
      "eval_logps/rejected": -83.2776107788086,
      "eval_loss": 1.2247282266616821,
      "eval_rewards/accuracies": 0.3214285671710968,
      "eval_rewards/chosen": 0.1936810314655304,
      "eval_rewards/margins": 2.0420379638671875,
      "eval_rewards/rejected": -1.848357081413269,
      "eval_runtime": 113.3979,
      "eval_samples_per_second": 17.637,
      "eval_steps_per_second": 0.556,
      "step": 500
    },
    {
      "epoch": 1.7770034843205575,
      "grad_norm": 22.381813019053016,
      "learning_rate": 1.1162790697674417e-07,
      "logits/chosen": -2.572063446044922,
      "logits/rejected": -2.551802635192871,
      "logps/chosen": -62.93878173828125,
      "logps/rejected": -80.62296295166016,
      "loss": 0.4109,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": 6.723170280456543,
      "rewards/margins": 16.216602325439453,
      "rewards/rejected": -9.49343204498291,
      "step": 510
    },
    {
      "epoch": 1.8118466898954704,
      "grad_norm": 34.84061860816185,
      "learning_rate": 9.418604651162791e-08,
      "logits/chosen": -2.5822184085845947,
      "logits/rejected": -2.5742483139038086,
      "logps/chosen": -66.30496978759766,
      "logps/rejected": -88.59693908691406,
      "loss": 0.3956,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": 6.850976467132568,
      "rewards/margins": 16.641326904296875,
      "rewards/rejected": -9.790349006652832,
      "step": 520
    },
    {
      "epoch": 1.8466898954703832,
      "grad_norm": 26.726116225430992,
      "learning_rate": 7.674418604651163e-08,
      "logits/chosen": -2.5606324672698975,
      "logits/rejected": -2.5552823543548584,
      "logps/chosen": -62.88715744018555,
      "logps/rejected": -84.60001373291016,
      "loss": 0.3962,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": 7.2661237716674805,
      "rewards/margins": 18.649456024169922,
      "rewards/rejected": -11.383334159851074,
      "step": 530
    },
    {
      "epoch": 1.8815331010452963,
      "grad_norm": 746.1912709837381,
      "learning_rate": 5.930232558139535e-08,
      "logits/chosen": -2.5763189792633057,
      "logits/rejected": -2.589749336242676,
      "logps/chosen": -54.900634765625,
      "logps/rejected": -89.78245544433594,
      "loss": 0.4804,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": 7.59392786026001,
      "rewards/margins": 21.733322143554688,
      "rewards/rejected": -14.139394760131836,
      "step": 540
    },
    {
      "epoch": 1.916376306620209,
      "grad_norm": 0.6251683362748027,
      "learning_rate": 4.1860465116279067e-08,
      "logits/chosen": -2.5647377967834473,
      "logits/rejected": -2.5423054695129395,
      "logps/chosen": -79.24058532714844,
      "logps/rejected": -106.3379898071289,
      "loss": 0.4013,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 9.971015930175781,
      "rewards/margins": 26.699609756469727,
      "rewards/rejected": -16.728595733642578,
      "step": 550
    },
    {
      "epoch": 1.951219512195122,
      "grad_norm": 252.84212758390947,
      "learning_rate": 2.441860465116279e-08,
      "logits/chosen": -2.560770034790039,
      "logits/rejected": -2.567601442337036,
      "logps/chosen": -57.37895584106445,
      "logps/rejected": -80.76757049560547,
      "loss": 0.4091,
      "rewards/accuracies": 0.4437499940395355,
      "rewards/chosen": 5.453306674957275,
      "rewards/margins": 14.458730697631836,
      "rewards/rejected": -9.005423545837402,
      "step": 560
    },
    {
      "epoch": 1.986062717770035,
      "grad_norm": 110.2396617228431,
      "learning_rate": 6.976744186046511e-09,
      "logits/chosen": -2.637852191925049,
      "logits/rejected": -2.608257293701172,
      "logps/chosen": -58.60048294067383,
      "logps/rejected": -71.56431579589844,
      "loss": 0.3934,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 5.686958312988281,
      "rewards/margins": 13.428431510925293,
      "rewards/rejected": -7.7414727210998535,
      "step": 570
    },
    {
      "epoch": 2.0,
      "step": 574,
      "total_flos": 0.0,
      "train_loss": 0.6794358088995106,
      "train_runtime": 6413.3344,
      "train_samples_per_second": 5.719,
      "train_steps_per_second": 0.09
    }
  ],
  "logging_steps": 10,
  "max_steps": 574,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}