{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 711,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.04,
      "grad_norm": 49.48526136809559,
      "learning_rate": 5.000000000000001e-07,
      "log_odds_chosen": 0.14267238974571228,
      "log_odds_ratio": -0.6910250782966614,
      "logits/chosen": -2.8734333515167236,
      "logits/rejected": -2.8792202472686768,
      "logps/chosen": -1.2169833183288574,
      "logps/rejected": -1.3137216567993164,
      "loss": 1.1778,
      "nll_loss": 1.1760419607162476,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.06084916740655899,
      "rewards/margins": 0.004836911801248789,
      "rewards/rejected": -0.06568608433008194,
      "step": 10
    },
    {
      "epoch": 0.08,
      "grad_norm": 13.42628442764115,
      "learning_rate": 1.0000000000000002e-06,
      "log_odds_chosen": 0.2992035746574402,
      "log_odds_ratio": -0.5965026617050171,
      "logits/chosen": -2.9965202808380127,
      "logits/rejected": -2.9929251670837402,
      "logps/chosen": -0.9654132723808289,
      "logps/rejected": -1.1591482162475586,
      "loss": 1.0,
      "nll_loss": 0.9463264346122742,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.04827066510915756,
      "rewards/margins": 0.009686745703220367,
      "rewards/rejected": -0.05795741081237793,
      "step": 20
    },
    {
      "epoch": 0.13,
      "grad_norm": 9.276480900326913,
      "learning_rate": 1.5e-06,
      "log_odds_chosen": 0.35957902669906616,
      "log_odds_ratio": -0.5845057368278503,
      "logits/chosen": -3.0192956924438477,
      "logits/rejected": -3.018825054168701,
      "logps/chosen": -0.8531354665756226,
      "logps/rejected": -1.0771838426589966,
      "loss": 0.9246,
      "nll_loss": 0.8811793327331543,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.04265677183866501,
      "rewards/margins": 0.011202414520084858,
      "rewards/rejected": -0.05385918542742729,
      "step": 30
    },
    {
      "epoch": 0.17,
      "grad_norm": 8.627451557441312,
      "learning_rate": 2.0000000000000003e-06,
      "log_odds_chosen": 0.25122153759002686,
      "log_odds_ratio": -0.6361113786697388,
      "logits/chosen": -2.915778160095215,
      "logits/rejected": -2.919457197189331,
      "logps/chosen": -0.8648379445075989,
      "logps/rejected": -0.9924612045288086,
      "loss": 0.974,
      "nll_loss": 0.9005156755447388,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.04324189946055412,
      "rewards/margins": 0.0063811675645411015,
      "rewards/rejected": -0.04962306469678879,
      "step": 40
    },
    {
      "epoch": 0.21,
      "grad_norm": 10.358595971591946,
      "learning_rate": 2.5e-06,
      "log_odds_chosen": 0.20706939697265625,
      "log_odds_ratio": -0.6479698419570923,
      "logits/chosen": -2.931480884552002,
      "logits/rejected": -2.941484212875366,
      "logps/chosen": -0.9367877244949341,
      "logps/rejected": -1.0564123392105103,
      "loss": 0.9112,
      "nll_loss": 0.9363681674003601,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.046839382499456406,
      "rewards/margins": 0.005981230176985264,
      "rewards/rejected": -0.052820611745119095,
      "step": 50
    },
    {
      "epoch": 0.25,
      "grad_norm": 10.184305920593003,
      "learning_rate": 3e-06,
      "log_odds_chosen": 0.274976909160614,
      "log_odds_ratio": -0.6372017860412598,
      "logits/chosen": -2.887624979019165,
      "logits/rejected": -2.902449369430542,
      "logps/chosen": -0.8582770228385925,
      "logps/rejected": -1.012649416923523,
      "loss": 0.9268,
      "nll_loss": 0.9307074546813965,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04291384667158127,
      "rewards/margins": 0.007718615233898163,
      "rewards/rejected": -0.05063246563076973,
      "step": 60
    },
    {
      "epoch": 0.3,
      "grad_norm": 10.247661529064816,
      "learning_rate": 3.5e-06,
      "log_odds_chosen": 0.29217422008514404,
      "log_odds_ratio": -0.6399118304252625,
      "logits/chosen": -2.9243764877319336,
      "logits/rejected": -2.9292612075805664,
      "logps/chosen": -0.8583773374557495,
      "logps/rejected": -1.0340784788131714,
      "loss": 0.9441,
      "nll_loss": 0.9250191450119019,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.042918868362903595,
      "rewards/margins": 0.008785055950284004,
      "rewards/rejected": -0.05170391872525215,
      "step": 70
    },
    {
      "epoch": 0.34,
      "grad_norm": 8.823827784258985,
      "learning_rate": 4.000000000000001e-06,
      "log_odds_chosen": 0.33998143672943115,
      "log_odds_ratio": -0.6087978482246399,
      "logits/chosen": -2.901061773300171,
      "logits/rejected": -2.909398078918457,
      "logps/chosen": -0.8597785234451294,
      "logps/rejected": -1.0702154636383057,
      "loss": 0.9265,
      "nll_loss": 0.883587658405304,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.04298892617225647,
      "rewards/margins": 0.010521845892071724,
      "rewards/rejected": -0.053510766476392746,
      "step": 80
    },
    {
      "epoch": 0.38,
      "grad_norm": 9.3837801640132,
      "learning_rate": 4.5e-06,
      "log_odds_chosen": 0.32301634550094604,
      "log_odds_ratio": -0.645144522190094,
      "logits/chosen": -2.9571635723114014,
      "logits/rejected": -2.9651668071746826,
      "logps/chosen": -0.8539797067642212,
      "logps/rejected": -1.0550405979156494,
      "loss": 0.9336,
      "nll_loss": 0.9078629612922668,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.04269898682832718,
      "rewards/margins": 0.010053041391074657,
      "rewards/rejected": -0.05275202915072441,
      "step": 90
    },
    {
      "epoch": 0.42,
      "grad_norm": 8.451957287235773,
      "learning_rate": 5e-06,
      "log_odds_chosen": 0.27936068177223206,
      "log_odds_ratio": -0.619978129863739,
      "logits/chosen": -2.9700396060943604,
      "logits/rejected": -2.9722213745117188,
      "logps/chosen": -0.892767608165741,
      "logps/rejected": -1.0630358457565308,
      "loss": 0.9536,
      "nll_loss": 0.9281327128410339,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.04463838413357735,
      "rewards/margins": 0.008513410575687885,
      "rewards/rejected": -0.05315179377794266,
      "step": 100
    },
    {
      "epoch": 0.46,
      "grad_norm": 10.228379998303796,
      "learning_rate": 4.767312946227961e-06,
      "log_odds_chosen": 0.18160709738731384,
      "log_odds_ratio": -0.7056410908699036,
      "logits/chosen": -2.980069637298584,
      "logits/rejected": -2.9743785858154297,
      "logps/chosen": -0.9338927268981934,
      "logps/rejected": -1.0694698095321655,
      "loss": 0.949,
      "nll_loss": 0.9810144305229187,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.046694640070199966,
      "rewards/margins": 0.006778855808079243,
      "rewards/rejected": -0.053473494946956635,
      "step": 110
    },
    {
      "epoch": 0.51,
      "grad_norm": 15.501078885228303,
      "learning_rate": 4.564354645876385e-06,
      "log_odds_chosen": 0.3752610683441162,
      "log_odds_ratio": -0.5817548036575317,
      "logits/chosen": -2.8941051959991455,
      "logits/rejected": -2.898369073867798,
      "logps/chosen": -0.8418710827827454,
      "logps/rejected": -1.0447475910186768,
      "loss": 0.9253,
      "nll_loss": 0.908086895942688,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.04209355264902115,
      "rewards/margins": 0.010143828578293324,
      "rewards/rejected": -0.0522373802959919,
      "step": 120
    },
    {
      "epoch": 0.55,
      "grad_norm": 8.598131028345922,
      "learning_rate": 4.385290096535147e-06,
      "log_odds_chosen": 0.2913978397846222,
      "log_odds_ratio": -0.6210304498672485,
      "logits/chosen": -2.8913044929504395,
      "logits/rejected": -2.900383234024048,
      "logps/chosen": -0.9610312581062317,
      "logps/rejected": -1.1445128917694092,
      "loss": 0.979,
      "nll_loss": 1.0131185054779053,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.048051562160253525,
      "rewards/margins": 0.00917408149689436,
      "rewards/rejected": -0.05722564458847046,
      "step": 130
    },
    {
      "epoch": 0.59,
      "grad_norm": 8.665950821616725,
      "learning_rate": 4.2257712736425835e-06,
      "log_odds_chosen": 0.31699851155281067,
      "log_odds_ratio": -0.6054994463920593,
      "logits/chosen": -2.866349697113037,
      "logits/rejected": -2.872480869293213,
      "logps/chosen": -0.8127169609069824,
      "logps/rejected": -1.005972146987915,
      "loss": 0.9264,
      "nll_loss": 0.8691350817680359,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.040635846555233,
      "rewards/margins": 0.009662764146924019,
      "rewards/rejected": -0.05029861256480217,
      "step": 140
    },
    {
      "epoch": 0.63,
      "grad_norm": 7.610661708462692,
      "learning_rate": 4.082482904638631e-06,
      "log_odds_chosen": 0.26200634241104126,
      "log_odds_ratio": -0.635285496711731,
      "logits/chosen": -2.813951253890991,
      "logits/rejected": -2.831160306930542,
      "logps/chosen": -0.8999479413032532,
      "logps/rejected": -1.0443265438079834,
      "loss": 0.9319,
      "nll_loss": 0.9246827363967896,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.04499739781022072,
      "rewards/margins": 0.0072189331986010075,
      "rewards/rejected": -0.05221632868051529,
      "step": 150
    },
    {
      "epoch": 0.68,
      "grad_norm": 7.926717564356759,
      "learning_rate": 3.952847075210474e-06,
      "log_odds_chosen": 0.4248151183128357,
      "log_odds_ratio": -0.5570122599601746,
      "logits/chosen": -2.8344883918762207,
      "logits/rejected": -2.8441624641418457,
      "logps/chosen": -0.8424416780471802,
      "logps/rejected": -1.1050217151641846,
      "loss": 0.9371,
      "nll_loss": 0.8941017985343933,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.04212208464741707,
      "rewards/margins": 0.01312900148332119,
      "rewards/rejected": -0.05525108426809311,
      "step": 160
    },
    {
      "epoch": 0.72,
      "grad_norm": 8.768416551022222,
      "learning_rate": 3.834824944236852e-06,
      "log_odds_chosen": 0.280124694108963,
      "log_odds_ratio": -0.6224346160888672,
      "logits/chosen": -2.8287386894226074,
      "logits/rejected": -2.83266019821167,
      "logps/chosen": -0.8510664105415344,
      "logps/rejected": -1.0343472957611084,
      "loss": 0.9355,
      "nll_loss": 0.8820816278457642,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.04255332052707672,
      "rewards/margins": 0.009164043702185154,
      "rewards/rejected": -0.0517173632979393,
      "step": 170
    },
    {
      "epoch": 0.76,
      "grad_norm": 7.46076847939469,
      "learning_rate": 3.72677996249965e-06,
      "log_odds_chosen": 0.17997188866138458,
      "log_odds_ratio": -0.6717725992202759,
      "logits/chosen": -2.8428828716278076,
      "logits/rejected": -2.8347556591033936,
      "logps/chosen": -0.9121964573860168,
      "logps/rejected": -0.9958919286727905,
      "loss": 0.946,
      "nll_loss": 0.9758418202400208,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04560982435941696,
      "rewards/margins": 0.004184772726148367,
      "rewards/rejected": -0.04979459568858147,
      "step": 180
    },
    {
      "epoch": 0.8,
      "grad_norm": 7.680949233012938,
      "learning_rate": 3.6273812505500587e-06,
      "log_odds_chosen": 0.35985735058784485,
      "log_odds_ratio": -0.5938600301742554,
      "logits/chosen": -2.859172821044922,
      "logits/rejected": -2.8654944896698,
      "logps/chosen": -0.847649097442627,
      "logps/rejected": -1.0266352891921997,
      "loss": 0.9285,
      "nll_loss": 0.8830558657646179,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.04238245636224747,
      "rewards/margins": 0.008949300274252892,
      "rewards/rejected": -0.05133175849914551,
      "step": 190
    },
    {
      "epoch": 0.84,
      "grad_norm": 9.078983143397938,
      "learning_rate": 3.5355339059327378e-06,
      "log_odds_chosen": 0.4591992497444153,
      "log_odds_ratio": -0.5660054087638855,
      "logits/chosen": -2.9353525638580322,
      "logits/rejected": -2.9381651878356934,
      "logps/chosen": -0.9107562303543091,
      "logps/rejected": -1.1601167917251587,
      "loss": 0.943,
      "nll_loss": 1.0148041248321533,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.045537810772657394,
      "rewards/margins": 0.012468031607568264,
      "rewards/rejected": -0.05800584703683853,
      "step": 200
    },
    {
      "epoch": 0.89,
      "grad_norm": 8.44546881476589,
      "learning_rate": 3.450327796711771e-06,
      "log_odds_chosen": 0.29461508989334106,
      "log_odds_ratio": -0.6495056748390198,
      "logits/chosen": -2.90964412689209,
      "logits/rejected": -2.9097769260406494,
      "logps/chosen": -0.8788071870803833,
      "logps/rejected": -1.0280598402023315,
      "loss": 0.9482,
      "nll_loss": 0.9459528923034668,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.043940357863903046,
      "rewards/margins": 0.007462637964636087,
      "rewards/rejected": -0.051402997225522995,
      "step": 210
    },
    {
      "epoch": 0.93,
      "grad_norm": 8.193251853529034,
      "learning_rate": 3.3709993123162106e-06,
      "log_odds_chosen": 0.4032968580722809,
      "log_odds_ratio": -0.568686842918396,
      "logits/chosen": -2.9146342277526855,
      "logits/rejected": -2.9228734970092773,
      "logps/chosen": -0.8045447468757629,
      "logps/rejected": -1.03438401222229,
      "loss": 0.9203,
      "nll_loss": 0.8637560606002808,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.04022723808884621,
      "rewards/margins": 0.011491960845887661,
      "rewards/rejected": -0.05171919986605644,
      "step": 220
    },
    {
      "epoch": 0.97,
      "grad_norm": 8.324361077294697,
      "learning_rate": 3.296902366978936e-06,
      "log_odds_chosen": 0.36563342809677124,
      "log_odds_ratio": -0.5865170955657959,
      "logits/chosen": -2.9637176990509033,
      "logits/rejected": -2.9688830375671387,
      "logps/chosen": -0.8463845252990723,
      "logps/rejected": -1.061700701713562,
      "loss": 0.9112,
      "nll_loss": 0.9032902717590332,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.04231923073530197,
      "rewards/margins": 0.010765810497105122,
      "rewards/rejected": -0.05308504030108452,
      "step": 230
    },
    {
      "epoch": 1.01,
      "grad_norm": 9.173894745062437,
      "learning_rate": 3.2274861218395142e-06,
      "log_odds_chosen": 0.5646923184394836,
      "log_odds_ratio": -0.550338625907898,
      "logits/chosen": -2.9580464363098145,
      "logits/rejected": -2.9572625160217285,
      "logps/chosen": -0.7748560309410095,
      "logps/rejected": -1.0799105167388916,
      "loss": 0.8136,
      "nll_loss": 0.8241481781005859,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.038742803037166595,
      "rewards/margins": 0.015252722427248955,
      "rewards/rejected": -0.0539955273270607,
      "step": 240
    },
    {
      "epoch": 1.05,
      "grad_norm": 9.099732749463024,
      "learning_rate": 3.1622776601683796e-06,
      "log_odds_chosen": 1.4823018312454224,
      "log_odds_ratio": -0.2659846544265747,
      "logits/chosen": -2.9596664905548096,
      "logits/rejected": -2.9568517208099365,
      "logps/chosen": -0.47842296957969666,
      "logps/rejected": -1.2101627588272095,
      "loss": 0.5233,
      "nll_loss": 0.5237408876419067,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.023921148851513863,
      "rewards/margins": 0.03658699244260788,
      "rewards/rejected": -0.06050813943147659,
      "step": 250
    },
    {
      "epoch": 1.1,
      "grad_norm": 9.707924912075288,
      "learning_rate": 3.1008683647302113e-06,
      "log_odds_chosen": 1.2276315689086914,
      "log_odds_ratio": -0.337185263633728,
      "logits/chosen": -2.9387476444244385,
      "logits/rejected": -2.947472333908081,
      "logps/chosen": -0.507585346698761,
      "logps/rejected": -1.0944607257843018,
      "loss": 0.5243,
      "nll_loss": 0.5015721321105957,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.02537926472723484,
      "rewards/margins": 0.02934376522898674,
      "rewards/rejected": -0.05472303181886673,
      "step": 260
    },
    {
      "epoch": 1.14,
      "grad_norm": 9.72287079567634,
      "learning_rate": 3.0429030972509227e-06,
      "log_odds_chosen": 1.6538162231445312,
      "log_odds_ratio": -0.23319308459758759,
      "logits/chosen": -3.0101027488708496,
      "logits/rejected": -3.0216686725616455,
      "logps/chosen": -0.4229302406311035,
      "logps/rejected": -1.2452623844146729,
      "loss": 0.4944,
      "nll_loss": 0.4760821759700775,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.021146511659026146,
      "rewards/margins": 0.04111660644412041,
      "rewards/rejected": -0.0622631199657917,
      "step": 270
    },
    {
      "epoch": 1.18,
      "grad_norm": 8.91121542892108,
      "learning_rate": 2.988071523335984e-06,
      "log_odds_chosen": 1.5969759225845337,
      "log_odds_ratio": -0.24297292530536652,
      "logits/chosen": -2.9810919761657715,
      "logits/rejected": -2.984095335006714,
      "logps/chosen": -0.4323214888572693,
      "logps/rejected": -1.1894398927688599,
      "loss": 0.4899,
      "nll_loss": 0.4894355237483978,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.021616075187921524,
      "rewards/margins": 0.03785592317581177,
      "rewards/rejected": -0.05947200208902359,
      "step": 280
    },
    {
      "epoch": 1.22,
      "grad_norm": 9.172297878492826,
      "learning_rate": 2.9361010975735177e-06,
      "log_odds_chosen": 1.5589730739593506,
      "log_odds_ratio": -0.2480790615081787,
      "logits/chosen": -2.936096668243408,
      "logits/rejected": -2.9368419647216797,
      "logps/chosen": -0.4527331292629242,
      "logps/rejected": -1.1788005828857422,
      "loss": 0.489,
      "nll_loss": 0.48634010553359985,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.0226366575807333,
      "rewards/margins": 0.03630336746573448,
      "rewards/rejected": -0.05894003063440323,
      "step": 290
    },
    {
      "epoch": 1.27,
      "grad_norm": 9.341330783727024,
      "learning_rate": 2.8867513459481293e-06,
      "log_odds_chosen": 1.5156631469726562,
      "log_odds_ratio": -0.2381705492734909,
      "logits/chosen": -2.949042320251465,
      "logits/rejected": -2.934840202331543,
      "logps/chosen": -0.43807506561279297,
      "logps/rejected": -1.1634423732757568,
      "loss": 0.5031,
      "nll_loss": 0.4857659935951233,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.0219037514179945,
      "rewards/margins": 0.03626837208867073,
      "rewards/rejected": -0.05817211791872978,
      "step": 300
    },
    {
      "epoch": 1.31,
      "grad_norm": 9.51833987789547,
      "learning_rate": 2.839809171235324e-06,
      "log_odds_chosen": 1.4176826477050781,
      "log_odds_ratio": -0.281759113073349,
      "logits/chosen": -2.953523874282837,
      "logits/rejected": -2.954210042953491,
      "logps/chosen": -0.47912734746932983,
      "logps/rejected": -1.2172271013259888,
      "loss": 0.5149,
      "nll_loss": 0.4759243428707123,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.02395637147128582,
      "rewards/margins": 0.03690498322248459,
      "rewards/rejected": -0.060861360281705856,
      "step": 310
    },
    {
      "epoch": 1.35,
      "grad_norm": 9.733927381742513,
      "learning_rate": 2.7950849718747376e-06,
      "log_odds_chosen": 1.33830726146698,
      "log_odds_ratio": -0.2881857454776764,
      "logits/chosen": -3.0379624366760254,
      "logits/rejected": -3.0340828895568848,
      "logps/chosen": -0.5138579607009888,
      "logps/rejected": -1.1911945343017578,
      "loss": 0.5378,
      "nll_loss": 0.5391749739646912,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.025692898780107498,
      "rewards/margins": 0.03386683017015457,
      "rewards/rejected": -0.05955972522497177,
      "step": 320
    },
    {
      "epoch": 1.39,
      "grad_norm": 8.811470336283001,
      "learning_rate": 2.752409412815902e-06,
      "log_odds_chosen": 1.485058307647705,
      "log_odds_ratio": -0.26495322585105896,
      "logits/chosen": -2.9619758129119873,
      "logits/rejected": -2.954977035522461,
      "logps/chosen": -0.440424382686615,
      "logps/rejected": -1.131378412246704,
      "loss": 0.4785,
      "nll_loss": 0.4669272303581238,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.0220212172716856,
      "rewards/margins": 0.034547705203294754,
      "rewards/rejected": -0.056568920612335205,
      "step": 330
    },
    {
      "epoch": 1.43,
      "grad_norm": 8.015826159950372,
      "learning_rate": 2.711630722733202e-06,
      "log_odds_chosen": 1.5126841068267822,
      "log_odds_ratio": -0.25113973021507263,
      "logits/chosen": -2.979628801345825,
      "logits/rejected": -2.9764034748077393,
      "logps/chosen": -0.42568206787109375,
      "logps/rejected": -1.1240874528884888,
      "loss": 0.5006,
      "nll_loss": 0.46929699182510376,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.021284103393554688,
      "rewards/margins": 0.03492026776075363,
      "rewards/rejected": -0.05620437115430832,
      "step": 340
    },
    {
      "epoch": 1.48,
      "grad_norm": 8.45163306773323,
      "learning_rate": 2.6726124191242444e-06,
      "log_odds_chosen": 1.3055660724639893,
      "log_odds_ratio": -0.3173283338546753,
      "logits/chosen": -2.961458683013916,
      "logits/rejected": -2.9598679542541504,
      "logps/chosen": -0.5269508957862854,
      "logps/rejected": -1.1968458890914917,
      "loss": 0.4854,
      "nll_loss": 0.5489671230316162,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.02634754404425621,
      "rewards/margins": 0.033494751900434494,
      "rewards/rejected": -0.059842299669981,
      "step": 350
    },
    {
      "epoch": 1.52,
      "grad_norm": 8.682519620016398,
      "learning_rate": 2.6352313834736496e-06,
      "log_odds_chosen": 1.644923210144043,
      "log_odds_ratio": -0.23544283211231232,
      "logits/chosen": -2.930689573287964,
      "logits/rejected": -2.928385019302368,
      "logps/chosen": -0.4198189675807953,
      "logps/rejected": -1.2138855457305908,
      "loss": 0.5002,
      "nll_loss": 0.45420289039611816,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.020990949124097824,
      "rewards/margins": 0.03970332816243172,
      "rewards/rejected": -0.06069428473711014,
      "step": 360
    },
    {
      "epoch": 1.56,
      "grad_norm": 9.273291108083583,
      "learning_rate": 2.599376224550182e-06,
      "log_odds_chosen": 1.5730557441711426,
      "log_odds_ratio": -0.2532302439212799,
      "logits/chosen": -2.9642691612243652,
      "logits/rejected": -2.9628634452819824,
      "logps/chosen": -0.4353133738040924,
      "logps/rejected": -1.1966689825057983,
      "loss": 0.5056,
      "nll_loss": 0.48967617750167847,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.02176566980779171,
      "rewards/margins": 0.0380677804350853,
      "rewards/rejected": -0.05983344465494156,
      "step": 370
    },
    {
      "epoch": 1.6,
      "grad_norm": 8.84508067219799,
      "learning_rate": 2.564945880212886e-06,
      "log_odds_chosen": 1.6267578601837158,
      "log_odds_ratio": -0.24944674968719482,
      "logits/chosen": -2.934312582015991,
      "logits/rejected": -2.9307126998901367,
      "logps/chosen": -0.44339004158973694,
      "logps/rejected": -1.2226251363754272,
      "loss": 0.4946,
      "nll_loss": 0.44669169187545776,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.022169504314661026,
      "rewards/margins": 0.038961753249168396,
      "rewards/rejected": -0.06113125756382942,
      "step": 380
    },
    {
      "epoch": 1.65,
      "grad_norm": 7.973489385408531,
      "learning_rate": 2.5318484177091667e-06,
      "log_odds_chosen": 1.4293986558914185,
      "log_odds_ratio": -0.26934128999710083,
      "logits/chosen": -2.927151918411255,
      "logits/rejected": -2.9325613975524902,
      "logps/chosen": -0.48316580057144165,
      "logps/rejected": -1.1627215147018433,
      "loss": 0.5043,
      "nll_loss": 0.5102673172950745,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.024158289656043053,
      "rewards/margins": 0.03397778421640396,
      "rewards/rejected": -0.05813607573509216,
      "step": 390
    },
    {
      "epoch": 1.69,
      "grad_norm": 8.824910828626702,
      "learning_rate": 2.5e-06,
      "log_odds_chosen": 1.6508582830429077,
      "log_odds_ratio": -0.24078328907489777,
      "logits/chosen": -2.9304981231689453,
      "logits/rejected": -2.9384522438049316,
      "logps/chosen": -0.4645829200744629,
      "logps/rejected": -1.2513707876205444,
      "loss": 0.528,
      "nll_loss": 0.5313082933425903,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.023229148238897324,
      "rewards/margins": 0.03933940455317497,
      "rewards/rejected": -0.0625685527920723,
      "step": 400
    },
    {
      "epoch": 1.73,
      "grad_norm": 9.94593557333145,
      "learning_rate": 2.4693239916239746e-06,
      "log_odds_chosen": 1.5409417152404785,
      "log_odds_ratio": -0.26054129004478455,
      "logits/chosen": -2.8831980228424072,
      "logits/rejected": -2.881943702697754,
      "logps/chosen": -0.42643943428993225,
      "logps/rejected": -1.1698906421661377,
      "loss": 0.5037,
      "nll_loss": 0.464557409286499,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.021321970969438553,
      "rewards/margins": 0.03717255964875221,
      "rewards/rejected": -0.05849452689290047,
      "step": 410
    },
    {
      "epoch": 1.77,
      "grad_norm": 8.290147408251284,
      "learning_rate": 2.4397501823713327e-06,
      "log_odds_chosen": 1.3443034887313843,
      "log_odds_ratio": -0.3111021816730499,
      "logits/chosen": -2.867410182952881,
      "logits/rejected": -2.867697238922119,
      "logps/chosen": -0.5130647420883179,
      "logps/rejected": -1.2247389554977417,
      "loss": 0.4997,
      "nll_loss": 0.5347863435745239,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.025653237476944923,
      "rewards/margins": 0.03558371216058731,
      "rewards/rejected": -0.06123695522546768,
      "step": 420
    },
    {
      "epoch": 1.81,
      "grad_norm": 10.095422116087253,
      "learning_rate": 2.411214110852061e-06,
      "log_odds_chosen": 1.538722276687622,
      "log_odds_ratio": -0.25320082902908325,
      "logits/chosen": -2.9051260948181152,
      "logits/rejected": -2.907191038131714,
      "logps/chosen": -0.49453073740005493,
      "logps/rejected": -1.2816953659057617,
      "loss": 0.5557,
      "nll_loss": 0.5553343892097473,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.024726541712880135,
      "rewards/margins": 0.0393582321703434,
      "rewards/rejected": -0.06408476829528809,
      "step": 430
    },
    {
      "epoch": 1.86,
      "grad_norm": 9.482272989808482,
      "learning_rate": 2.3836564731139807e-06,
      "log_odds_chosen": 1.701960802078247,
      "log_odds_ratio": -0.23092034459114075,
      "logits/chosen": -2.88254976272583,
      "logits/rejected": -2.880296230316162,
      "logps/chosen": -0.41986674070358276,
      "logps/rejected": -1.1859886646270752,
      "loss": 0.5171,
      "nll_loss": 0.4521302282810211,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.020993337035179138,
      "rewards/margins": 0.0383060947060585,
      "rewards/rejected": -0.05929943174123764,
      "step": 440
    },
    {
      "epoch": 1.9,
      "grad_norm": 10.287479798932107,
      "learning_rate": 2.357022603955159e-06,
      "log_odds_chosen": 1.5226378440856934,
      "log_odds_ratio": -0.26804596185684204,
      "logits/chosen": -2.9180657863616943,
      "logits/rejected": -2.9176905155181885,
      "logps/chosen": -0.4231870174407959,
      "logps/rejected": -1.1624265909194946,
      "loss": 0.4775,
      "nll_loss": 0.42117613554000854,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.021159350872039795,
      "rewards/margins": 0.036961980164051056,
      "rewards/rejected": -0.05812133103609085,
      "step": 450
    },
    {
      "epoch": 1.94,
      "grad_norm": 8.629730433025097,
      "learning_rate": 2.3312620206007847e-06,
      "log_odds_chosen": 1.3161282539367676,
      "log_odds_ratio": -0.3122040629386902,
      "logits/chosen": -2.9203555583953857,
      "logits/rejected": -2.9158670902252197,
      "logps/chosen": -0.46862277388572693,
      "logps/rejected": -1.085384726524353,
      "loss": 0.5357,
      "nll_loss": 0.4903482496738434,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.023431137204170227,
      "rewards/margins": 0.030838098376989365,
      "rewards/rejected": -0.05426923185586929,
      "step": 460
    },
    {
      "epoch": 1.98,
      "grad_norm": 9.462108983797318,
      "learning_rate": 2.3063280200722128e-06,
      "log_odds_chosen": 1.5001277923583984,
      "log_odds_ratio": -0.2745407521724701,
      "logits/chosen": -2.918792724609375,
      "logits/rejected": -2.920560598373413,
      "logps/chosen": -0.45306238532066345,
      "logps/rejected": -1.1488702297210693,
      "loss": 0.5047,
      "nll_loss": 0.46945127844810486,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.02265312150120735,
      "rewards/margins": 0.034790389239788055,
      "rewards/rejected": -0.05744351074099541,
      "step": 470
    },
    {
      "epoch": 2.03,
      "grad_norm": 12.033704149037684,
      "learning_rate": 2.2821773229381924e-06,
      "log_odds_chosen": 2.368060350418091,
      "log_odds_ratio": -0.13723622262477875,
      "logits/chosen": -2.896536111831665,
      "logits/rejected": -2.9012863636016846,
      "logps/chosen": -0.26962071657180786,
      "logps/rejected": -1.277259349822998,
      "loss": 0.2775,
      "nll_loss": 0.2636214792728424,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.013481035828590393,
      "rewards/margins": 0.05038193613290787,
      "rewards/rejected": -0.06386296451091766,
      "step": 480
    },
    {
      "epoch": 2.07,
      "grad_norm": 8.31587372487098,
      "learning_rate": 2.2587697572631284e-06,
      "log_odds_chosen": 3.0361766815185547,
      "log_odds_ratio": -0.0762939602136612,
      "logits/chosen": -2.872748374938965,
      "logits/rejected": -2.8818652629852295,
      "logps/chosen": -0.17086732387542725,
      "logps/rejected": -1.341355562210083,
      "loss": 0.1837,
      "nll_loss": 0.18102502822875977,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.008543366566300392,
      "rewards/margins": 0.05852441117167473,
      "rewards/rejected": -0.06706777215003967,
      "step": 490
    },
    {
      "epoch": 2.11,
      "grad_norm": 9.109087690728854,
      "learning_rate": 2.23606797749979e-06,
      "log_odds_chosen": 3.1204092502593994,
      "log_odds_ratio": -0.0752432718873024,
      "logits/chosen": -2.842435121536255,
      "logits/rejected": -2.851132869720459,
      "logps/chosen": -0.1684246063232422,
      "logps/rejected": -1.4159977436065674,
      "loss": 0.1865,
      "nll_loss": 0.1788669377565384,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.008421231061220169,
      "rewards/margins": 0.0623786523938179,
      "rewards/rejected": -0.07079987227916718,
      "step": 500
    },
    {
      "epoch": 2.15,
      "grad_norm": 11.342602042204765,
      "learning_rate": 2.2140372138502386e-06,
      "log_odds_chosen": 3.1781134605407715,
      "log_odds_ratio": -0.06865429133176804,
      "logits/chosen": -2.842808485031128,
      "logits/rejected": -2.8529012203216553,
      "logps/chosen": -0.15066179633140564,
      "logps/rejected": -1.4150397777557373,
      "loss": 0.1774,
      "nll_loss": 0.15856973826885223,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.007533090654760599,
      "rewards/margins": 0.06321889907121658,
      "rewards/rejected": -0.07075198739767075,
      "step": 510
    },
    {
      "epoch": 2.19,
      "grad_norm": 9.015217844997702,
      "learning_rate": 2.1926450482675734e-06,
      "log_odds_chosen": 3.1590893268585205,
      "log_odds_ratio": -0.05701795965433121,
      "logits/chosen": -2.8566677570343018,
      "logits/rejected": -2.8613815307617188,
      "logps/chosen": -0.17371872067451477,
      "logps/rejected": -1.5091571807861328,
      "loss": 0.1875,
      "nll_loss": 0.1826210618019104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.008685936219990253,
      "rewards/margins": 0.06677192449569702,
      "rewards/rejected": -0.0754578709602356,
      "step": 520
    },
    {
      "epoch": 2.24,
      "grad_norm": 7.705666021658087,
      "learning_rate": 2.1718612138153473e-06,
      "log_odds_chosen": 3.129737615585327,
      "log_odds_ratio": -0.07283337414264679,
      "logits/chosen": -2.857102632522583,
      "logits/rejected": -2.8588953018188477,
      "logps/chosen": -0.16752207279205322,
      "logps/rejected": -1.4430062770843506,
      "loss": 0.1864,
      "nll_loss": 0.18135443329811096,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.008376102894544601,
      "rewards/margins": 0.0637742131948471,
      "rewards/rejected": -0.072150319814682,
      "step": 530
    },
    {
      "epoch": 2.28,
      "grad_norm": 10.453820153776633,
      "learning_rate": 2.151657414559676e-06,
      "log_odds_chosen": 3.152796983718872,
      "log_odds_ratio": -0.07412304729223251,
      "logits/chosen": -2.8558595180511475,
      "logits/rejected": -2.8634510040283203,
      "logps/chosen": -0.1711394190788269,
      "logps/rejected": -1.419811725616455,
      "loss": 0.1867,
      "nll_loss": 0.18491069972515106,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.0085569703951478,
      "rewards/margins": 0.06243361160159111,
      "rewards/rejected": -0.07099058479070663,
      "step": 540
    },
    {
      "epoch": 2.32,
      "grad_norm": 9.218290542920707,
      "learning_rate": 2.132007163556104e-06,
      "log_odds_chosen": 2.85337233543396,
      "log_odds_ratio": -0.10947513580322266,
      "logits/chosen": -2.851966142654419,
      "logits/rejected": -2.8540444374084473,
      "logps/chosen": -0.19084951281547546,
      "logps/rejected": -1.3400590419769287,
      "loss": 0.1955,
      "nll_loss": 0.19958652555942535,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.009542474523186684,
      "rewards/margins": 0.057460468262434006,
      "rewards/rejected": -0.06700295209884644,
      "step": 550
    },
    {
      "epoch": 2.36,
      "grad_norm": 10.317365217804,
      "learning_rate": 2.1128856368212917e-06,
      "log_odds_chosen": 2.9810779094696045,
      "log_odds_ratio": -0.09573061764240265,
      "logits/chosen": -2.842881441116333,
      "logits/rejected": -2.8447251319885254,
      "logps/chosen": -0.19754521548748016,
      "logps/rejected": -1.4727751016616821,
      "loss": 0.1741,
      "nll_loss": 0.20112493634223938,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.009877260774374008,
      "rewards/margins": 0.06376149505376816,
      "rewards/rejected": -0.07363875210285187,
      "step": 560
    },
    {
      "epoch": 2.41,
      "grad_norm": 8.85810509927659,
      "learning_rate": 2.0942695414584777e-06,
      "log_odds_chosen": 3.2707130908966064,
      "log_odds_ratio": -0.05988941714167595,
      "logits/chosen": -2.8249504566192627,
      "logits/rejected": -2.835737705230713,
      "logps/chosen": -0.1614939272403717,
      "logps/rejected": -1.4856724739074707,
      "loss": 0.188,
      "nll_loss": 0.17816588282585144,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.008074694313108921,
      "rewards/margins": 0.06620893627405167,
      "rewards/rejected": -0.07428362965583801,
      "step": 570
    },
    {
      "epoch": 2.45,
      "grad_norm": 8.66523647376059,
      "learning_rate": 2.0761369963434992e-06,
      "log_odds_chosen": 2.9631845951080322,
      "log_odds_ratio": -0.08499319851398468,
      "logits/chosen": -2.785752058029175,
      "logits/rejected": -2.796713352203369,
      "logps/chosen": -0.18658441305160522,
      "logps/rejected": -1.3940823078155518,
      "loss": 0.1968,
      "nll_loss": 0.1999494731426239,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.009329221211373806,
      "rewards/margins": 0.060374896973371506,
      "rewards/rejected": -0.06970412284135818,
      "step": 580
    },
    {
      "epoch": 2.49,
      "grad_norm": 7.663377813440287,
      "learning_rate": 2.058467423981546e-06,
      "log_odds_chosen": 3.1809768676757812,
      "log_odds_ratio": -0.0660170167684555,
      "logits/chosen": -2.7790160179138184,
      "logits/rejected": -2.7909960746765137,
      "logps/chosen": -0.16522589325904846,
      "logps/rejected": -1.5188618898391724,
      "loss": 0.1828,
      "nll_loss": 0.18212279677391052,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.008261295035481453,
      "rewards/margins": 0.06768180429935455,
      "rewards/rejected": -0.07594309747219086,
      "step": 590
    },
    {
      "epoch": 2.53,
      "grad_norm": 9.43229140775389,
      "learning_rate": 2.0412414523193154e-06,
      "log_odds_chosen": 3.2196316719055176,
      "log_odds_ratio": -0.05672148987650871,
      "logits/chosen": -2.8662352561950684,
      "logits/rejected": -2.872044086456299,
      "logps/chosen": -0.16352586448192596,
      "logps/rejected": -1.5207058191299438,
      "loss": 0.1882,
      "nll_loss": 0.1710083782672882,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.008176293224096298,
      "rewards/margins": 0.06785898655653,
      "rewards/rejected": -0.0760352835059166,
      "step": 600
    },
    {
      "epoch": 2.57,
      "grad_norm": 9.251985707632262,
      "learning_rate": 2.0244408254472904e-06,
      "log_odds_chosen": 3.181028366088867,
      "log_odds_ratio": -0.06368444114923477,
      "logits/chosen": -2.8114702701568604,
      "logits/rejected": -2.827327013015747,
      "logps/chosen": -0.16818083822727203,
      "logps/rejected": -1.5173419713974,
      "loss": 0.1928,
      "nll_loss": 0.19379279017448425,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.008409040980041027,
      "rewards/margins": 0.06745805591344833,
      "rewards/rejected": -0.07586710155010223,
      "step": 610
    },
    {
      "epoch": 2.62,
      "grad_norm": 8.85560770094724,
      "learning_rate": 2.0080483222562476e-06,
      "log_odds_chosen": 3.0775837898254395,
      "log_odds_ratio": -0.08850983530282974,
      "logits/chosen": -2.8276491165161133,
      "logits/rejected": -2.8322935104370117,
      "logps/chosen": -0.17984047532081604,
      "logps/rejected": -1.4078468084335327,
      "loss": 0.1874,
      "nll_loss": 0.18593069911003113,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.008992023766040802,
      "rewards/margins": 0.061400316655635834,
      "rewards/rejected": -0.07039234042167664,
      "step": 620
    },
    {
      "epoch": 2.66,
      "grad_norm": 9.092588797512901,
      "learning_rate": 1.9920476822239895e-06,
      "log_odds_chosen": 3.452129364013672,
      "log_odds_ratio": -0.048345282673835754,
      "logits/chosen": -2.828294277191162,
      "logits/rejected": -2.8251712322235107,
      "logps/chosen": -0.15338510274887085,
      "logps/rejected": -1.581225872039795,
      "loss": 0.1829,
      "nll_loss": 0.18267366290092468,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0076692551374435425,
      "rewards/margins": 0.07139204442501068,
      "rewards/rejected": -0.07906129956245422,
      "step": 630
    },
    {
      "epoch": 2.7,
      "grad_norm": 8.85624863375054,
      "learning_rate": 1.976423537605237e-06,
      "log_odds_chosen": 3.381225109100342,
      "log_odds_ratio": -0.0466490238904953,
      "logits/chosen": -2.8369762897491455,
      "logits/rejected": -2.8300728797912598,
      "logps/chosen": -0.13137118518352509,
      "logps/rejected": -1.4386910200119019,
      "loss": 0.1705,
      "nll_loss": 0.1446365863084793,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.006568559445440769,
      "rewards/margins": 0.0653659924864769,
      "rewards/rejected": -0.07193455845117569,
      "step": 640
    },
    {
      "epoch": 2.74,
      "grad_norm": 9.126901201726705,
      "learning_rate": 1.961161351381841e-06,
      "log_odds_chosen": 3.2348713874816895,
      "log_odds_ratio": -0.059200920164585114,
      "logits/chosen": -2.8560492992401123,
      "logits/rejected": -2.8655595779418945,
      "logps/chosen": -0.15543992817401886,
      "logps/rejected": -1.4683492183685303,
      "loss": 0.1718,
      "nll_loss": 0.17798814177513123,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.007771995849907398,
      "rewards/margins": 0.06564547121524811,
      "rewards/rejected": -0.07341746240854263,
      "step": 650
    },
    {
      "epoch": 2.78,
      "grad_norm": 9.459348581011165,
      "learning_rate": 1.9462473604038077e-06,
      "log_odds_chosen": 3.150841236114502,
      "log_odds_ratio": -0.08523111045360565,
      "logits/chosen": -2.883835792541504,
      "logits/rejected": -2.880549669265747,
      "logps/chosen": -0.16044911742210388,
      "logps/rejected": -1.4550909996032715,
      "loss": 0.1827,
      "nll_loss": 0.1803114414215088,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.008022455498576164,
      "rewards/margins": 0.06473208963871002,
      "rewards/rejected": -0.07275454699993134,
      "step": 660
    },
    {
      "epoch": 2.83,
      "grad_norm": 8.641861937738083,
      "learning_rate": 1.9316685232156397e-06,
      "log_odds_chosen": 3.220848798751831,
      "log_odds_ratio": -0.06002454087138176,
      "logits/chosen": -2.8484086990356445,
      "logits/rejected": -2.8489136695861816,
      "logps/chosen": -0.15638573467731476,
      "logps/rejected": -1.4806524515151978,
      "loss": 0.1855,
      "nll_loss": 0.177657350897789,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.007819287478923798,
      "rewards/margins": 0.06621333211660385,
      "rewards/rejected": -0.07403261959552765,
      "step": 670
    },
    {
      "epoch": 2.87,
      "grad_norm": 9.590977394670045,
      "learning_rate": 1.917412472118426e-06,
      "log_odds_chosen": 3.190727949142456,
      "log_odds_ratio": -0.06804339587688446,
      "logits/chosen": -2.8769946098327637,
      "logits/rejected": -2.8714823722839355,
      "logps/chosen": -0.16566260159015656,
      "logps/rejected": -1.4919040203094482,
      "loss": 0.1888,
      "nll_loss": 0.1725473403930664,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.008283129893243313,
      "rewards/margins": 0.06631207466125488,
      "rewards/rejected": -0.07459519803524017,
      "step": 680
    },
    {
      "epoch": 2.91,
      "grad_norm": 8.352762820112602,
      "learning_rate": 1.9034674690672024e-06,
      "log_odds_chosen": 3.2079219818115234,
      "log_odds_ratio": -0.08390762656927109,
      "logits/chosen": -2.8269476890563965,
      "logits/rejected": -2.837005376815796,
      "logps/chosen": -0.18829426169395447,
      "logps/rejected": -1.5218613147735596,
      "loss": 0.1972,
      "nll_loss": 0.18950611352920532,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.009414712898433208,
      "rewards/margins": 0.06667836010456085,
      "rewards/rejected": -0.07609307020902634,
      "step": 690
    },
    {
      "epoch": 2.95,
      "grad_norm": 7.962144706780938,
      "learning_rate": 1.8898223650461362e-06,
      "log_odds_chosen": 3.216644763946533,
      "log_odds_ratio": -0.060684747993946075,
      "logits/chosen": -2.856433868408203,
      "logits/rejected": -2.8612186908721924,
      "logps/chosen": -0.16995587944984436,
      "logps/rejected": -1.478044867515564,
      "loss": 0.1852,
      "nll_loss": 0.18319742381572723,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.008497794158756733,
      "rewards/margins": 0.06540445238351822,
      "rewards/rejected": -0.07390224188566208,
      "step": 700
    },
    {
      "epoch": 3.0,
      "grad_norm": 8.826772322191102,
      "learning_rate": 1.876466562602004e-06,
      "log_odds_chosen": 3.2584176063537598,
      "log_odds_ratio": -0.05594608932733536,
      "logits/chosen": -2.808353900909424,
      "logits/rejected": -2.8018672466278076,
      "logps/chosen": -0.1637522280216217,
      "logps/rejected": -1.5413471460342407,
      "loss": 0.1864,
      "nll_loss": 0.16807685792446136,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.008187612518668175,
      "rewards/margins": 0.06887974590063095,
      "rewards/rejected": -0.07706736028194427,
      "step": 710
    },
    {
      "epoch": 3.0,
      "step": 711,
      "total_flos": 0.0,
      "train_loss": 0.5470885709871219,
      "train_runtime": 7181.4173,
      "train_samples_per_second": 3.159,
      "train_steps_per_second": 0.099
    }
  ],
  "logging_steps": 10,
  "max_steps": 711,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}