{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9995320542817033,
  "eval_steps": 500,
  "global_step": 1068,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.009358914365933552,
      "grad_norm": 35.0,
      "learning_rate": 4.672897196261682e-08,
      "log_odds_chosen": 0.028124790638685226,
      "log_odds_ratio": -0.7122673988342285,
      "logits/chosen": -2.177924633026123,
      "logits/rejected": -2.1597542762756348,
      "logps/chosen": -0.49789491295814514,
      "logps/rejected": -0.5033541321754456,
      "loss": 1.2,
      "nll_loss": 1.1897943019866943,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 10
    },
    {
      "epoch": 0.018717828731867104,
      "grad_norm": 31.25,
      "learning_rate": 9.345794392523364e-08,
      "log_odds_chosen": 0.10793106257915497,
      "log_odds_ratio": -0.6836065649986267,
      "logits/chosen": -2.2072064876556396,
      "logits/rejected": -2.1955819129943848,
      "logps/chosen": -0.509109377861023,
      "logps/rejected": -0.5578223466873169,
      "loss": 1.2711,
      "nll_loss": 1.2594499588012695,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 20
    },
    {
      "epoch": 0.028076743097800654,
      "grad_norm": 27.5,
      "learning_rate": 1.4018691588785045e-07,
      "log_odds_chosen": -0.06382028758525848,
      "log_odds_ratio": -0.7615261077880859,
      "logits/chosen": -2.1590800285339355,
      "logits/rejected": -2.1389636993408203,
      "logps/chosen": -0.5088254809379578,
      "logps/rejected": -0.48718294501304626,
      "loss": 1.2219,
      "nll_loss": 1.2155386209487915,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 30
    },
    {
      "epoch": 0.03743565746373421,
      "grad_norm": 32.5,
      "learning_rate": 1.8691588785046729e-07,
      "log_odds_chosen": 0.027226638048887253,
      "log_odds_ratio": -0.7158316969871521,
      "logits/chosen": -2.2030715942382812,
      "logits/rejected": -2.182612419128418,
      "logps/chosen": -0.47975221276283264,
      "logps/rejected": -0.4883168339729309,
      "loss": 1.2153,
      "nll_loss": 1.2491893768310547,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 40
    },
    {
      "epoch": 0.04679457182966776,
      "grad_norm": 25.0,
      "learning_rate": 2.336448598130841e-07,
      "log_odds_chosen": 0.07931121438741684,
      "log_odds_ratio": -0.7042826414108276,
      "logits/chosen": -2.1056649684906006,
      "logits/rejected": -2.101743459701538,
      "logps/chosen": -0.4837135672569275,
      "logps/rejected": -0.524016261100769,
      "loss": 1.1557,
      "nll_loss": 1.152883529663086,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 50
    },
    {
      "epoch": 0.05615348619560131,
      "grad_norm": 28.625,
      "learning_rate": 2.803738317757009e-07,
      "log_odds_chosen": 0.07345812022686005,
      "log_odds_ratio": -0.6917258501052856,
      "logits/chosen": -2.18230938911438,
      "logits/rejected": -2.1790032386779785,
      "logps/chosen": -0.4614621698856354,
      "logps/rejected": -0.4911852777004242,
      "loss": 1.203,
      "nll_loss": 1.1553771495819092,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 60
    },
    {
      "epoch": 0.06551240056153486,
      "grad_norm": 24.25,
      "learning_rate": 3.271028037383177e-07,
      "log_odds_chosen": -0.0254877507686615,
      "log_odds_ratio": -0.7420052886009216,
      "logits/chosen": -2.2310657501220703,
      "logits/rejected": -2.2102017402648926,
      "logps/chosen": -0.4837246537208557,
      "logps/rejected": -0.48194044828414917,
      "loss": 1.227,
      "nll_loss": 1.2863795757293701,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 70
    },
    {
      "epoch": 0.07487131492746842,
      "grad_norm": 23.875,
      "learning_rate": 3.7383177570093457e-07,
      "log_odds_chosen": 0.10875405371189117,
      "log_odds_ratio": -0.691827654838562,
      "logits/chosen": -2.2158102989196777,
      "logits/rejected": -2.198007345199585,
      "logps/chosen": -0.48060980439186096,
      "logps/rejected": -0.5459263324737549,
      "loss": 1.113,
      "nll_loss": 1.0872571468353271,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 80
    },
    {
      "epoch": 0.08423022929340196,
      "grad_norm": 19.75,
      "learning_rate": 4.205607476635514e-07,
      "log_odds_chosen": -0.004326423164457083,
      "log_odds_ratio": -0.7243191003799438,
      "logits/chosen": -2.2511162757873535,
      "logits/rejected": -2.2321133613586426,
      "logps/chosen": -0.49657949805259705,
      "logps/rejected": -0.49786829948425293,
      "loss": 1.1546,
      "nll_loss": 1.1654746532440186,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 90
    },
    {
      "epoch": 0.09358914365933552,
      "grad_norm": 19.375,
      "learning_rate": 4.672897196261682e-07,
      "log_odds_chosen": 0.0023641125299036503,
      "log_odds_ratio": -0.7194215059280396,
      "logits/chosen": -2.1953084468841553,
      "logits/rejected": -2.188227891921997,
      "logps/chosen": -0.4719774127006531,
      "logps/rejected": -0.46587830781936646,
      "loss": 1.109,
      "nll_loss": 1.142874836921692,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 100
    },
    {
      "epoch": 0.10294805802526907,
      "grad_norm": 14.8125,
      "learning_rate": 4.999879772999679e-07,
      "log_odds_chosen": 0.05830005556344986,
      "log_odds_ratio": -0.6932491660118103,
      "logits/chosen": -2.1755123138427734,
      "logits/rejected": -2.1791396141052246,
      "logps/chosen": -0.4585028290748596,
      "logps/rejected": -0.48959556221961975,
      "loss": 1.0518,
      "nll_loss": 1.0515038967132568,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 110
    },
    {
      "epoch": 0.11230697239120262,
      "grad_norm": 13.5625,
      "learning_rate": 4.997742725777528e-07,
      "log_odds_chosen": 0.05845099687576294,
      "log_odds_ratio": -0.692961573600769,
      "logits/chosen": -2.2289061546325684,
      "logits/rejected": -2.2106642723083496,
      "logps/chosen": -0.47383299469947815,
      "logps/rejected": -0.49634575843811035,
      "loss": 1.005,
      "nll_loss": 0.98981773853302,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 120
    },
    {
      "epoch": 0.12166588675713617,
      "grad_norm": 12.5,
      "learning_rate": 4.992936596071119e-07,
      "log_odds_chosen": 0.08409204334020615,
      "log_odds_ratio": -0.6914924383163452,
      "logits/chosen": -2.2284302711486816,
      "logits/rejected": -2.2103445529937744,
      "logps/chosen": -0.46478062868118286,
      "logps/rejected": -0.5053035020828247,
      "loss": 1.038,
      "nll_loss": 1.0772876739501953,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 130
    },
    {
      "epoch": 0.13102480112306972,
      "grad_norm": 12.0625,
      "learning_rate": 4.985466519700276e-07,
      "log_odds_chosen": 0.05864203721284866,
      "log_odds_ratio": -0.702600359916687,
      "logits/chosen": -2.2445855140686035,
      "logits/rejected": -2.23516845703125,
      "logps/chosen": -0.47693437337875366,
      "logps/rejected": -0.5074238777160645,
      "loss": 1.0134,
      "nll_loss": 1.022524118423462,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 140
    },
    {
      "epoch": 0.14038371548900327,
      "grad_norm": 12.625,
      "learning_rate": 4.975340479172484e-07,
      "log_odds_chosen": 0.02999536693096161,
      "log_odds_ratio": -0.7136459350585938,
      "logits/chosen": -2.2367188930511475,
      "logits/rejected": -2.206267833709717,
      "logps/chosen": -0.43298593163490295,
      "logps/rejected": -0.44292062520980835,
      "loss": 0.9101,
      "nll_loss": 0.905490517616272,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 150
    },
    {
      "epoch": 0.14974262985493683,
      "grad_norm": 13.1875,
      "learning_rate": 4.962569295152789e-07,
      "log_odds_chosen": 0.11916179955005646,
      "log_odds_ratio": -0.6726263165473938,
      "logits/chosen": -2.1630289554595947,
      "logits/rejected": -2.166447401046753,
      "logps/chosen": -0.46223917603492737,
      "logps/rejected": -0.49929705262184143,
      "loss": 1.0266,
      "nll_loss": 1.0065381526947021,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 160
    },
    {
      "epoch": 0.1591015442208704,
      "grad_norm": 12.3125,
      "learning_rate": 4.947166614900862e-07,
      "log_odds_chosen": 0.04568071290850639,
      "log_odds_ratio": -0.7053205966949463,
      "logits/chosen": -2.2215309143066406,
      "logits/rejected": -2.2159583568573,
      "logps/chosen": -0.46625766158103943,
      "logps/rejected": -0.4844442903995514,
      "loss": 0.9139,
      "nll_loss": 0.9563388824462891,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 170
    },
    {
      "epoch": 0.16846045858680392,
      "grad_norm": 12.625,
      "learning_rate": 4.929148897687566e-07,
      "log_odds_chosen": 0.20167502760887146,
      "log_odds_ratio": -0.688310980796814,
      "logits/chosen": -2.220001697540283,
      "logits/rejected": -2.2004828453063965,
      "logps/chosen": -0.4466385841369629,
      "logps/rejected": -0.5849505662918091,
      "loss": 0.9175,
      "nll_loss": 0.9583885073661804,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 180
    },
    {
      "epoch": 0.17781937295273748,
      "grad_norm": 11.0,
      "learning_rate": 4.908535397206616e-07,
      "log_odds_chosen": 0.12753407657146454,
      "log_odds_ratio": -0.6645318269729614,
      "logits/chosen": -2.1886050701141357,
      "logits/rejected": -2.1912922859191895,
      "logps/chosen": -0.4209999442100525,
      "logps/rejected": -0.46723008155822754,
      "loss": 0.9702,
      "nll_loss": 0.9464343786239624,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 190
    },
    {
      "epoch": 0.18717828731867103,
      "grad_norm": 11.125,
      "learning_rate": 4.885348141000122e-07,
      "log_odds_chosen": 0.10208719968795776,
      "log_odds_ratio": -0.6818111538887024,
      "logits/chosen": -2.190870761871338,
      "logits/rejected": -2.1749870777130127,
      "logps/chosen": -0.4555496275424957,
      "logps/rejected": -0.5055020451545715,
      "loss": 0.9374,
      "nll_loss": 0.9525081515312195,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 200
    },
    {
      "epoch": 0.1965372016846046,
      "grad_norm": 11.25,
      "learning_rate": 4.859611906920007e-07,
      "log_odds_chosen": 0.17131951451301575,
      "log_odds_ratio": -0.6447469592094421,
      "logits/chosen": -2.181173324584961,
      "logits/rejected": -2.1704306602478027,
      "logps/chosen": -0.4280853271484375,
      "logps/rejected": -0.49025410413742065,
      "loss": 0.9279,
      "nll_loss": 0.9838709831237793,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 210
    },
    {
      "epoch": 0.20589611605053815,
      "grad_norm": 11.9375,
      "learning_rate": 4.831354196650446e-07,
      "log_odds_chosen": 0.12637579441070557,
      "log_odds_ratio": -0.6647494435310364,
      "logits/chosen": -2.209960460662842,
      "logits/rejected": -2.190532922744751,
      "logps/chosen": -0.4298935532569885,
      "logps/rejected": -0.4736618399620056,
      "loss": 0.929,
      "nll_loss": 0.947419285774231,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 220
    },
    {
      "epoch": 0.21525503041647168,
      "grad_norm": 11.5,
      "learning_rate": 4.800605206319624e-07,
      "log_odds_chosen": 0.17757382988929749,
      "log_odds_ratio": -0.6481081247329712,
      "logits/chosen": -2.1775827407836914,
      "logits/rejected": -2.1693122386932373,
      "logps/chosen": -0.3929872512817383,
      "logps/rejected": -0.4595734477043152,
      "loss": 0.8857,
      "nll_loss": 0.8631205558776855,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 230
    },
    {
      "epoch": 0.22461394478240523,
      "grad_norm": 12.1875,
      "learning_rate": 4.767397794232225e-07,
      "log_odds_chosen": 0.11623436212539673,
      "log_odds_ratio": -0.6670342683792114,
      "logits/chosen": -2.2236437797546387,
      "logits/rejected": -2.198996067047119,
      "logps/chosen": -0.39739638566970825,
      "logps/rejected": -0.43490853905677795,
      "loss": 0.8506,
      "nll_loss": 0.8460358381271362,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 240
    },
    {
      "epoch": 0.2339728591483388,
      "grad_norm": 10.8125,
      "learning_rate": 4.731767445757111e-07,
      "log_odds_chosen": 0.1510472446680069,
      "log_odds_ratio": -0.6724820137023926,
      "logits/chosen": -2.1905274391174316,
      "logits/rejected": -2.170478343963623,
      "logps/chosen": -0.3974972665309906,
      "logps/rejected": -0.46447643637657166,
      "loss": 1.0051,
      "nll_loss": 0.9697535634040833,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 250
    },
    {
      "epoch": 0.24333177351427235,
      "grad_norm": 10.9375,
      "learning_rate": 4.6937522354077397e-07,
      "log_odds_chosen": 0.034748904407024384,
      "log_odds_ratio": -0.709030270576477,
      "logits/chosen": -2.2467408180236816,
      "logits/rejected": -2.23775315284729,
      "logps/chosen": -0.41722503304481506,
      "logps/rejected": -0.4315374791622162,
      "loss": 0.9569,
      "nll_loss": 0.9505971670150757,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 260
    },
    {
      "epoch": 0.2526906878802059,
      "grad_norm": 11.6875,
      "learning_rate": 4.6533927861558166e-07,
      "log_odds_chosen": 0.15775738656520844,
      "log_odds_ratio": -0.6476392149925232,
      "logits/chosen": -2.219115734100342,
      "logits/rejected": -2.200603485107422,
      "logps/chosen": -0.4007194936275482,
      "logps/rejected": -0.45418375730514526,
      "loss": 0.9836,
      "nll_loss": 0.9320660829544067,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 270
    },
    {
      "epoch": 0.26204960224613943,
      "grad_norm": 13.5,
      "learning_rate": 4.6107322260216787e-07,
      "log_odds_chosen": 0.15421156585216522,
      "log_odds_ratio": -0.6568454504013062,
      "logits/chosen": -2.1702048778533936,
      "logits/rejected": -2.15659499168396,
      "logps/chosen": -0.38430947065353394,
      "logps/rejected": -0.44216400384902954,
      "loss": 0.8788,
      "nll_loss": 0.8656591176986694,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 280
    },
    {
      "epoch": 0.271408516612073,
      "grad_norm": 12.4375,
      "learning_rate": 4.565816141987782e-07,
      "log_odds_chosen": 0.24107065796852112,
      "log_odds_ratio": -0.6283782720565796,
      "logits/chosen": -2.1911845207214355,
      "logits/rejected": -2.170578718185425,
      "logps/chosen": -0.38485488295555115,
      "logps/rejected": -0.4777398109436035,
      "loss": 0.9194,
      "nll_loss": 0.9205155372619629,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 290
    },
    {
      "epoch": 0.28076743097800655,
      "grad_norm": 12.125,
      "learning_rate": 4.518692531284555e-07,
      "log_odds_chosen": 0.11865685135126114,
      "log_odds_ratio": -0.6765984296798706,
      "logits/chosen": -2.1895639896392822,
      "logits/rejected": -2.1931369304656982,
      "logps/chosen": -0.4192424714565277,
      "logps/rejected": -0.46702760457992554,
      "loss": 0.9343,
      "nll_loss": 0.9279173612594604,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 300
    },
    {
      "epoch": 0.2901263453439401,
      "grad_norm": 12.0625,
      "learning_rate": 4.469411750100657e-07,
      "log_odds_chosen": 0.1528710126876831,
      "log_odds_ratio": -0.6519570350646973,
      "logits/chosen": -2.1230368614196777,
      "logits/rejected": -2.111417293548584,
      "logps/chosen": -0.41170787811279297,
      "logps/rejected": -0.4662221372127533,
      "loss": 0.9539,
      "nll_loss": 0.9637632369995117,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 310
    },
    {
      "epoch": 0.29948525970987366,
      "grad_norm": 12.0,
      "learning_rate": 4.418026459772465e-07,
      "log_odds_chosen": 0.24052071571350098,
      "log_odds_ratio": -0.6307070851325989,
      "logits/chosen": -2.1706483364105225,
      "logits/rejected": -2.166989803314209,
      "logps/chosen": -0.3876957893371582,
      "logps/rejected": -0.4977281093597412,
      "loss": 0.9242,
      "nll_loss": 0.8865762948989868,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 320
    },
    {
      "epoch": 0.3088441740758072,
      "grad_norm": 10.125,
      "learning_rate": 4.3645915705102875e-07,
      "log_odds_chosen": 0.12348359823226929,
      "log_odds_ratio": -0.6693438291549683,
      "logits/chosen": -2.2137680053710938,
      "logits/rejected": -2.221555709838867,
      "logps/chosen": -0.41634154319763184,
      "logps/rejected": -0.46320948004722595,
      "loss": 0.9491,
      "nll_loss": 0.9716747403144836,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 330
    },
    {
      "epoch": 0.3182030884417408,
      "grad_norm": 10.125,
      "learning_rate": 4.3091641827214255e-07,
      "log_odds_chosen": 0.18769888579845428,
      "log_odds_ratio": -0.6529528498649597,
      "logits/chosen": -2.2090022563934326,
      "logits/rejected": -2.2146294116973877,
      "logps/chosen": -0.3794373869895935,
      "logps/rejected": -0.45459240674972534,
      "loss": 0.865,
      "nll_loss": 0.8704261779785156,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 340
    },
    {
      "epoch": 0.32756200280767434,
      "grad_norm": 11.125,
      "learning_rate": 4.251803525992806e-07,
      "log_odds_chosen": 0.11162462085485458,
      "log_odds_ratio": -0.6723114252090454,
      "logits/chosen": -2.1960465908050537,
      "logits/rejected": -2.1942174434661865,
      "logps/chosen": -0.4046311378479004,
      "logps/rejected": -0.452523797750473,
      "loss": 0.8887,
      "nll_loss": 0.9013251066207886,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 350
    },
    {
      "epoch": 0.33692091717360784,
      "grad_norm": 11.5625,
      "learning_rate": 4.192570895798369e-07,
      "log_odds_chosen": 0.18315447866916656,
      "log_odds_ratio": -0.6423401832580566,
      "logits/chosen": -2.184838056564331,
      "logits/rejected": -2.1885063648223877,
      "logps/chosen": -0.39569249749183655,
      "logps/rejected": -0.4560003876686096,
      "loss": 0.9208,
      "nll_loss": 0.8708004951477051,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 360
    },
    {
      "epoch": 0.3462798315395414,
      "grad_norm": 12.1875,
      "learning_rate": 4.1315295879988603e-07,
      "log_odds_chosen": 0.13799132406711578,
      "log_odds_ratio": -0.6605507731437683,
      "logits/chosen": -2.1906182765960693,
      "logits/rejected": -2.183380603790283,
      "logps/chosen": -0.3925025463104248,
      "logps/rejected": -0.4469067454338074,
      "loss": 0.8882,
      "nll_loss": 0.89342200756073,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 370
    },
    {
      "epoch": 0.35563874590547495,
      "grad_norm": 12.4375,
      "learning_rate": 4.06874483120401e-07,
      "log_odds_chosen": 0.18038353323936462,
      "log_odds_ratio": -0.6446860432624817,
      "logits/chosen": -2.227576494216919,
      "logits/rejected": -2.209685802459717,
      "logps/chosen": -0.3906407356262207,
      "logps/rejected": -0.45715445280075073,
      "loss": 0.9863,
      "nll_loss": 0.9728641510009766,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 380
    },
    {
      "epoch": 0.3649976602714085,
      "grad_norm": 12.6875,
      "learning_rate": 4.00428371706938e-07,
      "log_odds_chosen": 0.16080859303474426,
      "log_odds_ratio": -0.6434266567230225,
      "logits/chosen": -2.237511396408081,
      "logits/rejected": -2.213977336883545,
      "logps/chosen": -0.39901459217071533,
      "logps/rejected": -0.461778461933136,
      "loss": 0.8935,
      "nll_loss": 0.9344221353530884,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 390
    },
    {
      "epoch": 0.37435657463734207,
      "grad_norm": 11.875,
      "learning_rate": 3.9382151286023644e-07,
      "log_odds_chosen": 0.13357871770858765,
      "log_odds_ratio": -0.6608735918998718,
      "logits/chosen": -2.2457404136657715,
      "logits/rejected": -2.2428553104400635,
      "logps/chosen": -0.39655953645706177,
      "logps/rejected": -0.4421748220920563,
      "loss": 0.8421,
      "nll_loss": 0.8112044334411621,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 400
    },
    {
      "epoch": 0.3837154890032756,
      "grad_norm": 12.375,
      "learning_rate": 3.8706096665539617e-07,
      "log_odds_chosen": 0.2025759220123291,
      "log_odds_ratio": -0.6549407839775085,
      "logits/chosen": -2.231397867202759,
      "logits/rejected": -2.211425304412842,
      "logps/chosen": -0.39163392782211304,
      "logps/rejected": -0.4965929090976715,
      "loss": 0.9318,
      "nll_loss": 0.9396621584892273,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 410
    },
    {
      "epoch": 0.3930744033692092,
      "grad_norm": 11.0625,
      "learning_rate": 3.801539573974959e-07,
      "log_odds_chosen": 0.30672961473464966,
      "log_odds_ratio": -0.6271880269050598,
      "logits/chosen": -2.259282112121582,
      "logits/rejected": -2.2307496070861816,
      "logps/chosen": -0.3660631775856018,
      "logps/rejected": -0.5345751047134399,
      "loss": 0.91,
      "nll_loss": 0.9085640907287598,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 420
    },
    {
      "epoch": 0.40243331773514274,
      "grad_norm": 10.125,
      "learning_rate": 3.7310786590171683e-07,
      "log_odds_chosen": 0.12312289327383041,
      "log_odds_ratio": -0.666767418384552,
      "logits/chosen": -2.2531981468200684,
      "logits/rejected": -2.2322800159454346,
      "logps/chosen": -0.4015491008758545,
      "logps/rejected": -0.4476936459541321,
      "loss": 0.9478,
      "nll_loss": 0.9238454699516296,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 430
    },
    {
      "epoch": 0.4117922321010763,
      "grad_norm": 12.875,
      "learning_rate": 3.659302216062191e-07,
      "log_odds_chosen": 0.18517343699932098,
      "log_odds_ratio": -0.6397743225097656,
      "logits/chosen": -2.199251890182495,
      "logits/rejected": -2.1922011375427246,
      "logps/chosen": -0.38667598366737366,
      "logps/rejected": -0.4541630148887634,
      "loss": 0.899,
      "nll_loss": 0.9328523874282837,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 440
    },
    {
      "epoch": 0.42115114646700985,
      "grad_norm": 10.5,
      "learning_rate": 3.586286945262007e-07,
      "log_odds_chosen": 0.19681106507778168,
      "log_odds_ratio": -0.632773756980896,
      "logits/chosen": -2.0825257301330566,
      "logits/rejected": -2.0917224884033203,
      "logps/chosen": -0.3770086467266083,
      "logps/rejected": -0.4443152844905853,
      "loss": 0.863,
      "nll_loss": 0.8851884007453918,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 450
    },
    {
      "epoch": 0.43051006083294335,
      "grad_norm": 11.375,
      "learning_rate": 3.512110870577357e-07,
      "log_odds_chosen": 0.18732145428657532,
      "log_odds_ratio": -0.6455721855163574,
      "logits/chosen": -2.168400287628174,
      "logits/rejected": -2.1601405143737793,
      "logps/chosen": -0.3884671926498413,
      "logps/rejected": -0.45501670241355896,
      "loss": 0.958,
      "nll_loss": 0.9003719091415405,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 460
    },
    {
      "epoch": 0.4398689751988769,
      "grad_norm": 9.0,
      "learning_rate": 3.436853256401506e-07,
      "log_odds_chosen": 0.18461090326309204,
      "log_odds_ratio": -0.6388983726501465,
      "logits/chosen": -2.175309896469116,
      "logits/rejected": -2.1828722953796387,
      "logps/chosen": -0.3819858431816101,
      "logps/rejected": -0.4427759647369385,
      "loss": 0.8924,
      "nll_loss": 0.861772358417511,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 470
    },
    {
      "epoch": 0.44922788956481047,
      "grad_norm": 11.625,
      "learning_rate": 3.3605945228584866e-07,
      "log_odds_chosen": 0.14212632179260254,
      "log_odds_ratio": -0.6561137437820435,
      "logits/chosen": -2.226238965988159,
      "logits/rejected": -2.214329242706299,
      "logps/chosen": -0.39799243211746216,
      "logps/rejected": -0.4503845274448395,
      "loss": 0.9181,
      "nll_loss": 0.9322090148925781,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 480
    },
    {
      "epoch": 0.458586803930744,
      "grad_norm": 10.5625,
      "learning_rate": 3.283416159866321e-07,
      "log_odds_chosen": 0.19436636567115784,
      "log_odds_ratio": -0.6323596239089966,
      "logits/chosen": -2.2073452472686768,
      "logits/rejected": -2.2207274436950684,
      "logps/chosen": -0.3859022557735443,
      "logps/rejected": -0.45602479577064514,
      "loss": 0.8872,
      "nll_loss": 0.866982102394104,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 490
    },
    {
      "epoch": 0.4679457182966776,
      "grad_norm": 12.875,
      "learning_rate": 3.205400640057081e-07,
      "log_odds_chosen": 0.07180626690387726,
      "log_odds_ratio": -0.7025886178016663,
      "logits/chosen": -2.2078473567962646,
      "logits/rejected": -2.205059289932251,
      "logps/chosen": -0.39877811074256897,
      "logps/rejected": -0.42040663957595825,
      "loss": 0.9405,
      "nll_loss": 0.9519411325454712,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 500
    },
    {
      "epoch": 0.47730463266261114,
      "grad_norm": 11.625,
      "learning_rate": 3.126631330646801e-07,
      "log_odds_chosen": 0.21832695603370667,
      "log_odds_ratio": -0.6433119177818298,
      "logits/chosen": -2.243464946746826,
      "logits/rejected": -2.245924234390259,
      "logps/chosen": -0.3781166076660156,
      "logps/rejected": -0.46226248145103455,
      "loss": 0.8928,
      "nll_loss": 0.8886586427688599,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 510
    },
    {
      "epoch": 0.4866635470285447,
      "grad_norm": 11.0625,
      "learning_rate": 3.0471924043494595e-07,
      "log_odds_chosen": 0.1995456963777542,
      "log_odds_ratio": -0.6375859975814819,
      "logits/chosen": -2.1644656658172607,
      "logits/rejected": -2.1902050971984863,
      "logps/chosen": -0.38845792412757874,
      "logps/rejected": -0.46764081716537476,
      "loss": 0.8949,
      "nll_loss": 0.872398853302002,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 520
    },
    {
      "epoch": 0.49602246139447825,
      "grad_norm": 10.0625,
      "learning_rate": 2.967168749430191e-07,
      "log_odds_chosen": 0.1926880180835724,
      "log_odds_ratio": -0.6435622572898865,
      "logits/chosen": -2.206993818283081,
      "logits/rejected": -2.191556215286255,
      "logps/chosen": -0.3731563985347748,
      "logps/rejected": -0.45363467931747437,
      "loss": 0.8595,
      "nll_loss": 0.8209236264228821,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 530
    },
    {
      "epoch": 0.5053813757604118,
      "grad_norm": 10.9375,
      "learning_rate": 2.8866458789938774e-07,
      "log_odds_chosen": 0.25284868478775024,
      "log_odds_ratio": -0.6184083819389343,
      "logits/chosen": -2.21002197265625,
      "logits/rejected": -2.218122720718384,
      "logps/chosen": -0.3761533200740814,
      "logps/rejected": -0.4766814708709717,
      "loss": 0.8866,
      "nll_loss": 0.8908084034919739,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 540
    },
    {
      "epoch": 0.5147402901263454,
      "grad_norm": 10.5,
      "learning_rate": 2.8057098396060196e-07,
      "log_odds_chosen": 0.07347196340560913,
      "log_odds_ratio": -0.6952499747276306,
      "logits/chosen": -2.2205636501312256,
      "logits/rejected": -2.2049291133880615,
      "logps/chosen": -0.39118486642837524,
      "logps/rejected": -0.4140162467956543,
      "loss": 0.867,
      "nll_loss": 0.8503815531730652,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 550
    },
    {
      "epoch": 0.5240992044922789,
      "grad_norm": 12.375,
      "learning_rate": 2.724447119343572e-07,
      "log_odds_chosen": 0.14831513166427612,
      "log_odds_ratio": -0.6715076565742493,
      "logits/chosen": -2.2129743099212646,
      "logits/rejected": -2.202531337738037,
      "logps/chosen": -0.3877745270729065,
      "logps/rejected": -0.4536987841129303,
      "loss": 0.8995,
      "nll_loss": 0.9369557499885559,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 560
    },
    {
      "epoch": 0.5334581188582125,
      "grad_norm": 10.8125,
      "learning_rate": 2.642944555373965e-07,
      "log_odds_chosen": 0.15156146883964539,
      "log_odds_ratio": -0.6531722545623779,
      "logits/chosen": -2.2276740074157715,
      "logits/rejected": -2.2312440872192383,
      "logps/chosen": -0.3907632827758789,
      "logps/rejected": -0.44457387924194336,
      "loss": 0.8901,
      "nll_loss": 0.9654369354248047,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 570
    },
    {
      "epoch": 0.542817033224146,
      "grad_norm": 12.375,
      "learning_rate": 2.561289241161095e-07,
      "log_odds_chosen": 0.16657111048698425,
      "log_odds_ratio": -0.6478754281997681,
      "logits/chosen": -2.2165768146514893,
      "logits/rejected": -2.218228340148926,
      "logps/chosen": -0.38118991255760193,
      "logps/rejected": -0.43258100748062134,
      "loss": 0.9585,
      "nll_loss": 0.9578523635864258,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 580
    },
    {
      "epoch": 0.5521759475900796,
      "grad_norm": 10.75,
      "learning_rate": 2.479568433397441e-07,
      "log_odds_chosen": 0.13426819443702698,
      "log_odds_ratio": -0.6695290207862854,
      "logits/chosen": -2.215862989425659,
      "logits/rejected": -2.2101454734802246,
      "logps/chosen": -0.3773101568222046,
      "logps/rejected": -0.4307495653629303,
      "loss": 0.8512,
      "nll_loss": 0.8539910316467285,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 590
    },
    {
      "epoch": 0.5615348619560131,
      "grad_norm": 12.3125,
      "learning_rate": 2.3978694587617473e-07,
      "log_odds_chosen": 0.15946467220783234,
      "log_odds_ratio": -0.6578431129455566,
      "logits/chosen": -2.1991055011749268,
      "logits/rejected": -2.2187142372131348,
      "logps/chosen": -0.3854084610939026,
      "logps/rejected": -0.43376803398132324,
      "loss": 0.8769,
      "nll_loss": 0.8557795286178589,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 600
    },
    {
      "epoch": 0.5708937763219466,
      "grad_norm": 11.4375,
      "learning_rate": 2.3162796206019266e-07,
      "log_odds_chosen": 0.19673123955726624,
      "log_odds_ratio": -0.6394155621528625,
      "logits/chosen": -2.1414363384246826,
      "logits/rejected": -2.137732982635498,
      "logps/chosen": -0.3869827091693878,
      "logps/rejected": -0.4562526345252991,
      "loss": 0.8573,
      "nll_loss": 0.8035401105880737,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 610
    },
    {
      "epoch": 0.5802526906878802,
      "grad_norm": 13.4375,
      "learning_rate": 2.2348861056428868e-07,
      "log_odds_chosen": 0.17950603365898132,
      "log_odds_ratio": -0.6448289752006531,
      "logits/chosen": -2.2693517208099365,
      "logits/rejected": -2.2487759590148926,
      "logps/chosen": -0.3937278389930725,
      "logps/rejected": -0.47679153084754944,
      "loss": 0.8618,
      "nll_loss": 0.8664249181747437,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 620
    },
    {
      "epoch": 0.5896116050538137,
      "grad_norm": 12.625,
      "learning_rate": 2.153775890818989e-07,
      "log_odds_chosen": 0.16706261038780212,
      "log_odds_ratio": -0.6368611454963684,
      "logits/chosen": -2.2624447345733643,
      "logits/rejected": -2.2497055530548096,
      "logps/chosen": -0.38325315713882446,
      "logps/rejected": -0.43356624245643616,
      "loss": 0.8918,
      "nll_loss": 0.9218624234199524,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 630
    },
    {
      "epoch": 0.5989705194197473,
      "grad_norm": 11.4375,
      "learning_rate": 2.0730356503306806e-07,
      "log_odds_chosen": 0.06589867919683456,
      "log_odds_ratio": -0.691096305847168,
      "logits/chosen": -2.210688352584839,
      "logits/rejected": -2.207019329071045,
      "logps/chosen": -0.3885454535484314,
      "logps/rejected": -0.4102134704589844,
      "loss": 0.877,
      "nll_loss": 0.8589447736740112,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 640
    },
    {
      "epoch": 0.6083294337856808,
      "grad_norm": 12.75,
      "learning_rate": 1.9927516630246335e-07,
      "log_odds_chosen": 0.1333846151828766,
      "log_odds_ratio": -0.6582024693489075,
      "logits/chosen": -2.276191473007202,
      "logits/rejected": -2.2583796977996826,
      "logps/chosen": -0.38965895771980286,
      "logps/rejected": -0.43281763792037964,
      "loss": 0.9212,
      "nll_loss": 0.8427213430404663,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 650
    },
    {
      "epoch": 0.6176883481516144,
      "grad_norm": 9.4375,
      "learning_rate": 1.9130097201963545e-07,
      "log_odds_chosen": 0.2855406403541565,
      "log_odds_ratio": -0.6136248707771301,
      "logits/chosen": -2.1809091567993164,
      "logits/rejected": -2.1479828357696533,
      "logps/chosen": -0.3822997212409973,
      "logps/rejected": -0.5103154182434082,
      "loss": 0.8373,
      "nll_loss": 0.8126581907272339,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 660
    },
    {
      "epoch": 0.6270472625175479,
      "grad_norm": 10.9375,
      "learning_rate": 1.833895033913789e-07,
      "log_odds_chosen": 0.2555854916572571,
      "log_odds_ratio": -0.6214571595191956,
      "logits/chosen": -2.1881003379821777,
      "logits/rejected": -2.2056005001068115,
      "logps/chosen": -0.38820981979370117,
      "logps/rejected": -0.49839335680007935,
      "loss": 0.882,
      "nll_loss": 0.8913403749465942,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 670
    },
    {
      "epoch": 0.6364061768834816,
      "grad_norm": 12.1875,
      "learning_rate": 1.755492145959896e-07,
      "log_odds_chosen": 0.27418556809425354,
      "log_odds_ratio": -0.6061091423034668,
      "logits/chosen": -2.237029552459717,
      "logits/rejected": -2.258998155593872,
      "logps/chosen": -0.36980870366096497,
      "logps/rejected": -0.47223013639450073,
      "loss": 0.897,
      "nll_loss": 0.9085220098495483,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 680
    },
    {
      "epoch": 0.6457650912494151,
      "grad_norm": 11.5625,
      "learning_rate": 1.6778848374914728e-07,
      "log_odds_chosen": 0.1495472937822342,
      "log_odds_ratio": -0.666257917881012,
      "logits/chosen": -2.232522964477539,
      "logits/rejected": -2.2000768184661865,
      "logps/chosen": -0.3772524297237396,
      "logps/rejected": -0.43047913908958435,
      "loss": 0.8478,
      "nll_loss": 0.8113440275192261,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 690
    },
    {
      "epoch": 0.6551240056153487,
      "grad_norm": 10.875,
      "learning_rate": 1.6011560395107998e-07,
      "log_odds_chosen": 0.1376083940267563,
      "log_odds_ratio": -0.6573753356933594,
      "logits/chosen": -2.1723008155822754,
      "logits/rejected": -2.152540683746338,
      "logps/chosen": -0.39665332436561584,
      "logps/rejected": -0.4487191140651703,
      "loss": 0.8878,
      "nll_loss": 0.8974568247795105,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 700
    },
    {
      "epoch": 0.6644829199812822,
      "grad_norm": 11.5,
      "learning_rate": 1.5253877442457446e-07,
      "log_odds_chosen": 0.14467862248420715,
      "log_odds_ratio": -0.6563876271247864,
      "logits/chosen": -2.2255406379699707,
      "logits/rejected": -2.2133126258850098,
      "logps/chosen": -0.3928825259208679,
      "logps/rejected": -0.441458523273468,
      "loss": 0.8912,
      "nll_loss": 0.8892416954040527,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 710
    },
    {
      "epoch": 0.6738418343472157,
      "grad_norm": 11.4375,
      "learning_rate": 1.450660917533048e-07,
      "log_odds_chosen": 0.16002854704856873,
      "log_odds_ratio": -0.6535741090774536,
      "logits/chosen": -2.152003049850464,
      "logits/rejected": -2.1638240814208984,
      "logps/chosen": -0.3956072926521301,
      "logps/rejected": -0.4611222743988037,
      "loss": 0.9267,
      "nll_loss": 0.9579558372497559,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 720
    },
    {
      "epoch": 0.6832007487131493,
      "grad_norm": 11.1875,
      "learning_rate": 1.377055412298402e-07,
      "log_odds_chosen": 0.19179068505764008,
      "log_odds_ratio": -0.6297602653503418,
      "logits/chosen": -2.2455074787139893,
      "logits/rejected": -2.249809741973877,
      "logps/chosen": -0.37641215324401855,
      "logps/rejected": -0.44604501128196716,
      "loss": 0.8778,
      "nll_loss": 0.8855358958244324,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 730
    },
    {
      "epoch": 0.6925596630790828,
      "grad_norm": 10.8125,
      "learning_rate": 1.3046498832257924e-07,
      "log_odds_chosen": 0.30156663060188293,
      "log_odds_ratio": -0.5952414870262146,
      "logits/chosen": -2.2041194438934326,
      "logits/rejected": -2.2143595218658447,
      "logps/chosen": -0.37330108880996704,
      "logps/rejected": -0.4847453236579895,
      "loss": 0.8899,
      "nll_loss": 0.8489750623703003,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 740
    },
    {
      "epoch": 0.7019185774450164,
      "grad_norm": 12.9375,
      "learning_rate": 1.233521702707264e-07,
      "log_odds_chosen": 0.12726518511772156,
      "log_odds_ratio": -0.6660614013671875,
      "logits/chosen": -2.2514965534210205,
      "logits/rejected": -2.2471282482147217,
      "logps/chosen": -0.3983449637889862,
      "logps/rejected": -0.4496460556983948,
      "loss": 0.8822,
      "nll_loss": 0.9095920324325562,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 750
    },
    {
      "epoch": 0.7112774918109499,
      "grad_norm": 9.3125,
      "learning_rate": 1.1637468781629567e-07,
      "log_odds_chosen": 0.221353679895401,
      "log_odds_ratio": -0.6361473798751831,
      "logits/chosen": -2.201655864715576,
      "logits/rejected": -2.2004904747009277,
      "logps/chosen": -0.37590572237968445,
      "logps/rejected": -0.46796751022338867,
      "loss": 0.8861,
      "nll_loss": 0.8721257448196411,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 760
    },
    {
      "epoch": 0.7206364061768835,
      "grad_norm": 10.375,
      "learning_rate": 1.0953999708197404e-07,
      "log_odds_chosen": 0.14084991812705994,
      "log_odds_ratio": -0.6663475632667542,
      "logits/chosen": -2.2368383407592773,
      "logits/rejected": -2.2399377822875977,
      "logps/chosen": -0.39043423533439636,
      "logps/rejected": -0.4488556981086731,
      "loss": 0.8736,
      "nll_loss": 0.8931602239608765,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 770
    },
    {
      "epoch": 0.729995320542817,
      "grad_norm": 10.25,
      "learning_rate": 1.0285540160352404e-07,
      "log_odds_chosen": 0.10203123092651367,
      "log_odds_ratio": -0.6751465797424316,
      "logits/chosen": -2.278337001800537,
      "logits/rejected": -2.264094591140747,
      "logps/chosen": -0.3964601159095764,
      "logps/rejected": -0.4286923408508301,
      "loss": 0.8769,
      "nll_loss": 0.9037150144577026,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 780
    },
    {
      "epoch": 0.7393542349087506,
      "grad_norm": 10.3125,
      "learning_rate": 9.632804452524256e-08,
      "log_odds_chosen": 0.17006394267082214,
      "log_odds_ratio": -0.647351861000061,
      "logits/chosen": -2.162752628326416,
      "logits/rejected": -2.163599729537964,
      "logps/chosen": -0.39570730924606323,
      "logps/rejected": -0.45942431688308716,
      "loss": 0.8123,
      "nll_loss": 0.8693070411682129,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 790
    },
    {
      "epoch": 0.7487131492746841,
      "grad_norm": 10.4375,
      "learning_rate": 8.996490096681109e-08,
      "log_odds_chosen": 0.13158640265464783,
      "log_odds_ratio": -0.67692631483078,
      "logits/chosen": -2.180257797241211,
      "logits/rejected": -2.1624608039855957,
      "logps/chosen": -0.40238919854164124,
      "logps/rejected": -0.46038302779197693,
      "loss": 0.8765,
      "nll_loss": 0.9388486742973328,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 800
    },
    {
      "epoch": 0.7580720636406177,
      "grad_norm": 11.0,
      "learning_rate": 8.377277056969842e-08,
      "log_odds_chosen": 0.20461300015449524,
      "log_odds_ratio": -0.6381627321243286,
      "logits/chosen": -2.2441630363464355,
      "logits/rejected": -2.2336671352386475,
      "logps/chosen": -0.39944490790367126,
      "logps/rejected": -0.47810035943984985,
      "loss": 0.9009,
      "nll_loss": 0.8954687118530273,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 810
    },
    {
      "epoch": 0.7674309780065512,
      "grad_norm": 12.3125,
      "learning_rate": 7.775827023107834e-08,
      "log_odds_chosen": 0.21859991550445557,
      "log_odds_ratio": -0.6190574765205383,
      "logits/chosen": -2.2584617137908936,
      "logits/rejected": -2.235044479370117,
      "logps/chosen": -0.3853353261947632,
      "logps/rejected": -0.46209225058555603,
      "loss": 0.8857,
      "nll_loss": 0.9639716148376465,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 820
    },
    {
      "epoch": 0.7767898923724847,
      "grad_norm": 13.75,
      "learning_rate": 7.192782703302785e-08,
      "log_odds_chosen": 0.23809826374053955,
      "log_odds_ratio": -0.6136423349380493,
      "logits/chosen": -2.222693920135498,
      "logits/rejected": -2.198784828186035,
      "logps/chosen": -0.3764819800853729,
      "logps/rejected": -0.45437926054000854,
      "loss": 0.8463,
      "nll_loss": 0.8469756841659546,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 830
    },
    {
      "epoch": 0.7861488067384184,
      "grad_norm": 9.875,
      "learning_rate": 6.628767137456067e-08,
      "log_odds_chosen": 0.1315256655216217,
      "log_odds_ratio": -0.6661390066146851,
      "logits/chosen": -2.1875691413879395,
      "logits/rejected": -2.193073034286499,
      "logps/chosen": -0.41907310485839844,
      "logps/rejected": -0.4673064649105072,
      "loss": 0.9208,
      "nll_loss": 0.9243777990341187,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 840
    },
    {
      "epoch": 0.7955077211043519,
      "grad_norm": 13.125,
      "learning_rate": 6.08438303138365e-08,
      "log_odds_chosen": 0.0879138857126236,
      "log_odds_ratio": -0.6789141893386841,
      "logits/chosen": -2.191913366317749,
      "logits/rejected": -2.1954355239868164,
      "logps/chosen": -0.38346320390701294,
      "logps/rejected": -0.415657103061676,
      "loss": 0.8978,
      "nll_loss": 0.9076964259147644,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 850
    },
    {
      "epoch": 0.8048666354702855,
      "grad_norm": 10.0625,
      "learning_rate": 5.560212112766011e-08,
      "log_odds_chosen": 0.14250405132770538,
      "log_odds_ratio": -0.6655804514884949,
      "logits/chosen": -2.218059539794922,
      "logits/rejected": -2.209329843521118,
      "logps/chosen": -0.386410653591156,
      "logps/rejected": -0.4412326216697693,
      "loss": 0.8719,
      "nll_loss": 0.8908575177192688,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 860
    },
    {
      "epoch": 0.814225549836219,
      "grad_norm": 11.125,
      "learning_rate": 5.056814509515092e-08,
      "log_odds_chosen": 0.20792749524116516,
      "log_odds_ratio": -0.6451287865638733,
      "logits/chosen": -2.1958823204040527,
      "logits/rejected": -2.227415084838867,
      "logps/chosen": -0.3777526021003723,
      "logps/rejected": -0.4553712010383606,
      "loss": 0.8686,
      "nll_loss": 0.8219059705734253,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 870
    },
    {
      "epoch": 0.8235844642021526,
      "grad_norm": 11.5625,
      "learning_rate": 4.57472815122294e-08,
      "log_odds_chosen": 0.12616851925849915,
      "log_odds_ratio": -0.6744897961616516,
      "logits/chosen": -2.239567279815674,
      "logits/rejected": -2.2245991230010986,
      "logps/chosen": -0.39220941066741943,
      "logps/rejected": -0.43573325872421265,
      "loss": 0.8534,
      "nll_loss": 0.8611897230148315,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 880
    },
    {
      "epoch": 0.8329433785680861,
      "grad_norm": 11.0,
      "learning_rate": 4.1144681943312135e-08,
      "log_odds_chosen": 0.18745501339435577,
      "log_odds_ratio": -0.6411550045013428,
      "logits/chosen": -2.1551897525787354,
      "logits/rejected": -2.1661386489868164,
      "logps/chosen": -0.3798125982284546,
      "logps/rejected": -0.4548855721950531,
      "loss": 0.9022,
      "nll_loss": 0.9224790334701538,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 890
    },
    {
      "epoch": 0.8423022929340197,
      "grad_norm": 10.875,
      "learning_rate": 3.676526471636168e-08,
      "log_odds_chosen": 0.20975276827812195,
      "log_odds_ratio": -0.6299919486045837,
      "logits/chosen": -2.2081241607666016,
      "logits/rejected": -2.2052078247070312,
      "logps/chosen": -0.3888050615787506,
      "logps/rejected": -0.47621068358421326,
      "loss": 0.8317,
      "nll_loss": 0.7904391884803772,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 900
    },
    {
      "epoch": 0.8516612072999532,
      "grad_norm": 11.0,
      "learning_rate": 3.2613709667171893e-08,
      "log_odds_chosen": 0.1670573353767395,
      "log_odds_ratio": -0.650361180305481,
      "logits/chosen": -2.1842589378356934,
      "logits/rejected": -2.193791151046753,
      "logps/chosen": -0.38958585262298584,
      "logps/rejected": -0.44640764594078064,
      "loss": 0.8918,
      "nll_loss": 0.8657588958740234,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 910
    },
    {
      "epoch": 0.8610201216658867,
      "grad_norm": 13.1875,
      "learning_rate": 2.8694453138505296e-08,
      "log_odds_chosen": 0.18690498173236847,
      "log_odds_ratio": -0.643862247467041,
      "logits/chosen": -2.155745267868042,
      "logits/rejected": -2.159320116043091,
      "logps/chosen": -0.379416286945343,
      "logps/rejected": -0.44661951065063477,
      "loss": 0.8589,
      "nll_loss": 0.8909047245979309,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 920
    },
    {
      "epoch": 0.8703790360318203,
      "grad_norm": 11.625,
      "learning_rate": 2.5011683239426847e-08,
      "log_odds_chosen": 0.23082701861858368,
      "log_odds_ratio": -0.6160436868667603,
      "logits/chosen": -2.1533544063568115,
      "logits/rejected": -2.12406587600708,
      "logps/chosen": -0.3944535255432129,
      "logps/rejected": -0.47682076692581177,
      "loss": 0.9318,
      "nll_loss": 0.9148885607719421,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 930
    },
    {
      "epoch": 0.8797379503977538,
      "grad_norm": 11.4375,
      "learning_rate": 2.1569335369899884e-08,
      "log_odds_chosen": 0.21969418227672577,
      "log_odds_ratio": -0.6254672408103943,
      "logits/chosen": -2.2102274894714355,
      "logits/rejected": -2.197098970413208,
      "logps/chosen": -0.3689967691898346,
      "logps/rejected": -0.4529111385345459,
      "loss": 0.9042,
      "nll_loss": 0.9422861933708191,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 940
    },
    {
      "epoch": 0.8890968647636874,
      "grad_norm": 10.3125,
      "learning_rate": 1.837108801542589e-08,
      "log_odds_chosen": 0.26736536622047424,
      "log_odds_ratio": -0.6209851503372192,
      "logits/chosen": -2.193885564804077,
      "logits/rejected": -2.1692655086517334,
      "logps/chosen": -0.3677137494087219,
      "logps/rejected": -0.49029093980789185,
      "loss": 0.903,
      "nll_loss": 0.9059442281723022,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 950
    },
    {
      "epoch": 0.8984557791296209,
      "grad_norm": 11.75,
      "learning_rate": 1.5420358816223e-08,
      "log_odds_chosen": 0.21792039275169373,
      "log_odds_ratio": -0.6334658861160278,
      "logits/chosen": -2.1850409507751465,
      "logits/rejected": -2.1796321868896484,
      "logps/chosen": -0.3942349851131439,
      "logps/rejected": -0.4783645272254944,
      "loss": 0.8978,
      "nll_loss": 0.8998034596443176,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 960
    },
    {
      "epoch": 0.9078146934955545,
      "grad_norm": 10.1875,
      "learning_rate": 1.2720300915142978e-08,
      "log_odds_chosen": 0.19287154078483582,
      "log_odds_ratio": -0.632893443107605,
      "logits/chosen": -2.238175630569458,
      "logits/rejected": -2.2145469188690186,
      "logps/chosen": -0.3847096562385559,
      "logps/rejected": -0.44970273971557617,
      "loss": 0.945,
      "nll_loss": 0.8885831832885742,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 970
    },
    {
      "epoch": 0.917173607861488,
      "grad_norm": 9.5625,
      "learning_rate": 1.0273799588229659e-08,
      "log_odds_chosen": 0.14185890555381775,
      "log_odds_ratio": -0.654270350933075,
      "logits/chosen": -2.204613208770752,
      "logits/rejected": -2.192824602127075,
      "logps/chosen": -0.3829004764556885,
      "logps/rejected": -0.43003320693969727,
      "loss": 0.969,
      "nll_loss": 0.9601060152053833,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 980
    },
    {
      "epoch": 0.9265325222274217,
      "grad_norm": 10.25,
      "learning_rate": 8.08346916151903e-09,
      "log_odds_chosen": 0.18503454327583313,
      "log_odds_ratio": -0.6544171571731567,
      "logits/chosen": -2.224045515060425,
      "logits/rejected": -2.1920361518859863,
      "logps/chosen": -0.39199355244636536,
      "logps/rejected": -0.47227340936660767,
      "loss": 0.8559,
      "nll_loss": 0.8463509678840637,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 990
    },
    {
      "epoch": 0.9358914365933552,
      "grad_norm": 11.0,
      "learning_rate": 6.151650217376347e-09,
      "log_odds_chosen": 0.2020350694656372,
      "log_odds_ratio": -0.6279042959213257,
      "logits/chosen": -2.1893532276153564,
      "logits/rejected": -2.1703667640686035,
      "logps/chosen": -0.39548614621162415,
      "logps/rejected": -0.47544950246810913,
      "loss": 0.8985,
      "nll_loss": 0.9111002683639526,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1000
    },
    {
      "epoch": 0.9452503509592888,
      "grad_norm": 10.0625,
      "learning_rate": 4.480407093354566e-09,
      "log_odds_chosen": 0.12327748537063599,
      "log_odds_ratio": -0.6677930951118469,
      "logits/chosen": -2.215139865875244,
      "logits/rejected": -2.2001559734344482,
      "logps/chosen": -0.39772313833236694,
      "logps/rejected": -0.4363015294075012,
      "loss": 0.9252,
      "nll_loss": 0.8623770475387573,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1010
    },
    {
      "epoch": 0.9546092653252223,
      "grad_norm": 8.8125,
      "learning_rate": 3.0715256762478825e-09,
      "log_odds_chosen": 0.24486811459064484,
      "log_odds_ratio": -0.6162024140357971,
      "logits/chosen": -2.1808042526245117,
      "logits/rejected": -2.1617355346679688,
      "logps/chosen": -0.3899694085121155,
      "logps/rejected": -0.4756743907928467,
      "loss": 0.887,
      "nll_loss": 0.8923400640487671,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1020
    },
    {
      "epoch": 0.9639681796911558,
      "grad_norm": 11.9375,
      "learning_rate": 1.926511493696936e-09,
      "log_odds_chosen": 0.11399779468774796,
      "log_odds_ratio": -0.6707941889762878,
      "logits/chosen": -2.2159488201141357,
      "logits/rejected": -2.221107244491577,
      "logps/chosen": -0.39392104744911194,
      "logps/rejected": -0.4452263414859772,
      "loss": 0.9244,
      "nll_loss": 0.8677811622619629,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1030
    },
    {
      "epoch": 0.9733270940570894,
      "grad_norm": 11.625,
      "learning_rate": 1.04658810538516e-09,
      "log_odds_chosen": 0.18175111711025238,
      "log_odds_ratio": -0.634798526763916,
      "logits/chosen": -2.1980602741241455,
      "logits/rejected": -2.200080394744873,
      "logps/chosen": -0.36757367849349976,
      "logps/rejected": -0.42325907945632935,
      "loss": 0.8552,
      "nll_loss": 0.8295245170593262,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1040
    },
    {
      "epoch": 0.9826860084230229,
      "grad_norm": 10.625,
      "learning_rate": 4.3269579554558455e-10,
      "log_odds_chosen": 0.10918863862752914,
      "log_odds_ratio": -0.6753650903701782,
      "logits/chosen": -2.2247250080108643,
      "logits/rejected": -2.2069859504699707,
      "logps/chosen": -0.4014809727668762,
      "logps/rejected": -0.4421558380126953,
      "loss": 0.9251,
      "nll_loss": 0.9922356605529785,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1050
    },
    {
      "epoch": 0.9920449227889565,
      "grad_norm": 10.625,
      "learning_rate": 8.549056817513944e-11,
      "log_odds_chosen": 0.2163115292787552,
      "log_odds_ratio": -0.6318882703781128,
      "logits/chosen": -2.2252681255340576,
      "logits/rejected": -2.2008399963378906,
      "logps/chosen": -0.39117246866226196,
      "logps/rejected": -0.4764987528324127,
      "loss": 0.893,
      "nll_loss": 0.927023708820343,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1060
    },
    {
      "epoch": 0.9995320542817033,
      "step": 1068,
      "total_flos": 0.0,
      "train_loss": 0.9322090081954271,
      "train_runtime": 16459.2443,
      "train_samples_per_second": 2.077,
      "train_steps_per_second": 0.065
    }
  ],
  "logging_steps": 10,
  "max_steps": 1068,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}