{
  "best_metric": 0.8629826903343201,
  "best_model_checkpoint": "saves/Mistral-7B-Instruct-v0.2/lora/orpo-salt/checkpoint-1500",
  "epoch": 2.9969690846635686,
  "eval_steps": 500,
  "global_step": 1854,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.01616488179430188,
      "grad_norm": 16.64879608154297,
      "learning_rate": 4.999648198770648e-06,
      "logits/chosen": -2.4989278316497803,
      "logits/rejected": -2.5208303928375244,
      "logps/chosen": -1.9139716625213623,
      "logps/rejected": -3.1082823276519775,
      "loss": 1.9977,
      "odds_ratio_loss": 0.8370735049247742,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.19139717519283295,
      "rewards/margins": 0.1194310411810875,
      "rewards/rejected": -0.31082823872566223,
      "sft_loss": 1.9139716625213623,
      "step": 10
    },
    {
      "epoch": 0.03232976358860376,
      "grad_norm": 13.894062042236328,
      "learning_rate": 4.998578646361359e-06,
      "logits/chosen": -2.5156219005584717,
      "logits/rejected": -2.51640248298645,
      "logps/chosen": -1.635488748550415,
      "logps/rejected": -2.132800817489624,
      "loss": 1.7095,
      "odds_ratio_loss": 0.7404953241348267,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.16354887187480927,
      "rewards/margins": 0.04973122477531433,
      "rewards/rejected": -0.2132801115512848,
      "sft_loss": 1.635488748550415,
      "step": 20
    },
    {
      "epoch": 0.04849464538290564,
      "grad_norm": 23.089773178100586,
      "learning_rate": 4.996791614004449e-06,
      "logits/chosen": -2.518998861312866,
      "logits/rejected": -2.544835090637207,
      "logps/chosen": -1.6531565189361572,
      "logps/rejected": -2.541318893432617,
      "loss": 1.7385,
      "odds_ratio_loss": 0.8539272546768188,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.165315642952919,
      "rewards/margins": 0.08881621062755585,
      "rewards/rejected": -0.25413185358047485,
      "sft_loss": 1.6531565189361572,
      "step": 30
    },
    {
      "epoch": 0.06465952717720752,
      "grad_norm": 13.833389282226562,
      "learning_rate": 4.994287614855618e-06,
      "logits/chosen": -2.518852472305298,
      "logits/rejected": -2.551032066345215,
      "logps/chosen": -1.7646430730819702,
      "logps/rejected": -2.508850574493408,
      "loss": 1.8742,
      "odds_ratio_loss": 1.0958486795425415,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.17646430432796478,
      "rewards/margins": 0.07442077249288559,
      "rewards/rejected": -0.2508850693702698,
      "sft_loss": 1.7646430730819702,
      "step": 40
    },
    {
      "epoch": 0.0808244089715094,
      "grad_norm": 28.34682846069336,
      "learning_rate": 4.991067367951343e-06,
      "logits/chosen": -2.5992355346679688,
      "logits/rejected": -2.5891082286834717,
      "logps/chosen": -1.345651388168335,
      "logps/rejected": -2.2306911945343018,
      "loss": 1.4115,
      "odds_ratio_loss": 0.6583842039108276,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.13456514477729797,
      "rewards/margins": 0.08850395679473877,
      "rewards/rejected": -0.22306910157203674,
      "sft_loss": 1.345651388168335,
      "step": 50
    },
    {
      "epoch": 0.09698929076581128,
      "grad_norm": 3.4724316596984863,
      "learning_rate": 4.987131798002389e-06,
      "logits/chosen": -2.539771556854248,
      "logits/rejected": -2.5456976890563965,
      "logps/chosen": -1.3674490451812744,
      "logps/rejected": -2.1061840057373047,
      "loss": 1.4542,
      "odds_ratio_loss": 0.8671566247940063,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.1367449164390564,
      "rewards/margins": 0.07387349754571915,
      "rewards/rejected": -0.21061840653419495,
      "sft_loss": 1.3674490451812744,
      "step": 60
    },
    {
      "epoch": 0.11315417256011315,
      "grad_norm": 46.33675003051758,
      "learning_rate": 4.982482035128285e-06,
      "logits/chosen": -2.5208637714385986,
      "logits/rejected": -2.528776168823242,
      "logps/chosen": -1.4248360395431519,
      "logps/rejected": -2.067411184310913,
      "loss": 1.5025,
      "odds_ratio_loss": 0.7764666676521301,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.14248362183570862,
      "rewards/margins": 0.06425751000642776,
      "rewards/rejected": -0.2067411243915558,
      "sft_loss": 1.4248360395431519,
      "step": 70
    },
    {
      "epoch": 0.12931905435441504,
      "grad_norm": 25.993545532226562,
      "learning_rate": 4.9771194145328e-06,
      "logits/chosen": -2.5788090229034424,
      "logits/rejected": -2.572688341140747,
      "logps/chosen": -1.0824676752090454,
      "logps/rejected": -1.7445621490478516,
      "loss": 1.1449,
      "odds_ratio_loss": 0.6242043972015381,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.10824675858020782,
      "rewards/margins": 0.06620947271585464,
      "rewards/rejected": -0.17445623874664307,
      "sft_loss": 1.0824676752090454,
      "step": 80
    },
    {
      "epoch": 0.1454839361487169,
      "grad_norm": 19.184228897094727,
      "learning_rate": 4.971045476120532e-06,
      "logits/chosen": -2.5863890647888184,
      "logits/rejected": -2.591404914855957,
      "logps/chosen": -1.080370306968689,
      "logps/rejected": -1.753382682800293,
      "loss": 1.1463,
      "odds_ratio_loss": 0.6591774821281433,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.10803703963756561,
      "rewards/margins": 0.06730123609304428,
      "rewards/rejected": -0.1753382831811905,
      "sft_loss": 1.080370306968689,
      "step": 90
    },
    {
      "epoch": 0.1616488179430188,
      "grad_norm": 5.7092084884643555,
      "learning_rate": 4.964261964054713e-06,
      "logits/chosen": -2.5851123332977295,
      "logits/rejected": -2.5928287506103516,
      "logps/chosen": -1.20145583152771,
      "logps/rejected": -1.920117735862732,
      "loss": 1.2771,
      "odds_ratio_loss": 0.7563266754150391,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12014558166265488,
      "rewards/margins": 0.07186620682477951,
      "rewards/rejected": -0.1920117884874344,
      "sft_loss": 1.20145583152771,
      "step": 100
    },
    {
      "epoch": 0.17781369973732067,
      "grad_norm": 4.211212635040283,
      "learning_rate": 4.956770826256372e-06,
      "logits/chosen": -2.6192798614501953,
      "logits/rejected": -2.6177656650543213,
      "logps/chosen": -1.1085783243179321,
      "logps/rejected": -1.4738147258758545,
      "loss": 1.1766,
      "odds_ratio_loss": 0.6805119514465332,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.11085782200098038,
      "rewards/margins": 0.03652365505695343,
      "rewards/rejected": -0.1473814696073532,
      "sft_loss": 1.1085783243179321,
      "step": 110
    },
    {
      "epoch": 0.19397858153162256,
      "grad_norm": 3.4872381687164307,
      "learning_rate": 4.94857421384497e-06,
      "logits/chosen": -2.602118968963623,
      "logits/rejected": -2.6089630126953125,
      "logps/chosen": -1.0341213941574097,
      "logps/rejected": -1.5845638513565063,
      "loss": 1.1041,
      "odds_ratio_loss": 0.6995517611503601,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.10341213643550873,
      "rewards/margins": 0.05504424497485161,
      "rewards/rejected": -0.15845640003681183,
      "sft_loss": 1.0341213941574097,
      "step": 120
    },
    {
      "epoch": 0.21014346332592443,
      "grad_norm": 5.468324661254883,
      "learning_rate": 4.939674480520701e-06,
      "logits/chosen": -2.6128063201904297,
      "logits/rejected": -2.6255507469177246,
      "logps/chosen": -0.9619969129562378,
      "logps/rejected": -1.390077829360962,
      "loss": 1.0297,
      "odds_ratio_loss": 0.6766607165336609,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.09619969129562378,
      "rewards/margins": 0.04280809685587883,
      "rewards/rejected": -0.1390077769756317,
      "sft_loss": 0.9619969129562378,
      "step": 130
    },
    {
      "epoch": 0.2263083451202263,
      "grad_norm": 5.18142032623291,
      "learning_rate": 4.930074181888613e-06,
      "logits/chosen": -2.6814427375793457,
      "logits/rejected": -2.7020936012268066,
      "logps/chosen": -0.9705274701118469,
      "logps/rejected": -1.315450668334961,
      "loss": 1.0341,
      "odds_ratio_loss": 0.636103630065918,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.09705274552106857,
      "rewards/margins": 0.03449232131242752,
      "rewards/rejected": -0.1315450817346573,
      "sft_loss": 0.9705274701118469,
      "step": 140
    },
    {
      "epoch": 0.2424732269145282,
      "grad_norm": 1.4752620458602905,
      "learning_rate": 4.91977607472475e-06,
      "logits/chosen": -2.704951524734497,
      "logits/rejected": -2.7246315479278564,
      "logps/chosen": -1.0248619318008423,
      "logps/rejected": -1.4426223039627075,
      "loss": 1.0895,
      "odds_ratio_loss": 0.6460444331169128,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.10248619318008423,
      "rewards/margins": 0.04177603870630264,
      "rewards/rejected": -0.14426222443580627,
      "sft_loss": 1.0248619318008423,
      "step": 150
    },
    {
      "epoch": 0.2586381087088301,
      "grad_norm": 2.9540135860443115,
      "learning_rate": 4.908783116184534e-06,
      "logits/chosen": -2.671297550201416,
      "logits/rejected": -2.676952838897705,
      "logps/chosen": -0.9303582906723022,
      "logps/rejected": -1.28878653049469,
      "loss": 0.991,
      "odds_ratio_loss": 0.6061214208602905,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.09303583949804306,
      "rewards/margins": 0.03584280610084534,
      "rewards/rejected": -0.1288786381483078,
      "sft_loss": 0.9303582906723022,
      "step": 160
    },
    {
      "epoch": 0.27480299050313195,
      "grad_norm": 2.913118839263916,
      "learning_rate": 4.897098462953598e-06,
      "logits/chosen": -2.7513809204101562,
      "logits/rejected": -2.7600345611572266,
      "logps/chosen": -0.8939758539199829,
      "logps/rejected": -1.4527159929275513,
      "loss": 0.9601,
      "odds_ratio_loss": 0.661632239818573,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.08939759433269501,
      "rewards/margins": 0.05587399750947952,
      "rewards/rejected": -0.14527159929275513,
      "sft_loss": 0.8939758539199829,
      "step": 170
    },
    {
      "epoch": 0.2909678722974338,
      "grad_norm": 1.985352635383606,
      "learning_rate": 4.884725470341331e-06,
      "logits/chosen": -2.7102103233337402,
      "logits/rejected": -2.739673137664795,
      "logps/chosen": -0.8302527666091919,
      "logps/rejected": -1.2092260122299194,
      "loss": 0.8851,
      "odds_ratio_loss": 0.5487207174301147,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.08302526921033859,
      "rewards/margins": 0.03789733722805977,
      "rewards/rejected": -0.12092261016368866,
      "sft_loss": 0.8302527666091919,
      "step": 180
    },
    {
      "epoch": 0.3071327540917357,
      "grad_norm": 8.031681060791016,
      "learning_rate": 4.871667691317377e-06,
      "logits/chosen": -2.764559745788574,
      "logits/rejected": -2.767064332962036,
      "logps/chosen": -1.0171376466751099,
      "logps/rejected": -1.1592780351638794,
      "loss": 1.0939,
      "odds_ratio_loss": 0.7678386569023132,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.10171377658843994,
      "rewards/margins": 0.01421402208507061,
      "rewards/rejected": -0.1159278005361557,
      "sft_loss": 1.0171376466751099,
      "step": 190
    },
    {
      "epoch": 0.3232976358860376,
      "grad_norm": 4.448939323425293,
      "learning_rate": 4.857928875491392e-06,
      "logits/chosen": -2.750746965408325,
      "logits/rejected": -2.7596051692962646,
      "logps/chosen": -0.8164304494857788,
      "logps/rejected": -1.0888216495513916,
      "loss": 0.8794,
      "odds_ratio_loss": 0.6294754147529602,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.08164305239915848,
      "rewards/margins": 0.027239132672548294,
      "rewards/rejected": -0.10888218879699707,
      "sft_loss": 0.8164304494857788,
      "step": 200
    },
    {
      "epoch": 0.33946251768033947,
      "grad_norm": 2.216554641723633,
      "learning_rate": 4.843512968036314e-06,
      "logits/chosen": -2.7625343799591064,
      "logits/rejected": -2.7599010467529297,
      "logps/chosen": -0.833400547504425,
      "logps/rejected": -1.0677030086517334,
      "loss": 0.8944,
      "odds_ratio_loss": 0.6096410751342773,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.08334006369113922,
      "rewards/margins": 0.0234302319586277,
      "rewards/rejected": -0.10677029192447662,
      "sft_loss": 0.833400547504425,
      "step": 210
    },
    {
      "epoch": 0.35562739947464134,
      "grad_norm": 1.4112659692764282,
      "learning_rate": 4.828424108555486e-06,
      "logits/chosen": -2.807507276535034,
      "logits/rejected": -2.803765296936035,
      "logps/chosen": -1.0460469722747803,
      "logps/rejected": -1.4173492193222046,
      "loss": 1.1091,
      "odds_ratio_loss": 0.6301766037940979,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.10460470616817474,
      "rewards/margins": 0.037130214273929596,
      "rewards/rejected": -0.14173491299152374,
      "sft_loss": 1.0460469722747803,
      "step": 220
    },
    {
      "epoch": 0.3717922812689432,
      "grad_norm": 0.9852223992347717,
      "learning_rate": 4.812666629893957e-06,
      "logits/chosen": -2.795703649520874,
      "logits/rejected": -2.8211073875427246,
      "logps/chosen": -0.891126275062561,
      "logps/rejected": -1.0855722427368164,
      "loss": 0.9626,
      "odds_ratio_loss": 0.7152143716812134,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.08911262452602386,
      "rewards/margins": 0.019444596022367477,
      "rewards/rejected": -0.10855722427368164,
      "sft_loss": 0.891126275062561,
      "step": 230
    },
    {
      "epoch": 0.3879571630632451,
      "grad_norm": 2.482409954071045,
      "learning_rate": 4.796245056894273e-06,
      "logits/chosen": -2.757913112640381,
      "logits/rejected": -2.794553518295288,
      "logps/chosen": -0.9089745283126831,
      "logps/rejected": -1.3391778469085693,
      "loss": 0.9804,
      "odds_ratio_loss": 0.7146768569946289,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.09089745581150055,
      "rewards/margins": 0.0430203452706337,
      "rewards/rejected": -0.13391780853271484,
      "sft_loss": 0.9089745283126831,
      "step": 240
    },
    {
      "epoch": 0.404122044857547,
      "grad_norm": 1.3123791217803955,
      "learning_rate": 4.779164105097148e-06,
      "logits/chosen": -2.796814441680908,
      "logits/rejected": -2.8013055324554443,
      "logps/chosen": -0.8589127659797668,
      "logps/rejected": -1.3229057788848877,
      "loss": 0.9186,
      "odds_ratio_loss": 0.5965861082077026,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.08589127659797668,
      "rewards/margins": 0.046399302780628204,
      "rewards/rejected": -0.1322905719280243,
      "sft_loss": 0.8589127659797668,
      "step": 250
    },
    {
      "epoch": 0.42028692665184886,
      "grad_norm": 2.171173095703125,
      "learning_rate": 4.761428679387373e-06,
      "logits/chosen": -2.790588617324829,
      "logits/rejected": -2.7970798015594482,
      "logps/chosen": -0.8536098599433899,
      "logps/rejected": -1.0807464122772217,
      "loss": 0.9168,
      "odds_ratio_loss": 0.6316367387771606,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.08536098152399063,
      "rewards/margins": 0.022713668644428253,
      "rewards/rejected": -0.10807464271783829,
      "sft_loss": 0.8536098599433899,
      "step": 260
    },
    {
      "epoch": 0.4364518084461507,
      "grad_norm": 3.753523111343384,
      "learning_rate": 4.7430438725853515e-06,
      "logits/chosen": -2.7550888061523438,
      "logits/rejected": -2.766615629196167,
      "logps/chosen": -0.913661003112793,
      "logps/rejected": -1.41799795627594,
      "loss": 0.9739,
      "odds_ratio_loss": 0.6024969816207886,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.09136610478162766,
      "rewards/margins": 0.0504336841404438,
      "rewards/rejected": -0.14179977774620056,
      "sft_loss": 0.913661003112793,
      "step": 270
    },
    {
      "epoch": 0.4526166902404526,
      "grad_norm": 1.5982986688613892,
      "learning_rate": 4.724014963984669e-06,
      "logits/chosen": -2.798797130584717,
      "logits/rejected": -2.8145482540130615,
      "logps/chosen": -0.8752357363700867,
      "logps/rejected": -1.1694762706756592,
      "loss": 0.9358,
      "odds_ratio_loss": 0.6060217618942261,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.08752357959747314,
      "rewards/margins": 0.029424061998724937,
      "rewards/rejected": -0.11694763600826263,
      "sft_loss": 0.8752357363700867,
      "step": 280
    },
    {
      "epoch": 0.4687815720347545,
      "grad_norm": 3.8735010623931885,
      "learning_rate": 4.704347417836116e-06,
      "logits/chosen": -2.7753589153289795,
      "logits/rejected": -2.829224109649658,
      "logps/chosen": -0.7804813385009766,
      "logps/rejected": -1.1957075595855713,
      "loss": 0.8432,
      "odds_ratio_loss": 0.6271591186523438,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07804813235998154,
      "rewards/margins": 0.04152262955904007,
      "rewards/rejected": -0.1195707693696022,
      "sft_loss": 0.7804813385009766,
      "step": 290
    },
    {
      "epoch": 0.4849464538290564,
      "grad_norm": 2.0640830993652344,
      "learning_rate": 4.684046881778603e-06,
      "logits/chosen": -2.8023476600646973,
      "logits/rejected": -2.8235526084899902,
      "logps/chosen": -0.8398802876472473,
      "logps/rejected": -0.9978183507919312,
      "loss": 0.9045,
      "odds_ratio_loss": 0.6464654803276062,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.08398802578449249,
      "rewards/margins": 0.015793804079294205,
      "rewards/rejected": -0.099781833589077,
      "sft_loss": 0.8398802876472473,
      "step": 300
    },
    {
      "epoch": 0.5011113356233583,
      "grad_norm": 1.626105785369873,
      "learning_rate": 4.663119185217409e-06,
      "logits/chosen": -2.796461343765259,
      "logits/rejected": -2.8225197792053223,
      "logps/chosen": -0.8273599743843079,
      "logps/rejected": -1.096482515335083,
      "loss": 0.8875,
      "odds_ratio_loss": 0.6016198396682739,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.08273600041866302,
      "rewards/margins": 0.026912260800600052,
      "rewards/rejected": -0.10964826494455338,
      "sft_loss": 0.8273599743843079,
      "step": 310
    },
    {
      "epoch": 0.5172762174176602,
      "grad_norm": 1.5098748207092285,
      "learning_rate": 4.641570337650232e-06,
      "logits/chosen": -2.847539186477661,
      "logits/rejected": -2.85341215133667,
      "logps/chosen": -0.7699432969093323,
      "logps/rejected": -1.0820213556289673,
      "loss": 0.8268,
      "odds_ratio_loss": 0.5688191652297974,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.07699433714151382,
      "rewards/margins": 0.03120780549943447,
      "rewards/rejected": -0.10820214450359344,
      "sft_loss": 0.7699432969093323,
      "step": 320
    },
    {
      "epoch": 0.533441099211962,
      "grad_norm": 1.3477349281311035,
      "learning_rate": 4.61940652694154e-06,
      "logits/chosen": -2.7625374794006348,
      "logits/rejected": -2.8054728507995605,
      "logps/chosen": -0.8576439023017883,
      "logps/rejected": -1.2374662160873413,
      "loss": 0.9224,
      "odds_ratio_loss": 0.6476989984512329,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.08576439321041107,
      "rewards/margins": 0.03798223286867142,
      "rewards/rejected": -0.12374663352966309,
      "sft_loss": 0.8576439023017883,
      "step": 330
    },
    {
      "epoch": 0.5496059810062639,
      "grad_norm": 2.094233274459839,
      "learning_rate": 4.596634117545689e-06,
      "logits/chosen": -2.8440895080566406,
      "logits/rejected": -2.8477485179901123,
      "logps/chosen": -0.8450831174850464,
      "logps/rejected": -1.1874289512634277,
      "loss": 0.9084,
      "odds_ratio_loss": 0.6333492994308472,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.08450832217931747,
      "rewards/margins": 0.03423457592725754,
      "rewards/rejected": -0.11874288320541382,
      "sft_loss": 0.8450831174850464,
      "step": 340
    },
    {
      "epoch": 0.5657708628005658,
      "grad_norm": 1.2610398530960083,
      "learning_rate": 4.573259648679335e-06,
      "logits/chosen": -2.8393020629882812,
      "logits/rejected": -2.8172850608825684,
      "logps/chosen": -0.8293860554695129,
      "logps/rejected": -1.1484854221343994,
      "loss": 0.8924,
      "odds_ratio_loss": 0.6304416060447693,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.08293859660625458,
      "rewards/margins": 0.03190993517637253,
      "rewards/rejected": -0.1148485392332077,
      "sft_loss": 0.8293860554695129,
      "step": 350
    },
    {
      "epoch": 0.5819357445948676,
      "grad_norm": 7.934630870819092,
      "learning_rate": 4.549289832443663e-06,
      "logits/chosen": -2.8159756660461426,
      "logits/rejected": -2.8409628868103027,
      "logps/chosen": -0.885659396648407,
      "logps/rejected": -1.2282092571258545,
      "loss": 0.9498,
      "odds_ratio_loss": 0.641811192035675,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.08856594562530518,
      "rewards/margins": 0.03425499051809311,
      "rewards/rejected": -0.12282093614339828,
      "sft_loss": 0.885659396648407,
      "step": 360
    },
    {
      "epoch": 0.5981006263891695,
      "grad_norm": 1.7960658073425293,
      "learning_rate": 4.524731551896978e-06,
      "logits/chosen": -2.8090755939483643,
      "logits/rejected": -2.825777292251587,
      "logps/chosen": -0.7784116864204407,
      "logps/rejected": -0.9700002670288086,
      "loss": 0.8424,
      "odds_ratio_loss": 0.6396910548210144,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07784116268157959,
      "rewards/margins": 0.01915885880589485,
      "rewards/rejected": -0.09700002521276474,
      "sft_loss": 0.7784116864204407,
      "step": 370
    },
    {
      "epoch": 0.6142655081834714,
      "grad_norm": 3.268920421600342,
      "learning_rate": 4.4995918590781925e-06,
      "logits/chosen": -2.853820562362671,
      "logits/rejected": -2.8512935638427734,
      "logps/chosen": -0.8428764343261719,
      "logps/rejected": -1.0172072649002075,
      "loss": 0.9104,
      "odds_ratio_loss": 0.6751636266708374,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.08428764343261719,
      "rewards/margins": 0.01743307337164879,
      "rewards/rejected": -0.10172072798013687,
      "sft_loss": 0.8428764343261719,
      "step": 380
    },
    {
      "epoch": 0.6304303899777733,
      "grad_norm": 1.0598444938659668,
      "learning_rate": 4.473877972981797e-06,
      "logits/chosen": -2.7993013858795166,
      "logits/rejected": -2.789777994155884,
      "logps/chosen": -0.8297500610351562,
      "logps/rejected": -1.0850985050201416,
      "loss": 0.8895,
      "odds_ratio_loss": 0.5971348881721497,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.08297501504421234,
      "rewards/margins": 0.025534838438034058,
      "rewards/rejected": -0.1085098534822464,
      "sft_loss": 0.8297500610351562,
      "step": 390
    },
    {
      "epoch": 0.6465952717720752,
      "grad_norm": 1.9357444047927856,
      "learning_rate": 4.447597277484894e-06,
      "logits/chosen": -2.7699055671691895,
      "logits/rejected": -2.798750400543213,
      "logps/chosen": -0.7733790874481201,
      "logps/rejected": -0.9783531427383423,
      "loss": 0.8347,
      "odds_ratio_loss": 0.6135808825492859,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07733791321516037,
      "rewards/margins": 0.020497407764196396,
      "rewards/rejected": -0.09783531725406647,
      "sft_loss": 0.7733790874481201,
      "step": 400
    },
    {
      "epoch": 0.6627601535663771,
      "grad_norm": 1.4025357961654663,
      "learning_rate": 4.42075731922687e-06,
      "logits/chosen": -2.8587729930877686,
      "logits/rejected": -2.87328839302063,
      "logps/chosen": -0.9505017995834351,
      "logps/rejected": -1.1930662393569946,
      "loss": 1.0132,
      "odds_ratio_loss": 0.6273903250694275,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.09505018591880798,
      "rewards/margins": 0.024256447330117226,
      "rewards/rejected": -0.11930663883686066,
      "sft_loss": 0.9505017995834351,
      "step": 410
    },
    {
      "epoch": 0.6789250353606789,
      "grad_norm": 5.174298286437988,
      "learning_rate": 4.3933658054423465e-06,
      "logits/chosen": -2.8345279693603516,
      "logits/rejected": -2.83827543258667,
      "logps/chosen": -0.80866539478302,
      "logps/rejected": -1.174803614616394,
      "loss": 0.8664,
      "odds_ratio_loss": 0.5777753591537476,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08086653053760529,
      "rewards/margins": 0.036613818258047104,
      "rewards/rejected": -0.11748035252094269,
      "sft_loss": 0.80866539478302,
      "step": 420
    },
    {
      "epoch": 0.6950899171549808,
      "grad_norm": 2.207981586456299,
      "learning_rate": 4.365430601748003e-06,
      "logits/chosen": -2.8343446254730225,
      "logits/rejected": -2.857731342315674,
      "logps/chosen": -0.9037211537361145,
      "logps/rejected": -1.0559289455413818,
      "loss": 0.9705,
      "odds_ratio_loss": 0.6677287817001343,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.09037211537361145,
      "rewards/margins": 0.01522077340632677,
      "rewards/rejected": -0.10559289157390594,
      "sft_loss": 0.9037211537361145,
      "step": 430
    },
    {
      "epoch": 0.7112547989492827,
      "grad_norm": 15.49936580657959,
      "learning_rate": 4.336959729883925e-06,
      "logits/chosen": -2.8130838871002197,
      "logits/rejected": -2.8357608318328857,
      "logps/chosen": -0.8217814564704895,
      "logps/rejected": -0.9188777804374695,
      "loss": 0.8923,
      "odds_ratio_loss": 0.7047211527824402,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.08217814564704895,
      "rewards/margins": 0.00970962829887867,
      "rewards/rejected": -0.09188777953386307,
      "sft_loss": 0.8217814564704895,
      "step": 440
    },
    {
      "epoch": 0.7274196807435845,
      "grad_norm": 1.7557275295257568,
      "learning_rate": 4.307961365410118e-06,
      "logits/chosen": -2.790027379989624,
      "logits/rejected": -2.809622049331665,
      "logps/chosen": -0.840091347694397,
      "logps/rejected": -1.0152480602264404,
      "loss": 0.9039,
      "odds_ratio_loss": 0.6380866169929504,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.08400914072990417,
      "rewards/margins": 0.017515674233436584,
      "rewards/rejected": -0.10152481496334076,
      "sft_loss": 0.840091347694397,
      "step": 450
    },
    {
      "epoch": 0.7435845625378864,
      "grad_norm": 2.8990914821624756,
      "learning_rate": 4.278443835358854e-06,
      "logits/chosen": -2.812924861907959,
      "logits/rejected": -2.811110734939575,
      "logps/chosen": -0.8139681816101074,
      "logps/rejected": -1.0690581798553467,
      "loss": 0.8748,
      "odds_ratio_loss": 0.6082891225814819,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.08139681816101074,
      "rewards/margins": 0.025509005412459373,
      "rewards/rejected": -0.10690581798553467,
      "sft_loss": 0.8139681816101074,
      "step": 460
    },
    {
      "epoch": 0.7597494443321883,
      "grad_norm": 2.2395644187927246,
      "learning_rate": 4.248415615843523e-06,
      "logits/chosen": -2.8422694206237793,
      "logits/rejected": -2.850648880004883,
      "logps/chosen": -0.8527294993400574,
      "logps/rejected": -1.0392307043075562,
      "loss": 0.9183,
      "odds_ratio_loss": 0.6552284359931946,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.08527294546365738,
      "rewards/margins": 0.01865011267364025,
      "rewards/rejected": -0.10392306745052338,
      "sft_loss": 0.8527294993400574,
      "step": 470
    },
    {
      "epoch": 0.7759143261264903,
      "grad_norm": 2.0075857639312744,
      "learning_rate": 4.217885329624666e-06,
      "logits/chosen": -2.8313276767730713,
      "logits/rejected": -2.8245348930358887,
      "logps/chosen": -0.790324330329895,
      "logps/rejected": -1.0767412185668945,
      "loss": 0.8498,
      "odds_ratio_loss": 0.5943514108657837,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07903242856264114,
      "rewards/margins": 0.02864169515669346,
      "rewards/rejected": -0.10767412185668945,
      "sft_loss": 0.790324330329895,
      "step": 480
    },
    {
      "epoch": 0.7920792079207921,
      "grad_norm": 1.7681854963302612,
      "learning_rate": 4.186861743633911e-06,
      "logits/chosen": -2.8171868324279785,
      "logits/rejected": -2.8480162620544434,
      "logps/chosen": -0.7983497381210327,
      "logps/rejected": -1.1061131954193115,
      "loss": 0.8646,
      "odds_ratio_loss": 0.6626344919204712,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.07983498275279999,
      "rewards/margins": 0.03077634610235691,
      "rewards/rejected": -0.11061131954193115,
      "sft_loss": 0.7983497381210327,
      "step": 490
    },
    {
      "epoch": 0.808244089715094,
      "grad_norm": 1.5298829078674316,
      "learning_rate": 4.155353766456497e-06,
      "logits/chosen": -2.874368190765381,
      "logits/rejected": -2.8658576011657715,
      "logps/chosen": -0.8663871884346008,
      "logps/rejected": -1.0296813249588013,
      "loss": 0.93,
      "odds_ratio_loss": 0.636117160320282,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.08663871884346008,
      "rewards/margins": 0.016329411417245865,
      "rewards/rejected": -0.10296813398599625,
      "sft_loss": 0.8663871884346008,
      "step": 500
    },
    {
      "epoch": 0.808244089715094,
      "eval_logits/chosen": -2.836843729019165,
      "eval_logits/rejected": -2.8441994190216064,
      "eval_logps/chosen": -0.8278239965438843,
      "eval_logps/rejected": -1.0567275285720825,
      "eval_loss": 0.8927881121635437,
      "eval_odds_ratio_loss": 0.6496399641036987,
      "eval_rewards/accuracies": 0.5772727131843567,
      "eval_rewards/chosen": -0.08278240263462067,
      "eval_rewards/margins": 0.02289034053683281,
      "eval_rewards/rejected": -0.10567274689674377,
      "eval_runtime": 194.5311,
      "eval_samples_per_second": 5.655,
      "eval_sft_loss": 0.8278239965438843,
      "eval_steps_per_second": 2.827,
      "step": 500
    },
    {
      "epoch": 0.8244089715093958,
      "grad_norm": 1.6909329891204834,
      "learning_rate": 4.123370445773134e-06,
      "logits/chosen": -2.8691649436950684,
      "logits/rejected": -2.8811800479888916,
      "logps/chosen": -0.8283156156539917,
      "logps/rejected": -0.9291037321090698,
      "loss": 0.8973,
      "odds_ratio_loss": 0.689969539642334,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.08283156156539917,
      "rewards/margins": 0.01007880363613367,
      "rewards/rejected": -0.09291036427021027,
      "sft_loss": 0.8283156156539917,
      "step": 510
    },
    {
      "epoch": 0.8405738533036977,
      "grad_norm": 4.33729362487793,
      "learning_rate": 4.090920965761906e-06,
      "logits/chosen": -2.808586597442627,
      "logits/rejected": -2.8186278343200684,
      "logps/chosen": -0.8606308698654175,
      "logps/rejected": -1.0332623720169067,
      "loss": 0.9284,
      "odds_ratio_loss": 0.6780760884284973,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.08606309443712234,
      "rewards/margins": 0.0172631423920393,
      "rewards/rejected": -0.1033262237906456,
      "sft_loss": 0.8606308698654175,
      "step": 520
    },
    {
      "epoch": 0.8567387350979996,
      "grad_norm": 6.002406120300293,
      "learning_rate": 4.058014644460991e-06,
      "logits/chosen": -2.833061456680298,
      "logits/rejected": -2.8458170890808105,
      "logps/chosen": -0.8242424726486206,
      "logps/rejected": -0.9793018102645874,
      "loss": 0.8862,
      "odds_ratio_loss": 0.6198969483375549,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.08242423832416534,
      "rewards/margins": 0.015505945309996605,
      "rewards/rejected": -0.0979301929473877,
      "sft_loss": 0.8242424726486206,
      "step": 530
    },
    {
      "epoch": 0.8729036168923014,
      "grad_norm": 1.998780608177185,
      "learning_rate": 4.024660931092939e-06,
      "logits/chosen": -2.81856369972229,
      "logits/rejected": -2.8293251991271973,
      "logps/chosen": -0.8208298683166504,
      "logps/rejected": -1.0441166162490845,
      "loss": 0.8828,
      "odds_ratio_loss": 0.6198452115058899,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.08208298683166504,
      "rewards/margins": 0.022328665480017662,
      "rewards/rejected": -0.10441166162490845,
      "sft_loss": 0.8208298683166504,
      "step": 540
    },
    {
      "epoch": 0.8890684986866033,
      "grad_norm": 2.4577414989471436,
      "learning_rate": 3.990869403351272e-06,
      "logits/chosen": -2.8507511615753174,
      "logits/rejected": -2.8566970825195312,
      "logps/chosen": -0.8117038011550903,
      "logps/rejected": -1.0751911401748657,
      "loss": 0.8674,
      "odds_ratio_loss": 0.5573362112045288,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.08117038011550903,
      "rewards/margins": 0.026348743587732315,
      "rewards/rejected": -0.10751912742853165,
      "sft_loss": 0.8117038011550903,
      "step": 550
    },
    {
      "epoch": 0.9052333804809052,
      "grad_norm": 3.4686763286590576,
      "learning_rate": 3.956649764650206e-06,
      "logits/chosen": -2.881647825241089,
      "logits/rejected": -2.8819093704223633,
      "logps/chosen": -0.840446949005127,
      "logps/rejected": -1.052137017250061,
      "loss": 0.907,
      "odds_ratio_loss": 0.6658841967582703,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0840446949005127,
      "rewards/margins": 0.02116900309920311,
      "rewards/rejected": -0.1052137017250061,
      "sft_loss": 0.840446949005127,
      "step": 560
    },
    {
      "epoch": 0.9213982622752072,
      "grad_norm": 2.2446658611297607,
      "learning_rate": 3.92201184133826e-06,
      "logits/chosen": -2.864419460296631,
      "logits/rejected": -2.8783581256866455,
      "logps/chosen": -0.7979758381843567,
      "logps/rejected": -1.0608371496200562,
      "loss": 0.858,
      "odds_ratio_loss": 0.5999220609664917,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07979758828878403,
      "rewards/margins": 0.026286140084266663,
      "rewards/rejected": -0.1060837134718895,
      "sft_loss": 0.7979758381843567,
      "step": 570
    },
    {
      "epoch": 0.937563144069509,
      "grad_norm": 1.9976744651794434,
      "learning_rate": 3.886965579876572e-06,
      "logits/chosen": -2.900329351425171,
      "logits/rejected": -2.90751051902771,
      "logps/chosen": -0.8153482675552368,
      "logps/rejected": -0.9346411824226379,
      "loss": 0.8816,
      "odds_ratio_loss": 0.6620460748672485,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.08153482526540756,
      "rewards/margins": 0.011929300613701344,
      "rewards/rejected": -0.09346412122249603,
      "sft_loss": 0.8153482675552368,
      "step": 580
    },
    {
      "epoch": 0.9537280258638109,
      "grad_norm": 1.6091820001602173,
      "learning_rate": 3.851521043982716e-06,
      "logits/chosen": -2.8917582035064697,
      "logits/rejected": -2.902100086212158,
      "logps/chosen": -0.8334836959838867,
      "logps/rejected": -1.004950761795044,
      "loss": 0.9,
      "odds_ratio_loss": 0.6651790738105774,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.0833483636379242,
      "rewards/margins": 0.017146697267889977,
      "rewards/rejected": -0.10049506276845932,
      "sft_loss": 0.8334836959838867,
      "step": 590
    },
    {
      "epoch": 0.9698929076581128,
      "grad_norm": 5.672989845275879,
      "learning_rate": 3.81568841174086e-06,
      "logits/chosen": -2.861603021621704,
      "logits/rejected": -2.876756191253662,
      "logps/chosen": -0.7806357145309448,
      "logps/rejected": -1.1542575359344482,
      "loss": 0.8442,
      "odds_ratio_loss": 0.6360144019126892,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07806357741355896,
      "rewards/margins": 0.03736215457320213,
      "rewards/rejected": -0.1154257282614708,
      "sft_loss": 0.7806357145309448,
      "step": 600
    },
    {
      "epoch": 0.9860577894524146,
      "grad_norm": 1.4658279418945312,
      "learning_rate": 3.7794779726790664e-06,
      "logits/chosen": -2.845876455307007,
      "logits/rejected": -2.8574581146240234,
      "logps/chosen": -0.7789396047592163,
      "logps/rejected": -1.1114189624786377,
      "loss": 0.8409,
      "odds_ratio_loss": 0.6194978952407837,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07789396494626999,
      "rewards/margins": 0.0332479402422905,
      "rewards/rejected": -0.11114190518856049,
      "sft_loss": 0.7789396047592163,
      "step": 610
    },
    {
      "epoch": 1.0022226712467166,
      "grad_norm": 2.5747179985046387,
      "learning_rate": 3.7429001248146096e-06,
      "logits/chosen": -2.8244144916534424,
      "logits/rejected": -2.832597494125366,
      "logps/chosen": -0.7860082387924194,
      "logps/rejected": -1.0231492519378662,
      "loss": 0.8435,
      "odds_ratio_loss": 0.5752763748168945,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07860083132982254,
      "rewards/margins": 0.023714100942015648,
      "rewards/rejected": -0.10231492668390274,
      "sft_loss": 0.7860082387924194,
      "step": 620
    },
    {
      "epoch": 1.0183875530410185,
      "grad_norm": 1.24222993850708,
      "learning_rate": 3.7059653716681227e-06,
      "logits/chosen": -2.8329997062683105,
      "logits/rejected": -2.8287994861602783,
      "logps/chosen": -0.8590106964111328,
      "logps/rejected": -1.0588136911392212,
      "loss": 0.9265,
      "odds_ratio_loss": 0.6749905347824097,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.08590107411146164,
      "rewards/margins": 0.01998029835522175,
      "rewards/rejected": -0.10588137060403824,
      "sft_loss": 0.8590106964111328,
      "step": 630
    },
    {
      "epoch": 1.0345524348353203,
      "grad_norm": 1.6466968059539795,
      "learning_rate": 3.668684319247463e-06,
      "logits/chosen": -2.8495888710021973,
      "logits/rejected": -2.872880220413208,
      "logps/chosen": -0.7487844824790955,
      "logps/rejected": -1.0430450439453125,
      "loss": 0.8035,
      "odds_ratio_loss": 0.5467280149459839,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.07487844675779343,
      "rewards/margins": 0.02942606247961521,
      "rewards/rejected": -0.10430450737476349,
      "sft_loss": 0.7487844824790955,
      "step": 640
    },
    {
      "epoch": 1.0507173166296222,
      "grad_norm": 1.1547085046768188,
      "learning_rate": 3.6310676730021373e-06,
      "logits/chosen": -2.8986639976501465,
      "logits/rejected": -2.900839328765869,
      "logps/chosen": -0.7881689071655273,
      "logps/rejected": -0.9517928957939148,
      "loss": 0.8509,
      "odds_ratio_loss": 0.6268683075904846,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.07881689816713333,
      "rewards/margins": 0.016362406313419342,
      "rewards/rejected": -0.09517930448055267,
      "sft_loss": 0.7881689071655273,
      "step": 650
    },
    {
      "epoch": 1.066882198423924,
      "grad_norm": 3.282292604446411,
      "learning_rate": 3.593126234749178e-06,
      "logits/chosen": -2.8645131587982178,
      "logits/rejected": -2.898613929748535,
      "logps/chosen": -0.9009162187576294,
      "logps/rejected": -1.1612458229064941,
      "loss": 0.9648,
      "odds_ratio_loss": 0.6383681297302246,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.09009162336587906,
      "rewards/margins": 0.026032963767647743,
      "rewards/rejected": -0.11612458527088165,
      "sft_loss": 0.9009162187576294,
      "step": 660
    },
    {
      "epoch": 1.083047080218226,
      "grad_norm": 1.7910722494125366,
      "learning_rate": 3.554870899571343e-06,
      "logits/chosen": -2.8563625812530518,
      "logits/rejected": -2.8744523525238037,
      "logps/chosen": -0.8285778760910034,
      "logps/rejected": -1.0025149583816528,
      "loss": 0.8927,
      "odds_ratio_loss": 0.6415389776229858,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.08285778015851974,
      "rewards/margins": 0.01739371195435524,
      "rewards/rejected": -0.10025149583816528,
      "sft_loss": 0.8285778760910034,
      "step": 670
    },
    {
      "epoch": 1.0992119620125278,
      "grad_norm": 3.5774316787719727,
      "learning_rate": 3.5163126526885373e-06,
      "logits/chosen": -2.8437960147857666,
      "logits/rejected": -2.870513916015625,
      "logps/chosen": -0.7732303142547607,
      "logps/rejected": -1.0101302862167358,
      "loss": 0.8343,
      "odds_ratio_loss": 0.6102721095085144,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07732303440570831,
      "rewards/margins": 0.023690002039074898,
      "rewards/rejected": -0.10101302713155746,
      "sft_loss": 0.7732303142547607,
      "step": 680
    },
    {
      "epoch": 1.1153768438068297,
      "grad_norm": 2.30400013923645,
      "learning_rate": 3.4774625663033484e-06,
      "logits/chosen": -2.849010467529297,
      "logits/rejected": -2.8660061359405518,
      "logps/chosen": -0.7853142619132996,
      "logps/rejected": -0.9644325971603394,
      "loss": 0.8466,
      "odds_ratio_loss": 0.6127563714981079,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.0785314291715622,
      "rewards/margins": 0.017911842092871666,
      "rewards/rejected": -0.09644327312707901,
      "sft_loss": 0.7853142619132996,
      "step": 690
    },
    {
      "epoch": 1.1315417256011315,
      "grad_norm": 1.4719704389572144,
      "learning_rate": 3.4383317964216067e-06,
      "logits/chosen": -2.8511626720428467,
      "logits/rejected": -2.881286382675171,
      "logps/chosen": -0.7790023684501648,
      "logps/rejected": -0.9076374173164368,
      "loss": 0.8484,
      "odds_ratio_loss": 0.6935282945632935,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.0779002457857132,
      "rewards/margins": 0.012863497249782085,
      "rewards/rejected": -0.09076374769210815,
      "sft_loss": 0.7790023684501648,
      "step": 700
    },
    {
      "epoch": 1.1477066073954334,
      "grad_norm": 2.1927425861358643,
      "learning_rate": 3.398931579648877e-06,
      "logits/chosen": -2.8756051063537598,
      "logits/rejected": -2.880699872970581,
      "logps/chosen": -0.8047206997871399,
      "logps/rejected": -1.1634694337844849,
      "loss": 0.8667,
      "odds_ratio_loss": 0.6202768087387085,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.08047207444906235,
      "rewards/margins": 0.0358748659491539,
      "rewards/rejected": -0.11634693294763565,
      "sft_loss": 0.8047206997871399,
      "step": 710
    },
    {
      "epoch": 1.1638714891897353,
      "grad_norm": 1.4328726530075073,
      "learning_rate": 3.359273229963813e-06,
      "logits/chosen": -2.8490045070648193,
      "logits/rejected": -2.8502037525177,
      "logps/chosen": -0.7575694918632507,
      "logps/rejected": -0.9301745295524597,
      "loss": 0.821,
      "odds_ratio_loss": 0.6343931555747986,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07575695216655731,
      "rewards/margins": 0.017260495573282242,
      "rewards/rejected": -0.09301744401454926,
      "sft_loss": 0.7575694918632507,
      "step": 720
    },
    {
      "epoch": 1.1800363709840371,
      "grad_norm": 1.3170576095581055,
      "learning_rate": 3.319368135469285e-06,
      "logits/chosen": -2.8658504486083984,
      "logits/rejected": -2.8875842094421387,
      "logps/chosen": -0.8195670247077942,
      "logps/rejected": -1.1535929441452026,
      "loss": 0.8841,
      "odds_ratio_loss": 0.6450805068016052,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.08195669949054718,
      "rewards/margins": 0.03340259566903114,
      "rewards/rejected": -0.11535929143428802,
      "sft_loss": 0.8195670247077942,
      "step": 730
    },
    {
      "epoch": 1.196201252778339,
      "grad_norm": 4.55858850479126,
      "learning_rate": 3.279227755122228e-06,
      "logits/chosen": -2.858372211456299,
      "logits/rejected": -2.860966205596924,
      "logps/chosen": -0.7807797193527222,
      "logps/rejected": -1.1492526531219482,
      "loss": 0.839,
      "odds_ratio_loss": 0.5826634764671326,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.07807797938585281,
      "rewards/margins": 0.03684728592634201,
      "rewards/rejected": -0.11492526531219482,
      "sft_loss": 0.7807797193527222,
      "step": 740
    },
    {
      "epoch": 1.2123661345726409,
      "grad_norm": 2.330960988998413,
      "learning_rate": 3.2388636154431417e-06,
      "logits/chosen": -2.868211507797241,
      "logits/rejected": -2.898150682449341,
      "logps/chosen": -0.8243536949157715,
      "logps/rejected": -1.195150375366211,
      "loss": 0.883,
      "odds_ratio_loss": 0.5866126418113708,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.08243536949157715,
      "rewards/margins": 0.03707967326045036,
      "rewards/rejected": -0.11951503902673721,
      "sft_loss": 0.8243536949157715,
      "step": 750
    },
    {
      "epoch": 1.2285310163669427,
      "grad_norm": 2.7208411693573,
      "learning_rate": 3.198287307206192e-06,
      "logits/chosen": -2.844311237335205,
      "logits/rejected": -2.8444716930389404,
      "logps/chosen": -0.7780786752700806,
      "logps/rejected": -0.9966138005256653,
      "loss": 0.8378,
      "odds_ratio_loss": 0.5971704721450806,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07780785858631134,
      "rewards/margins": 0.021853512153029442,
      "rewards/rejected": -0.09966136515140533,
      "sft_loss": 0.7780786752700806,
      "step": 760
    },
    {
      "epoch": 1.2446958981612446,
      "grad_norm": 1.3042361736297607,
      "learning_rate": 3.157510482110856e-06,
      "logits/chosen": -2.9084322452545166,
      "logits/rejected": -2.905463933944702,
      "logps/chosen": -0.7917675971984863,
      "logps/rejected": -1.0798178911209106,
      "loss": 0.8557,
      "odds_ratio_loss": 0.6388932466506958,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07917676120996475,
      "rewards/margins": 0.02880503609776497,
      "rewards/rejected": -0.10798178613185883,
      "sft_loss": 0.7917675971984863,
      "step": 770
    },
    {
      "epoch": 1.2608607799555465,
      "grad_norm": 1.315172553062439,
      "learning_rate": 3.116544849436077e-06,
      "logits/chosen": -2.828716993331909,
      "logits/rejected": -2.8282887935638428,
      "logps/chosen": -0.8439006805419922,
      "logps/rejected": -1.2042268514633179,
      "loss": 0.9037,
      "odds_ratio_loss": 0.5979124307632446,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0843900591135025,
      "rewards/margins": 0.03603263571858406,
      "rewards/rejected": -0.12042269855737686,
      "sft_loss": 0.8439006805419922,
      "step": 780
    },
    {
      "epoch": 1.2770256617498483,
      "grad_norm": 1.611197829246521,
      "learning_rate": 3.0754021726778848e-06,
      "logits/chosen": -2.84073543548584,
      "logits/rejected": -2.832176685333252,
      "logps/chosen": -0.7603198885917664,
      "logps/rejected": -1.202492117881775,
      "loss": 0.8152,
      "odds_ratio_loss": 0.5489572882652283,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07603198289871216,
      "rewards/margins": 0.044217221438884735,
      "rewards/rejected": -0.1202491968870163,
      "sft_loss": 0.7603198885917664,
      "step": 790
    },
    {
      "epoch": 1.2931905435441502,
      "grad_norm": 1.179275631904602,
      "learning_rate": 3.0340942661714463e-06,
      "logits/chosen": -2.877725839614868,
      "logits/rejected": -2.891244411468506,
      "logps/chosen": -0.8281265497207642,
      "logps/rejected": -1.0409139394760132,
      "loss": 0.8904,
      "odds_ratio_loss": 0.6231717467308044,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.08281265199184418,
      "rewards/margins": 0.021278750151395798,
      "rewards/rejected": -0.10409140586853027,
      "sft_loss": 0.8281265497207642,
      "step": 800
    },
    {
      "epoch": 1.3093554253384523,
      "grad_norm": 2.1846208572387695,
      "learning_rate": 2.992632991698512e-06,
      "logits/chosen": -2.8389461040496826,
      "logits/rejected": -2.85896635055542,
      "logps/chosen": -0.8289766311645508,
      "logps/rejected": -1.0603488683700562,
      "loss": 0.8918,
      "odds_ratio_loss": 0.6286410093307495,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08289766311645508,
      "rewards/margins": 0.02313724346458912,
      "rewards/rejected": -0.10603491216897964,
      "sft_loss": 0.8289766311645508,
      "step": 810
    },
    {
      "epoch": 1.3255203071327541,
      "grad_norm": 1.583357334136963,
      "learning_rate": 2.9510302550812537e-06,
      "logits/chosen": -2.845541000366211,
      "logits/rejected": -2.8782455921173096,
      "logps/chosen": -0.7186457514762878,
      "logps/rejected": -1.0902959108352661,
      "loss": 0.776,
      "odds_ratio_loss": 0.573469340801239,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07186457514762878,
      "rewards/margins": 0.03716501593589783,
      "rewards/rejected": -0.10902959108352661,
      "sft_loss": 0.7186457514762878,
      "step": 820
    },
    {
      "epoch": 1.341685188927056,
      "grad_norm": 2.841128349304199,
      "learning_rate": 2.9092980027634325e-06,
      "logits/chosen": -2.8583426475524902,
      "logits/rejected": -2.874774217605591,
      "logps/chosen": -0.7276403903961182,
      "logps/rejected": -1.0125164985656738,
      "loss": 0.788,
      "odds_ratio_loss": 0.6034457683563232,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07276404649019241,
      "rewards/margins": 0.028487607836723328,
      "rewards/rejected": -0.10125164687633514,
      "sft_loss": 0.7276403903961182,
      "step": 830
    },
    {
      "epoch": 1.3578500707213579,
      "grad_norm": 1.7055377960205078,
      "learning_rate": 2.867448218379927e-06,
      "logits/chosen": -2.8610100746154785,
      "logits/rejected": -2.8836147785186768,
      "logps/chosen": -0.8485835790634155,
      "logps/rejected": -1.0031511783599854,
      "loss": 0.9172,
      "odds_ratio_loss": 0.6861482858657837,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.08485837280750275,
      "rewards/margins": 0.015456756576895714,
      "rewards/rejected": -0.10031511634588242,
      "sft_loss": 0.8485835790634155,
      "step": 840
    },
    {
      "epoch": 1.3740149525156597,
      "grad_norm": 9.629118919372559,
      "learning_rate": 2.825492919315559e-06,
      "logits/chosen": -2.8479480743408203,
      "logits/rejected": -2.8763227462768555,
      "logps/chosen": -0.8768585324287415,
      "logps/rejected": -0.999729335308075,
      "loss": 0.9437,
      "odds_ratio_loss": 0.668052613735199,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.08768586814403534,
      "rewards/margins": 0.012287073768675327,
      "rewards/rejected": -0.0999729260802269,
      "sft_loss": 0.8768585324287415,
      "step": 850
    },
    {
      "epoch": 1.3901798343099616,
      "grad_norm": 2.416870594024658,
      "learning_rate": 2.7834441532542482e-06,
      "logits/chosen": -2.8881735801696777,
      "logits/rejected": -2.9063100814819336,
      "logps/chosen": -0.7879316210746765,
      "logps/rejected": -1.023233413696289,
      "loss": 0.8456,
      "odds_ratio_loss": 0.5766496658325195,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07879316806793213,
      "rewards/margins": 0.023530183359980583,
      "rewards/rejected": -0.10232335329055786,
      "sft_loss": 0.7879316210746765,
      "step": 860
    },
    {
      "epoch": 1.4063447161042635,
      "grad_norm": 1.5628215074539185,
      "learning_rate": 2.74131399471945e-06,
      "logits/chosen": -2.855931520462036,
      "logits/rejected": -2.8686752319335938,
      "logps/chosen": -0.7991023063659668,
      "logps/rejected": -0.9908691644668579,
      "loss": 0.8644,
      "odds_ratio_loss": 0.6528818607330322,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07991023361682892,
      "rewards/margins": 0.01917668618261814,
      "rewards/rejected": -0.09908691793680191,
      "sft_loss": 0.7991023063659668,
      "step": 870
    },
    {
      "epoch": 1.4225095978985653,
      "grad_norm": 2.0555615425109863,
      "learning_rate": 2.6991145416068947e-06,
      "logits/chosen": -2.846782922744751,
      "logits/rejected": -2.8673818111419678,
      "logps/chosen": -0.8078680038452148,
      "logps/rejected": -0.9619809985160828,
      "loss": 0.8714,
      "odds_ratio_loss": 0.6356260180473328,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.0807868018746376,
      "rewards/margins": 0.01541130244731903,
      "rewards/rejected": -0.09619811177253723,
      "sft_loss": 0.8078680038452148,
      "step": 880
    },
    {
      "epoch": 1.4386744796928672,
      "grad_norm": 0.9378024339675903,
      "learning_rate": 2.6568579117106143e-06,
      "logits/chosen": -2.8469960689544678,
      "logits/rejected": -2.850614070892334,
      "logps/chosen": -0.7744920253753662,
      "logps/rejected": -1.0393074750900269,
      "loss": 0.8347,
      "odds_ratio_loss": 0.6018751859664917,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07744920998811722,
      "rewards/margins": 0.026481550186872482,
      "rewards/rejected": -0.1039307564496994,
      "sft_loss": 0.7744920253753662,
      "step": 890
    },
    {
      "epoch": 1.454839361487169,
      "grad_norm": 0.9352036118507385,
      "learning_rate": 2.6145562392432544e-06,
      "logits/chosen": -2.875109910964966,
      "logits/rejected": -2.887655735015869,
      "logps/chosen": -0.8057360649108887,
      "logps/rejected": -0.9923427700996399,
      "loss": 0.8708,
      "odds_ratio_loss": 0.6502856016159058,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.08057360351085663,
      "rewards/margins": 0.01866067573428154,
      "rewards/rejected": -0.09923428297042847,
      "sft_loss": 0.8057360649108887,
      "step": 900
    },
    {
      "epoch": 1.471004243281471,
      "grad_norm": 2.6385111808776855,
      "learning_rate": 2.5722216713516682e-06,
      "logits/chosen": -2.8550915718078613,
      "logits/rejected": -2.8972582817077637,
      "logps/chosen": -0.7460139989852905,
      "logps/rejected": -0.9863673448562622,
      "loss": 0.8057,
      "odds_ratio_loss": 0.5972028374671936,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07460139691829681,
      "rewards/margins": 0.024035323411226273,
      "rewards/rejected": -0.09863673150539398,
      "sft_loss": 0.7460139989852905,
      "step": 910
    },
    {
      "epoch": 1.4871691250757728,
      "grad_norm": 1.7198817729949951,
      "learning_rate": 2.5298663646288064e-06,
      "logits/chosen": -2.8807036876678467,
      "logits/rejected": -2.888306140899658,
      "logps/chosen": -0.7764211893081665,
      "logps/rejected": -1.0312559604644775,
      "loss": 0.8377,
      "odds_ratio_loss": 0.6123490333557129,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07764211297035217,
      "rewards/margins": 0.025483474135398865,
      "rewards/rejected": -0.10312558710575104,
      "sft_loss": 0.7764211893081665,
      "step": 920
    },
    {
      "epoch": 1.503334006870075,
      "grad_norm": 2.7615318298339844,
      "learning_rate": 2.487502481622879e-06,
      "logits/chosen": -2.8637490272521973,
      "logits/rejected": -2.874497652053833,
      "logps/chosen": -0.8163179159164429,
      "logps/rejected": -0.9841713905334473,
      "loss": 0.8791,
      "odds_ratio_loss": 0.6274018287658691,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.08163177967071533,
      "rewards/margins": 0.01678534969687462,
      "rewards/rejected": -0.09841714054346085,
      "sft_loss": 0.8163179159164429,
      "step": 930
    },
    {
      "epoch": 1.5194988886643768,
      "grad_norm": 1.7173594236373901,
      "learning_rate": 2.4451421873448253e-06,
      "logits/chosen": -2.8568150997161865,
      "logits/rejected": -2.879917621612549,
      "logps/chosen": -0.8009888529777527,
      "logps/rejected": -0.9833795428276062,
      "loss": 0.8678,
      "odds_ratio_loss": 0.667960524559021,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.08009888231754303,
      "rewards/margins": 0.018239066004753113,
      "rewards/rejected": -0.09833794832229614,
      "sft_loss": 0.8009888529777527,
      "step": 940
    },
    {
      "epoch": 1.5356637704586786,
      "grad_norm": 3.4001808166503906,
      "learning_rate": 2.40279764577506e-06,
      "logits/chosen": -2.885816812515259,
      "logits/rejected": -2.9209980964660645,
      "logps/chosen": -0.8259257078170776,
      "logps/rejected": -0.9810823202133179,
      "loss": 0.8903,
      "odds_ratio_loss": 0.6437360048294067,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.08259257674217224,
      "rewards/margins": 0.015515660867094994,
      "rewards/rejected": -0.09810823202133179,
      "sft_loss": 0.8259257078170776,
      "step": 950
    },
    {
      "epoch": 1.5518286522529805,
      "grad_norm": 3.7369155883789062,
      "learning_rate": 2.3604810163705242e-06,
      "logits/chosen": -2.878312587738037,
      "logits/rejected": -2.9087862968444824,
      "logps/chosen": -0.7468287944793701,
      "logps/rejected": -0.999441921710968,
      "loss": 0.8033,
      "odds_ratio_loss": 0.5650970339775085,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07468288391828537,
      "rewards/margins": 0.02526130899786949,
      "rewards/rejected": -0.09994419664144516,
      "sft_loss": 0.7468287944793701,
      "step": 960
    },
    {
      "epoch": 1.5679935340472824,
      "grad_norm": 1.2655407190322876,
      "learning_rate": 2.3182044505730364e-06,
      "logits/chosen": -2.872468948364258,
      "logits/rejected": -2.873964309692383,
      "logps/chosen": -0.7006109952926636,
      "logps/rejected": -0.9527314901351929,
      "loss": 0.7581,
      "odds_ratio_loss": 0.5752806067466736,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.0700611025094986,
      "rewards/margins": 0.025212040171027184,
      "rewards/rejected": -0.09527313709259033,
      "sft_loss": 0.7006109952926636,
      "step": 970
    },
    {
      "epoch": 1.5841584158415842,
      "grad_norm": 2.9336001873016357,
      "learning_rate": 2.275980088319941e-06,
      "logits/chosen": -2.8779749870300293,
      "logits/rejected": -2.8763155937194824,
      "logps/chosen": -0.7721344232559204,
      "logps/rejected": -0.9309911727905273,
      "loss": 0.8406,
      "odds_ratio_loss": 0.6845985651016235,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07721343636512756,
      "rewards/margins": 0.01588568463921547,
      "rewards/rejected": -0.09309910982847214,
      "sft_loss": 0.7721344232559204,
      "step": 980
    },
    {
      "epoch": 1.600323297635886,
      "grad_norm": 2.434041738510132,
      "learning_rate": 2.2338200545580577e-06,
      "logits/chosen": -2.849057674407959,
      "logits/rejected": -2.873142957687378,
      "logps/chosen": -0.7509113550186157,
      "logps/rejected": -1.0347163677215576,
      "loss": 0.8135,
      "odds_ratio_loss": 0.6254162788391113,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07509114593267441,
      "rewards/margins": 0.02838050201535225,
      "rewards/rejected": -0.10347163677215576,
      "sft_loss": 0.7509113550186157,
      "step": 990
    },
    {
      "epoch": 1.616488179430188,
      "grad_norm": 1.686830997467041,
      "learning_rate": 2.191736455761947e-06,
      "logits/chosen": -2.8971669673919678,
      "logits/rejected": -2.9139630794525146,
      "logps/chosen": -0.7013322114944458,
      "logps/rejected": -0.8860443234443665,
      "loss": 0.7571,
      "odds_ratio_loss": 0.5572749972343445,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07013322412967682,
      "rewards/margins": 0.01847122237086296,
      "rewards/rejected": -0.08860443532466888,
      "sft_loss": 0.7013322114944458,
      "step": 1000
    },
    {
      "epoch": 1.616488179430188,
      "eval_logits/chosen": -2.8644185066223145,
      "eval_logits/rejected": -2.8728911876678467,
      "eval_logps/chosen": -0.8028324842453003,
      "eval_logps/rejected": -1.0336546897888184,
      "eval_loss": 0.8679323792457581,
      "eval_odds_ratio_loss": 0.6509982943534851,
      "eval_rewards/accuracies": 0.5699999928474426,
      "eval_rewards/chosen": -0.08028324693441391,
      "eval_rewards/margins": 0.02308221347630024,
      "eval_rewards/rejected": -0.1033654510974884,
      "eval_runtime": 194.7336,
      "eval_samples_per_second": 5.649,
      "eval_sft_loss": 0.8028324842453003,
      "eval_steps_per_second": 2.824,
      "step": 1000
    },
    {
      "epoch": 1.6326530612244898,
      "grad_norm": 1.7911335229873657,
      "learning_rate": 2.1497413764574673e-06,
      "logits/chosen": -2.8975167274475098,
      "logits/rejected": -2.8892812728881836,
      "logps/chosen": -0.7816007137298584,
      "logps/rejected": -1.069588541984558,
      "loss": 0.8393,
      "odds_ratio_loss": 0.5774248242378235,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07816006988286972,
      "rewards/margins": 0.02879878506064415,
      "rewards/rejected": -0.10695885121822357,
      "sft_loss": 0.7816007137298584,
      "step": 1010
    },
    {
      "epoch": 1.6488179430187917,
      "grad_norm": 1.912550687789917,
      "learning_rate": 2.1078468757516395e-06,
      "logits/chosen": -2.8402116298675537,
      "logits/rejected": -2.8773112297058105,
      "logps/chosen": -0.7441704273223877,
      "logps/rejected": -0.9479702115058899,
      "loss": 0.8035,
      "odds_ratio_loss": 0.5933586955070496,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07441703975200653,
      "rewards/margins": 0.02037998102605343,
      "rewards/rejected": -0.0947970300912857,
      "sft_loss": 0.7441704273223877,
      "step": 1020
    },
    {
      "epoch": 1.6649828248130936,
      "grad_norm": 2.0232253074645996,
      "learning_rate": 2.0660649838698145e-06,
      "logits/chosen": -2.8627827167510986,
      "logits/rejected": -2.882736921310425,
      "logps/chosen": -0.7718713283538818,
      "logps/rejected": -1.1140234470367432,
      "loss": 0.832,
      "odds_ratio_loss": 0.6009626984596252,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07718713581562042,
      "rewards/margins": 0.03421521559357643,
      "rewards/rejected": -0.11140235513448715,
      "sft_loss": 0.7718713283538818,
      "step": 1030
    },
    {
      "epoch": 1.6811477066073954,
      "grad_norm": 1.9653966426849365,
      "learning_rate": 2.0244076987011284e-06,
      "logits/chosen": -2.905303716659546,
      "logits/rejected": -2.9009556770324707,
      "logps/chosen": -0.827530562877655,
      "logps/rejected": -1.0324897766113281,
      "loss": 0.8888,
      "odds_ratio_loss": 0.6124246716499329,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.08275305479764938,
      "rewards/margins": 0.020495926961302757,
      "rewards/rejected": -0.1032489761710167,
      "sft_loss": 0.827530562877655,
      "step": 1040
    },
    {
      "epoch": 1.6973125884016973,
      "grad_norm": 1.4363154172897339,
      "learning_rate": 1.982886982353251e-06,
      "logits/chosen": -2.888767957687378,
      "logits/rejected": -2.8874547481536865,
      "logps/chosen": -0.7899632453918457,
      "logps/rejected": -1.1214802265167236,
      "loss": 0.8526,
      "odds_ratio_loss": 0.6266939640045166,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07899631559848785,
      "rewards/margins": 0.03315168619155884,
      "rewards/rejected": -0.11214800179004669,
      "sft_loss": 0.7899632453918457,
      "step": 1050
    },
    {
      "epoch": 1.7134774701959992,
      "grad_norm": 1.8043084144592285,
      "learning_rate": 1.941514757717392e-06,
      "logits/chosen": -2.866079330444336,
      "logits/rejected": -2.879364490509033,
      "logps/chosen": -0.8468548655509949,
      "logps/rejected": -1.1184252500534058,
      "loss": 0.9022,
      "odds_ratio_loss": 0.552977442741394,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.08468548208475113,
      "rewards/margins": 0.02715705707669258,
      "rewards/rejected": -0.11184253543615341,
      "sft_loss": 0.8468548655509949,
      "step": 1060
    },
    {
      "epoch": 1.729642351990301,
      "grad_norm": 3.669512987136841,
      "learning_rate": 1.9003029050445953e-06,
      "logits/chosen": -2.8407020568847656,
      "logits/rejected": -2.8639755249023438,
      "logps/chosen": -0.8030735850334167,
      "logps/rejected": -0.9715849757194519,
      "loss": 0.8692,
      "odds_ratio_loss": 0.660782516002655,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.08030736446380615,
      "rewards/margins": 0.016851136460900307,
      "rewards/rejected": -0.0971585065126419,
      "sft_loss": 0.8030735850334167,
      "step": 1070
    },
    {
      "epoch": 1.745807233784603,
      "grad_norm": 1.9885250329971313,
      "learning_rate": 1.8592632585342523e-06,
      "logits/chosen": -2.849134922027588,
      "logits/rejected": -2.8679654598236084,
      "logps/chosen": -0.7700011730194092,
      "logps/rejected": -1.0313342809677124,
      "loss": 0.8306,
      "odds_ratio_loss": 0.6062373518943787,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07700012624263763,
      "rewards/margins": 0.02613331377506256,
      "rewards/rejected": -0.1031334400177002,
      "sft_loss": 0.7700011730194092,
      "step": 1080
    },
    {
      "epoch": 1.7619721155789048,
      "grad_norm": 4.0624895095825195,
      "learning_rate": 1.8184076029358527e-06,
      "logits/chosen": -2.840611457824707,
      "logits/rejected": -2.8494577407836914,
      "logps/chosen": -0.7611902952194214,
      "logps/rejected": -0.9082427024841309,
      "loss": 0.8272,
      "odds_ratio_loss": 0.6598888635635376,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07611902803182602,
      "rewards/margins": 0.014705238863825798,
      "rewards/rejected": -0.09082427620887756,
      "sft_loss": 0.7611902952194214,
      "step": 1090
    },
    {
      "epoch": 1.7781369973732066,
      "grad_norm": 1.7686785459518433,
      "learning_rate": 1.7777476701649318e-06,
      "logits/chosen": -2.8446550369262695,
      "logits/rejected": -2.85874342918396,
      "logps/chosen": -0.7774368524551392,
      "logps/rejected": -1.0228512287139893,
      "loss": 0.8388,
      "odds_ratio_loss": 0.6141053438186646,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07774369418621063,
      "rewards/margins": 0.024541418999433517,
      "rewards/rejected": -0.10228510946035385,
      "sft_loss": 0.7774368524551392,
      "step": 1100
    },
    {
      "epoch": 1.7943018791675085,
      "grad_norm": 2.743757724761963,
      "learning_rate": 1.7372951359341925e-06,
      "logits/chosen": -2.8636326789855957,
      "logits/rejected": -2.8647377490997314,
      "logps/chosen": -0.750954806804657,
      "logps/rejected": -0.9340154528617859,
      "loss": 0.814,
      "odds_ratio_loss": 0.6307731866836548,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.07509546726942062,
      "rewards/margins": 0.018306076526641846,
      "rewards/rejected": -0.09340154379606247,
      "sft_loss": 0.750954806804657,
      "step": 1110
    },
    {
      "epoch": 1.8104667609618104,
      "grad_norm": 3.9680521488189697,
      "learning_rate": 1.6970616164007547e-06,
      "logits/chosen": -2.8542914390563965,
      "logits/rejected": -2.8552489280700684,
      "logps/chosen": -0.7380022406578064,
      "logps/rejected": -0.9561580419540405,
      "loss": 0.801,
      "odds_ratio_loss": 0.6301542520523071,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0738002210855484,
      "rewards/margins": 0.0218155849725008,
      "rewards/rejected": -0.09561581164598465,
      "sft_loss": 0.7380022406578064,
      "step": 1120
    },
    {
      "epoch": 1.8266316427561122,
      "grad_norm": 2.8756582736968994,
      "learning_rate": 1.6570586648305276e-06,
      "logits/chosen": -2.8676905632019043,
      "logits/rejected": -2.895289897918701,
      "logps/chosen": -0.7943655252456665,
      "logps/rejected": -1.0809084177017212,
      "loss": 0.8591,
      "odds_ratio_loss": 0.6475063562393188,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07943655550479889,
      "rewards/margins": 0.028654297813773155,
      "rewards/rejected": -0.1080908551812172,
      "sft_loss": 0.7943655252456665,
      "step": 1130
    },
    {
      "epoch": 1.842796524550414,
      "grad_norm": 1.8805325031280518,
      "learning_rate": 1.6172977682806151e-06,
      "logits/chosen": -2.8678653240203857,
      "logits/rejected": -2.900193214416504,
      "logps/chosen": -0.7862238883972168,
      "logps/rejected": -1.0396199226379395,
      "loss": 0.8453,
      "odds_ratio_loss": 0.5909398198127747,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07862239331007004,
      "rewards/margins": 0.02533959411084652,
      "rewards/rejected": -0.1039619892835617,
      "sft_loss": 0.7862238883972168,
      "step": 1140
    },
    {
      "epoch": 1.858961406344716,
      "grad_norm": 1.586294174194336,
      "learning_rate": 1.5777903443007586e-06,
      "logits/chosen": -2.8388750553131104,
      "logits/rejected": -2.838686466217041,
      "logps/chosen": -0.7984446883201599,
      "logps/rejected": -1.093590497970581,
      "loss": 0.8601,
      "odds_ratio_loss": 0.6163803935050964,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07984446734189987,
      "rewards/margins": 0.029514577239751816,
      "rewards/rejected": -0.10935904830694199,
      "sft_loss": 0.7984446883201599,
      "step": 1150
    },
    {
      "epoch": 1.8751262881390178,
      "grad_norm": 3.058032751083374,
      "learning_rate": 1.5385477376547226e-06,
      "logits/chosen": -2.853109121322632,
      "logits/rejected": -2.863646984100342,
      "logps/chosen": -0.7820562124252319,
      "logps/rejected": -1.004570484161377,
      "loss": 0.8417,
      "odds_ratio_loss": 0.5969026684761047,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07820562273263931,
      "rewards/margins": 0.022251427173614502,
      "rewards/rejected": -0.10045703500509262,
      "sft_loss": 0.7820562124252319,
      "step": 1160
    },
    {
      "epoch": 1.89129116993332,
      "grad_norm": 3.296496868133545,
      "learning_rate": 1.4995812170625845e-06,
      "logits/chosen": -2.8537023067474365,
      "logits/rejected": -2.8620083332061768,
      "logps/chosen": -0.7803040742874146,
      "logps/rejected": -1.1614640951156616,
      "loss": 0.8383,
      "odds_ratio_loss": 0.5798701047897339,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07803040742874146,
      "rewards/margins": 0.038116004317998886,
      "rewards/rejected": -0.11614640802145004,
      "sft_loss": 0.7803040742874146,
      "step": 1170
    },
    {
      "epoch": 1.9074560517276218,
      "grad_norm": 2.4982151985168457,
      "learning_rate": 1.4609019719648666e-06,
      "logits/chosen": -2.8664259910583496,
      "logits/rejected": -2.880103826522827,
      "logps/chosen": -0.7934621572494507,
      "logps/rejected": -1.0411931276321411,
      "loss": 0.8522,
      "odds_ratio_loss": 0.5876864194869995,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07934621721506119,
      "rewards/margins": 0.024773094803094864,
      "rewards/rejected": -0.10411931574344635,
      "sft_loss": 0.7934621572494507,
      "step": 1180
    },
    {
      "epoch": 1.9236209335219236,
      "grad_norm": 4.357522964477539,
      "learning_rate": 1.42252110930943e-06,
      "logits/chosen": -2.8305060863494873,
      "logits/rejected": -2.850817918777466,
      "logps/chosen": -0.7121320962905884,
      "logps/rejected": -0.97893887758255,
      "loss": 0.7723,
      "odds_ratio_loss": 0.6020933389663696,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07121320813894272,
      "rewards/margins": 0.02668066881597042,
      "rewards/rejected": -0.09789387881755829,
      "sft_loss": 0.7121320962905884,
      "step": 1190
    },
    {
      "epoch": 1.9397858153162255,
      "grad_norm": 3.2690622806549072,
      "learning_rate": 1.3844496503620493e-06,
      "logits/chosen": -2.855846881866455,
      "logits/rejected": -2.885960817337036,
      "logps/chosen": -0.7993025779724121,
      "logps/rejected": -1.008312702178955,
      "loss": 0.8606,
      "odds_ratio_loss": 0.6124933362007141,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.07993026077747345,
      "rewards/margins": 0.020901009440422058,
      "rewards/rejected": -0.10083127021789551,
      "sft_loss": 0.7993025779724121,
      "step": 1200
    },
    {
      "epoch": 1.9559506971105274,
      "grad_norm": 3.07012677192688,
      "learning_rate": 1.3466985275416081e-06,
      "logits/chosen": -2.8368687629699707,
      "logits/rejected": -2.8513948917388916,
      "logps/chosen": -0.8561896085739136,
      "logps/rejected": -1.0195033550262451,
      "loss": 0.9234,
      "odds_ratio_loss": 0.6718183159828186,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.08561895787715912,
      "rewards/margins": 0.016331372782588005,
      "rewards/rejected": -0.10195034742355347,
      "sft_loss": 0.8561896085739136,
      "step": 1210
    },
    {
      "epoch": 1.9721155789048292,
      "grad_norm": 4.26687479019165,
      "learning_rate": 1.309278581280791e-06,
      "logits/chosen": -2.8606760501861572,
      "logits/rejected": -2.868224620819092,
      "logps/chosen": -0.7406347990036011,
      "logps/rejected": -1.0179945230484009,
      "loss": 0.7986,
      "odds_ratio_loss": 0.5793353319168091,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.07406347990036011,
      "rewards/margins": 0.02773597277700901,
      "rewards/rejected": -0.10179946571588516,
      "sft_loss": 0.7406347990036011,
      "step": 1220
    },
    {
      "epoch": 1.9882804606991311,
      "grad_norm": 1.2442247867584229,
      "learning_rate": 1.272200556913199e-06,
      "logits/chosen": -2.8689868450164795,
      "logits/rejected": -2.8818325996398926,
      "logps/chosen": -0.812061607837677,
      "logps/rejected": -1.029280424118042,
      "loss": 0.8795,
      "odds_ratio_loss": 0.6747404336929321,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.08120616525411606,
      "rewards/margins": 0.02172188088297844,
      "rewards/rejected": -0.102928027510643,
      "sft_loss": 0.812061607837677,
      "step": 1230
    },
    {
      "epoch": 2.004445342493433,
      "grad_norm": 2.5222415924072266,
      "learning_rate": 1.2354751015877698e-06,
      "logits/chosen": -2.842041015625,
      "logits/rejected": -2.861173629760742,
      "logps/chosen": -0.7999058961868286,
      "logps/rejected": -1.1007378101348877,
      "loss": 0.86,
      "odds_ratio_loss": 0.6008915305137634,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07999058067798615,
      "rewards/margins": 0.030083194375038147,
      "rewards/rejected": -0.11007378250360489,
      "sft_loss": 0.7999058961868286,
      "step": 1240
    },
    {
      "epoch": 2.020610224287735,
      "grad_norm": 3.1796367168426514,
      "learning_rate": 1.1991127612113945e-06,
      "logits/chosen": -2.860217571258545,
      "logits/rejected": -2.8857686519622803,
      "logps/chosen": -0.7788959741592407,
      "logps/rejected": -1.0279576778411865,
      "loss": 0.8366,
      "odds_ratio_loss": 0.5771896839141846,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0778895914554596,
      "rewards/margins": 0.02490617148578167,
      "rewards/rejected": -0.10279576480388641,
      "sft_loss": 0.7788959741592407,
      "step": 1250
    },
    {
      "epoch": 2.036775106082037,
      "grad_norm": 2.174238681793213,
      "learning_rate": 1.1631239774206035e-06,
      "logits/chosen": -2.8261468410491943,
      "logits/rejected": -2.8276760578155518,
      "logps/chosen": -0.7623487114906311,
      "logps/rejected": -1.0154896974563599,
      "loss": 0.8249,
      "odds_ratio_loss": 0.6253183484077454,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.0762348622083664,
      "rewards/margins": 0.02531411312520504,
      "rewards/rejected": -0.10154898464679718,
      "sft_loss": 0.7623487114906311,
      "step": 1260
    },
    {
      "epoch": 2.052939987876339,
      "grad_norm": 3.220973253250122,
      "learning_rate": 1.1275190845831978e-06,
      "logits/chosen": -2.8474819660186768,
      "logits/rejected": -2.8597018718719482,
      "logps/chosen": -0.730771541595459,
      "logps/rejected": -1.0029503107070923,
      "loss": 0.7858,
      "odds_ratio_loss": 0.550129234790802,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.07307715713977814,
      "rewards/margins": 0.02721787989139557,
      "rewards/rejected": -0.10029502958059311,
      "sft_loss": 0.730771541595459,
      "step": 1270
    },
    {
      "epoch": 2.0691048696706407,
      "grad_norm": 2.44575834274292,
      "learning_rate": 1.0923083068306778e-06,
      "logits/chosen": -2.8472275733947754,
      "logits/rejected": -2.8387467861175537,
      "logps/chosen": -0.7656749486923218,
      "logps/rejected": -1.1094231605529785,
      "loss": 0.8236,
      "odds_ratio_loss": 0.5792102813720703,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07656749337911606,
      "rewards/margins": 0.03437482565641403,
      "rewards/rejected": -0.11094231903553009,
      "sft_loss": 0.7656749486923218,
      "step": 1280
    },
    {
      "epoch": 2.0852697514649425,
      "grad_norm": 1.4943968057632446,
      "learning_rate": 1.0575017551223348e-06,
      "logits/chosen": -2.829378128051758,
      "logits/rejected": -2.8376450538635254,
      "logps/chosen": -0.7342156171798706,
      "logps/rejected": -0.9912710189819336,
      "loss": 0.7958,
      "odds_ratio_loss": 0.6156936883926392,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07342156767845154,
      "rewards/margins": 0.02570553496479988,
      "rewards/rejected": -0.09912709891796112,
      "sft_loss": 0.7342156171798706,
      "step": 1290
    },
    {
      "epoch": 2.1014346332592444,
      "grad_norm": 2.5311193466186523,
      "learning_rate": 1.023109424341833e-06,
      "logits/chosen": -2.8397974967956543,
      "logits/rejected": -2.8779385089874268,
      "logps/chosen": -0.7779219746589661,
      "logps/rejected": -1.1433827877044678,
      "loss": 0.8376,
      "odds_ratio_loss": 0.5970156192779541,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.0777921974658966,
      "rewards/margins": 0.03654608502984047,
      "rewards/rejected": -0.11433827877044678,
      "sft_loss": 0.7779219746589661,
      "step": 1300
    },
    {
      "epoch": 2.1175995150535463,
      "grad_norm": 2.6538310050964355,
      "learning_rate": 9.891411904271273e-07,
      "logits/chosen": -2.856947422027588,
      "logits/rejected": -2.86110782623291,
      "logps/chosen": -0.7499477863311768,
      "logps/rejected": -0.9801033139228821,
      "loss": 0.8093,
      "odds_ratio_loss": 0.593558669090271,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.0749947652220726,
      "rewards/margins": 0.023015562444925308,
      "rewards/rejected": -0.0980103388428688,
      "sft_loss": 0.7499477863311768,
      "step": 1310
    },
    {
      "epoch": 2.133764396847848,
      "grad_norm": 1.2850011587142944,
      "learning_rate": 9.556068075345363e-07,
      "logits/chosen": -2.8736729621887207,
      "logits/rejected": -2.8673884868621826,
      "logps/chosen": -0.7692313194274902,
      "logps/rejected": -0.9742280840873718,
      "loss": 0.8271,
      "odds_ratio_loss": 0.5790851712226868,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07692314684391022,
      "rewards/margins": 0.02049967274069786,
      "rewards/rejected": -0.09742281585931778,
      "sft_loss": 0.7692313194274902,
      "step": 1320
    },
    {
      "epoch": 2.14992927864215,
      "grad_norm": 1.7034938335418701,
      "learning_rate": 9.225159052377838e-07,
      "logits/chosen": -2.834965944290161,
      "logits/rejected": -2.8684887886047363,
      "logps/chosen": -0.796667218208313,
      "logps/rejected": -1.1322475671768188,
      "loss": 0.8554,
      "odds_ratio_loss": 0.587177574634552,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07966671884059906,
      "rewards/margins": 0.03355802968144417,
      "rewards/rejected": -0.11322475969791412,
      "sft_loss": 0.796667218208313,
      "step": 1330
    },
    {
      "epoch": 2.166094160436452,
      "grad_norm": 2.5143074989318848,
      "learning_rate": 8.898779857628184e-07,
      "logits/chosen": -2.8322224617004395,
      "logits/rejected": -2.8632161617279053,
      "logps/chosen": -0.6862845420837402,
      "logps/rejected": -0.923437774181366,
      "loss": 0.7449,
      "odds_ratio_loss": 0.5857266783714294,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.0686284601688385,
      "rewards/margins": 0.02371532842516899,
      "rewards/rejected": -0.0923437848687172,
      "sft_loss": 0.6862845420837402,
      "step": 1340
    },
    {
      "epoch": 2.1822590422307537,
      "grad_norm": 1.7262011766433716,
      "learning_rate": 8.577024212591975e-07,
      "logits/chosen": -2.8671224117279053,
      "logits/rejected": -2.867626428604126,
      "logps/chosen": -0.7982193231582642,
      "logps/rejected": -0.9524084329605103,
      "loss": 0.862,
      "odds_ratio_loss": 0.6382196545600891,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.07982192933559418,
      "rewards/margins": 0.01541891973465681,
      "rewards/rejected": -0.09524084627628326,
      "sft_loss": 0.7982193231582642,
      "step": 1350
    },
    {
      "epoch": 2.1984239240250556,
      "grad_norm": 1.9137386083602905,
      "learning_rate": 8.259984511088276e-07,
      "logits/chosen": -2.8300180435180664,
      "logits/rejected": -2.8534936904907227,
      "logps/chosen": -0.7877185940742493,
      "logps/rejected": -1.0415524244308472,
      "loss": 0.8505,
      "odds_ratio_loss": 0.6278126239776611,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07877186685800552,
      "rewards/margins": 0.025383388623595238,
      "rewards/rejected": -0.10415525734424591,
      "sft_loss": 0.7877185940742493,
      "step": 1360
    },
    {
      "epoch": 2.2145888058193575,
      "grad_norm": 2.398965835571289,
      "learning_rate": 7.947751792728237e-07,
      "logits/chosen": -2.8527517318725586,
      "logits/rejected": -2.8384506702423096,
      "logps/chosen": -0.7678119540214539,
      "logps/rejected": -1.105531930923462,
      "loss": 0.8275,
      "odds_ratio_loss": 0.5968826413154602,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.07678119093179703,
      "rewards/margins": 0.03377201408147812,
      "rewards/rejected": -0.11055320501327515,
      "sft_loss": 0.7678119540214539,
      "step": 1370
    },
    {
      "epoch": 2.2307536876136593,
      "grad_norm": 2.101724147796631,
      "learning_rate": 7.640415716772626e-07,
      "logits/chosen": -2.8620262145996094,
      "logits/rejected": -2.881200075149536,
      "logps/chosen": -0.7912808656692505,
      "logps/rejected": -1.0620834827423096,
      "loss": 0.8546,
      "odds_ratio_loss": 0.6336351633071899,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07912809401750565,
      "rewards/margins": 0.027080247178673744,
      "rewards/rejected": -0.10620833933353424,
      "sft_loss": 0.7912808656692505,
      "step": 1380
    },
    {
      "epoch": 2.246918569407961,
      "grad_norm": 1.2350420951843262,
      "learning_rate": 7.338064536385722e-07,
      "logits/chosen": -2.839816093444824,
      "logits/rejected": -2.84806489944458,
      "logps/chosen": -0.7491471171379089,
      "logps/rejected": -1.098024606704712,
      "loss": 0.8078,
      "odds_ratio_loss": 0.5867569446563721,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07491471618413925,
      "rewards/margins": 0.03488774597644806,
      "rewards/rejected": -0.10980246961116791,
      "sft_loss": 0.7491471171379089,
      "step": 1390
    },
    {
      "epoch": 2.263083451202263,
      "grad_norm": 3.2553515434265137,
      "learning_rate": 7.040785073292883e-07,
      "logits/chosen": -2.795974016189575,
      "logits/rejected": -2.812316417694092,
      "logps/chosen": -0.8446899652481079,
      "logps/rejected": -1.1183385848999023,
      "loss": 0.9119,
      "odds_ratio_loss": 0.6722968220710754,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.08446899801492691,
      "rewards/margins": 0.02736486867070198,
      "rewards/rejected": -0.111833855509758,
      "sft_loss": 0.8446899652481079,
      "step": 1400
    },
    {
      "epoch": 2.279248332996565,
      "grad_norm": 1.5375083684921265,
      "learning_rate": 6.748662692849297e-07,
      "logits/chosen": -2.8378682136535645,
      "logits/rejected": -2.8527588844299316,
      "logps/chosen": -0.7140767574310303,
      "logps/rejected": -1.1210377216339111,
      "loss": 0.7679,
      "odds_ratio_loss": 0.5377554893493652,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.07140768319368362,
      "rewards/margins": 0.04069609194993973,
      "rewards/rejected": -0.11210376024246216,
      "sft_loss": 0.7140767574310303,
      "step": 1410
    },
    {
      "epoch": 2.295413214790867,
      "grad_norm": 3.371690273284912,
      "learning_rate": 6.46178127952686e-07,
      "logits/chosen": -2.8596229553222656,
      "logits/rejected": -2.86143159866333,
      "logps/chosen": -0.7527777552604675,
      "logps/rejected": -1.0262553691864014,
      "loss": 0.8073,
      "odds_ratio_loss": 0.5452762842178345,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.07527776062488556,
      "rewards/margins": 0.02734777331352234,
      "rewards/rejected": -0.10262554883956909,
      "sft_loss": 0.7527777552604675,
      "step": 1420
    },
    {
      "epoch": 2.3115780965851687,
      "grad_norm": 5.5002760887146,
      "learning_rate": 6.180223212826289e-07,
      "logits/chosen": -2.8466854095458984,
      "logits/rejected": -2.84420108795166,
      "logps/chosen": -0.760028600692749,
      "logps/rejected": -1.0010223388671875,
      "loss": 0.8196,
      "odds_ratio_loss": 0.595847487449646,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07600285112857819,
      "rewards/margins": 0.024099376052618027,
      "rewards/rejected": -0.10010223090648651,
      "sft_loss": 0.760028600692749,
      "step": 1430
    },
    {
      "epoch": 2.3277429783794705,
      "grad_norm": 2.094597339630127,
      "learning_rate": 5.904069343621443e-07,
      "logits/chosen": -2.8559889793395996,
      "logits/rejected": -2.843318462371826,
      "logps/chosen": -0.7583047747612,
      "logps/rejected": -1.0201733112335205,
      "loss": 0.8157,
      "odds_ratio_loss": 0.5739010572433472,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07583048194646835,
      "rewards/margins": 0.02618684433400631,
      "rewards/rejected": -0.10201732814311981,
      "sft_loss": 0.7583047747612,
      "step": 1440
    },
    {
      "epoch": 2.3439078601737724,
      "grad_norm": 3.256753444671631,
      "learning_rate": 5.633398970942544e-07,
      "logits/chosen": -2.8187243938446045,
      "logits/rejected": -2.8463759422302246,
      "logps/chosen": -0.763822078704834,
      "logps/rejected": -0.9972942471504211,
      "loss": 0.8274,
      "odds_ratio_loss": 0.6356968283653259,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07638221234083176,
      "rewards/margins": 0.023347217589616776,
      "rewards/rejected": -0.09972943365573883,
      "sft_loss": 0.763822078704834,
      "step": 1450
    },
    {
      "epoch": 2.3600727419680743,
      "grad_norm": 2.1988418102264404,
      "learning_rate": 5.368289819205069e-07,
      "logits/chosen": -2.8621747493743896,
      "logits/rejected": -2.8629798889160156,
      "logps/chosen": -0.699676513671875,
      "logps/rejected": -0.9881321787834167,
      "loss": 0.7602,
      "odds_ratio_loss": 0.6056861877441406,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.06996765732765198,
      "rewards/margins": 0.028845559805631638,
      "rewards/rejected": -0.09881322085857391,
      "sft_loss": 0.699676513671875,
      "step": 1460
    },
    {
      "epoch": 2.376237623762376,
      "grad_norm": 2.666426181793213,
      "learning_rate": 5.108818015890785e-07,
      "logits/chosen": -2.8656005859375,
      "logits/rejected": -2.889970302581787,
      "logps/chosen": -0.8437716364860535,
      "logps/rejected": -1.0408810377120972,
      "loss": 0.9052,
      "odds_ratio_loss": 0.6140363216400146,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.08437716960906982,
      "rewards/margins": 0.01971094310283661,
      "rewards/rejected": -0.10408811271190643,
      "sft_loss": 0.8437716364860535,
      "step": 1470
    },
    {
      "epoch": 2.392402505556678,
      "grad_norm": 2.2777225971221924,
      "learning_rate": 4.855058069687291e-07,
      "logits/chosen": -2.834155559539795,
      "logits/rejected": -2.8524587154388428,
      "logps/chosen": -0.7329773306846619,
      "logps/rejected": -1.1425807476043701,
      "loss": 0.7861,
      "odds_ratio_loss": 0.5314901471138,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.07329773157835007,
      "rewards/margins": 0.04096033796668053,
      "rewards/rejected": -0.11425807327032089,
      "sft_loss": 0.7329773306846619,
      "step": 1480
    },
    {
      "epoch": 2.40856738735098,
      "grad_norm": 2.6650478839874268,
      "learning_rate": 4.607082849092523e-07,
      "logits/chosen": -2.862356662750244,
      "logits/rejected": -2.864802598953247,
      "logps/chosen": -0.829633891582489,
      "logps/rejected": -1.0255271196365356,
      "loss": 0.8935,
      "odds_ratio_loss": 0.638370156288147,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.08296339213848114,
      "rewards/margins": 0.019589336588978767,
      "rewards/rejected": -0.10255272686481476,
      "sft_loss": 0.829633891582489,
      "step": 1490
    },
    {
      "epoch": 2.4247322691452817,
      "grad_norm": 3.085514783859253,
      "learning_rate": 4.3649635614901405e-07,
      "logits/chosen": -2.8451571464538574,
      "logits/rejected": -2.8950095176696777,
      "logps/chosen": -0.7389890551567078,
      "logps/rejected": -0.8802745938301086,
      "loss": 0.8035,
      "odds_ratio_loss": 0.6446704864501953,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.07389890402555466,
      "rewards/margins": 0.014128552749752998,
      "rewards/rejected": -0.0880274623632431,
      "sft_loss": 0.7389890551567078,
      "step": 1500
    },
    {
      "epoch": 2.4247322691452817,
      "eval_logits/chosen": -2.8472585678100586,
      "eval_logits/rejected": -2.8558220863342285,
      "eval_logps/chosen": -0.7975095510482788,
      "eval_logps/rejected": -1.0328320264816284,
      "eval_loss": 0.8629826903343201,
      "eval_odds_ratio_loss": 0.6547309160232544,
      "eval_rewards/accuracies": 0.5618181824684143,
      "eval_rewards/chosen": -0.07975095510482788,
      "eval_rewards/margins": 0.02353225089609623,
      "eval_rewards/rejected": -0.10328320413827896,
      "eval_runtime": 194.6849,
      "eval_samples_per_second": 5.65,
      "eval_sft_loss": 0.7975095510482788,
      "eval_steps_per_second": 2.825,
      "step": 1500
    },
    {
      "epoch": 2.4408971509395836,
      "grad_norm": 1.7019646167755127,
      "learning_rate": 4.128769732701973e-07,
      "logits/chosen": -2.82879638671875,
      "logits/rejected": -2.832578420639038,
      "logps/chosen": -0.7700603604316711,
      "logps/rejected": -0.9951756596565247,
      "loss": 0.8304,
      "odds_ratio_loss": 0.6030290722846985,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.077006034553051,
      "rewards/margins": 0.022511538118124008,
      "rewards/rejected": -0.0995175689458847,
      "sft_loss": 0.7700603604316711,
      "step": 1510
    },
    {
      "epoch": 2.4570620327338855,
      "grad_norm": 2.5611681938171387,
      "learning_rate": 3.8985691870233046e-07,
      "logits/chosen": -2.882220506668091,
      "logits/rejected": -2.880516529083252,
      "logps/chosen": -0.7692660689353943,
      "logps/rejected": -1.0380921363830566,
      "loss": 0.8284,
      "odds_ratio_loss": 0.5917290449142456,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07692660391330719,
      "rewards/margins": 0.026882609352469444,
      "rewards/rejected": -0.10380921512842178,
      "sft_loss": 0.7692660689353943,
      "step": 1520
    },
    {
      "epoch": 2.4732269145281873,
      "grad_norm": 2.6633763313293457,
      "learning_rate": 3.6744280277467904e-07,
      "logits/chosen": -2.8530020713806152,
      "logits/rejected": -2.8719234466552734,
      "logps/chosen": -0.7769867181777954,
      "logps/rejected": -1.0218976736068726,
      "loss": 0.8392,
      "odds_ratio_loss": 0.6218123435974121,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07769867032766342,
      "rewards/margins": 0.024491112679243088,
      "rewards/rejected": -0.10218977928161621,
      "sft_loss": 0.7769867181777954,
      "step": 1530
    },
    {
      "epoch": 2.489391796322489,
      "grad_norm": 2.7384212017059326,
      "learning_rate": 3.456410618180503e-07,
      "logits/chosen": -2.832824468612671,
      "logits/rejected": -2.856114149093628,
      "logps/chosen": -0.7060586810112,
      "logps/rejected": -1.0986192226409912,
      "loss": 0.7646,
      "odds_ratio_loss": 0.5853801965713501,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.07060587406158447,
      "rewards/margins": 0.03925605118274689,
      "rewards/rejected": -0.10986192524433136,
      "sft_loss": 0.7060586810112,
      "step": 1540
    },
    {
      "epoch": 2.5055566781167915,
      "grad_norm": 1.9465371370315552,
      "learning_rate": 3.244579563165753e-07,
      "logits/chosen": -2.8586621284484863,
      "logits/rejected": -2.869255542755127,
      "logps/chosen": -0.7589577436447144,
      "logps/rejected": -1.1315686702728271,
      "loss": 0.8173,
      "odds_ratio_loss": 0.5836090445518494,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.0758957713842392,
      "rewards/margins": 0.03726109117269516,
      "rewards/rejected": -0.11315685510635376,
      "sft_loss": 0.7589577436447144,
      "step": 1550
    },
    {
      "epoch": 2.521721559911093,
      "grad_norm": 1.2344708442687988,
      "learning_rate": 3.038995691099697e-07,
      "logits/chosen": -2.8416831493377686,
      "logits/rejected": -2.85313081741333,
      "logps/chosen": -0.7924615144729614,
      "logps/rejected": -1.2077696323394775,
      "loss": 0.8503,
      "odds_ratio_loss": 0.5783108472824097,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.0792461559176445,
      "rewards/margins": 0.041530806571245193,
      "rewards/rejected": -0.1207769513130188,
      "sft_loss": 0.7924615144729614,
      "step": 1560
    },
    {
      "epoch": 2.5378864417053952,
      "grad_norm": 12.726688385009766,
      "learning_rate": 2.839718036468192e-07,
      "logits/chosen": -2.8868002891540527,
      "logits/rejected": -2.9153692722320557,
      "logps/chosen": -0.884573757648468,
      "logps/rejected": -1.0609769821166992,
      "loss": 0.9513,
      "odds_ratio_loss": 0.6675292253494263,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.0884573832154274,
      "rewards/margins": 0.01764032617211342,
      "rewards/rejected": -0.10609769821166992,
      "sft_loss": 0.884573757648468,
      "step": 1570
    },
    {
      "epoch": 2.5540513234996967,
      "grad_norm": 2.5232503414154053,
      "learning_rate": 2.646803822893723e-07,
      "logits/chosen": -2.8850457668304443,
      "logits/rejected": -2.894557476043701,
      "logps/chosen": -0.8000026941299438,
      "logps/rejected": -1.0157983303070068,
      "loss": 0.8627,
      "odds_ratio_loss": 0.6269931793212891,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.08000027388334274,
      "rewards/margins": 0.02157955802977085,
      "rewards/rejected": -0.10157983005046844,
      "sft_loss": 0.8000026941299438,
      "step": 1580
    },
    {
      "epoch": 2.570216205293999,
      "grad_norm": 2.3380508422851562,
      "learning_rate": 2.460308446703341e-07,
      "logits/chosen": -2.8933000564575195,
      "logits/rejected": -2.8834781646728516,
      "logps/chosen": -0.791167676448822,
      "logps/rejected": -0.9255102276802063,
      "loss": 0.8556,
      "odds_ratio_loss": 0.6445525884628296,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07911677658557892,
      "rewards/margins": 0.013434251770377159,
      "rewards/rejected": -0.09255101531744003,
      "sft_loss": 0.791167676448822,
      "step": 1590
    },
    {
      "epoch": 2.5863810870883004,
      "grad_norm": 3.6344377994537354,
      "learning_rate": 2.2802854610213143e-07,
      "logits/chosen": -2.8420848846435547,
      "logits/rejected": -2.8515543937683105,
      "logps/chosen": -0.6993797421455383,
      "logps/rejected": -1.0781666040420532,
      "loss": 0.7531,
      "odds_ratio_loss": 0.5369757413864136,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06993797421455383,
      "rewards/margins": 0.03787868469953537,
      "rewards/rejected": -0.1078166589140892,
      "sft_loss": 0.6993797421455383,
      "step": 1600
    },
    {
      "epoch": 2.6025459688826027,
      "grad_norm": 2.515239715576172,
      "learning_rate": 2.106786560391072e-07,
      "logits/chosen": -2.8365635871887207,
      "logits/rejected": -2.8803467750549316,
      "logps/chosen": -0.8032782673835754,
      "logps/rejected": -1.0168392658233643,
      "loss": 0.8638,
      "odds_ratio_loss": 0.6049396395683289,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.0803278312087059,
      "rewards/margins": 0.021356089040637016,
      "rewards/rejected": -0.10168392956256866,
      "sft_loss": 0.8032782673835754,
      "step": 1610
    },
    {
      "epoch": 2.6187108506769046,
      "grad_norm": 1.520639181137085,
      "learning_rate": 1.9398615659308255e-07,
      "logits/chosen": -2.861687183380127,
      "logits/rejected": -2.89752459526062,
      "logps/chosen": -0.7549802660942078,
      "logps/rejected": -0.9435558319091797,
      "loss": 0.8181,
      "odds_ratio_loss": 0.6309365034103394,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07549802213907242,
      "rewards/margins": 0.018857568502426147,
      "rewards/rejected": -0.09435557574033737,
      "sft_loss": 0.7549802660942078,
      "step": 1620
    },
    {
      "epoch": 2.6348757324712064,
      "grad_norm": 2.2465171813964844,
      "learning_rate": 1.7795584110272184e-07,
      "logits/chosen": -2.8905723094940186,
      "logits/rejected": -2.877936840057373,
      "logps/chosen": -0.7934287786483765,
      "logps/rejected": -1.0050441026687622,
      "loss": 0.8594,
      "odds_ratio_loss": 0.6593586802482605,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07934287935495377,
      "rewards/margins": 0.021161522716283798,
      "rewards/rejected": -0.10050439834594727,
      "sft_loss": 0.7934287786483765,
      "step": 1630
    },
    {
      "epoch": 2.6510406142655083,
      "grad_norm": 4.033486366271973,
      "learning_rate": 1.6259231275709636e-07,
      "logits/chosen": -2.8982126712799072,
      "logits/rejected": -2.8980660438537598,
      "logps/chosen": -0.7681853175163269,
      "logps/rejected": -0.9490568041801453,
      "loss": 0.8356,
      "odds_ratio_loss": 0.6740620732307434,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0768185406923294,
      "rewards/margins": 0.018087133765220642,
      "rewards/rejected": -0.09490568190813065,
      "sft_loss": 0.7681853175163269,
      "step": 1640
    },
    {
      "epoch": 2.66720549605981,
      "grad_norm": 1.5368350744247437,
      "learning_rate": 1.478999832738548e-07,
      "logits/chosen": -2.8781023025512695,
      "logits/rejected": -2.8767361640930176,
      "logps/chosen": -0.7599083185195923,
      "logps/rejected": -1.0983332395553589,
      "loss": 0.82,
      "odds_ratio_loss": 0.601204514503479,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07599084079265594,
      "rewards/margins": 0.033842481672763824,
      "rewards/rejected": -0.10983331501483917,
      "sft_loss": 0.7599083185195923,
      "step": 1650
    },
    {
      "epoch": 2.683370377854112,
      "grad_norm": 1.8103063106536865,
      "learning_rate": 1.338830716323769e-07,
      "logits/chosen": -2.8456664085388184,
      "logits/rejected": -2.8552403450012207,
      "logps/chosen": -0.8041807413101196,
      "logps/rejected": -0.9866863489151001,
      "loss": 0.8687,
      "odds_ratio_loss": 0.6454349756240845,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.08041806519031525,
      "rewards/margins": 0.018250569701194763,
      "rewards/rejected": -0.0986686423420906,
      "sft_loss": 0.8041807413101196,
      "step": 1660
    },
    {
      "epoch": 2.699535259648414,
      "grad_norm": 3.796130657196045,
      "learning_rate": 1.205456028622723e-07,
      "logits/chosen": -2.8858485221862793,
      "logits/rejected": -2.883568286895752,
      "logps/chosen": -0.7273125648498535,
      "logps/rejected": -1.0116485357284546,
      "loss": 0.7835,
      "odds_ratio_loss": 0.5615276098251343,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07273125648498535,
      "rewards/margins": 0.02843359112739563,
      "rewards/rejected": -0.10116485506296158,
      "sft_loss": 0.7273125648498535,
      "step": 1670
    },
    {
      "epoch": 2.7157001414427158,
      "grad_norm": 1.619040608406067,
      "learning_rate": 1.0789140688756805e-07,
      "logits/chosen": -2.8932971954345703,
      "logits/rejected": -2.8933002948760986,
      "logps/chosen": -0.7631897926330566,
      "logps/rejected": -1.0072143077850342,
      "loss": 0.8217,
      "odds_ratio_loss": 0.5846946239471436,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.07631897926330566,
      "rewards/margins": 0.024402452632784843,
      "rewards/rejected": -0.10072143375873566,
      "sft_loss": 0.7631897926330566,
      "step": 1680
    },
    {
      "epoch": 2.7318650232370176,
      "grad_norm": 4.591987133026123,
      "learning_rate": 9.592411742693098e-08,
      "logits/chosen": -2.8280813694000244,
      "logits/rejected": -2.832314968109131,
      "logps/chosen": -0.7757545709609985,
      "logps/rejected": -0.9772068858146667,
      "loss": 0.845,
      "odds_ratio_loss": 0.6925373673439026,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.0775754451751709,
      "rewards/margins": 0.020145252346992493,
      "rewards/rejected": -0.09772069752216339,
      "sft_loss": 0.7757545709609985,
      "step": 1690
    },
    {
      "epoch": 2.7480299050313195,
      "grad_norm": 2.0528857707977295,
      "learning_rate": 8.464717095022168e-08,
      "logits/chosen": -2.8116049766540527,
      "logits/rejected": -2.8237504959106445,
      "logps/chosen": -0.7476006746292114,
      "logps/rejected": -1.0309717655181885,
      "loss": 0.805,
      "odds_ratio_loss": 0.574048638343811,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.0747600644826889,
      "rewards/margins": 0.02833711728453636,
      "rewards/rejected": -0.10309717804193497,
      "sft_loss": 0.7476006746292114,
      "step": 1700
    },
    {
      "epoch": 2.7641947868256214,
      "grad_norm": 2.445467233657837,
      "learning_rate": 7.406380569169841e-08,
      "logits/chosen": -2.860349178314209,
      "logits/rejected": -2.8944199085235596,
      "logps/chosen": -0.7957582473754883,
      "logps/rejected": -0.9725676774978638,
      "loss": 0.8593,
      "odds_ratio_loss": 0.6357892155647278,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.07957582920789719,
      "rewards/margins": 0.01768093928694725,
      "rewards/rejected": -0.09725676476955414,
      "sft_loss": 0.7957582473754883,
      "step": 1710
    },
    {
      "epoch": 2.7803596686199232,
      "grad_norm": 11.543617248535156,
      "learning_rate": 6.417706072013808e-08,
      "logits/chosen": -2.8683581352233887,
      "logits/rejected": -2.894205331802368,
      "logps/chosen": -0.7598998546600342,
      "logps/rejected": -0.9663190841674805,
      "loss": 0.8231,
      "odds_ratio_loss": 0.6316258907318115,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07598999887704849,
      "rewards/margins": 0.02064192108809948,
      "rewards/rejected": -0.09663191437721252,
      "sft_loss": 0.7598998546600342,
      "step": 1720
    },
    {
      "epoch": 2.796524550414225,
      "grad_norm": 3.360384941101074,
      "learning_rate": 5.498977506615294e-08,
      "logits/chosen": -2.8601443767547607,
      "logits/rejected": -2.898664712905884,
      "logps/chosen": -0.790396511554718,
      "logps/rejected": -0.9606446027755737,
      "loss": 0.8544,
      "odds_ratio_loss": 0.6396982073783875,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.07903965562582016,
      "rewards/margins": 0.01702481135725975,
      "rewards/rejected": -0.09606447070837021,
      "sft_loss": 0.790396511554718,
      "step": 1730
    },
    {
      "epoch": 2.812689432208527,
      "grad_norm": 2.132490873336792,
      "learning_rate": 4.6504586906947756e-08,
      "logits/chosen": -2.8836772441864014,
      "logits/rejected": -2.9003067016601562,
      "logps/chosen": -0.8166056871414185,
      "logps/rejected": -0.9932202100753784,
      "loss": 0.8767,
      "odds_ratio_loss": 0.6010292768478394,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.08166056871414185,
      "rewards/margins": 0.01766144670546055,
      "rewards/rejected": -0.09932202100753784,
      "sft_loss": 0.8166056871414185,
      "step": 1740
    },
    {
      "epoch": 2.828854314002829,
      "grad_norm": 7.204352855682373,
      "learning_rate": 3.8723932808754914e-08,
      "logits/chosen": -2.887660503387451,
      "logits/rejected": -2.9059557914733887,
      "logps/chosen": -0.8569768667221069,
      "logps/rejected": -0.9907077550888062,
      "loss": 0.9219,
      "odds_ratio_loss": 0.6491862535476685,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.08569768816232681,
      "rewards/margins": 0.01337310392409563,
      "rewards/rejected": -0.09907079488039017,
      "sft_loss": 0.8569768667221069,
      "step": 1750
    },
    {
      "epoch": 2.8450191957971307,
      "grad_norm": 3.7778828144073486,
      "learning_rate": 3.1650047027158014e-08,
      "logits/chosen": -2.8876945972442627,
      "logits/rejected": -2.9152872562408447,
      "logps/chosen": -0.7689987421035767,
      "logps/rejected": -0.981308102607727,
      "loss": 0.828,
      "odds_ratio_loss": 0.5896368622779846,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.07689988613128662,
      "rewards/margins": 0.02123093418776989,
      "rewards/rejected": -0.09813080728054047,
      "sft_loss": 0.7689987421035767,
      "step": 1760
    },
    {
      "epoch": 2.8611840775914326,
      "grad_norm": 1.726138949394226,
      "learning_rate": 2.5284960865517848e-08,
      "logits/chosen": -2.851304769515991,
      "logits/rejected": -2.871598243713379,
      "logps/chosen": -0.7240949273109436,
      "logps/rejected": -1.0288841724395752,
      "loss": 0.7798,
      "odds_ratio_loss": 0.5571027994155884,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.0724094957113266,
      "rewards/margins": 0.030478913336992264,
      "rewards/rejected": -0.10288842022418976,
      "sft_loss": 0.7240949273109436,
      "step": 1770
    },
    {
      "epoch": 2.8773489593857344,
      "grad_norm": 2.2119297981262207,
      "learning_rate": 1.9630502091670388e-08,
      "logits/chosen": -2.8459057807922363,
      "logits/rejected": -2.866259813308716,
      "logps/chosen": -0.7477800250053406,
      "logps/rejected": -1.0080687999725342,
      "loss": 0.8054,
      "odds_ratio_loss": 0.5758811235427856,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.0747780054807663,
      "rewards/margins": 0.026028871536254883,
      "rewards/rejected": -0.10080687701702118,
      "sft_loss": 0.7477800250053406,
      "step": 1780
    },
    {
      "epoch": 2.8935138411800363,
      "grad_norm": 2.910409450531006,
      "learning_rate": 1.4688294413074677e-08,
      "logits/chosen": -2.850733757019043,
      "logits/rejected": -2.8780460357666016,
      "logps/chosen": -0.6847941279411316,
      "logps/rejected": -1.00661301612854,
      "loss": 0.7411,
      "odds_ratio_loss": 0.5632899403572083,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": -0.06847941130399704,
      "rewards/margins": 0.03218189254403114,
      "rewards/rejected": -0.10066130012273788,
      "sft_loss": 0.6847941279411316,
      "step": 1790
    },
    {
      "epoch": 2.909678722974338,
      "grad_norm": 2.044072389602661,
      "learning_rate": 1.0459757010556626e-08,
      "logits/chosen": -2.856724262237549,
      "logits/rejected": -2.877833366394043,
      "logps/chosen": -0.7718300223350525,
      "logps/rejected": -0.9458082914352417,
      "loss": 0.8346,
      "odds_ratio_loss": 0.6273509860038757,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.07718300819396973,
      "rewards/margins": 0.017397824674844742,
      "rewards/rejected": -0.09458083659410477,
      "sft_loss": 0.7718300223350525,
      "step": 1800
    },
    {
      "epoch": 2.92584360476864,
      "grad_norm": 1.9232614040374756,
      "learning_rate": 6.94610413078306e-09,
      "logits/chosen": -2.8028831481933594,
      "logits/rejected": -2.8568198680877686,
      "logps/chosen": -0.8266820907592773,
      "logps/rejected": -1.2092140913009644,
      "loss": 0.8869,
      "odds_ratio_loss": 0.6017346382141113,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.08266820758581161,
      "rewards/margins": 0.03825319558382034,
      "rewards/rejected": -0.12092139571905136,
      "sft_loss": 0.8266820907592773,
      "step": 1810
    },
    {
      "epoch": 2.942008486562942,
      "grad_norm": 1.0960156917572021,
      "learning_rate": 4.14834473758563e-09,
      "logits/chosen": -2.8286824226379395,
      "logits/rejected": -2.838784694671631,
      "logps/chosen": -0.7189845442771912,
      "logps/rejected": -0.9857820272445679,
      "loss": 0.7756,
      "odds_ratio_loss": 0.5664829015731812,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.0718984454870224,
      "rewards/margins": 0.02667975425720215,
      "rewards/rejected": -0.09857820719480515,
      "sft_loss": 0.7189845442771912,
      "step": 1820
    },
    {
      "epoch": 2.9581733683572438,
      "grad_norm": 1.63419771194458,
      "learning_rate": 2.067282222230349e-09,
      "logits/chosen": -2.8597445487976074,
      "logits/rejected": -2.8696541786193848,
      "logps/chosen": -0.7367098331451416,
      "logps/rejected": -1.0127137899398804,
      "loss": 0.7943,
      "odds_ratio_loss": 0.5762413740158081,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.07367098331451416,
      "rewards/margins": 0.027600402012467384,
      "rewards/rejected": -0.1012713760137558,
      "sft_loss": 0.7367098331451416,
      "step": 1830
    },
    {
      "epoch": 2.9743382501515456,
      "grad_norm": 2.9457271099090576,
      "learning_rate": 7.035141727212979e-10,
      "logits/chosen": -2.8564071655273438,
      "logits/rejected": -2.8889355659484863,
      "logps/chosen": -0.7218343615531921,
      "logps/rejected": -1.0010156631469727,
      "loss": 0.7784,
      "odds_ratio_loss": 0.5654899477958679,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.07218344509601593,
      "rewards/margins": 0.02791813388466835,
      "rewards/rejected": -0.10010156780481339,
      "sft_loss": 0.7218343615531921,
      "step": 1840
    },
    {
      "epoch": 2.9905031319458475,
      "grad_norm": 4.486654758453369,
      "learning_rate": 5.743220219761592e-11,
      "logits/chosen": -2.8505501747131348,
      "logits/rejected": -2.870176076889038,
      "logps/chosen": -0.8715106248855591,
      "logps/rejected": -1.054720401763916,
      "loss": 0.9404,
      "odds_ratio_loss": 0.6889584064483643,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.08715107291936874,
      "rewards/margins": 0.0183209627866745,
      "rewards/rejected": -0.10547204315662384,
      "sft_loss": 0.8715106248855591,
      "step": 1850
    },
    {
      "epoch": 2.9969690846635686,
      "step": 1854,
      "total_flos": 2.1013894560546816e+18,
      "train_loss": 0.9013287582572352,
      "train_runtime": 18144.1457,
      "train_samples_per_second": 1.637,
      "train_steps_per_second": 0.102
    }
  ],
  "logging_steps": 10,
  "max_steps": 1854,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "total_flos": 2.1013894560546816e+18,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}