{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.9765925925925925,
  "eval_steps": 500,
  "global_step": 315,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.047407407407407405,
      "grad_norm": 1149.4111328125,
      "learning_rate": 7.8125e-06,
      "log_odds_chosen": -1.058286428451538,
      "log_odds_ratio": -11.003866195678711,
      "logps/chosen": -22.701828002929688,
      "logps/rejected": -21.64377212524414,
      "loss": 487.7823,
      "nll_loss": 9.837206840515137,
      "po_loss": 5.405989170074463,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -11.350914001464844,
      "rewards/margins": -0.5290289521217346,
      "rewards/rejected": -10.82188606262207,
      "step": 5
    },
    {
      "epoch": 0.09481481481481481,
      "grad_norm": 597.1365966796875,
      "learning_rate": 1.5625e-05,
      "log_odds_chosen": -3.132328510284424,
      "log_odds_ratio": -11.042044639587402,
      "logps/chosen": -21.983291625976562,
      "logps/rejected": -18.8513126373291,
      "loss": 453.2121,
      "nll_loss": 8.745321273803711,
      "po_loss": 5.417555809020996,
      "rewards/accuracies": 0.4437499940395355,
      "rewards/chosen": -10.991645812988281,
      "rewards/margins": -1.5659898519515991,
      "rewards/rejected": -9.42565631866455,
      "step": 10
    },
    {
      "epoch": 0.14222222222222222,
      "grad_norm": 527.598388671875,
      "learning_rate": 2.34375e-05,
      "log_odds_chosen": -1.3104137182235718,
      "log_odds_ratio": -11.265727996826172,
      "logps/chosen": -21.87612533569336,
      "logps/rejected": -20.564207077026367,
      "loss": 432.0127,
      "nll_loss": 7.955958366394043,
      "po_loss": 5.544436931610107,
      "rewards/accuracies": 0.515625,
      "rewards/chosen": -10.93806266784668,
      "rewards/margins": -0.6559588313102722,
      "rewards/rejected": -10.282103538513184,
      "step": 15
    },
    {
      "epoch": 0.18962962962962962,
      "grad_norm": 2963.93408203125,
      "learning_rate": 3.125e-05,
      "log_odds_chosen": -3.6984353065490723,
      "log_odds_ratio": -8.447766304016113,
      "logps/chosen": -15.567995071411133,
      "logps/rejected": -11.872796058654785,
      "loss": 317.4233,
      "nll_loss": 5.814067840576172,
      "po_loss": 4.105409622192383,
      "rewards/accuracies": 0.4312500059604645,
      "rewards/chosen": -7.783997535705566,
      "rewards/margins": -1.8475996255874634,
      "rewards/rejected": -5.936398029327393,
      "step": 20
    },
    {
      "epoch": 0.23703703703703705,
      "grad_norm": 273.48370361328125,
      "learning_rate": 3.90625e-05,
      "log_odds_chosen": -0.019201982766389847,
      "log_odds_ratio": -1.406181812286377,
      "logps/chosen": -3.055501937866211,
      "logps/rejected": -3.0181398391723633,
      "loss": 95.9863,
      "nll_loss": 2.533766269683838,
      "po_loss": 0.46580663323402405,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -1.5277509689331055,
      "rewards/margins": -0.018681075423955917,
      "rewards/rejected": -1.5090699195861816,
      "step": 25
    },
    {
      "epoch": 0.28444444444444444,
      "grad_norm": 216.39308166503906,
      "learning_rate": 4.6875e-05,
      "log_odds_chosen": 0.09772934764623642,
      "log_odds_ratio": -0.8396835327148438,
      "logps/chosen": -1.784551978111267,
      "logps/rejected": -1.8702020645141602,
      "loss": 68.8806,
      "nll_loss": 2.0184409618377686,
      "po_loss": 0.13407745957374573,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.8922759890556335,
      "rewards/margins": 0.0428251177072525,
      "rewards/rejected": -0.9351010322570801,
      "step": 30
    },
    {
      "epoch": 0.33185185185185184,
      "grad_norm": 280.289794921875,
      "learning_rate": 4.998613757348784e-05,
      "log_odds_chosen": 0.25464916229248047,
      "log_odds_ratio": -0.7872657775878906,
      "logps/chosen": -1.6759769916534424,
      "logps/rejected": -1.8980681896209717,
      "loss": 62.3358,
      "nll_loss": 1.86691415309906,
      "po_loss": 0.0810796245932579,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.8379884958267212,
      "rewards/margins": 0.11104562133550644,
      "rewards/rejected": -0.9490340948104858,
      "step": 35
    },
    {
      "epoch": 0.37925925925925924,
      "grad_norm": 708.8033447265625,
      "learning_rate": 4.990147841143462e-05,
      "log_odds_chosen": 0.31906959414482117,
      "log_odds_ratio": -0.6981687545776367,
      "logps/chosen": -1.4914863109588623,
      "logps/rejected": -1.7542707920074463,
      "loss": 57.8745,
      "nll_loss": 1.7927372455596924,
      "po_loss": 0.015842173248529434,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.7457431554794312,
      "rewards/margins": 0.13139230012893677,
      "rewards/rejected": -0.8771353960037231,
      "step": 40
    },
    {
      "epoch": 0.4266666666666667,
      "grad_norm": 216.06666564941406,
      "learning_rate": 4.97401218720448e-05,
      "log_odds_chosen": 0.29898008704185486,
      "log_odds_ratio": -0.6774553060531616,
      "logps/chosen": -1.4114806652069092,
      "logps/rejected": -1.6652767658233643,
      "loss": 55.5358,
      "nll_loss": 1.7187302112579346,
      "po_loss": 0.01676475629210472,
      "rewards/accuracies": 0.6156250238418579,
      "rewards/chosen": -0.7057403326034546,
      "rewards/margins": 0.12689796090126038,
      "rewards/rejected": -0.8326383829116821,
      "step": 45
    },
    {
      "epoch": 0.4740740740740741,
      "grad_norm": 134.64588928222656,
      "learning_rate": 4.9502564938797946e-05,
      "log_odds_chosen": 0.3125649392604828,
      "log_odds_ratio": -0.6582598686218262,
      "logps/chosen": -1.3025346994400024,
      "logps/rejected": -1.5579352378845215,
      "loss": 50.6171,
      "nll_loss": 1.5993175506591797,
      "po_loss": -0.01753159798681736,
      "rewards/accuracies": 0.6156250238418579,
      "rewards/chosen": -0.6512673497200012,
      "rewards/margins": 0.1277003139257431,
      "rewards/rejected": -0.7789676189422607,
      "step": 50
    },
    {
      "epoch": 0.5214814814814814,
      "grad_norm": 125.86614227294922,
      "learning_rate": 4.918953929490768e-05,
      "log_odds_chosen": 0.32853975892066956,
      "log_odds_ratio": -0.6720215082168579,
      "logps/chosen": -1.3117344379425049,
      "logps/rejected": -1.5763285160064697,
      "loss": 51.7501,
      "nll_loss": 1.6202409267425537,
      "po_loss": -0.0030506025068461895,
      "rewards/accuracies": 0.6031249761581421,
      "rewards/chosen": -0.6558672189712524,
      "rewards/margins": 0.13229700922966003,
      "rewards/rejected": -0.7881642580032349,
      "step": 55
    },
    {
      "epoch": 0.5688888888888889,
      "grad_norm": 223.18544006347656,
      "learning_rate": 4.88020090697132e-05,
      "log_odds_chosen": 0.35989266633987427,
      "log_odds_ratio": -0.6441806554794312,
      "logps/chosen": -1.3117573261260986,
      "logps/rejected": -1.609619379043579,
      "loss": 51.8589,
      "nll_loss": 1.6353687047958374,
      "po_loss": -0.014778362587094307,
      "rewards/accuracies": 0.6031249761581421,
      "rewards/chosen": -0.6558786630630493,
      "rewards/margins": 0.1489311009645462,
      "rewards/rejected": -0.8048096895217896,
      "step": 60
    },
    {
      "epoch": 0.6162962962962963,
      "grad_norm": 195.27818298339844,
      "learning_rate": 4.834116786912897e-05,
      "log_odds_chosen": 0.36024975776672363,
      "log_odds_ratio": -0.6309642195701599,
      "logps/chosen": -1.305149793624878,
      "logps/rejected": -1.584517478942871,
      "loss": 50.395,
      "nll_loss": 1.5889090299606323,
      "po_loss": -0.014065178111195564,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.652574896812439,
      "rewards/margins": 0.13968387246131897,
      "rewards/rejected": -0.7922587394714355,
      "step": 65
    },
    {
      "epoch": 0.6637037037037037,
      "grad_norm": 90.99329376220703,
      "learning_rate": 4.7808435099299045e-05,
      "log_odds_chosen": 0.49140042066574097,
      "log_odds_ratio": -0.5946860313415527,
      "logps/chosen": -1.2680964469909668,
      "logps/rejected": -1.666022539138794,
      "loss": 48.3341,
      "nll_loss": 1.567561388015747,
      "po_loss": -0.05712021514773369,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.6340482234954834,
      "rewards/margins": 0.19896303117275238,
      "rewards/rejected": -0.833011269569397,
      "step": 70
    },
    {
      "epoch": 0.7111111111111111,
      "grad_norm": 311.4484558105469,
      "learning_rate": 4.720545159477922e-05,
      "log_odds_chosen": 0.4681640565395355,
      "log_odds_ratio": -0.6271704435348511,
      "logps/chosen": -1.2391738891601562,
      "logps/rejected": -1.6162137985229492,
      "loss": 48.3516,
      "nll_loss": 1.547349214553833,
      "po_loss": -0.03636247664690018,
      "rewards/accuracies": 0.6781250238418579,
      "rewards/chosen": -0.6195869445800781,
      "rewards/margins": 0.1885199397802353,
      "rewards/rejected": -0.8081068992614746,
      "step": 75
    },
    {
      "epoch": 0.7585185185185185,
      "grad_norm": 290.2730407714844,
      "learning_rate": 4.653407456471222e-05,
      "log_odds_chosen": 0.37080585956573486,
      "log_odds_ratio": -0.645098090171814,
      "logps/chosen": -1.2255287170410156,
      "logps/rejected": -1.5139461755752563,
      "loss": 46.785,
      "nll_loss": 1.5034220218658447,
      "po_loss": -0.04139198362827301,
      "rewards/accuracies": 0.6031249761581421,
      "rewards/chosen": -0.6127643585205078,
      "rewards/margins": 0.14420875906944275,
      "rewards/rejected": -0.7569730877876282,
      "step": 80
    },
    {
      "epoch": 0.8059259259259259,
      "grad_norm": 123.97457885742188,
      "learning_rate": 4.579637187256222e-05,
      "log_odds_chosen": 0.3362444341182709,
      "log_odds_ratio": -0.6288684010505676,
      "logps/chosen": -1.1935008764266968,
      "logps/rejected": -1.456559419631958,
      "loss": 47.5166,
      "nll_loss": 1.5174460411071777,
      "po_loss": -0.03255213052034378,
      "rewards/accuracies": 0.628125011920929,
      "rewards/chosen": -0.5967504382133484,
      "rewards/margins": 0.1315293163061142,
      "rewards/rejected": -0.728279709815979,
      "step": 85
    },
    {
      "epoch": 0.8533333333333334,
      "grad_norm": 215.05145263671875,
      "learning_rate": 4.499461566702685e-05,
      "log_odds_chosen": 0.5429065823554993,
      "log_odds_ratio": -0.5807454586029053,
      "logps/chosen": -1.1128449440002441,
      "logps/rejected": -1.5253103971481323,
      "loss": 44.2251,
      "nll_loss": 1.449202537536621,
      "po_loss": -0.06716945022344589,
      "rewards/accuracies": 0.6781250238418579,
      "rewards/chosen": -0.5564224720001221,
      "rewards/margins": 0.20623275637626648,
      "rewards/rejected": -0.7626551985740662,
      "step": 90
    },
    {
      "epoch": 0.9007407407407407,
      "grad_norm": 107.29192352294922,
      "learning_rate": 4.413127538374411e-05,
      "log_odds_chosen": 0.4139605462551117,
      "log_odds_ratio": -0.6369166374206543,
      "logps/chosen": -1.1876246929168701,
      "logps/rejected": -1.5150048732757568,
      "loss": 46.2734,
      "nll_loss": 1.4956274032592773,
      "po_loss": -0.049582015722990036,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.5938123464584351,
      "rewards/margins": 0.16369004547595978,
      "rewards/rejected": -0.7575024366378784,
      "step": 95
    },
    {
      "epoch": 0.9481481481481482,
      "grad_norm": 88.76741027832031,
      "learning_rate": 4.320901013934887e-05,
      "log_odds_chosen": 0.49218830466270447,
      "log_odds_ratio": -0.5807846188545227,
      "logps/chosen": -1.1516714096069336,
      "logps/rejected": -1.5277314186096191,
      "loss": 44.1828,
      "nll_loss": 1.4526407718658447,
      "po_loss": -0.07192973792552948,
      "rewards/accuracies": 0.6781250238418579,
      "rewards/chosen": -0.5758357048034668,
      "rewards/margins": 0.18803004920482635,
      "rewards/rejected": -0.7638657093048096,
      "step": 100
    },
    {
      "epoch": 0.9955555555555555,
      "grad_norm": 205.42556762695312,
      "learning_rate": 4.223066054130568e-05,
      "log_odds_chosen": 0.6228845119476318,
      "log_odds_ratio": -0.5975527763366699,
      "logps/chosen": -1.139916181564331,
      "logps/rejected": -1.6632674932479858,
      "loss": 44.3594,
      "nll_loss": 1.4790844917297363,
      "po_loss": -0.09285450726747513,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.5699580907821655,
      "rewards/margins": 0.26167553663253784,
      "rewards/rejected": -0.8316337466239929,
      "step": 105
    },
    {
      "epoch": 1.037925925925926,
      "grad_norm": 264.0072326660156,
      "learning_rate": 4.1199239938743797e-05,
      "log_odds_chosen": 0.9932449460029602,
      "log_odds_ratio": -0.44668498635292053,
      "logps/chosen": -1.02596116065979,
      "logps/rejected": -1.7672877311706543,
      "loss": 30.7868,
      "nll_loss": 1.3024835586547852,
      "po_loss": -0.2260211557149887,
      "rewards/accuracies": 0.7902097702026367,
      "rewards/chosen": -0.512980580329895,
      "rewards/margins": 0.3706631660461426,
      "rewards/rejected": -0.8836438655853271,
      "step": 110
    },
    {
      "epoch": 1.0853333333333333,
      "grad_norm": 156.9835968017578,
      "learning_rate": 4.0117925141242174e-05,
      "log_odds_chosen": 1.619765043258667,
      "log_odds_ratio": -0.3932790756225586,
      "logps/chosen": -0.9797142148017883,
      "logps/rejected": -2.256946563720703,
      "loss": 29.6065,
      "nll_loss": 1.299898386001587,
      "po_loss": -0.3746953308582306,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.48985710740089417,
      "rewards/margins": 0.6386162042617798,
      "rewards/rejected": -1.1284732818603516,
      "step": 115
    },
    {
      "epoch": 1.1327407407407408,
      "grad_norm": 388.80926513671875,
      "learning_rate": 3.899004663415084e-05,
      "log_odds_chosen": 1.9765598773956299,
      "log_odds_ratio": -0.3769186735153198,
      "logps/chosen": -1.0536551475524902,
      "logps/rejected": -2.6980745792388916,
      "loss": 28.527,
      "nll_loss": 1.3548568487167358,
      "po_loss": -0.4633867144584656,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.5268275737762451,
      "rewards/margins": 0.8222096562385559,
      "rewards/rejected": -1.3490372896194458,
      "step": 120
    },
    {
      "epoch": 1.1801481481481482,
      "grad_norm": 701.78466796875,
      "learning_rate": 3.781907832058587e-05,
      "log_odds_chosen": 2.3174455165863037,
      "log_odds_ratio": -0.4550691545009613,
      "logps/chosen": -1.3317890167236328,
      "logps/rejected": -3.3481040000915527,
      "loss": 29.2023,
      "nll_loss": 1.430572748184204,
      "po_loss": -0.5180009007453918,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.6658945083618164,
      "rewards/margins": 1.00815749168396,
      "rewards/rejected": -1.6740520000457764,
      "step": 125
    },
    {
      "epoch": 1.2275555555555555,
      "grad_norm": 284.3743896484375,
      "learning_rate": 3.660862682169282e-05,
      "log_odds_chosen": 1.8650093078613281,
      "log_odds_ratio": -0.3967917561531067,
      "logps/chosen": -1.0224735736846924,
      "logps/rejected": -2.5611350536346436,
      "loss": 31.3418,
      "nll_loss": 1.3258826732635498,
      "po_loss": -0.34645071625709534,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.5112367868423462,
      "rewards/margins": 0.7693307995796204,
      "rewards/rejected": -1.2805675268173218,
      "step": 130
    },
    {
      "epoch": 1.274962962962963,
      "grad_norm": 514.7438354492188,
      "learning_rate": 3.5362420368134356e-05,
      "log_odds_chosen": 3.2559561729431152,
      "log_odds_ratio": -0.4518875479698181,
      "logps/chosen": -1.2612740993499756,
      "logps/rejected": -4.1930646896362305,
      "loss": 29.2929,
      "nll_loss": 1.4387086629867554,
      "po_loss": -0.5233055949211121,
      "rewards/accuracies": 0.8343750238418579,
      "rewards/chosen": -0.6306370496749878,
      "rewards/margins": 1.4658952951431274,
      "rewards/rejected": -2.0965323448181152,
      "step": 135
    },
    {
      "epoch": 1.3223703703703704,
      "grad_norm": 581.5865478515625,
      "learning_rate": 3.408429731701635e-05,
      "log_odds_chosen": 2.8635735511779785,
      "log_odds_ratio": -0.4325293004512787,
      "logps/chosen": -1.1112406253814697,
      "logps/rejected": -3.690396785736084,
      "loss": 27.7038,
      "nll_loss": 1.3551785945892334,
      "po_loss": -0.48943454027175903,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.5556203126907349,
      "rewards/margins": 1.2895781993865967,
      "rewards/rejected": -1.845198392868042,
      "step": 140
    },
    {
      "epoch": 1.3697777777777778,
      "grad_norm": 112.59197998046875,
      "learning_rate": 3.2778194329621104e-05,
      "log_odds_chosen": 2.2916512489318848,
      "log_odds_ratio": -0.41769519448280334,
      "logps/chosen": -1.020237684249878,
      "logps/rejected": -3.0198616981506348,
      "loss": 27.0053,
      "nll_loss": 1.3402351140975952,
      "po_loss": -0.4963192939758301,
      "rewards/accuracies": 0.8218749761581421,
      "rewards/chosen": -0.510118842124939,
      "rewards/margins": 0.9998119473457336,
      "rewards/rejected": -1.5099308490753174,
      "step": 145
    },
    {
      "epoch": 1.417185185185185,
      "grad_norm": 805.1986694335938,
      "learning_rate": 3.144813424636031e-05,
      "log_odds_chosen": 6.398291110992432,
      "log_odds_ratio": -0.40958553552627563,
      "logps/chosen": -1.284738302230835,
      "logps/rejected": -7.353111267089844,
      "loss": 18.0904,
      "nll_loss": 1.4131615161895752,
      "po_loss": -0.8478377461433411,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -0.6423691511154175,
      "rewards/margins": 3.034186601638794,
      "rewards/rejected": -3.676555633544922,
      "step": 150
    },
    {
      "epoch": 1.4645925925925927,
      "grad_norm": 219.5271759033203,
      "learning_rate": 3.0098213696293542e-05,
      "log_odds_chosen": 6.040364742279053,
      "log_odds_ratio": -0.49418535828590393,
      "logps/chosen": -1.2788912057876587,
      "logps/rejected": -7.029244899749756,
      "loss": 29.1762,
      "nll_loss": 1.5258408784866333,
      "po_loss": -0.6140841245651245,
      "rewards/accuracies": 0.809374988079071,
      "rewards/chosen": -0.6394456028938293,
      "rewards/margins": 2.8751773834228516,
      "rewards/rejected": -3.514622449874878,
      "step": 155
    },
    {
      "epoch": 1.512,
      "grad_norm": 485.47576904296875,
      "learning_rate": 2.8732590479375165e-05,
      "log_odds_chosen": 3.2111659049987793,
      "log_odds_ratio": -0.4072226583957672,
      "logps/chosen": -0.9738284945487976,
      "logps/rejected": -3.8901615142822266,
      "loss": 27.3374,
      "nll_loss": 1.3175849914550781,
      "po_loss": -0.4632897973060608,
      "rewards/accuracies": 0.8218749761581421,
      "rewards/chosen": -0.4869142472743988,
      "rewards/margins": 1.458166480064392,
      "rewards/rejected": -1.9450807571411133,
      "step": 160
    },
    {
      "epoch": 1.5594074074074074,
      "grad_norm": 835.3743286132812,
      "learning_rate": 2.7355470760292956e-05,
      "log_odds_chosen": 3.777056932449341,
      "log_odds_ratio": -0.535234808921814,
      "logps/chosen": -1.136812448501587,
      "logps/rejected": -4.577279090881348,
      "loss": 28.2855,
      "nll_loss": 1.4577716588974,
      "po_loss": -0.5738505125045776,
      "rewards/accuracies": 0.840624988079071,
      "rewards/chosen": -0.5684062242507935,
      "rewards/margins": 1.7202333211898804,
      "rewards/rejected": -2.288639545440674,
      "step": 165
    },
    {
      "epoch": 1.6068148148148147,
      "grad_norm": 275.0011291503906,
      "learning_rate": 2.597109611334169e-05,
      "log_odds_chosen": 3.221364974975586,
      "log_odds_ratio": -0.6203993558883667,
      "logps/chosen": -1.2395914793014526,
      "logps/rejected": -4.143717288970947,
      "loss": 36.586,
      "nll_loss": 1.4404484033584595,
      "po_loss": -0.2971368730068207,
      "rewards/accuracies": 0.8343750238418579,
      "rewards/chosen": -0.6197957396507263,
      "rewards/margins": 1.4520628452301025,
      "rewards/rejected": -2.0718586444854736,
      "step": 170
    },
    {
      "epoch": 1.6542222222222223,
      "grad_norm": 454.6676940917969,
      "learning_rate": 2.458373045823404e-05,
      "log_odds_chosen": 3.51635479927063,
      "log_odds_ratio": -0.4349392354488373,
      "logps/chosen": -1.2439143657684326,
      "logps/rejected": -4.460102081298828,
      "loss": 20.5614,
      "nll_loss": 1.4349825382232666,
      "po_loss": -0.7924402356147766,
      "rewards/accuracies": 0.8218749761581421,
      "rewards/chosen": -0.6219571828842163,
      "rewards/margins": 1.6080938577651978,
      "rewards/rejected": -2.230051040649414,
      "step": 175
    },
    {
      "epoch": 1.7016296296296296,
      "grad_norm": 1382.2955322265625,
      "learning_rate": 2.3197646927086697e-05,
      "log_odds_chosen": 2.992593288421631,
      "log_odds_ratio": -0.4546445906162262,
      "logps/chosen": -1.1713831424713135,
      "logps/rejected": -3.8302390575408936,
      "loss": 25.5293,
      "nll_loss": 1.433226227760315,
      "po_loss": -0.6354348659515381,
      "rewards/accuracies": 0.8218749761581421,
      "rewards/chosen": -0.5856915712356567,
      "rewards/margins": 1.32942795753479,
      "rewards/rejected": -1.9151195287704468,
      "step": 180
    },
    {
      "epoch": 1.749037037037037,
      "grad_norm": 828.3840942382812,
      "learning_rate": 2.1817114703032176e-05,
      "log_odds_chosen": 2.797692060470581,
      "log_odds_ratio": -0.43407559394836426,
      "logps/chosen": -1.1162699460983276,
      "logps/rejected": -3.64747953414917,
      "loss": 25.3097,
      "nll_loss": 1.3284680843353271,
      "po_loss": -0.5375403165817261,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.5581349730491638,
      "rewards/margins": 1.265605092048645,
      "rewards/rejected": -1.823739767074585,
      "step": 185
    },
    {
      "epoch": 1.7964444444444445,
      "grad_norm": 1088.3760986328125,
      "learning_rate": 2.0446385870993467e-05,
      "log_odds_chosen": 3.665976047515869,
      "log_odds_ratio": -0.43087244033813477,
      "logps/chosen": -0.9914714097976685,
      "logps/rejected": -4.345323085784912,
      "loss": 20.7913,
      "nll_loss": 1.3063815832138062,
      "po_loss": -0.656653106212616,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.49573570489883423,
      "rewards/margins": 1.6769258975982666,
      "rewards/rejected": -2.172661542892456,
      "step": 190
    },
    {
      "epoch": 1.8438518518518519,
      "grad_norm": 1150.76123046875,
      "learning_rate": 1.9089682321121834e-05,
      "log_odds_chosen": 6.179993629455566,
      "log_odds_ratio": -1.0467485189437866,
      "logps/chosen": -1.6919925212860107,
      "logps/rejected": -7.563102722167969,
      "loss": 44.1491,
      "nll_loss": 1.8505232334136963,
      "po_loss": -0.4708642065525055,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.8459962606430054,
      "rewards/margins": 2.9355549812316895,
      "rewards/rejected": -3.7815513610839844,
      "step": 195
    },
    {
      "epoch": 1.8912592592592592,
      "grad_norm": 454.659912109375,
      "learning_rate": 1.775118274523545e-05,
      "log_odds_chosen": 4.30472469329834,
      "log_odds_ratio": -0.5107940435409546,
      "logps/chosen": -1.1776766777038574,
      "logps/rejected": -5.171801567077637,
      "loss": 24.2982,
      "nll_loss": 1.444457769393921,
      "po_loss": -0.685138463973999,
      "rewards/accuracies": 0.846875011920929,
      "rewards/chosen": -0.5888383388519287,
      "rewards/margins": 1.9970623254776,
      "rewards/rejected": -2.5859007835388184,
      "step": 200
    },
    {
      "epoch": 1.9386666666666668,
      "grad_norm": 800.92578125,
      "learning_rate": 1.643500976631037e-05,
      "log_odds_chosen": 2.9392075538635254,
      "log_odds_ratio": -0.5851477384567261,
      "logps/chosen": -1.1809542179107666,
      "logps/rejected": -3.785405397415161,
      "loss": 31.8492,
      "nll_loss": 1.4079935550689697,
      "po_loss": -0.412706196308136,
      "rewards/accuracies": 0.840624988079071,
      "rewards/chosen": -0.5904771089553833,
      "rewards/margins": 1.3022253513336182,
      "rewards/rejected": -1.8927026987075806,
      "step": 205
    },
    {
      "epoch": 1.986074074074074,
      "grad_norm": 717.8953247070312,
      "learning_rate": 1.514521724066537e-05,
      "log_odds_chosen": 3.531224012374878,
      "log_odds_ratio": -0.637366771697998,
      "logps/chosen": -1.275508165359497,
      "logps/rejected": -4.51492977142334,
      "loss": 26.7864,
      "nll_loss": 1.5013716220855713,
      "po_loss": -0.6642956733703613,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.6377540826797485,
      "rewards/margins": 1.6197106838226318,
      "rewards/rejected": -2.25746488571167,
      "step": 210
    },
    {
      "epoch": 2.0284444444444443,
      "grad_norm": 1063.2196044921875,
      "learning_rate": 1.3885777771950348e-05,
      "log_odds_chosen": 7.453225612640381,
      "log_odds_ratio": -0.5666939616203308,
      "logps/chosen": -1.2951644659042358,
      "logps/rejected": -8.28040885925293,
      "loss": 0.3911,
      "nll_loss": 1.4602717161178589,
      "po_loss": -1.4465978145599365,
      "rewards/accuracies": 0.8741258978843689,
      "rewards/chosen": -0.6475822329521179,
      "rewards/margins": 3.492622137069702,
      "rewards/rejected": -4.140204429626465,
      "step": 215
    },
    {
      "epoch": 2.075851851851852,
      "grad_norm": 1035.703369140625,
      "learning_rate": 1.2660570475395683e-05,
      "log_odds_chosen": 8.555649757385254,
      "log_odds_ratio": -0.9756455421447754,
      "logps/chosen": -2.252915859222412,
      "logps/rejected": -10.40683364868164,
      "loss": 26.9416,
      "nll_loss": 2.1952455043792725,
      "po_loss": -1.353319764137268,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -1.126457929611206,
      "rewards/margins": 4.076959133148193,
      "rewards/rejected": -5.20341682434082,
      "step": 220
    },
    {
      "epoch": 2.1232592592592594,
      "grad_norm": 983.8046875,
      "learning_rate": 1.1473369030008974e-05,
      "log_odds_chosen": 6.684991359710693,
      "log_odds_ratio": -0.562995433807373,
      "logps/chosen": -1.3902976512908936,
      "logps/rejected": -7.629068851470947,
      "loss": -8.0422,
      "nll_loss": 1.5127547979354858,
      "po_loss": -1.7640743255615234,
      "rewards/accuracies": 0.8968750238418579,
      "rewards/chosen": -0.6951488256454468,
      "rewards/margins": 3.119385242462158,
      "rewards/rejected": -3.8145344257354736,
      "step": 225
    },
    {
      "epoch": 2.1706666666666665,
      "grad_norm": 1034.22802734375,
      "learning_rate": 1.0327830055518842e-05,
      "log_odds_chosen": 6.934300899505615,
      "log_odds_ratio": -0.2921414077281952,
      "logps/chosen": -0.9763369560241699,
      "logps/rejected": -7.44844913482666,
      "loss": -18.2469,
      "nll_loss": 1.264357328414917,
      "po_loss": -1.8345714807510376,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -0.48816847801208496,
      "rewards/margins": 3.236056089401245,
      "rewards/rejected": -3.72422456741333,
      "step": 230
    },
    {
      "epoch": 2.218074074074074,
      "grad_norm": 3417.23193359375,
      "learning_rate": 9.227481849865235e-06,
      "log_odds_chosen": 10.725831031799316,
      "log_odds_ratio": -0.9306958317756653,
      "logps/chosen": -1.6473735570907593,
      "logps/rejected": -11.90982437133789,
      "loss": -0.3702,
      "nll_loss": 1.8238227367401123,
      "po_loss": -1.8353925943374634,
      "rewards/accuracies": 0.871874988079071,
      "rewards/chosen": -0.8236867785453796,
      "rewards/margins": 5.131225109100342,
      "rewards/rejected": -5.954912185668945,
      "step": 235
    },
    {
      "epoch": 2.2654814814814817,
      "grad_norm": 6067.5625,
      "learning_rate": 8.175713521924978e-06,
      "log_odds_chosen": 13.598230361938477,
      "log_odds_ratio": -0.6408636569976807,
      "logps/chosen": -1.4218194484710693,
      "logps/rejected": -14.529757499694824,
      "loss": -35.3649,
      "nll_loss": 1.5829206705093384,
      "po_loss": -2.688075304031372,
      "rewards/accuracies": 0.909375011920929,
      "rewards/chosen": -0.7109097242355347,
      "rewards/margins": 6.553968906402588,
      "rewards/rejected": -7.264878749847412,
      "step": 240
    },
    {
      "epoch": 2.3128888888888888,
      "grad_norm": 2778.763671875,
      "learning_rate": 7.1757645529443665e-06,
      "log_odds_chosen": 12.886404037475586,
      "log_odds_ratio": -0.262265145778656,
      "logps/chosen": -1.1175204515457153,
      "logps/rejected": -13.538418769836426,
      "loss": -32.4414,
      "nll_loss": 1.350950002670288,
      "po_loss": -2.3647446632385254,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.5587602257728577,
      "rewards/margins": 6.210448265075684,
      "rewards/rejected": -6.769209384918213,
      "step": 245
    },
    {
      "epoch": 2.3602962962962963,
      "grad_norm": 2757.852783203125,
      "learning_rate": 6.230714818829733e-06,
      "log_odds_chosen": 12.95228385925293,
      "log_odds_ratio": -0.26469850540161133,
      "logps/chosen": -0.9851976633071899,
      "logps/rejected": -13.434306144714355,
      "loss": -44.9231,
      "nll_loss": 1.3150312900543213,
      "po_loss": -2.7188773155212402,
      "rewards/accuracies": 0.903124988079071,
      "rewards/chosen": -0.49259883165359497,
      "rewards/margins": 6.224554538726807,
      "rewards/rejected": -6.717153072357178,
      "step": 250
    },
    {
      "epoch": 2.407703703703704,
      "grad_norm": 2125.9111328125,
      "learning_rate": 5.343475104027743e-06,
      "log_odds_chosen": 12.030647277832031,
      "log_odds_ratio": -0.5396193265914917,
      "logps/chosen": -1.6967741250991821,
      "logps/rejected": -13.254587173461914,
      "loss": -11.2409,
      "nll_loss": 1.7866249084472656,
      "po_loss": -2.1379032135009766,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.8483870625495911,
      "rewards/margins": 5.77890682220459,
      "rewards/rejected": -6.627293586730957,
      "step": 255
    },
    {
      "epoch": 2.455111111111111,
      "grad_norm": 2060.32958984375,
      "learning_rate": 4.516778136213037e-06,
      "log_odds_chosen": 13.848899841308594,
      "log_odds_ratio": -0.3086540997028351,
      "logps/chosen": -1.331648349761963,
      "logps/rejected": -14.704687118530273,
      "loss": -44.157,
      "nll_loss": 1.3919355869293213,
      "po_loss": -2.7718422412872314,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": -0.6658241748809814,
      "rewards/margins": 6.686519622802734,
      "rewards/rejected": -7.352343559265137,
      "step": 260
    },
    {
      "epoch": 2.5025185185185186,
      "grad_norm": 5455.78369140625,
      "learning_rate": 3.7531701693965554e-06,
      "log_odds_chosen": 16.588672637939453,
      "log_odds_ratio": -0.3408772945404053,
      "logps/chosen": -1.6753642559051514,
      "logps/rejected": -17.813495635986328,
      "loss": -37.1963,
      "nll_loss": 1.6350977420806885,
      "po_loss": -2.7974817752838135,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.8376821279525757,
      "rewards/margins": 8.069067001342773,
      "rewards/rejected": -8.906747817993164,
      "step": 265
    },
    {
      "epoch": 2.549925925925926,
      "grad_norm": 4511.32470703125,
      "learning_rate": 3.055003141378948e-06,
      "log_odds_chosen": 14.822712898254395,
      "log_odds_ratio": -0.9747039079666138,
      "logps/chosen": -2.418854236602783,
      "logps/rejected": -16.78946304321289,
      "loss": -6.2935,
      "nll_loss": 2.2066473960876465,
      "po_loss": -2.403317451477051,
      "rewards/accuracies": 0.878125011920929,
      "rewards/chosen": -1.2094271183013916,
      "rewards/margins": 7.185305118560791,
      "rewards/rejected": -8.394731521606445,
      "step": 270
    },
    {
      "epoch": 2.5973333333333333,
      "grad_norm": 2031.630859375,
      "learning_rate": 2.424427429704365e-06,
      "log_odds_chosen": 12.37581729888916,
      "log_odds_ratio": -0.4414401650428772,
      "logps/chosen": -1.395058035850525,
      "logps/rejected": -13.333681106567383,
      "loss": -32.9732,
      "nll_loss": 1.4321500062942505,
      "po_loss": -2.4625613689422607,
      "rewards/accuracies": 0.909375011920929,
      "rewards/chosen": -0.6975290179252625,
      "rewards/margins": 5.969311714172363,
      "rewards/rejected": -6.666840553283691,
      "step": 275
    },
    {
      "epoch": 2.644740740740741,
      "grad_norm": 2148.434326171875,
      "learning_rate": 1.8633852284264508e-06,
      "log_odds_chosen": 12.9013671875,
      "log_odds_ratio": -0.5055217742919922,
      "logps/chosen": -1.2132359743118286,
      "logps/rejected": -13.65849494934082,
      "loss": -46.2362,
      "nll_loss": 1.4333760738372803,
      "po_loss": -2.8782572746276855,
      "rewards/accuracies": 0.871874988079071,
      "rewards/chosen": -0.6066179871559143,
      "rewards/margins": 6.222629547119141,
      "rewards/rejected": -6.82924747467041,
      "step": 280
    },
    {
      "epoch": 2.6921481481481484,
      "grad_norm": 1996.1431884765625,
      "learning_rate": 1.3736045660864034e-06,
      "log_odds_chosen": 9.410332679748535,
      "log_odds_ratio": -0.5190314054489136,
      "logps/chosen": -1.217825174331665,
      "logps/rejected": -10.21954345703125,
      "loss": -21.9301,
      "nll_loss": 1.4607911109924316,
      "po_loss": -2.1461057662963867,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6089125871658325,
      "rewards/margins": 4.500859260559082,
      "rewards/rejected": -5.109771728515625,
      "step": 285
    },
    {
      "epoch": 2.7395555555555555,
      "grad_norm": 2126.190673828125,
      "learning_rate": 9.565939833279192e-07,
      "log_odds_chosen": 9.630215644836426,
      "log_odds_ratio": -0.6117982268333435,
      "logps/chosen": -1.3845949172973633,
      "logps/rejected": -10.577371597290039,
      "loss": -7.386,
      "nll_loss": 1.5414915084838867,
      "po_loss": -1.7723032236099243,
      "rewards/accuracies": 0.8843749761581421,
      "rewards/chosen": -0.6922974586486816,
      "rewards/margins": 4.596388816833496,
      "rewards/rejected": -5.2886857986450195,
      "step": 290
    },
    {
      "epoch": 2.786962962962963,
      "grad_norm": 1371.4443359375,
      "learning_rate": 6.136378865420872e-07,
      "log_odds_chosen": 9.735053062438965,
      "log_odds_ratio": -0.34574881196022034,
      "logps/chosen": -1.0634145736694336,
      "logps/rejected": -10.3881254196167,
      "loss": -27.8217,
      "nll_loss": 1.3093498945236206,
      "po_loss": -2.1787781715393066,
      "rewards/accuracies": 0.856249988079071,
      "rewards/chosen": -0.5317072868347168,
      "rewards/margins": 4.662355899810791,
      "rewards/rejected": -5.19406270980835,
      "step": 295
    },
    {
      "epoch": 2.83437037037037,
      "grad_norm": 1635.2408447265625,
      "learning_rate": 3.45792591853214e-07,
      "log_odds_chosen": 12.451967239379883,
      "log_odds_ratio": -0.2706998884677887,
      "logps/chosen": -0.9222389459609985,
      "logps/rejected": -12.897394180297852,
      "loss": -45.3531,
      "nll_loss": 1.2098591327667236,
      "po_loss": -2.6271443367004395,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.46111947298049927,
      "rewards/margins": 5.987576961517334,
      "rewards/rejected": -6.448697090148926,
      "step": 300
    },
    {
      "epoch": 2.8817777777777778,
      "grad_norm": 1768.5748291015625,
      "learning_rate": 1.538830716302092e-07,
      "log_odds_chosen": 12.523870468139648,
      "log_odds_ratio": -0.49474939703941345,
      "logps/chosen": -1.1987851858139038,
      "logps/rejected": -13.224016189575195,
      "loss": -39.2375,
      "nll_loss": 1.4008312225341797,
      "po_loss": -2.6270015239715576,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.5993925929069519,
      "rewards/margins": 6.012616157531738,
      "rewards/rejected": -6.612008094787598,
      "step": 305
    },
    {
      "epoch": 2.9291851851851853,
      "grad_norm": 2095.619140625,
      "learning_rate": 3.8500413544415025e-08,
      "log_odds_chosen": 13.987775802612305,
      "log_odds_ratio": -0.43595457077026367,
      "logps/chosen": -1.334578514099121,
      "logps/rejected": -14.836097717285156,
      "loss": -48.9085,
      "nll_loss": 1.5135501623153687,
      "po_loss": -3.041940689086914,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.6672892570495605,
      "rewards/margins": 6.750760078430176,
      "rewards/rejected": -7.418048858642578,
      "step": 310
    },
    {
      "epoch": 2.9765925925925925,
      "grad_norm": 4473.79345703125,
      "learning_rate": 0.0,
      "log_odds_chosen": 12.231574058532715,
      "log_odds_ratio": -0.3107563853263855,
      "logps/chosen": -1.1671451330184937,
      "logps/rejected": -12.893664360046387,
      "loss": -49.9097,
      "nll_loss": 1.2714515924453735,
      "po_loss": -2.8311288356781006,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.5835725665092468,
      "rewards/margins": 5.863259315490723,
      "rewards/rejected": -6.446832180023193,
      "step": 315
    },
    {
      "epoch": 2.9765925925925925,
      "step": 315,
      "total_flos": 0.0,
      "train_loss": 42.33665879264711,
      "train_runtime": 9540.6023,
      "train_samples_per_second": 2.123,
      "train_steps_per_second": 0.033
    }
  ],
  "logging_steps": 5,
  "max_steps": 315,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 100000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}