{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.9765925925925925,
  "eval_steps": 500,
  "global_step": 315,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.047407407407407405,
      "grad_norm": 1199.2445068359375,
      "learning_rate": 7.8125e-06,
      "log_odds_chosen": -1.0599775314331055,
      "log_odds_ratio": -11.002888679504395,
      "logps/chosen": -22.7018985748291,
      "logps/rejected": -21.64215660095215,
      "loss": 482.2551,
      "nll_loss": 9.837621688842773,
      "po_loss": 5.232847690582275,
      "rewards/accuracies": 0.4781250059604645,
      "rewards/chosen": -11.35094928741455,
      "rewards/margins": -0.5298711061477661,
      "rewards/rejected": -10.821078300476074,
      "step": 5
    },
    {
      "epoch": 0.09481481481481481,
      "grad_norm": 597.969970703125,
      "learning_rate": 1.5625e-05,
      "log_odds_chosen": -3.1238608360290527,
      "log_odds_ratio": -11.037403106689453,
      "logps/chosen": -21.978214263916016,
      "logps/rejected": -18.854707717895508,
      "loss": 448.5505,
      "nll_loss": 8.742585182189941,
      "po_loss": 5.274618148803711,
      "rewards/accuracies": 0.4468750059604645,
      "rewards/chosen": -10.989107131958008,
      "rewards/margins": -1.5617557764053345,
      "rewards/rejected": -9.427353858947754,
      "step": 10
    },
    {
      "epoch": 0.14222222222222222,
      "grad_norm": 562.1139526367188,
      "learning_rate": 2.34375e-05,
      "log_odds_chosen": -1.3050577640533447,
      "log_odds_ratio": -11.273977279663086,
      "logps/chosen": -21.88943862915039,
      "logps/rejected": -20.582889556884766,
      "loss": 423.8629,
      "nll_loss": 7.963566780090332,
      "po_loss": 5.2821502685546875,
      "rewards/accuracies": 0.515625,
      "rewards/chosen": -10.944719314575195,
      "rewards/margins": -0.6532740592956543,
      "rewards/rejected": -10.291444778442383,
      "step": 15
    },
    {
      "epoch": 0.18962962962962962,
      "grad_norm": 3476.161865234375,
      "learning_rate": 3.125e-05,
      "log_odds_chosen": -3.9486403465270996,
      "log_odds_ratio": -8.879361152648926,
      "logps/chosen": -16.218509674072266,
      "logps/rejected": -12.272605895996094,
      "loss": 321.7442,
      "nll_loss": 5.964278221130371,
      "po_loss": 4.09022855758667,
      "rewards/accuracies": 0.4312500059604645,
      "rewards/chosen": -8.109254837036133,
      "rewards/margins": -1.972952127456665,
      "rewards/rejected": -6.136302947998047,
      "step": 20
    },
    {
      "epoch": 0.23703703703703705,
      "grad_norm": 253.1682586669922,
      "learning_rate": 3.90625e-05,
      "log_odds_chosen": -0.06654863059520721,
      "log_odds_ratio": -1.5640277862548828,
      "logps/chosen": -3.363724946975708,
      "logps/rejected": -3.2796008586883545,
      "loss": 89.1255,
      "nll_loss": 2.5614538192749023,
      "po_loss": 0.223719522356987,
      "rewards/accuracies": 0.5531250238418579,
      "rewards/chosen": -1.681862473487854,
      "rewards/margins": -0.042061954736709595,
      "rewards/rejected": -1.6398004293441772,
      "step": 25
    },
    {
      "epoch": 0.28444444444444444,
      "grad_norm": 201.95008850097656,
      "learning_rate": 4.6875e-05,
      "log_odds_chosen": 0.09966959804296494,
      "log_odds_ratio": -0.8472908735275269,
      "logps/chosen": -1.7985613346099854,
      "logps/rejected": -1.8834549188613892,
      "loss": 62.7189,
      "nll_loss": 2.001840591430664,
      "po_loss": -0.04187442362308502,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.8992806673049927,
      "rewards/margins": 0.04244700074195862,
      "rewards/rejected": -0.9417274594306946,
      "step": 30
    },
    {
      "epoch": 0.33185185185185184,
      "grad_norm": 226.0881805419922,
      "learning_rate": 4.998613757348784e-05,
      "log_odds_chosen": 0.3407457172870636,
      "log_odds_ratio": -0.7940031886100769,
      "logps/chosen": -1.766862154006958,
      "logps/rejected": -2.0710058212280273,
      "loss": 56.0033,
      "nll_loss": 1.9001868963241577,
      "po_loss": -0.15008527040481567,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.883431077003479,
      "rewards/margins": 0.15207180380821228,
      "rewards/rejected": -1.0355029106140137,
      "step": 35
    },
    {
      "epoch": 0.37925925925925924,
      "grad_norm": 266.1627197265625,
      "learning_rate": 4.990147841143462e-05,
      "log_odds_chosen": 0.5402604341506958,
      "log_odds_ratio": -0.7131898999214172,
      "logps/chosen": -1.7148605585098267,
      "logps/rejected": -2.1804628372192383,
      "loss": 53.4684,
      "nll_loss": 1.883787751197815,
      "po_loss": -0.21290016174316406,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.8574302792549133,
      "rewards/margins": 0.2328011691570282,
      "rewards/rejected": -1.0902314186096191,
      "step": 40
    },
    {
      "epoch": 0.4266666666666667,
      "grad_norm": 335.4718322753906,
      "learning_rate": 4.97401218720448e-05,
      "log_odds_chosen": 1.1245437860488892,
      "log_odds_ratio": -0.6130434274673462,
      "logps/chosen": -1.6581999063491821,
      "logps/rejected": -2.69628643989563,
      "loss": 45.3042,
      "nll_loss": 1.8416335582733154,
      "po_loss": -0.4258776307106018,
      "rewards/accuracies": 0.715624988079071,
      "rewards/chosen": -0.8290999531745911,
      "rewards/margins": 0.5190432667732239,
      "rewards/rejected": -1.348143219947815,
      "step": 45
    },
    {
      "epoch": 0.4740740740740741,
      "grad_norm": 170.5546875,
      "learning_rate": 4.9502564938797946e-05,
      "log_odds_chosen": 0.9557003974914551,
      "log_odds_ratio": -0.6759911775588989,
      "logps/chosen": -1.6203289031982422,
      "logps/rejected": -2.4938411712646484,
      "loss": 47.41,
      "nll_loss": 1.8312504291534424,
      "po_loss": -0.34968656301498413,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.8101644515991211,
      "rewards/margins": 0.4367561340332031,
      "rewards/rejected": -1.2469205856323242,
      "step": 50
    },
    {
      "epoch": 0.5214814814814814,
      "grad_norm": 225.52430725097656,
      "learning_rate": 4.918953929490768e-05,
      "log_odds_chosen": 1.1236488819122314,
      "log_odds_ratio": -0.6741882562637329,
      "logps/chosen": -1.6877021789550781,
      "logps/rejected": -2.727043390274048,
      "loss": 45.2223,
      "nll_loss": 1.8263689279556274,
      "po_loss": -0.41317272186279297,
      "rewards/accuracies": 0.6781250238418579,
      "rewards/chosen": -0.8438510894775391,
      "rewards/margins": 0.5196704864501953,
      "rewards/rejected": -1.363521695137024,
      "step": 55
    },
    {
      "epoch": 0.5688888888888889,
      "grad_norm": 355.39068603515625,
      "learning_rate": 4.88020090697132e-05,
      "log_odds_chosen": 1.1692771911621094,
      "log_odds_ratio": -0.6426931619644165,
      "logps/chosen": -1.5684707164764404,
      "logps/rejected": -2.65775728225708,
      "loss": 44.6871,
      "nll_loss": 1.8004539012908936,
      "po_loss": -0.40398064255714417,
      "rewards/accuracies": 0.653124988079071,
      "rewards/chosen": -0.7842353582382202,
      "rewards/margins": 0.544643223285675,
      "rewards/rejected": -1.32887864112854,
      "step": 60
    },
    {
      "epoch": 0.6162962962962963,
      "grad_norm": 162.62863159179688,
      "learning_rate": 4.834116786912897e-05,
      "log_odds_chosen": 1.382725715637207,
      "log_odds_ratio": -0.705934464931488,
      "logps/chosen": -1.7009910345077515,
      "logps/rejected": -2.989063024520874,
      "loss": 43.7142,
      "nll_loss": 1.8565829992294312,
      "po_loss": -0.49051398038864136,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.8504955172538757,
      "rewards/margins": 0.6440361142158508,
      "rewards/rejected": -1.494531512260437,
      "step": 65
    },
    {
      "epoch": 0.6637037037037037,
      "grad_norm": 889.6281127929688,
      "learning_rate": 4.7808435099299045e-05,
      "log_odds_chosen": 3.606097459793091,
      "log_odds_ratio": -0.7058836221694946,
      "logps/chosen": -2.0993778705596924,
      "logps/rejected": -5.576085090637207,
      "loss": 33.5164,
      "nll_loss": 2.1465699672698975,
      "po_loss": -1.0991829633712769,
      "rewards/accuracies": 0.7593749761581421,
      "rewards/chosen": -1.0496889352798462,
      "rewards/margins": 1.7383534908294678,
      "rewards/rejected": -2.7880425453186035,
      "step": 70
    },
    {
      "epoch": 0.7111111111111111,
      "grad_norm": 218.24794006347656,
      "learning_rate": 4.720545159477922e-05,
      "log_odds_chosen": 2.234952926635742,
      "log_odds_ratio": -0.8444352149963379,
      "logps/chosen": -1.784437894821167,
      "logps/rejected": -3.926750659942627,
      "loss": 35.6981,
      "nll_loss": 1.8649476766586304,
      "po_loss": -0.7493817806243896,
      "rewards/accuracies": 0.690625011920929,
      "rewards/chosen": -0.8922189474105835,
      "rewards/margins": 1.071156620979309,
      "rewards/rejected": -1.9633753299713135,
      "step": 75
    },
    {
      "epoch": 0.7585185185185185,
      "grad_norm": 1788.55126953125,
      "learning_rate": 4.653407456471222e-05,
      "log_odds_chosen": 1.6183449029922485,
      "log_odds_ratio": -0.846682071685791,
      "logps/chosen": -1.728458046913147,
      "logps/rejected": -3.26391339302063,
      "loss": 42.0319,
      "nll_loss": 1.8288921117782593,
      "po_loss": -0.5153962969779968,
      "rewards/accuracies": 0.621874988079071,
      "rewards/chosen": -0.8642290234565735,
      "rewards/margins": 0.7677275538444519,
      "rewards/rejected": -1.631956696510315,
      "step": 80
    },
    {
      "epoch": 0.8059259259259259,
      "grad_norm": 608.4232788085938,
      "learning_rate": 4.579637187256222e-05,
      "log_odds_chosen": 2.1726527214050293,
      "log_odds_ratio": -0.8868170976638794,
      "logps/chosen": -2.386735200881958,
      "logps/rejected": -4.454265594482422,
      "loss": 40.3963,
      "nll_loss": 2.1039323806762695,
      "po_loss": -0.8415490388870239,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -1.193367600440979,
      "rewards/margins": 1.0337653160095215,
      "rewards/rejected": -2.227132797241211,
      "step": 85
    },
    {
      "epoch": 0.8533333333333334,
      "grad_norm": 2436.2255859375,
      "learning_rate": 4.499461566702685e-05,
      "log_odds_chosen": 3.146548271179199,
      "log_odds_ratio": -0.9941431283950806,
      "logps/chosen": -2.020655393600464,
      "logps/rejected": -5.0438432693481445,
      "loss": 36.2214,
      "nll_loss": 2.022157669067383,
      "po_loss": -0.890238881111145,
      "rewards/accuracies": 0.715624988079071,
      "rewards/chosen": -1.010327696800232,
      "rewards/margins": 1.5115936994552612,
      "rewards/rejected": -2.5219216346740723,
      "step": 90
    },
    {
      "epoch": 0.9007407407407407,
      "grad_norm": 1364.458251953125,
      "learning_rate": 4.413127538374411e-05,
      "log_odds_chosen": 2.372877597808838,
      "log_odds_ratio": -0.8819985389709473,
      "logps/chosen": -1.8147878646850586,
      "logps/rejected": -4.103789806365967,
      "loss": 35.7507,
      "nll_loss": 1.7868406772613525,
      "po_loss": -0.6696311235427856,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -0.9073939323425293,
      "rewards/margins": 1.144500970840454,
      "rewards/rejected": -2.0518949031829834,
      "step": 95
    },
    {
      "epoch": 0.9481481481481482,
      "grad_norm": 1274.0474853515625,
      "learning_rate": 4.320901013934887e-05,
      "log_odds_chosen": 2.14143705368042,
      "log_odds_ratio": -0.786421000957489,
      "logps/chosen": -1.7358782291412354,
      "logps/rejected": -3.7556350231170654,
      "loss": 33.5542,
      "nll_loss": 1.854790449142456,
      "po_loss": -0.8062206506729126,
      "rewards/accuracies": 0.6968749761581421,
      "rewards/chosen": -0.8679391145706177,
      "rewards/margins": 1.0098785161972046,
      "rewards/rejected": -1.8778175115585327,
      "step": 100
    },
    {
      "epoch": 0.9955555555555555,
      "grad_norm": 568.3045043945312,
      "learning_rate": 4.223066054130568e-05,
      "log_odds_chosen": 2.152684450149536,
      "log_odds_ratio": -0.7120019197463989,
      "logps/chosen": -1.5902575254440308,
      "logps/rejected": -3.6419010162353516,
      "loss": 31.711,
      "nll_loss": 1.7339718341827393,
      "po_loss": -0.7430030107498169,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.7951287627220154,
      "rewards/margins": 1.0258219242095947,
      "rewards/rejected": -1.8209505081176758,
      "step": 105
    },
    {
      "epoch": 1.037925925925926,
      "grad_norm": 628.1900634765625,
      "learning_rate": 4.1199239938743797e-05,
      "log_odds_chosen": 3.4075300693511963,
      "log_odds_ratio": -0.5813586115837097,
      "logps/chosen": -1.8704923391342163,
      "logps/rejected": -5.093435287475586,
      "loss": 16.708,
      "nll_loss": 1.6286418437957764,
      "po_loss": -1.0444445610046387,
      "rewards/accuracies": 0.7552447319030762,
      "rewards/chosen": -0.9352461695671082,
      "rewards/margins": 1.6114716529846191,
      "rewards/rejected": -2.546717643737793,
      "step": 110
    },
    {
      "epoch": 1.0853333333333333,
      "grad_norm": 1226.1982421875,
      "learning_rate": 4.0117925141242174e-05,
      "log_odds_chosen": 3.840623378753662,
      "log_odds_ratio": -0.8993012309074402,
      "logps/chosen": -1.9903583526611328,
      "logps/rejected": -5.609218597412109,
      "loss": 14.4574,
      "nll_loss": 1.720370888710022,
      "po_loss": -1.2685784101486206,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.9951791763305664,
      "rewards/margins": 1.8094301223754883,
      "rewards/rejected": -2.8046092987060547,
      "step": 115
    },
    {
      "epoch": 1.1327407407407408,
      "grad_norm": 1661.664794921875,
      "learning_rate": 3.899004663415084e-05,
      "log_odds_chosen": 3.6831917762756348,
      "log_odds_ratio": -0.5217957496643066,
      "logps/chosen": -1.4346517324447632,
      "logps/rejected": -4.917321681976318,
      "loss": 14.9163,
      "nll_loss": 1.5880098342895508,
      "po_loss": -1.1218748092651367,
      "rewards/accuracies": 0.7593749761581421,
      "rewards/chosen": -0.7173258662223816,
      "rewards/margins": 1.7413349151611328,
      "rewards/rejected": -2.458660840988159,
      "step": 120
    },
    {
      "epoch": 1.1801481481481482,
      "grad_norm": 571.31982421875,
      "learning_rate": 3.781907832058587e-05,
      "log_odds_chosen": 6.352164268493652,
      "log_odds_ratio": -0.6368833780288696,
      "logps/chosen": -1.6029341220855713,
      "logps/rejected": -7.754278659820557,
      "loss": 11.4892,
      "nll_loss": 1.7506815195083618,
      "po_loss": -1.3916442394256592,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.8014670610427856,
      "rewards/margins": 3.075672149658203,
      "rewards/rejected": -3.8771393299102783,
      "step": 125
    },
    {
      "epoch": 1.2275555555555555,
      "grad_norm": 831.4838256835938,
      "learning_rate": 3.660862682169282e-05,
      "log_odds_chosen": 5.919846534729004,
      "log_odds_ratio": -0.5129478573799133,
      "logps/chosen": -1.4314855337142944,
      "logps/rejected": -7.115793704986572,
      "loss": 9.806,
      "nll_loss": 1.6523067951202393,
      "po_loss": -1.3458693027496338,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.7157427668571472,
      "rewards/margins": 2.842153549194336,
      "rewards/rejected": -3.557896852493286,
      "step": 130
    },
    {
      "epoch": 1.274962962962963,
      "grad_norm": 586.3059692382812,
      "learning_rate": 3.5362420368134356e-05,
      "log_odds_chosen": 5.599277019500732,
      "log_odds_ratio": -0.624258279800415,
      "logps/chosen": -1.7905899286270142,
      "logps/rejected": -7.161185264587402,
      "loss": 12.6168,
      "nll_loss": 1.760801076889038,
      "po_loss": -1.3665275573730469,
      "rewards/accuracies": 0.778124988079071,
      "rewards/chosen": -0.8952949643135071,
      "rewards/margins": 2.6852974891662598,
      "rewards/rejected": -3.580592632293701,
      "step": 135
    },
    {
      "epoch": 1.3223703703703704,
      "grad_norm": 880.5230102539062,
      "learning_rate": 3.408429731701635e-05,
      "log_odds_chosen": 3.740973711013794,
      "log_odds_ratio": -0.5780373215675354,
      "logps/chosen": -1.4464441537857056,
      "logps/rejected": -4.967782020568848,
      "loss": 15.2271,
      "nll_loss": 1.6292192935943604,
      "po_loss": -1.1533715724945068,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.7232220768928528,
      "rewards/margins": 1.7606691122055054,
      "rewards/rejected": -2.483891010284424,
      "step": 140
    },
    {
      "epoch": 1.3697777777777778,
      "grad_norm": 663.084228515625,
      "learning_rate": 3.2778194329621104e-05,
      "log_odds_chosen": 6.911275386810303,
      "log_odds_ratio": -0.8403680920600891,
      "logps/chosen": -1.9615440368652344,
      "logps/rejected": -8.633264541625977,
      "loss": 8.7443,
      "nll_loss": 1.8974125385284424,
      "po_loss": -1.624152421951294,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.9807720184326172,
      "rewards/margins": 3.335860013961792,
      "rewards/rejected": -4.316632270812988,
      "step": 145
    },
    {
      "epoch": 1.417185185185185,
      "grad_norm": 902.1053466796875,
      "learning_rate": 3.144813424636031e-05,
      "log_odds_chosen": 7.530759334564209,
      "log_odds_ratio": -0.5000367164611816,
      "logps/chosen": -1.5284488201141357,
      "logps/rejected": -8.809328079223633,
      "loss": -5.3961,
      "nll_loss": 1.633239984512329,
      "po_loss": -1.8018696308135986,
      "rewards/accuracies": 0.809374988079071,
      "rewards/chosen": -0.7642244100570679,
      "rewards/margins": 3.640439510345459,
      "rewards/rejected": -4.404664039611816,
      "step": 150
    },
    {
      "epoch": 1.4645925925925927,
      "grad_norm": 641.8519897460938,
      "learning_rate": 3.0098213696293542e-05,
      "log_odds_chosen": 6.644770622253418,
      "log_odds_ratio": -1.1336581707000732,
      "logps/chosen": -2.2592227458953857,
      "logps/rejected": -8.666966438293457,
      "loss": 19.4596,
      "nll_loss": 2.1993162631988525,
      "po_loss": -1.591204047203064,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -1.1296113729476929,
      "rewards/margins": 3.203871488571167,
      "rewards/rejected": -4.3334832191467285,
      "step": 155
    },
    {
      "epoch": 1.512,
      "grad_norm": 991.56591796875,
      "learning_rate": 2.8732590479375165e-05,
      "log_odds_chosen": 7.27960205078125,
      "log_odds_ratio": -0.6811318397521973,
      "logps/chosen": -1.9771201610565186,
      "logps/rejected": -8.984837532043457,
      "loss": -4.91,
      "nll_loss": 1.8900007009506226,
      "po_loss": -2.043437957763672,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.9885600805282593,
      "rewards/margins": 3.5038580894470215,
      "rewards/rejected": -4.4924187660217285,
      "step": 160
    },
    {
      "epoch": 1.5594074074074074,
      "grad_norm": 593.0498657226562,
      "learning_rate": 2.7355470760292956e-05,
      "log_odds_chosen": 6.110047340393066,
      "log_odds_ratio": -0.8749097585678101,
      "logps/chosen": -2.053755283355713,
      "logps/rejected": -7.891977787017822,
      "loss": 7.4888,
      "nll_loss": 2.0199038982391357,
      "po_loss": -1.7858794927597046,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -1.0268776416778564,
      "rewards/margins": 2.9191110134124756,
      "rewards/rejected": -3.945988893508911,
      "step": 165
    },
    {
      "epoch": 1.6068148148148147,
      "grad_norm": 883.1884765625,
      "learning_rate": 2.597109611334169e-05,
      "log_odds_chosen": 7.358190059661865,
      "log_odds_ratio": -0.6941929459571838,
      "logps/chosen": -1.9749581813812256,
      "logps/rejected": -9.061105728149414,
      "loss": -1.458,
      "nll_loss": 1.907533049583435,
      "po_loss": -1.9530951976776123,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.9874790906906128,
      "rewards/margins": 3.5430731773376465,
      "rewards/rejected": -4.530552864074707,
      "step": 170
    },
    {
      "epoch": 1.6542222222222223,
      "grad_norm": 776.2947387695312,
      "learning_rate": 2.458373045823404e-05,
      "log_odds_chosen": 7.158226013183594,
      "log_odds_ratio": -0.9282143712043762,
      "logps/chosen": -2.6594738960266113,
      "logps/rejected": -9.560735702514648,
      "loss": 7.5769,
      "nll_loss": 2.254758596420288,
      "po_loss": -2.0179810523986816,
      "rewards/accuracies": 0.784375011920929,
      "rewards/chosen": -1.3297369480133057,
      "rewards/margins": 3.4506313800811768,
      "rewards/rejected": -4.780367851257324,
      "step": 175
    },
    {
      "epoch": 1.7016296296296296,
      "grad_norm": 2007.3345947265625,
      "learning_rate": 2.3197646927086697e-05,
      "log_odds_chosen": 7.059978485107422,
      "log_odds_ratio": -0.91358882188797,
      "logps/chosen": -2.309866428375244,
      "logps/rejected": -9.11004638671875,
      "loss": -8.0333,
      "nll_loss": 1.9352056980133057,
      "po_loss": -2.1862475872039795,
      "rewards/accuracies": 0.7718750238418579,
      "rewards/chosen": -1.154933214187622,
      "rewards/margins": 3.400089979171753,
      "rewards/rejected": -4.555023193359375,
      "step": 180
    },
    {
      "epoch": 1.749037037037037,
      "grad_norm": 846.7615356445312,
      "learning_rate": 2.1817114703032176e-05,
      "log_odds_chosen": 7.948319435119629,
      "log_odds_ratio": -1.0959177017211914,
      "logps/chosen": -2.901278495788574,
      "logps/rejected": -10.634871482849121,
      "loss": -11.2961,
      "nll_loss": 2.0984649658203125,
      "po_loss": -2.451467514038086,
      "rewards/accuracies": 0.7718750238418579,
      "rewards/chosen": -1.450639247894287,
      "rewards/margins": 3.8667964935302734,
      "rewards/rejected": -5.3174357414245605,
      "step": 185
    },
    {
      "epoch": 1.7964444444444445,
      "grad_norm": 817.7583618164062,
      "learning_rate": 2.0446385870993467e-05,
      "log_odds_chosen": 6.724404335021973,
      "log_odds_ratio": -0.5374380350112915,
      "logps/chosen": -1.7088407278060913,
      "logps/rejected": -8.153546333312988,
      "loss": -27.5871,
      "nll_loss": 1.6332111358642578,
      "po_loss": -2.495307207107544,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.8544203639030457,
      "rewards/margins": 3.2223522663116455,
      "rewards/rejected": -4.076773166656494,
      "step": 190
    },
    {
      "epoch": 1.8438518518518519,
      "grad_norm": 2679.26513671875,
      "learning_rate": 1.9089682321121834e-05,
      "log_odds_chosen": 9.504817008972168,
      "log_odds_ratio": -1.4054466485977173,
      "logps/chosen": -4.4236674308776855,
      "logps/rejected": -13.693811416625977,
      "loss": 20.7869,
      "nll_loss": 3.1105668544769287,
      "po_loss": -2.460975170135498,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -2.2118337154388428,
      "rewards/margins": 4.635071754455566,
      "rewards/rejected": -6.846905708312988,
      "step": 195
    },
    {
      "epoch": 1.8912592592592592,
      "grad_norm": 896.3246459960938,
      "learning_rate": 1.775118274523545e-05,
      "log_odds_chosen": 6.952999114990234,
      "log_odds_ratio": -1.0420355796813965,
      "logps/chosen": -2.362539768218994,
      "logps/rejected": -9.074200630187988,
      "loss": -3.5533,
      "nll_loss": 2.0847651958465576,
      "po_loss": -2.1958060264587402,
      "rewards/accuracies": 0.753125011920929,
      "rewards/chosen": -1.181269884109497,
      "rewards/margins": 3.355830430984497,
      "rewards/rejected": -4.537100315093994,
      "step": 200
    },
    {
      "epoch": 1.9386666666666668,
      "grad_norm": 602.3760986328125,
      "learning_rate": 1.643500976631037e-05,
      "log_odds_chosen": 4.505127429962158,
      "log_odds_ratio": -0.7301878333091736,
      "logps/chosen": -1.551640272140503,
      "logps/rejected": -5.79341983795166,
      "loss": 2.1605,
      "nll_loss": 1.6449111700057983,
      "po_loss": -1.5773966312408447,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.7758201360702515,
      "rewards/margins": 2.120889663696289,
      "rewards/rejected": -2.89670991897583,
      "step": 205
    },
    {
      "epoch": 1.986074074074074,
      "grad_norm": 831.9571533203125,
      "learning_rate": 1.514521724066537e-05,
      "log_odds_chosen": 4.619518280029297,
      "log_odds_ratio": -0.8430620431900024,
      "logps/chosen": -1.8811943531036377,
      "logps/rejected": -6.274863243103027,
      "loss": 7.1653,
      "nll_loss": 1.774757742881775,
      "po_loss": -1.5508434772491455,
      "rewards/accuracies": 0.746874988079071,
      "rewards/chosen": -0.9405971765518188,
      "rewards/margins": 2.1968345642089844,
      "rewards/rejected": -3.1374316215515137,
      "step": 210
    },
    {
      "epoch": 2.0284444444444443,
      "grad_norm": 1437.5135498046875,
      "learning_rate": 1.3885777771950348e-05,
      "log_odds_chosen": 6.645725727081299,
      "log_odds_ratio": -0.6343217492103577,
      "logps/chosen": -1.605512022972107,
      "logps/rejected": -7.886458396911621,
      "loss": -23.0718,
      "nll_loss": 1.6514700651168823,
      "po_loss": -2.4581761360168457,
      "rewards/accuracies": 0.8216783404350281,
      "rewards/chosen": -0.8027560114860535,
      "rewards/margins": 3.1404736042022705,
      "rewards/rejected": -3.9432291984558105,
      "step": 215
    },
    {
      "epoch": 2.075851851851852,
      "grad_norm": 878.4225463867188,
      "learning_rate": 1.2660570475395683e-05,
      "log_odds_chosen": 7.114099025726318,
      "log_odds_ratio": -0.7919400930404663,
      "logps/chosen": -1.9719266891479492,
      "logps/rejected": -8.785611152648926,
      "loss": -21.0157,
      "nll_loss": 1.8605375289916992,
      "po_loss": -2.517279624938965,
      "rewards/accuracies": 0.8031250238418579,
      "rewards/chosen": -0.9859633445739746,
      "rewards/margins": 3.4068424701690674,
      "rewards/rejected": -4.392805576324463,
      "step": 220
    },
    {
      "epoch": 2.1232592592592594,
      "grad_norm": 1384.3018798828125,
      "learning_rate": 1.1473369030008974e-05,
      "log_odds_chosen": 9.657078742980957,
      "log_odds_ratio": -0.717284083366394,
      "logps/chosen": -2.1501874923706055,
      "logps/rejected": -11.44202995300293,
      "loss": -49.1628,
      "nll_loss": 1.9243361949920654,
      "po_loss": -3.460672378540039,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -1.0750937461853027,
      "rewards/margins": 4.645921230316162,
      "rewards/rejected": -5.721014976501465,
      "step": 225
    },
    {
      "epoch": 2.1706666666666665,
      "grad_norm": 1059.046142578125,
      "learning_rate": 1.0327830055518842e-05,
      "log_odds_chosen": 9.652680397033691,
      "log_odds_ratio": -0.5430900454521179,
      "logps/chosen": -1.8017524480819702,
      "logps/rejected": -11.105250358581543,
      "loss": -54.9112,
      "nll_loss": 1.7806711196899414,
      "po_loss": -3.496647596359253,
      "rewards/accuracies": 0.8343750238418579,
      "rewards/chosen": -0.9008762240409851,
      "rewards/margins": 4.651749610900879,
      "rewards/rejected": -5.5526251792907715,
      "step": 230
    },
    {
      "epoch": 2.218074074074074,
      "grad_norm": 934.3763427734375,
      "learning_rate": 9.227481849865235e-06,
      "log_odds_chosen": 8.37989616394043,
      "log_odds_ratio": -0.548344075679779,
      "logps/chosen": -1.6562755107879639,
      "logps/rejected": -9.679137229919434,
      "loss": -44.0365,
      "nll_loss": 1.69879949092865,
      "po_loss": -3.074941635131836,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.8281377553939819,
      "rewards/margins": 4.011431694030762,
      "rewards/rejected": -4.839568614959717,
      "step": 235
    },
    {
      "epoch": 2.2654814814814817,
      "grad_norm": 1470.9927978515625,
      "learning_rate": 8.175713521924978e-06,
      "log_odds_chosen": 9.9420166015625,
      "log_odds_ratio": -0.5666526556015015,
      "logps/chosen": -1.586470365524292,
      "logps/rejected": -11.144021987915039,
      "loss": -54.6849,
      "nll_loss": 1.6270078420639038,
      "po_loss": -3.335911989212036,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.793235182762146,
      "rewards/margins": 4.778776168823242,
      "rewards/rejected": -5.5720109939575195,
      "step": 240
    },
    {
      "epoch": 2.3128888888888888,
      "grad_norm": 2696.906982421875,
      "learning_rate": 7.1757645529443665e-06,
      "log_odds_chosen": 11.02066421508789,
      "log_odds_ratio": -0.523024320602417,
      "logps/chosen": -1.5870704650878906,
      "logps/rejected": -12.236452102661133,
      "loss": -57.4956,
      "nll_loss": 1.6896440982818604,
      "po_loss": -3.4863803386688232,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.7935352325439453,
      "rewards/margins": 5.324690341949463,
      "rewards/rejected": -6.118226051330566,
      "step": 245
    },
    {
      "epoch": 2.3602962962962963,
      "grad_norm": 2218.82373046875,
      "learning_rate": 6.230714818829733e-06,
      "log_odds_chosen": 12.59239673614502,
      "log_odds_ratio": -0.5458945035934448,
      "logps/chosen": -2.0517947673797607,
      "logps/rejected": -14.282119750976562,
      "loss": -61.3005,
      "nll_loss": 1.7380138635635376,
      "po_loss": -3.6536529064178467,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -1.0258973836898804,
      "rewards/margins": 6.115162372589111,
      "rewards/rejected": -7.141059875488281,
      "step": 250
    },
    {
      "epoch": 2.407703703703704,
      "grad_norm": 3879.0703125,
      "learning_rate": 5.343475104027743e-06,
      "log_odds_chosen": 13.796175956726074,
      "log_odds_ratio": -0.7049432992935181,
      "logps/chosen": -2.48822283744812,
      "logps/rejected": -15.938923835754395,
      "loss": -54.8681,
      "nll_loss": 2.094301700592041,
      "po_loss": -3.8089308738708496,
      "rewards/accuracies": 0.878125011920929,
      "rewards/chosen": -1.24411141872406,
      "rewards/margins": 6.725350379943848,
      "rewards/rejected": -7.969461917877197,
      "step": 255
    },
    {
      "epoch": 2.455111111111111,
      "grad_norm": 2860.3720703125,
      "learning_rate": 4.516778136213037e-06,
      "log_odds_chosen": 12.339117050170898,
      "log_odds_ratio": -0.8312928080558777,
      "logps/chosen": -2.22261381149292,
      "logps/rejected": -14.213083267211914,
      "loss": -51.4931,
      "nll_loss": 2.0115973949432373,
      "po_loss": -3.6207566261291504,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -1.11130690574646,
      "rewards/margins": 5.995234489440918,
      "rewards/rejected": -7.106541633605957,
      "step": 260
    },
    {
      "epoch": 2.5025185185185186,
      "grad_norm": 1744.2803955078125,
      "learning_rate": 3.7531701693965554e-06,
      "log_odds_chosen": 10.812795639038086,
      "log_odds_ratio": -0.5689637064933777,
      "logps/chosen": -1.7130670547485352,
      "logps/rejected": -12.172760009765625,
      "loss": -62.0607,
      "nll_loss": 1.6393451690673828,
      "po_loss": -3.5787429809570312,
      "rewards/accuracies": 0.840624988079071,
      "rewards/chosen": -0.8565335273742676,
      "rewards/margins": 5.229846000671387,
      "rewards/rejected": -6.0863800048828125,
      "step": 265
    },
    {
      "epoch": 2.549925925925926,
      "grad_norm": 1753.0069580078125,
      "learning_rate": 3.055003141378948e-06,
      "log_odds_chosen": 11.131937026977539,
      "log_odds_ratio": -0.6443347930908203,
      "logps/chosen": -1.8285239934921265,
      "logps/rejected": -12.587448120117188,
      "loss": -64.4668,
      "nll_loss": 1.797912836074829,
      "po_loss": -3.8124988079071045,
      "rewards/accuracies": 0.8531249761581421,
      "rewards/chosen": -0.9142619967460632,
      "rewards/margins": 5.379462242126465,
      "rewards/rejected": -6.293724060058594,
      "step": 270
    },
    {
      "epoch": 2.5973333333333333,
      "grad_norm": 2948.94775390625,
      "learning_rate": 2.424427429704365e-06,
      "log_odds_chosen": 10.50131893157959,
      "log_odds_ratio": -0.5303918123245239,
      "logps/chosen": -1.9819530248641968,
      "logps/rejected": -12.131158828735352,
      "loss": -54.0842,
      "nll_loss": 1.8271331787109375,
      "po_loss": -3.5172641277313232,
      "rewards/accuracies": 0.878125011920929,
      "rewards/chosen": -0.9909765124320984,
      "rewards/margins": 5.0746026039123535,
      "rewards/rejected": -6.065579414367676,
      "step": 275
    },
    {
      "epoch": 2.644740740740741,
      "grad_norm": 2592.339111328125,
      "learning_rate": 1.8633852284264508e-06,
      "log_odds_chosen": 12.00327205657959,
      "log_odds_ratio": -0.676723837852478,
      "logps/chosen": -1.9051532745361328,
      "logps/rejected": -13.546533584594727,
      "loss": -61.6846,
      "nll_loss": 1.8986514806747437,
      "po_loss": -3.826294422149658,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.9525766372680664,
      "rewards/margins": 5.820690155029297,
      "rewards/rejected": -6.773266792297363,
      "step": 280
    },
    {
      "epoch": 2.6921481481481484,
      "grad_norm": 2699.58544921875,
      "learning_rate": 1.3736045660864034e-06,
      "log_odds_chosen": 11.029436111450195,
      "log_odds_ratio": -0.5692117810249329,
      "logps/chosen": -1.7028005123138428,
      "logps/rejected": -12.390884399414062,
      "loss": -66.3219,
      "nll_loss": 1.6979774236679077,
      "po_loss": -3.770538330078125,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -0.8514002561569214,
      "rewards/margins": 5.344042778015137,
      "rewards/rejected": -6.195442199707031,
      "step": 285
    },
    {
      "epoch": 2.7395555555555555,
      "grad_norm": 2166.333984375,
      "learning_rate": 9.565939833279192e-07,
      "log_odds_chosen": 10.880716323852539,
      "log_odds_ratio": -0.7237260341644287,
      "logps/chosen": -2.316985607147217,
      "logps/rejected": -12.853070259094238,
      "loss": -52.5932,
      "nll_loss": 1.9568628072738647,
      "po_loss": -3.60040020942688,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -1.1584928035736084,
      "rewards/margins": 5.26804256439209,
      "rewards/rejected": -6.426535129547119,
      "step": 290
    },
    {
      "epoch": 2.786962962962963,
      "grad_norm": 2485.855712890625,
      "learning_rate": 6.136378865420872e-07,
      "log_odds_chosen": 10.605603218078613,
      "log_odds_ratio": -0.8477842211723328,
      "logps/chosen": -2.2415826320648193,
      "logps/rejected": -12.530155181884766,
      "loss": -47.1903,
      "nll_loss": 1.9998095035552979,
      "po_loss": -3.474505662918091,
      "rewards/accuracies": 0.7906249761581421,
      "rewards/chosen": -1.1207913160324097,
      "rewards/margins": 5.144285678863525,
      "rewards/rejected": -6.265077590942383,
      "step": 295
    },
    {
      "epoch": 2.83437037037037,
      "grad_norm": 2430.19970703125,
      "learning_rate": 3.45792591853214e-07,
      "log_odds_chosen": 11.173027992248535,
      "log_odds_ratio": -0.5032259821891785,
      "logps/chosen": -1.4856630563735962,
      "logps/rejected": -12.28561019897461,
      "loss": -71.7924,
      "nll_loss": 1.597579002380371,
      "po_loss": -3.8410916328430176,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.7428315281867981,
      "rewards/margins": 5.399973392486572,
      "rewards/rejected": -6.142805099487305,
      "step": 300
    },
    {
      "epoch": 2.8817777777777778,
      "grad_norm": 1150.9241943359375,
      "learning_rate": 1.538830716302092e-07,
      "log_odds_chosen": 11.992634773254395,
      "log_odds_ratio": -0.6470831632614136,
      "logps/chosen": -1.8577916622161865,
      "logps/rejected": -13.447807312011719,
      "loss": -69.0062,
      "nll_loss": 1.8080154657363892,
      "po_loss": -3.9644579887390137,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.9288958311080933,
      "rewards/margins": 5.795008182525635,
      "rewards/rejected": -6.723903656005859,
      "step": 305
    },
    {
      "epoch": 2.9291851851851853,
      "grad_norm": 1115.533447265625,
      "learning_rate": 3.8500413544415025e-08,
      "log_odds_chosen": 11.34982967376709,
      "log_odds_ratio": -0.5919870734214783,
      "logps/chosen": -1.6738865375518799,
      "logps/rejected": -12.641853332519531,
      "loss": -65.5454,
      "nll_loss": 1.8010927438735962,
      "po_loss": -3.849386692047119,
      "rewards/accuracies": 0.8656250238418579,
      "rewards/chosen": -0.8369432687759399,
      "rewards/margins": 5.483983993530273,
      "rewards/rejected": -6.320926666259766,
      "step": 310
    },
    {
      "epoch": 2.9765925925925925,
      "grad_norm": 2356.891845703125,
      "learning_rate": 0.0,
      "log_odds_chosen": 11.529512405395508,
      "log_odds_ratio": -0.520513653755188,
      "logps/chosen": -1.613694429397583,
      "logps/rejected": -12.730021476745605,
      "loss": -73.7153,
      "nll_loss": 1.6499922275543213,
      "po_loss": -3.9535961151123047,
      "rewards/accuracies": 0.840624988079071,
      "rewards/chosen": -0.8068472146987915,
      "rewards/margins": 5.558164119720459,
      "rewards/rejected": -6.365010738372803,
      "step": 315
    },
    {
      "epoch": 2.9765925925925925,
      "step": 315,
      "total_flos": 0.0,
      "train_loss": 22.20340297638424,
      "train_runtime": 9567.4219,
      "train_samples_per_second": 2.117,
      "train_steps_per_second": 0.033
    }
  ],
  "logging_steps": 5,
  "max_steps": 315,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 100000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}