{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.9989289539450197,
  "eval_steps": 500,
  "global_step": 4200,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.01,
      "grad_norm": 6.3125,
      "learning_rate": 4.997501873438867e-06,
      "log_odds_chosen": 0.027651220560073853,
      "log_odds_ratio": -0.7304251194000244,
      "logits/chosen": -3.0225749015808105,
      "logits/rejected": -3.0150020122528076,
      "logps/chosen": -0.815263569355011,
      "logps/rejected": -0.8314116597175598,
      "loss": 1.153,
      "nll_loss": 1.0209743976593018,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.1630527228116989,
      "rewards/margins": 0.0032296099234372377,
      "rewards/rejected": -0.16628232598304749,
      "step": 10
    },
    {
      "epoch": 0.01,
      "grad_norm": 5.40625,
      "learning_rate": 4.995007487521836e-06,
      "log_odds_chosen": 0.12160005420446396,
      "log_odds_ratio": -0.6930473446846008,
      "logits/chosen": -3.121931791305542,
      "logits/rejected": -3.1250596046447754,
      "logps/chosen": -0.7088677883148193,
      "logps/rejected": -0.7795363664627075,
      "loss": 0.6108,
      "nll_loss": 0.46274280548095703,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.14177358150482178,
      "rewards/margins": 0.014133691787719727,
      "rewards/rejected": -0.15590724349021912,
      "step": 20
    },
    {
      "epoch": 0.02,
      "grad_norm": 6.9375,
      "learning_rate": 4.992516832922945e-06,
      "log_odds_chosen": 0.03587382286787033,
      "log_odds_ratio": -0.7343233227729797,
      "logits/chosen": -3.130347728729248,
      "logits/rejected": -3.114506244659424,
      "logps/chosen": -0.7532758116722107,
      "logps/rejected": -0.7803520560264587,
      "loss": 0.6097,
      "nll_loss": 0.462548166513443,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.15065516531467438,
      "rewards/margins": 0.005415262188762426,
      "rewards/rejected": -0.15607044100761414,
      "step": 30
    },
    {
      "epoch": 0.03,
      "grad_norm": 5.65625,
      "learning_rate": 4.990029900348746e-06,
      "log_odds_chosen": 0.06624144315719604,
      "log_odds_ratio": -0.7134107351303101,
      "logits/chosen": -3.125946521759033,
      "logits/rejected": -3.1197519302368164,
      "logps/chosen": -0.7543585896492004,
      "logps/rejected": -0.8100128173828125,
      "loss": 0.595,
      "nll_loss": 0.47750869393348694,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.15087170898914337,
      "rewards/margins": 0.011130847968161106,
      "rewards/rejected": -0.1620025634765625,
      "step": 40
    },
    {
      "epoch": 0.04,
      "grad_norm": 5.21875,
      "learning_rate": 4.987546680538165e-06,
      "log_odds_chosen": 0.14277830719947815,
      "log_odds_ratio": -0.6764869093894958,
      "logits/chosen": -3.073333263397217,
      "logits/rejected": -3.081392288208008,
      "logps/chosen": -0.7409430146217346,
      "logps/rejected": -0.831874668598175,
      "loss": 0.6027,
      "nll_loss": 0.4561688005924225,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.14818862080574036,
      "rewards/margins": 0.018186338245868683,
      "rewards/rejected": -0.16637495160102844,
      "step": 50
    },
    {
      "epoch": 0.04,
      "grad_norm": 5.65625,
      "learning_rate": 4.985067164262359e-06,
      "log_odds_chosen": 0.09204573929309845,
      "log_odds_ratio": -0.7011358737945557,
      "logits/chosen": -3.172628879547119,
      "logits/rejected": -3.173214912414551,
      "logps/chosen": -0.7226709127426147,
      "logps/rejected": -0.7736715078353882,
      "loss": 0.5972,
      "nll_loss": 0.4593353867530823,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.1445341557264328,
      "rewards/margins": 0.010200141929090023,
      "rewards/rejected": -0.15473432838916779,
      "step": 60
    },
    {
      "epoch": 0.05,
      "grad_norm": 5.59375,
      "learning_rate": 4.98259134232457e-06,
      "log_odds_chosen": 0.09953074157238007,
      "log_odds_ratio": -0.6920744776725769,
      "logits/chosen": -3.1694746017456055,
      "logits/rejected": -3.185354471206665,
      "logps/chosen": -0.7267014980316162,
      "logps/rejected": -0.7774870991706848,
      "loss": 0.6119,
      "nll_loss": 0.46830782294273376,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.14534030854701996,
      "rewards/margins": 0.010157106444239616,
      "rewards/rejected": -0.15549740195274353,
      "step": 70
    },
    {
      "epoch": 0.06,
      "grad_norm": 5.53125,
      "learning_rate": 4.980119205559974e-06,
      "log_odds_chosen": 0.07614605873823166,
      "log_odds_ratio": -0.7038400769233704,
      "logits/chosen": -3.1640992164611816,
      "logits/rejected": -3.1609697341918945,
      "logps/chosen": -0.7791558504104614,
      "logps/rejected": -0.8135568499565125,
      "loss": 0.6066,
      "nll_loss": 0.461737722158432,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.15583118796348572,
      "rewards/margins": 0.006880161818116903,
      "rewards/rejected": -0.16271135210990906,
      "step": 80
    },
    {
      "epoch": 0.06,
      "grad_norm": 5.3125,
      "learning_rate": 4.977650744835555e-06,
      "log_odds_chosen": 0.07529701292514801,
      "log_odds_ratio": -0.6971887946128845,
      "logits/chosen": -3.1437430381774902,
      "logits/rejected": -3.14058518409729,
      "logps/chosen": -0.7142292261123657,
      "logps/rejected": -0.7589999437332153,
      "loss": 0.5784,
      "nll_loss": 0.4407345652580261,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.14284583926200867,
      "rewards/margins": 0.008954131975769997,
      "rewards/rejected": -0.1517999768257141,
      "step": 90
    },
    {
      "epoch": 0.07,
      "grad_norm": 5.375,
      "learning_rate": 4.975185951049947e-06,
      "log_odds_chosen": 0.06737220287322998,
      "log_odds_ratio": -0.7220064401626587,
      "logits/chosen": -3.1887149810791016,
      "logits/rejected": -3.1891226768493652,
      "logps/chosen": -0.7800450921058655,
      "logps/rejected": -0.8231565356254578,
      "loss": 0.551,
      "nll_loss": 0.4563975930213928,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.1560090035200119,
      "rewards/margins": 0.008622287772595882,
      "rewards/rejected": -0.16463130712509155,
      "step": 100
    },
    {
      "epoch": 0.08,
      "grad_norm": 5.9375,
      "learning_rate": 4.972724815133302e-06,
      "log_odds_chosen": 0.09414532780647278,
      "log_odds_ratio": -0.7080024480819702,
      "logits/chosen": -3.1148273944854736,
      "logits/rejected": -3.1315975189208984,
      "logps/chosen": -0.7162569761276245,
      "logps/rejected": -0.7553704977035522,
      "loss": 0.5325,
      "nll_loss": 0.43158403038978577,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.14325138926506042,
      "rewards/margins": 0.00782269798219204,
      "rewards/rejected": -0.1510740965604782,
      "step": 110
    },
    {
      "epoch": 0.09,
      "grad_norm": 5.8125,
      "learning_rate": 4.970267328047151e-06,
      "log_odds_chosen": 0.10901321470737457,
      "log_odds_ratio": -0.6950778961181641,
      "logits/chosen": -3.14462947845459,
      "logits/rejected": -3.137329578399658,
      "logps/chosen": -0.7042136192321777,
      "logps/rejected": -0.7720807194709778,
      "loss": 0.5508,
      "nll_loss": 0.40320760011672974,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.1408427208662033,
      "rewards/margins": 0.0135734211653471,
      "rewards/rejected": -0.15441615879535675,
      "step": 120
    },
    {
      "epoch": 0.09,
      "grad_norm": 5.65625,
      "learning_rate": 4.9678134807842575e-06,
      "log_odds_chosen": 0.07895953953266144,
      "log_odds_ratio": -0.7133638262748718,
      "logits/chosen": -3.125828266143799,
      "logits/rejected": -3.1391282081604004,
      "logps/chosen": -0.7350977063179016,
      "logps/rejected": -0.7833020091056824,
      "loss": 0.5682,
      "nll_loss": 0.41112464666366577,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.14701953530311584,
      "rewards/margins": 0.009640867821872234,
      "rewards/rejected": -0.15666040778160095,
      "step": 130
    },
    {
      "epoch": 0.1,
      "grad_norm": 6.84375,
      "learning_rate": 4.965363264368484e-06,
      "log_odds_chosen": 0.12207289040088654,
      "log_odds_ratio": -0.6769169569015503,
      "logits/chosen": -3.14564847946167,
      "logits/rejected": -3.1439242362976074,
      "logps/chosen": -0.6801249980926514,
      "logps/rejected": -0.7512098550796509,
      "loss": 0.5738,
      "nll_loss": 0.4280276298522949,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.13602499663829803,
      "rewards/margins": 0.014216972514986992,
      "rewards/rejected": -0.15024198591709137,
      "step": 140
    },
    {
      "epoch": 0.11,
      "grad_norm": 5.71875,
      "learning_rate": 4.962916669854652e-06,
      "log_odds_chosen": 0.1224343553185463,
      "log_odds_ratio": -0.6913517713546753,
      "logits/chosen": -3.1227478981018066,
      "logits/rejected": -3.1305549144744873,
      "logps/chosen": -0.7392982840538025,
      "logps/rejected": -0.8002193570137024,
      "loss": 0.5956,
      "nll_loss": 0.43565186858177185,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.14785966277122498,
      "rewards/margins": 0.01218421570956707,
      "rewards/rejected": -0.1600438803434372,
      "step": 150
    },
    {
      "epoch": 0.11,
      "grad_norm": 5.96875,
      "learning_rate": 4.960473688328407e-06,
      "log_odds_chosen": 0.0026047558058053255,
      "log_odds_ratio": -0.7348340153694153,
      "logits/chosen": -3.0425188541412354,
      "logits/rejected": -3.0503063201904297,
      "logps/chosen": -0.7252613306045532,
      "logps/rejected": -0.7286208271980286,
      "loss": 0.5688,
      "nll_loss": 0.41286152601242065,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.1450522541999817,
      "rewards/margins": 0.0006719084340147674,
      "rewards/rejected": -0.14572417736053467,
      "step": 160
    },
    {
      "epoch": 0.12,
      "grad_norm": 5.1875,
      "learning_rate": 4.95803431090608e-06,
      "log_odds_chosen": 0.16818314790725708,
      "log_odds_ratio": -0.6584939956665039,
      "logits/chosen": -3.142646074295044,
      "logits/rejected": -3.161475419998169,
      "logps/chosen": -0.7002934217453003,
      "logps/rejected": -0.7836463451385498,
      "loss": 0.5958,
      "nll_loss": 0.4563008248806,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.14005866646766663,
      "rewards/margins": 0.016670597717165947,
      "rewards/rejected": -0.15672926604747772,
      "step": 170
    },
    {
      "epoch": 0.13,
      "grad_norm": 5.4375,
      "learning_rate": 4.955598528734554e-06,
      "log_odds_chosen": 0.1538088172674179,
      "log_odds_ratio": -0.6675732135772705,
      "logits/chosen": -3.100310802459717,
      "logits/rejected": -3.117645740509033,
      "logps/chosen": -0.6987863183021545,
      "logps/rejected": -0.7786335945129395,
      "loss": 0.5698,
      "nll_loss": 0.45257002115249634,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.13975724577903748,
      "rewards/margins": 0.01596945896744728,
      "rewards/rejected": -0.15572671592235565,
      "step": 180
    },
    {
      "epoch": 0.14,
      "grad_norm": 6.28125,
      "learning_rate": 4.953166332991125e-06,
      "log_odds_chosen": 0.12078976631164551,
      "log_odds_ratio": -0.6859411001205444,
      "logits/chosen": -3.134028434753418,
      "logits/rejected": -3.137901782989502,
      "logps/chosen": -0.6982489824295044,
      "logps/rejected": -0.7687091827392578,
      "loss": 0.5676,
      "nll_loss": 0.43058595061302185,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.13964977860450745,
      "rewards/margins": 0.014092043042182922,
      "rewards/rejected": -0.15374182164669037,
      "step": 190
    },
    {
      "epoch": 0.14,
      "grad_norm": 6.28125,
      "learning_rate": 4.950737714883372e-06,
      "log_odds_chosen": 0.17639026045799255,
      "log_odds_ratio": -0.655586838722229,
      "logits/chosen": -3.102017641067505,
      "logits/rejected": -3.1128413677215576,
      "logps/chosen": -0.6888889670372009,
      "logps/rejected": -0.7917199730873108,
      "loss": 0.5431,
      "nll_loss": 0.39507898688316345,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.13777779042720795,
      "rewards/margins": 0.020566195249557495,
      "rewards/rejected": -0.15834400057792664,
      "step": 200
    },
    {
      "epoch": 0.15,
      "grad_norm": 6.21875,
      "learning_rate": 4.948312665649022e-06,
      "log_odds_chosen": 0.00438351184129715,
      "log_odds_ratio": -0.731238603591919,
      "logits/chosen": -3.072948932647705,
      "logits/rejected": -3.0700042247772217,
      "logps/chosen": -0.7648705244064331,
      "logps/rejected": -0.7703164219856262,
      "loss": 0.5744,
      "nll_loss": 0.41652363538742065,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.15297409892082214,
      "rewards/margins": 0.001089166384190321,
      "rewards/rejected": -0.15406326949596405,
      "step": 210
    },
    {
      "epoch": 0.16,
      "grad_norm": 5.0,
      "learning_rate": 4.945891176555817e-06,
      "log_odds_chosen": 0.09887596219778061,
      "log_odds_ratio": -0.6929382085800171,
      "logits/chosen": -3.109269618988037,
      "logits/rejected": -3.0964112281799316,
      "logps/chosen": -0.7023528218269348,
      "logps/rejected": -0.7567712068557739,
      "loss": 0.5629,
      "nll_loss": 0.4561777710914612,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.14047057926654816,
      "rewards/margins": 0.010883677750825882,
      "rewards/rejected": -0.15135423839092255,
      "step": 220
    },
    {
      "epoch": 0.16,
      "grad_norm": 6.5,
      "learning_rate": 4.943473238901383e-06,
      "log_odds_chosen": 0.12996645271778107,
      "log_odds_ratio": -0.6799687147140503,
      "logits/chosen": -3.0217673778533936,
      "logits/rejected": -3.0064945220947266,
      "logps/chosen": -0.6958892345428467,
      "logps/rejected": -0.7603468298912048,
      "loss": 0.5877,
      "nll_loss": 0.43383994698524475,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.1391778290271759,
      "rewards/margins": 0.012891518883407116,
      "rewards/rejected": -0.15206937491893768,
      "step": 230
    },
    {
      "epoch": 0.17,
      "grad_norm": 6.40625,
      "learning_rate": 4.941058844013094e-06,
      "log_odds_chosen": 0.15019556879997253,
      "log_odds_ratio": -0.665575385093689,
      "logits/chosen": -3.062437057495117,
      "logits/rejected": -3.067084789276123,
      "logps/chosen": -0.7049697041511536,
      "logps/rejected": -0.7816177606582642,
      "loss": 0.6147,
      "nll_loss": 0.48391756415367126,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.14099393784999847,
      "rewards/margins": 0.015329602174460888,
      "rewards/rejected": -0.15632352232933044,
      "step": 240
    },
    {
      "epoch": 0.18,
      "grad_norm": 6.9375,
      "learning_rate": 4.938647983247949e-06,
      "log_odds_chosen": 0.0968349426984787,
      "log_odds_ratio": -0.6998175382614136,
      "logits/chosen": -2.946869134902954,
      "logits/rejected": -2.954613208770752,
      "logps/chosen": -0.7149877548217773,
      "logps/rejected": -0.7582697868347168,
      "loss": 0.592,
      "nll_loss": 0.4492533206939697,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.14299754798412323,
      "rewards/margins": 0.008656383492052555,
      "rewards/rejected": -0.1516539454460144,
      "step": 250
    },
    {
      "epoch": 0.19,
      "grad_norm": 5.4375,
      "learning_rate": 4.936240647992436e-06,
      "log_odds_chosen": 0.13414113223552704,
      "log_odds_ratio": -0.670508861541748,
      "logits/chosen": -3.0646378993988037,
      "logits/rejected": -3.0543510913848877,
      "logps/chosen": -0.686463475227356,
      "logps/rejected": -0.7646616697311401,
      "loss": 0.577,
      "nll_loss": 0.4495324194431305,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.13729269802570343,
      "rewards/margins": 0.015639634802937508,
      "rewards/rejected": -0.1529323160648346,
      "step": 260
    },
    {
      "epoch": 0.19,
      "grad_norm": 5.65625,
      "learning_rate": 4.933836829662409e-06,
      "log_odds_chosen": 0.11647520214319229,
      "log_odds_ratio": -0.6881713271141052,
      "logits/chosen": -2.9797616004943848,
      "logits/rejected": -2.9717679023742676,
      "logps/chosen": -0.6882697939872742,
      "logps/rejected": -0.7442315220832825,
      "loss": 0.5775,
      "nll_loss": 0.42454642057418823,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.13765396177768707,
      "rewards/margins": 0.01119234412908554,
      "rewards/rejected": -0.14884629845619202,
      "step": 270
    },
    {
      "epoch": 0.2,
      "grad_norm": 5.40625,
      "learning_rate": 4.9314365197029475e-06,
      "log_odds_chosen": 0.12243346869945526,
      "log_odds_ratio": -0.6820363998413086,
      "logits/chosen": -3.0487539768218994,
      "logits/rejected": -3.0383288860321045,
      "logps/chosen": -0.6958305239677429,
      "logps/rejected": -0.7610360980033875,
      "loss": 0.5734,
      "nll_loss": 0.4472366273403168,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.13916611671447754,
      "rewards/margins": 0.013041108846664429,
      "rewards/rejected": -0.15220721065998077,
      "step": 280
    },
    {
      "epoch": 0.21,
      "grad_norm": 5.46875,
      "learning_rate": 4.9290397095882446e-06,
      "log_odds_chosen": 0.02017252705991268,
      "log_odds_ratio": -0.7348771095275879,
      "logits/chosen": -2.9625377655029297,
      "logits/rejected": -2.9633870124816895,
      "logps/chosen": -0.7289597988128662,
      "logps/rejected": -0.7401062250137329,
      "loss": 0.5777,
      "nll_loss": 0.47131404280662537,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.14579197764396667,
      "rewards/margins": 0.0022292803041636944,
      "rewards/rejected": -0.14802125096321106,
      "step": 290
    },
    {
      "epoch": 0.21,
      "grad_norm": 5.625,
      "learning_rate": 4.9266463908214664e-06,
      "log_odds_chosen": 0.13380487263202667,
      "log_odds_ratio": -0.6815378069877625,
      "logits/chosen": -3.0238821506500244,
      "logits/rejected": -3.012429714202881,
      "logps/chosen": -0.6980360746383667,
      "logps/rejected": -0.772484540939331,
      "loss": 0.5416,
      "nll_loss": 0.3923744261264801,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.13960722088813782,
      "rewards/margins": 0.014889700338244438,
      "rewards/rejected": -0.1544969230890274,
      "step": 300
    },
    {
      "epoch": 0.22,
      "grad_norm": 4.8125,
      "learning_rate": 4.924256554934632e-06,
      "log_odds_chosen": 0.10960634052753448,
      "log_odds_ratio": -0.6801453828811646,
      "logits/chosen": -2.9736227989196777,
      "logits/rejected": -2.9606356620788574,
      "logps/chosen": -0.7132256031036377,
      "logps/rejected": -0.7729824781417847,
      "loss": 0.5434,
      "nll_loss": 0.38369157910346985,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.14264513552188873,
      "rewards/margins": 0.01195137295871973,
      "rewards/rejected": -0.1545964926481247,
      "step": 310
    },
    {
      "epoch": 0.23,
      "grad_norm": 5.84375,
      "learning_rate": 4.9218701934884865e-06,
      "log_odds_chosen": 0.15664125978946686,
      "log_odds_ratio": -0.6632459759712219,
      "logits/chosen": -3.026071071624756,
      "logits/rejected": -3.011920690536499,
      "logps/chosen": -0.7300186157226562,
      "logps/rejected": -0.8273404836654663,
      "loss": 0.57,
      "nll_loss": 0.42472705245018005,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.14600373804569244,
      "rewards/margins": 0.01946437731385231,
      "rewards/rejected": -0.16546811163425446,
      "step": 320
    },
    {
      "epoch": 0.24,
      "grad_norm": 6.65625,
      "learning_rate": 4.919487298072377e-06,
      "log_odds_chosen": 0.16137509047985077,
      "log_odds_ratio": -0.672458291053772,
      "logits/chosen": -2.949476718902588,
      "logits/rejected": -2.9505724906921387,
      "logps/chosen": -0.6964123249053955,
      "logps/rejected": -0.7866235971450806,
      "loss": 0.5429,
      "nll_loss": 0.4337022304534912,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.1392824798822403,
      "rewards/margins": 0.018042229115962982,
      "rewards/rejected": -0.15732471644878387,
      "step": 330
    },
    {
      "epoch": 0.24,
      "grad_norm": 6.25,
      "learning_rate": 4.917107860304125e-06,
      "log_odds_chosen": 0.12605342268943787,
      "log_odds_ratio": -0.672644317150116,
      "logits/chosen": -3.1077914237976074,
      "logits/rejected": -3.0998547077178955,
      "logps/chosen": -0.6887668967247009,
      "logps/rejected": -0.7507665753364563,
      "loss": 0.5602,
      "nll_loss": 0.40362483263015747,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.13775338232517242,
      "rewards/margins": 0.012399917468428612,
      "rewards/rejected": -0.15015330910682678,
      "step": 340
    },
    {
      "epoch": 0.25,
      "grad_norm": 5.625,
      "learning_rate": 4.914731871829905e-06,
      "log_odds_chosen": 0.0800851434469223,
      "log_odds_ratio": -0.6991033554077148,
      "logits/chosen": -3.1771771907806396,
      "logits/rejected": -3.183584451675415,
      "logps/chosen": -0.7016042470932007,
      "logps/rejected": -0.7426560521125793,
      "loss": 0.5623,
      "nll_loss": 0.38557925820350647,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.14032086730003357,
      "rewards/margins": 0.008210362866520882,
      "rewards/rejected": -0.1485312283039093,
      "step": 350
    },
    {
      "epoch": 0.26,
      "grad_norm": 5.625,
      "learning_rate": 4.912359324324121e-06,
      "log_odds_chosen": 0.17609532177448273,
      "log_odds_ratio": -0.6462699174880981,
      "logits/chosen": -3.1602416038513184,
      "logits/rejected": -3.1689493656158447,
      "logps/chosen": -0.6804074048995972,
      "logps/rejected": -0.7664049863815308,
      "loss": 0.5685,
      "nll_loss": 0.4449498653411865,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1360814869403839,
      "rewards/margins": 0.01719951257109642,
      "rewards/rejected": -0.15328100323677063,
      "step": 360
    },
    {
      "epoch": 0.26,
      "grad_norm": 5.84375,
      "learning_rate": 4.909990209489284e-06,
      "log_odds_chosen": 0.09661266207695007,
      "log_odds_ratio": -0.6989172697067261,
      "logits/chosen": -3.1702992916107178,
      "logits/rejected": -3.1816422939300537,
      "logps/chosen": -0.6909779906272888,
      "logps/rejected": -0.7484806776046753,
      "loss": 0.5448,
      "nll_loss": 0.3755747377872467,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.13819561898708344,
      "rewards/margins": 0.011500529944896698,
      "rewards/rejected": -0.14969615638256073,
      "step": 370
    },
    {
      "epoch": 0.27,
      "grad_norm": 5.4375,
      "learning_rate": 4.907624519055888e-06,
      "log_odds_chosen": 0.053890157490968704,
      "log_odds_ratio": -0.7154654860496521,
      "logits/chosen": -3.1567158699035645,
      "logits/rejected": -3.165017604827881,
      "logps/chosen": -0.7759397625923157,
      "logps/rejected": -0.8097392320632935,
      "loss": 0.5697,
      "nll_loss": 0.42835354804992676,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.1551879346370697,
      "rewards/margins": 0.0067598940804600716,
      "rewards/rejected": -0.1619478464126587,
      "step": 380
    },
    {
      "epoch": 0.28,
      "grad_norm": 5.75,
      "learning_rate": 4.905262244782294e-06,
      "log_odds_chosen": 0.10774964094161987,
      "log_odds_ratio": -0.6780427694320679,
      "logits/chosen": -3.1538772583007812,
      "logits/rejected": -3.139371395111084,
      "logps/chosen": -0.72501140832901,
      "logps/rejected": -0.7887372970581055,
      "loss": 0.5558,
      "nll_loss": 0.4034719467163086,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.14500227570533752,
      "rewards/margins": 0.01274518109858036,
      "rewards/rejected": -0.15774747729301453,
      "step": 390
    },
    {
      "epoch": 0.29,
      "grad_norm": 6.4375,
      "learning_rate": 4.902903378454601e-06,
      "log_odds_chosen": 0.013013715855777264,
      "log_odds_ratio": -0.7487560510635376,
      "logits/chosen": -3.1208155155181885,
      "logits/rejected": -3.1211535930633545,
      "logps/chosen": -0.7125126719474792,
      "logps/rejected": -0.7153986692428589,
      "loss": 0.5383,
      "nll_loss": 0.3862176537513733,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.142502561211586,
      "rewards/margins": 0.0005772082367911935,
      "rewards/rejected": -0.1430797278881073,
      "step": 400
    },
    {
      "epoch": 0.29,
      "grad_norm": 4.53125,
      "learning_rate": 4.900547911886537e-06,
      "log_odds_chosen": 0.14288419485092163,
      "log_odds_ratio": -0.6765764951705933,
      "logits/chosen": -3.208242416381836,
      "logits/rejected": -3.2231342792510986,
      "logps/chosen": -0.7180224657058716,
      "logps/rejected": -0.7839994430541992,
      "loss": 0.5571,
      "nll_loss": 0.4236370921134949,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.14360451698303223,
      "rewards/margins": 0.013195383362472057,
      "rewards/rejected": -0.15679989755153656,
      "step": 410
    },
    {
      "epoch": 0.3,
      "grad_norm": 4.875,
      "learning_rate": 4.898195836919327e-06,
      "log_odds_chosen": 0.03629889339208603,
      "log_odds_ratio": -0.7236992716789246,
      "logits/chosen": -3.2515082359313965,
      "logits/rejected": -3.2554900646209717,
      "logps/chosen": -0.7256291508674622,
      "logps/rejected": -0.7619522213935852,
      "loss": 0.5552,
      "nll_loss": 0.41900959610939026,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.14512582123279572,
      "rewards/margins": 0.0072646364569664,
      "rewards/rejected": -0.1523904651403427,
      "step": 420
    },
    {
      "epoch": 0.31,
      "grad_norm": 5.21875,
      "learning_rate": 4.895847145421587e-06,
      "log_odds_chosen": 0.18708305060863495,
      "log_odds_ratio": -0.6497930288314819,
      "logits/chosen": -3.263317108154297,
      "logits/rejected": -3.263544797897339,
      "logps/chosen": -0.6555394530296326,
      "logps/rejected": -0.7548009753227234,
      "loss": 0.5175,
      "nll_loss": 0.386501282453537,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1311078816652298,
      "rewards/margins": 0.01985231228172779,
      "rewards/rejected": -0.15096020698547363,
      "step": 430
    },
    {
      "epoch": 0.31,
      "grad_norm": 5.46875,
      "learning_rate": 4.893501829289195e-06,
      "log_odds_chosen": 0.10671345144510269,
      "log_odds_ratio": -0.696262001991272,
      "logits/chosen": -3.2451579570770264,
      "logits/rejected": -3.2367336750030518,
      "logps/chosen": -0.711258590221405,
      "logps/rejected": -0.7748275995254517,
      "loss": 0.5643,
      "nll_loss": 0.435871422290802,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.14225171506404877,
      "rewards/margins": 0.012713788077235222,
      "rewards/rejected": -0.15496549010276794,
      "step": 440
    },
    {
      "epoch": 0.32,
      "grad_norm": 5.15625,
      "learning_rate": 4.891159880445185e-06,
      "log_odds_chosen": -0.009761805646121502,
      "log_odds_ratio": -0.7526240348815918,
      "logits/chosen": -3.3140366077423096,
      "logits/rejected": -3.3279337882995605,
      "logps/chosen": -0.7342787981033325,
      "logps/rejected": -0.7224690318107605,
      "loss": 0.5655,
      "nll_loss": 0.3994046151638031,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -0.14685577154159546,
      "rewards/margins": -0.002361953491345048,
      "rewards/rejected": -0.14449380338191986,
      "step": 450
    },
    {
      "epoch": 0.33,
      "grad_norm": 5.3125,
      "learning_rate": 4.888821290839617e-06,
      "log_odds_chosen": 0.08875492960214615,
      "log_odds_ratio": -0.7100639343261719,
      "logits/chosen": -3.2668724060058594,
      "logits/rejected": -3.266616106033325,
      "logps/chosen": -0.6991187334060669,
      "logps/rejected": -0.7543323040008545,
      "loss": 0.5456,
      "nll_loss": 0.39074766635894775,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.13982374966144562,
      "rewards/margins": 0.011042733676731586,
      "rewards/rejected": -0.15086647868156433,
      "step": 460
    },
    {
      "epoch": 0.34,
      "grad_norm": 4.84375,
      "learning_rate": 4.886486052449469e-06,
      "log_odds_chosen": 0.04209429770708084,
      "log_odds_ratio": -0.7359583973884583,
      "logits/chosen": -3.2799735069274902,
      "logits/rejected": -3.2890267372131348,
      "logps/chosen": -0.7149407267570496,
      "logps/rejected": -0.7467728853225708,
      "loss": 0.5785,
      "nll_loss": 0.3849171996116638,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.1429881602525711,
      "rewards/margins": 0.006366416811943054,
      "rewards/rejected": -0.14935457706451416,
      "step": 470
    },
    {
      "epoch": 0.34,
      "grad_norm": 5.0,
      "learning_rate": 4.8841541572785224e-06,
      "log_odds_chosen": 0.08705852925777435,
      "log_odds_ratio": -0.7012760043144226,
      "logits/chosen": -3.229048490524292,
      "logits/rejected": -3.2325732707977295,
      "logps/chosen": -0.7453920245170593,
      "logps/rejected": -0.7918086051940918,
      "loss": 0.6028,
      "nll_loss": 0.49805039167404175,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.14907841384410858,
      "rewards/margins": 0.009283310733735561,
      "rewards/rejected": -0.15836171805858612,
      "step": 480
    },
    {
      "epoch": 0.35,
      "grad_norm": 5.71875,
      "learning_rate": 4.881825597357242e-06,
      "log_odds_chosen": 0.12286486476659775,
      "log_odds_ratio": -0.6923102736473083,
      "logits/chosen": -3.2311508655548096,
      "logits/rejected": -3.241130828857422,
      "logps/chosen": -0.7206076979637146,
      "logps/rejected": -0.7855247259140015,
      "loss": 0.5515,
      "nll_loss": 0.4401041865348816,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.14412155747413635,
      "rewards/margins": 0.012983402237296104,
      "rewards/rejected": -0.15710493922233582,
      "step": 490
    },
    {
      "epoch": 0.36,
      "grad_norm": 6.03125,
      "learning_rate": 4.8795003647426654e-06,
      "log_odds_chosen": 0.08800844103097916,
      "log_odds_ratio": -0.71002596616745,
      "logits/chosen": -3.240417957305908,
      "logits/rejected": -3.249009609222412,
      "logps/chosen": -0.7202855348587036,
      "logps/rejected": -0.7685345411300659,
      "loss": 0.593,
      "nll_loss": 0.4520091116428375,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.14405712485313416,
      "rewards/margins": 0.00964980386197567,
      "rewards/rejected": -0.15370693802833557,
      "step": 500
    },
    {
      "epoch": 0.36,
      "grad_norm": 5.46875,
      "learning_rate": 4.877178451518289e-06,
      "log_odds_chosen": 0.017683375626802444,
      "log_odds_ratio": -0.7400511503219604,
      "logits/chosen": -3.2630767822265625,
      "logits/rejected": -3.2636375427246094,
      "logps/chosen": -0.7481754422187805,
      "logps/rejected": -0.7609735727310181,
      "loss": 0.5502,
      "nll_loss": 0.403683602809906,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.14963507652282715,
      "rewards/margins": 0.0025596513878554106,
      "rewards/rejected": -0.15219472348690033,
      "step": 510
    },
    {
      "epoch": 0.37,
      "grad_norm": 6.5,
      "learning_rate": 4.8748598497939494e-06,
      "log_odds_chosen": 0.08491934835910797,
      "log_odds_ratio": -0.6992539763450623,
      "logits/chosen": -3.2163689136505127,
      "logits/rejected": -3.2333626747131348,
      "logps/chosen": -0.7194134593009949,
      "logps/rejected": -0.7636415958404541,
      "loss": 0.5908,
      "nll_loss": 0.4157852232456207,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.14388270676136017,
      "rewards/margins": 0.008845620788633823,
      "rewards/rejected": -0.15272831916809082,
      "step": 520
    },
    {
      "epoch": 0.38,
      "grad_norm": 4.84375,
      "learning_rate": 4.872544551705718e-06,
      "log_odds_chosen": 0.12621939182281494,
      "log_odds_ratio": -0.6822875738143921,
      "logits/chosen": -3.219284772872925,
      "logits/rejected": -3.2353568077087402,
      "logps/chosen": -0.7198256254196167,
      "logps/rejected": -0.770844578742981,
      "loss": 0.5348,
      "nll_loss": 0.38905125856399536,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.14396515488624573,
      "rewards/margins": 0.010203775018453598,
      "rewards/rejected": -0.15416888892650604,
      "step": 530
    },
    {
      "epoch": 0.39,
      "grad_norm": 5.8125,
      "learning_rate": 4.870232549415787e-06,
      "log_odds_chosen": 0.039147090166807175,
      "log_odds_ratio": -0.7419449687004089,
      "logits/chosen": -3.215195894241333,
      "logits/rejected": -3.216196060180664,
      "logps/chosen": -0.7427582144737244,
      "logps/rejected": -0.7526665925979614,
      "loss": 0.5612,
      "nll_loss": 0.4239347577095032,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.14855164289474487,
      "rewards/margins": 0.001981673762202263,
      "rewards/rejected": -0.15053331851959229,
      "step": 540
    },
    {
      "epoch": 0.39,
      "grad_norm": 5.1875,
      "learning_rate": 4.867923835112355e-06,
      "log_odds_chosen": 0.12316079437732697,
      "log_odds_ratio": -0.6851805448532104,
      "logits/chosen": -3.174053907394409,
      "logits/rejected": -3.1798911094665527,
      "logps/chosen": -0.6940979957580566,
      "logps/rejected": -0.7675840258598328,
      "loss": 0.5704,
      "nll_loss": 0.46909135580062866,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.1388196051120758,
      "rewards/margins": 0.01469721831381321,
      "rewards/rejected": -0.15351681411266327,
      "step": 550
    },
    {
      "epoch": 0.4,
      "grad_norm": 5.4375,
      "learning_rate": 4.865618401009519e-06,
      "log_odds_chosen": 0.06827814131975174,
      "log_odds_ratio": -0.7105705738067627,
      "logits/chosen": -3.1506597995758057,
      "logits/rejected": -3.1666228771209717,
      "logps/chosen": -0.749689519405365,
      "logps/rejected": -0.7806918621063232,
      "loss": 0.5884,
      "nll_loss": 0.44312652945518494,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.14993791282176971,
      "rewards/margins": 0.0062004560604691505,
      "rewards/rejected": -0.1561383754014969,
      "step": 560
    },
    {
      "epoch": 0.41,
      "grad_norm": 4.96875,
      "learning_rate": 4.863316239347163e-06,
      "log_odds_chosen": 0.12389856576919556,
      "log_odds_ratio": -0.6848828196525574,
      "logits/chosen": -3.1486778259277344,
      "logits/rejected": -3.143662452697754,
      "logps/chosen": -0.6603912115097046,
      "logps/rejected": -0.7258492112159729,
      "loss": 0.5353,
      "nll_loss": 0.3620496392250061,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.13207824528217316,
      "rewards/margins": 0.013091601431369781,
      "rewards/rejected": -0.14516983926296234,
      "step": 570
    },
    {
      "epoch": 0.41,
      "grad_norm": 5.0,
      "learning_rate": 4.861017342390847e-06,
      "log_odds_chosen": 0.14408716559410095,
      "log_odds_ratio": -0.6706200242042542,
      "logits/chosen": -3.1252732276916504,
      "logits/rejected": -3.120234966278076,
      "logps/chosen": -0.700475811958313,
      "logps/rejected": -0.7832350730895996,
      "loss": 0.5601,
      "nll_loss": 0.41865190863609314,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.14009517431259155,
      "rewards/margins": 0.016551848500967026,
      "rewards/rejected": -0.15664701163768768,
      "step": 580
    },
    {
      "epoch": 0.42,
      "grad_norm": 5.4375,
      "learning_rate": 4.858721702431704e-06,
      "log_odds_chosen": 0.19907112419605255,
      "log_odds_ratio": -0.6365267634391785,
      "logits/chosen": -3.1869750022888184,
      "logits/rejected": -3.1916017532348633,
      "logps/chosen": -0.6768631339073181,
      "logps/rejected": -0.7860314249992371,
      "loss": 0.5486,
      "nll_loss": 0.4141230583190918,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.13537263870239258,
      "rewards/margins": 0.02183363400399685,
      "rewards/rejected": -0.15720628201961517,
      "step": 590
    },
    {
      "epoch": 0.43,
      "grad_norm": 5.1875,
      "learning_rate": 4.856429311786322e-06,
      "log_odds_chosen": 0.20475217700004578,
      "log_odds_ratio": -0.6470059752464294,
      "logits/chosen": -3.0911927223205566,
      "logits/rejected": -3.088820219039917,
      "logps/chosen": -0.7005254030227661,
      "logps/rejected": -0.8171119689941406,
      "loss": 0.5372,
      "nll_loss": 0.4075283110141754,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.14010508358478546,
      "rewards/margins": 0.023317309096455574,
      "rewards/rejected": -0.1634223908185959,
      "step": 600
    },
    {
      "epoch": 0.44,
      "grad_norm": 5.3125,
      "learning_rate": 4.8541401627966426e-06,
      "log_odds_chosen": 0.16298583149909973,
      "log_odds_ratio": -0.6658545732498169,
      "logits/chosen": -3.041769504547119,
      "logits/rejected": -3.043297290802002,
      "logps/chosen": -0.689085066318512,
      "logps/rejected": -0.7849712371826172,
      "loss": 0.5575,
      "nll_loss": 0.4643743634223938,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13781702518463135,
      "rewards/margins": 0.01917722448706627,
      "rewards/rejected": -0.15699425339698792,
      "step": 610
    },
    {
      "epoch": 0.44,
      "grad_norm": 4.78125,
      "learning_rate": 4.85185424782985e-06,
      "log_odds_chosen": 0.12417051941156387,
      "log_odds_ratio": -0.68137526512146,
      "logits/chosen": -2.961411714553833,
      "logits/rejected": -2.9629132747650146,
      "logps/chosen": -0.6715599298477173,
      "logps/rejected": -0.7414580583572388,
      "loss": 0.5462,
      "nll_loss": 0.3696933686733246,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.1343119889497757,
      "rewards/margins": 0.013979624025523663,
      "rewards/rejected": -0.14829161763191223,
      "step": 620
    },
    {
      "epoch": 0.45,
      "grad_norm": 5.28125,
      "learning_rate": 4.8495715592782715e-06,
      "log_odds_chosen": 0.129594087600708,
      "log_odds_ratio": -0.6814179420471191,
      "logits/chosen": -3.0220203399658203,
      "logits/rejected": -3.007021427154541,
      "logps/chosen": -0.7019616365432739,
      "logps/rejected": -0.7783767580986023,
      "loss": 0.581,
      "nll_loss": 0.43684515357017517,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.14039233326911926,
      "rewards/margins": 0.0152830109000206,
      "rewards/rejected": -0.15567535161972046,
      "step": 630
    },
    {
      "epoch": 0.46,
      "grad_norm": 5.34375,
      "learning_rate": 4.847292089559258e-06,
      "log_odds_chosen": 0.14772093296051025,
      "log_odds_ratio": -0.6776904463768005,
      "logits/chosen": -3.015796661376953,
      "logits/rejected": -2.998617649078369,
      "logps/chosen": -0.6798470616340637,
      "logps/rejected": -0.7485564351081848,
      "loss": 0.5124,
      "nll_loss": 0.3927190601825714,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.13596941530704498,
      "rewards/margins": 0.01374187134206295,
      "rewards/rejected": -0.14971129596233368,
      "step": 640
    },
    {
      "epoch": 0.46,
      "grad_norm": 5.8125,
      "learning_rate": 4.845015831115093e-06,
      "log_odds_chosen": 0.22233247756958008,
      "log_odds_ratio": -0.6407415866851807,
      "logits/chosen": -3.0206289291381836,
      "logits/rejected": -2.993927478790283,
      "logps/chosen": -0.7096582055091858,
      "logps/rejected": -0.8297877311706543,
      "loss": 0.5503,
      "nll_loss": 0.4199894070625305,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.1419316530227661,
      "rewards/margins": 0.024025900289416313,
      "rewards/rejected": -0.16595754027366638,
      "step": 650
    },
    {
      "epoch": 0.47,
      "grad_norm": 7.5,
      "learning_rate": 4.842742776412874e-06,
      "log_odds_chosen": 0.19071760773658752,
      "log_odds_ratio": -0.6609346270561218,
      "logits/chosen": -2.93975567817688,
      "logits/rejected": -2.927455425262451,
      "logps/chosen": -0.6851130723953247,
      "logps/rejected": -0.7924318313598633,
      "loss": 0.5098,
      "nll_loss": 0.39896029233932495,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.13702259957790375,
      "rewards/margins": 0.02146376296877861,
      "rewards/rejected": -0.15848635137081146,
      "step": 660
    },
    {
      "epoch": 0.48,
      "grad_norm": 5.03125,
      "learning_rate": 4.840472917944417e-06,
      "log_odds_chosen": 0.035539038479328156,
      "log_odds_ratio": -0.7433319687843323,
      "logits/chosen": -2.91545033454895,
      "logits/rejected": -2.904900074005127,
      "logps/chosen": -0.7592023015022278,
      "logps/rejected": -0.778312087059021,
      "loss": 0.5748,
      "nll_loss": 0.43543463945388794,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.151840478181839,
      "rewards/margins": 0.0038219629786908627,
      "rewards/rejected": -0.1556624323129654,
      "step": 670
    },
    {
      "epoch": 0.49,
      "grad_norm": 5.0,
      "learning_rate": 4.838206248226147e-06,
      "log_odds_chosen": 0.12000226974487305,
      "log_odds_ratio": -0.6970423460006714,
      "logits/chosen": -2.918330430984497,
      "logits/rejected": -2.909175395965576,
      "logps/chosen": -0.7081884145736694,
      "logps/rejected": -0.7808458209037781,
      "loss": 0.5926,
      "nll_loss": 0.4864211976528168,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.1416376680135727,
      "rewards/margins": 0.01453147642314434,
      "rewards/rejected": -0.15616916120052338,
      "step": 680
    },
    {
      "epoch": 0.49,
      "grad_norm": 5.125,
      "learning_rate": 4.835942759799002e-06,
      "log_odds_chosen": 0.12994246184825897,
      "log_odds_ratio": -0.679962694644928,
      "logits/chosen": -2.9056477546691895,
      "logits/rejected": -2.907723903656006,
      "logps/chosen": -0.6788502335548401,
      "logps/rejected": -0.7463597059249878,
      "loss": 0.5609,
      "nll_loss": 0.4340883791446686,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.1357700526714325,
      "rewards/margins": 0.013501903042197227,
      "rewards/rejected": -0.14927193522453308,
      "step": 690
    },
    {
      "epoch": 0.5,
      "grad_norm": 4.6875,
      "learning_rate": 4.833682445228318e-06,
      "log_odds_chosen": 0.1527804434299469,
      "log_odds_ratio": -0.6715155243873596,
      "logits/chosen": -2.8825762271881104,
      "logits/rejected": -2.878715991973877,
      "logps/chosen": -0.7160866856575012,
      "logps/rejected": -0.7908951044082642,
      "loss": 0.52,
      "nll_loss": 0.3986133933067322,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.14321734011173248,
      "rewards/margins": 0.014961689710617065,
      "rewards/rejected": -0.15817902982234955,
      "step": 700
    },
    {
      "epoch": 0.51,
      "grad_norm": 4.84375,
      "learning_rate": 4.831425297103738e-06,
      "log_odds_chosen": 0.07408870756626129,
      "log_odds_ratio": -0.7036418318748474,
      "logits/chosen": -2.915374279022217,
      "logits/rejected": -2.8941776752471924,
      "logps/chosen": -0.7258971929550171,
      "logps/rejected": -0.776419460773468,
      "loss": 0.5574,
      "nll_loss": 0.4164521098136902,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.14517942070960999,
      "rewards/margins": 0.010104473680257797,
      "rewards/rejected": -0.1552838832139969,
      "step": 710
    },
    {
      "epoch": 0.51,
      "grad_norm": 5.25,
      "learning_rate": 4.829171308039099e-06,
      "log_odds_chosen": 0.11705788224935532,
      "log_odds_ratio": -0.6934496164321899,
      "logits/chosen": -2.8689353466033936,
      "logits/rejected": -2.855515480041504,
      "logps/chosen": -0.7816129326820374,
      "logps/rejected": -0.8508423566818237,
      "loss": 0.5618,
      "nll_loss": 0.41237854957580566,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.15632258355617523,
      "rewards/margins": 0.013845880515873432,
      "rewards/rejected": -0.1701684594154358,
      "step": 720
    },
    {
      "epoch": 0.52,
      "grad_norm": 4.90625,
      "learning_rate": 4.826920470672344e-06,
      "log_odds_chosen": 0.14946161210536957,
      "log_odds_ratio": -0.6791409254074097,
      "logits/chosen": -2.920727252960205,
      "logits/rejected": -2.9093258380889893,
      "logps/chosen": -0.721591055393219,
      "logps/rejected": -0.8128422498703003,
      "loss": 0.5434,
      "nll_loss": 0.4294136166572571,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.14431820809841156,
      "rewards/margins": 0.018250230699777603,
      "rewards/rejected": -0.16256846487522125,
      "step": 730
    },
    {
      "epoch": 0.53,
      "grad_norm": 4.96875,
      "learning_rate": 4.824672777665406e-06,
      "log_odds_chosen": 0.1518145054578781,
      "log_odds_ratio": -0.6733521819114685,
      "logits/chosen": -2.8987679481506348,
      "logits/rejected": -2.8861145973205566,
      "logps/chosen": -0.7268251180648804,
      "logps/rejected": -0.8102623224258423,
      "loss": 0.592,
      "nll_loss": 0.4659670889377594,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.14536501467227936,
      "rewards/margins": 0.01668746955692768,
      "rewards/rejected": -0.1620524823665619,
      "step": 740
    },
    {
      "epoch": 0.54,
      "grad_norm": 5.46875,
      "learning_rate": 4.822428221704122e-06,
      "log_odds_chosen": 0.10895228385925293,
      "log_odds_ratio": -0.6847666501998901,
      "logits/chosen": -2.9176247119903564,
      "logits/rejected": -2.9068551063537598,
      "logps/chosen": -0.6662013530731201,
      "logps/rejected": -0.7180477976799011,
      "loss": 0.5466,
      "nll_loss": 0.4058578908443451,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.13324028253555298,
      "rewards/margins": 0.010369287803769112,
      "rewards/rejected": -0.14360955357551575,
      "step": 750
    },
    {
      "epoch": 0.54,
      "grad_norm": 5.5,
      "learning_rate": 4.820186795498119e-06,
      "log_odds_chosen": 0.07982759177684784,
      "log_odds_ratio": -0.6936464905738831,
      "logits/chosen": -2.900437831878662,
      "logits/rejected": -2.9012367725372314,
      "logps/chosen": -0.684615433216095,
      "logps/rejected": -0.7419982552528381,
      "loss": 0.5572,
      "nll_loss": 0.4167535901069641,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.13692307472229004,
      "rewards/margins": 0.011476578190922737,
      "rewards/rejected": -0.14839965105056763,
      "step": 760
    },
    {
      "epoch": 0.55,
      "grad_norm": 5.5625,
      "learning_rate": 4.817948491780728e-06,
      "log_odds_chosen": 0.07513515651226044,
      "log_odds_ratio": -0.700663685798645,
      "logits/chosen": -2.92631196975708,
      "logits/rejected": -2.9068970680236816,
      "logps/chosen": -0.701633095741272,
      "logps/rejected": -0.7442451119422913,
      "loss": 0.5573,
      "nll_loss": 0.41745367646217346,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.1403266191482544,
      "rewards/margins": 0.008522395975887775,
      "rewards/rejected": -0.1488490104675293,
      "step": 770
    },
    {
      "epoch": 0.56,
      "grad_norm": 5.5625,
      "learning_rate": 4.815713303308872e-06,
      "log_odds_chosen": 0.07833166420459747,
      "log_odds_ratio": -0.7058992385864258,
      "logits/chosen": -2.877963066101074,
      "logits/rejected": -2.8587679862976074,
      "logps/chosen": -0.7175201773643494,
      "logps/rejected": -0.7572144865989685,
      "loss": 0.558,
      "nll_loss": 0.4125480055809021,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.1435040533542633,
      "rewards/margins": 0.00793885625898838,
      "rewards/rejected": -0.15144291520118713,
      "step": 780
    },
    {
      "epoch": 0.56,
      "grad_norm": 4.84375,
      "learning_rate": 4.813481222862981e-06,
      "log_odds_chosen": 0.11849744617938995,
      "log_odds_ratio": -0.6830799579620361,
      "logits/chosen": -2.9524035453796387,
      "logits/rejected": -2.9341073036193848,
      "logps/chosen": -0.6966606378555298,
      "logps/rejected": -0.7586569786071777,
      "loss": 0.5619,
      "nll_loss": 0.43152493238449097,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.1393321305513382,
      "rewards/margins": 0.012399254366755486,
      "rewards/rejected": -0.15173138678073883,
      "step": 790
    },
    {
      "epoch": 0.57,
      "grad_norm": 5.53125,
      "learning_rate": 4.811252243246881e-06,
      "log_odds_chosen": 0.135534405708313,
      "log_odds_ratio": -0.6746788620948792,
      "logits/chosen": -2.8978843688964844,
      "logits/rejected": -2.8979437351226807,
      "logps/chosen": -0.7271771430969238,
      "logps/rejected": -0.7996577024459839,
      "loss": 0.5428,
      "nll_loss": 0.4260443150997162,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.14543543756008148,
      "rewards/margins": 0.01449611783027649,
      "rewards/rejected": -0.15993155539035797,
      "step": 800
    },
    {
      "epoch": 0.58,
      "grad_norm": 4.3125,
      "learning_rate": 4.809026357287709e-06,
      "log_odds_chosen": 0.11346729844808578,
      "log_odds_ratio": -0.6924766898155212,
      "logits/chosen": -2.9434874057769775,
      "logits/rejected": -2.931755781173706,
      "logps/chosen": -0.6964095830917358,
      "logps/rejected": -0.763741135597229,
      "loss": 0.5738,
      "nll_loss": 0.4434167444705963,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.13928191363811493,
      "rewards/margins": 0.013466304168105125,
      "rewards/rejected": -0.1527482271194458,
      "step": 810
    },
    {
      "epoch": 0.59,
      "grad_norm": 5.65625,
      "learning_rate": 4.806803557835802e-06,
      "log_odds_chosen": 0.053010594099760056,
      "log_odds_ratio": -0.7078055143356323,
      "logits/chosen": -2.951209545135498,
      "logits/rejected": -2.9389567375183105,
      "logps/chosen": -0.7215791344642639,
      "logps/rejected": -0.7523022890090942,
      "loss": 0.5626,
      "nll_loss": 0.45763856172561646,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.14431582391262054,
      "rewards/margins": 0.0061446288600564,
      "rewards/rejected": -0.15046045184135437,
      "step": 820
    },
    {
      "epoch": 0.59,
      "grad_norm": 5.0625,
      "learning_rate": 4.804583837764616e-06,
      "log_odds_chosen": 0.12964418530464172,
      "log_odds_ratio": -0.6815978288650513,
      "logits/chosen": -2.977019786834717,
      "logits/rejected": -2.9383797645568848,
      "logps/chosen": -0.7318437099456787,
      "logps/rejected": -0.8120439648628235,
      "loss": 0.5538,
      "nll_loss": 0.42555293440818787,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.14636874198913574,
      "rewards/margins": 0.01604006253182888,
      "rewards/rejected": -0.16240879893302917,
      "step": 830
    },
    {
      "epoch": 0.6,
      "grad_norm": 5.75,
      "learning_rate": 4.802367189970616e-06,
      "log_odds_chosen": 0.12411677837371826,
      "log_odds_ratio": -0.6854800581932068,
      "logits/chosen": -2.913602828979492,
      "logits/rejected": -2.901832342147827,
      "logps/chosen": -0.7199221849441528,
      "logps/rejected": -0.792190432548523,
      "loss": 0.5405,
      "nll_loss": 0.42834392189979553,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.14398446679115295,
      "rewards/margins": 0.014453647658228874,
      "rewards/rejected": -0.15843810141086578,
      "step": 840
    },
    {
      "epoch": 0.61,
      "grad_norm": 5.3125,
      "learning_rate": 4.8001536073731936e-06,
      "log_odds_chosen": 0.1903192102909088,
      "log_odds_ratio": -0.6434763669967651,
      "logits/chosen": -2.9723453521728516,
      "logits/rejected": -2.952861785888672,
      "logps/chosen": -0.6776723265647888,
      "logps/rejected": -0.7700678110122681,
      "loss": 0.5276,
      "nll_loss": 0.4004877209663391,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13553445041179657,
      "rewards/margins": 0.018479080870747566,
      "rewards/rejected": -0.15401355922222137,
      "step": 850
    },
    {
      "epoch": 0.61,
      "grad_norm": 4.84375,
      "learning_rate": 4.797943082914558e-06,
      "log_odds_chosen": 0.1074991226196289,
      "log_odds_ratio": -0.6918960809707642,
      "logits/chosen": -2.931121349334717,
      "logits/rejected": -2.9277279376983643,
      "logps/chosen": -0.7141292691230774,
      "logps/rejected": -0.7773590087890625,
      "loss": 0.5386,
      "nll_loss": 0.4104904532432556,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.1428258717060089,
      "rewards/margins": 0.012645942158997059,
      "rewards/rejected": -0.1554718017578125,
      "step": 860
    },
    {
      "epoch": 0.62,
      "grad_norm": 5.0625,
      "learning_rate": 4.795735609559657e-06,
      "log_odds_chosen": 0.1122872605919838,
      "log_odds_ratio": -0.6973878145217896,
      "logits/chosen": -2.918083429336548,
      "logits/rejected": -2.9120912551879883,
      "logps/chosen": -0.749970555305481,
      "logps/rejected": -0.8086145520210266,
      "loss": 0.5636,
      "nll_loss": 0.4259462356567383,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.1499941051006317,
      "rewards/margins": 0.011728787794709206,
      "rewards/rejected": -0.16172286868095398,
      "step": 870
    },
    {
      "epoch": 0.63,
      "grad_norm": 5.375,
      "learning_rate": 4.793531180296065e-06,
      "log_odds_chosen": 0.17277751863002777,
      "log_odds_ratio": -0.6646739840507507,
      "logits/chosen": -2.9382827281951904,
      "logits/rejected": -2.936652421951294,
      "logps/chosen": -0.7452108860015869,
      "logps/rejected": -0.8357378840446472,
      "loss": 0.5698,
      "nll_loss": 0.42103928327560425,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.14904220402240753,
      "rewards/margins": 0.018105393275618553,
      "rewards/rejected": -0.16714756190776825,
      "step": 880
    },
    {
      "epoch": 0.64,
      "grad_norm": 5.3125,
      "learning_rate": 4.7913297881339085e-06,
      "log_odds_chosen": 0.23507757484912872,
      "log_odds_ratio": -0.6391969323158264,
      "logits/chosen": -2.9719607830047607,
      "logits/rejected": -2.9441778659820557,
      "logps/chosen": -0.6970862746238708,
      "logps/rejected": -0.8258684873580933,
      "loss": 0.5789,
      "nll_loss": 0.4702020287513733,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.13941726088523865,
      "rewards/margins": 0.02575644478201866,
      "rewards/rejected": -0.1651737093925476,
      "step": 890
    },
    {
      "epoch": 0.64,
      "grad_norm": 5.40625,
      "learning_rate": 4.789131426105757e-06,
      "log_odds_chosen": 0.1419156789779663,
      "log_odds_ratio": -0.681240975856781,
      "logits/chosen": -2.9537596702575684,
      "logits/rejected": -2.9550156593322754,
      "logps/chosen": -0.6770733594894409,
      "logps/rejected": -0.7640056610107422,
      "loss": 0.5749,
      "nll_loss": 0.40316563844680786,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.13541468977928162,
      "rewards/margins": 0.01738644763827324,
      "rewards/rejected": -0.15280112624168396,
      "step": 900
    },
    {
      "epoch": 0.65,
      "grad_norm": 5.1875,
      "learning_rate": 4.786936087266542e-06,
      "log_odds_chosen": 0.16488385200500488,
      "log_odds_ratio": -0.6757220029830933,
      "logits/chosen": -2.9237289428710938,
      "logits/rejected": -2.918750047683716,
      "logps/chosen": -0.6665478944778442,
      "logps/rejected": -0.7528942823410034,
      "loss": 0.5518,
      "nll_loss": 0.4029026925563812,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.13330957293510437,
      "rewards/margins": 0.017269287258386612,
      "rewards/rejected": -0.15057885646820068,
      "step": 910
    },
    {
      "epoch": 0.66,
      "grad_norm": 4.53125,
      "learning_rate": 4.784743764693455e-06,
      "log_odds_chosen": 0.1091703400015831,
      "log_odds_ratio": -0.6948757767677307,
      "logits/chosen": -2.9501147270202637,
      "logits/rejected": -2.9290926456451416,
      "logps/chosen": -0.7138906717300415,
      "logps/rejected": -0.7786569595336914,
      "loss": 0.5813,
      "nll_loss": 0.45604902505874634,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.14277812838554382,
      "rewards/margins": 0.012953246012330055,
      "rewards/rejected": -0.15573139488697052,
      "step": 920
    },
    {
      "epoch": 0.66,
      "grad_norm": 4.78125,
      "learning_rate": 4.7825544514858655e-06,
      "log_odds_chosen": 0.12012086063623428,
      "log_odds_ratio": -0.6921867728233337,
      "logits/chosen": -2.9667954444885254,
      "logits/rejected": -2.9569287300109863,
      "logps/chosen": -0.7008910179138184,
      "logps/rejected": -0.7661630511283875,
      "loss": 0.5731,
      "nll_loss": 0.44391852617263794,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.14017818868160248,
      "rewards/margins": 0.013054423034191132,
      "rewards/rejected": -0.153232604265213,
      "step": 930
    },
    {
      "epoch": 0.67,
      "grad_norm": 5.875,
      "learning_rate": 4.780368140765222e-06,
      "log_odds_chosen": 0.0706237182021141,
      "log_odds_ratio": -0.714358925819397,
      "logits/chosen": -2.932490348815918,
      "logits/rejected": -2.9120583534240723,
      "logps/chosen": -0.6620159149169922,
      "logps/rejected": -0.7102792859077454,
      "loss": 0.5259,
      "nll_loss": 0.40104350447654724,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.1324031949043274,
      "rewards/margins": 0.009652670472860336,
      "rewards/rejected": -0.14205586910247803,
      "step": 940
    },
    {
      "epoch": 0.68,
      "grad_norm": 4.875,
      "learning_rate": 4.778184825674966e-06,
      "log_odds_chosen": 0.17702895402908325,
      "log_odds_ratio": -0.6546781063079834,
      "logits/chosen": -2.94228196144104,
      "logits/rejected": -2.9221675395965576,
      "logps/chosen": -0.6707956194877625,
      "logps/rejected": -0.7657662630081177,
      "loss": 0.5465,
      "nll_loss": 0.39552420377731323,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.1341591328382492,
      "rewards/margins": 0.01899414323270321,
      "rewards/rejected": -0.15315327048301697,
      "step": 950
    },
    {
      "epoch": 0.69,
      "grad_norm": 5.09375,
      "learning_rate": 4.776004499380439e-06,
      "log_odds_chosen": 0.019583452492952347,
      "log_odds_ratio": -0.7346916794776917,
      "logits/chosen": -2.9015583992004395,
      "logits/rejected": -2.9088730812072754,
      "logps/chosen": -0.7036567330360413,
      "logps/rejected": -0.7168689966201782,
      "loss": 0.5455,
      "nll_loss": 0.3951405882835388,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.14073136448860168,
      "rewards/margins": 0.002642437582835555,
      "rewards/rejected": -0.1433737874031067,
      "step": 960
    },
    {
      "epoch": 0.69,
      "grad_norm": 5.40625,
      "learning_rate": 4.773827155068793e-06,
      "log_odds_chosen": 0.06732301414012909,
      "log_odds_ratio": -0.7026981115341187,
      "logits/chosen": -2.9094488620758057,
      "logits/rejected": -2.902775287628174,
      "logps/chosen": -0.7069900631904602,
      "logps/rejected": -0.7331331372261047,
      "loss": 0.5518,
      "nll_loss": 0.3954068422317505,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.14139801263809204,
      "rewards/margins": 0.005228628404438496,
      "rewards/rejected": -0.14662663638591766,
      "step": 970
    },
    {
      "epoch": 0.7,
      "grad_norm": 4.9375,
      "learning_rate": 4.771652785948902e-06,
      "log_odds_chosen": 0.11821585893630981,
      "log_odds_ratio": -0.6919859647750854,
      "logits/chosen": -2.9587864875793457,
      "logits/rejected": -2.948873519897461,
      "logps/chosen": -0.7012097835540771,
      "logps/rejected": -0.7723513841629028,
      "loss": 0.5381,
      "nll_loss": 0.38139674067497253,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.14024195075035095,
      "rewards/margins": 0.014228323474526405,
      "rewards/rejected": -0.154470294713974,
      "step": 980
    },
    {
      "epoch": 0.71,
      "grad_norm": 5.09375,
      "learning_rate": 4.769481385251275e-06,
      "log_odds_chosen": 0.10404877364635468,
      "log_odds_ratio": -0.7043722867965698,
      "logits/chosen": -2.9584336280822754,
      "logits/rejected": -2.9519834518432617,
      "logps/chosen": -0.6719237565994263,
      "logps/rejected": -0.7380385994911194,
      "loss": 0.5493,
      "nll_loss": 0.43441280722618103,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.13438475131988525,
      "rewards/margins": 0.013222972862422466,
      "rewards/rejected": -0.1476077288389206,
      "step": 990
    },
    {
      "epoch": 0.71,
      "grad_norm": 5.53125,
      "learning_rate": 4.767312946227961e-06,
      "log_odds_chosen": 0.12437696754932404,
      "log_odds_ratio": -0.6797572374343872,
      "logits/chosen": -2.9542737007141113,
      "logits/rejected": -2.944174289703369,
      "logps/chosen": -0.6895853877067566,
      "logps/rejected": -0.7687771916389465,
      "loss": 0.5247,
      "nll_loss": 0.397632360458374,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.13791708648204803,
      "rewards/margins": 0.01583835482597351,
      "rewards/rejected": -0.15375544130802155,
      "step": 1000
    },
    {
      "epoch": 0.72,
      "grad_norm": 5.59375,
      "learning_rate": 4.765147462152471e-06,
      "log_odds_chosen": 0.03156871721148491,
      "log_odds_ratio": -0.7198628187179565,
      "logits/chosen": -2.9656319618225098,
      "logits/rejected": -2.9585325717926025,
      "logps/chosen": -0.7327839732170105,
      "logps/rejected": -0.7419841885566711,
      "loss": 0.5595,
      "nll_loss": 0.45351147651672363,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.1465567946434021,
      "rewards/margins": 0.0018400519620627165,
      "rewards/rejected": -0.148396834731102,
      "step": 1010
    },
    {
      "epoch": 0.73,
      "grad_norm": 6.03125,
      "learning_rate": 4.762984926319677e-06,
      "log_odds_chosen": 0.1008574515581131,
      "log_odds_ratio": -0.7012881636619568,
      "logits/chosen": -2.900951623916626,
      "logits/rejected": -2.913351058959961,
      "logps/chosen": -0.7476619482040405,
      "logps/rejected": -0.7988759279251099,
      "loss": 0.5313,
      "nll_loss": 0.3712484538555145,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.14953239262104034,
      "rewards/margins": 0.01024280209094286,
      "rewards/rejected": -0.15977518260478973,
      "step": 1020
    },
    {
      "epoch": 0.74,
      "grad_norm": 4.875,
      "learning_rate": 4.760825332045738e-06,
      "log_odds_chosen": 0.1647627353668213,
      "log_odds_ratio": -0.6773719787597656,
      "logits/chosen": -2.916574239730835,
      "logits/rejected": -2.9114716053009033,
      "logps/chosen": -0.732185959815979,
      "logps/rejected": -0.820814311504364,
      "loss": 0.563,
      "nll_loss": 0.40981560945510864,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.14643718302249908,
      "rewards/margins": 0.01772570051252842,
      "rewards/rejected": -0.16416288912296295,
      "step": 1030
    },
    {
      "epoch": 0.74,
      "grad_norm": 4.59375,
      "learning_rate": 4.758668672668006e-06,
      "log_odds_chosen": -0.007136444561183453,
      "log_odds_ratio": -0.7459918260574341,
      "logits/chosen": -2.950622797012329,
      "logits/rejected": -2.935281276702881,
      "logps/chosen": -0.7756951451301575,
      "logps/rejected": -0.7845336198806763,
      "loss": 0.5498,
      "nll_loss": 0.4428860545158386,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.1551390290260315,
      "rewards/margins": 0.001767703564837575,
      "rewards/rejected": -0.15690675377845764,
      "step": 1040
    },
    {
      "epoch": 0.75,
      "grad_norm": 5.3125,
      "learning_rate": 4.756514941544941e-06,
      "log_odds_chosen": 0.22850975394248962,
      "log_odds_ratio": -0.6434667110443115,
      "logits/chosen": -2.916316509246826,
      "logits/rejected": -2.900111675262451,
      "logps/chosen": -0.7090237736701965,
      "logps/rejected": -0.8277368545532227,
      "loss": 0.5347,
      "nll_loss": 0.39760535955429077,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.1418047398328781,
      "rewards/margins": 0.02374262548983097,
      "rewards/rejected": -0.16554740071296692,
      "step": 1050
    },
    {
      "epoch": 0.76,
      "grad_norm": 5.0,
      "learning_rate": 4.754364132056025e-06,
      "log_odds_chosen": 0.23557403683662415,
      "log_odds_ratio": -0.6366820931434631,
      "logits/chosen": -2.9666476249694824,
      "logits/rejected": -2.964963436126709,
      "logps/chosen": -0.6721023917198181,
      "logps/rejected": -0.7975376844406128,
      "loss": 0.5546,
      "nll_loss": 0.43326544761657715,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.1344204843044281,
      "rewards/margins": 0.025087062269449234,
      "rewards/rejected": -0.15950754284858704,
      "step": 1060
    },
    {
      "epoch": 0.76,
      "grad_norm": 4.375,
      "learning_rate": 4.752216237601676e-06,
      "log_odds_chosen": 0.17871122062206268,
      "log_odds_ratio": -0.6607886552810669,
      "logits/chosen": -2.9594788551330566,
      "logits/rejected": -2.949906349182129,
      "logps/chosen": -0.6754817366600037,
      "logps/rejected": -0.761304497718811,
      "loss": 0.5284,
      "nll_loss": 0.41694697737693787,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.13509634137153625,
      "rewards/margins": 0.0171645637601614,
      "rewards/rejected": -0.1522609144449234,
      "step": 1070
    },
    {
      "epoch": 0.77,
      "grad_norm": 5.21875,
      "learning_rate": 4.750071251603165e-06,
      "log_odds_chosen": 0.10223817825317383,
      "log_odds_ratio": -0.6892545819282532,
      "logits/chosen": -2.91923189163208,
      "logits/rejected": -2.9072306156158447,
      "logps/chosen": -0.700354814529419,
      "logps/rejected": -0.7621926069259644,
      "loss": 0.5481,
      "nll_loss": 0.4226892590522766,
      "rewards/accuracies": 0.606249988079071,
      "rewards/chosen": -0.14007095992565155,
      "rewards/margins": 0.01236753724515438,
      "rewards/rejected": -0.15243850648403168,
      "step": 1080
    },
    {
      "epoch": 0.78,
      "grad_norm": 5.5625,
      "learning_rate": 4.7479291675025314e-06,
      "log_odds_chosen": 0.16114802658557892,
      "log_odds_ratio": -0.6601124405860901,
      "logits/chosen": -2.9457671642303467,
      "logits/rejected": -2.9389617443084717,
      "logps/chosen": -0.6818099617958069,
      "logps/rejected": -0.7652055025100708,
      "loss": 0.5778,
      "nll_loss": 0.4147875905036926,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.1363619863986969,
      "rewards/margins": 0.01667911931872368,
      "rewards/rejected": -0.15304109454154968,
      "step": 1090
    },
    {
      "epoch": 0.79,
      "grad_norm": 4.625,
      "learning_rate": 4.745789978762496e-06,
      "log_odds_chosen": 0.08631005138158798,
      "log_odds_ratio": -0.7082911729812622,
      "logits/chosen": -2.927237033843994,
      "logits/rejected": -2.929316997528076,
      "logps/chosen": -0.7573009729385376,
      "logps/rejected": -0.8193367719650269,
      "loss": 0.5399,
      "nll_loss": 0.4314785897731781,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.151460200548172,
      "rewards/margins": 0.012407159432768822,
      "rewards/rejected": -0.16386735439300537,
      "step": 1100
    },
    {
      "epoch": 0.79,
      "grad_norm": 4.5625,
      "learning_rate": 4.7436536788663765e-06,
      "log_odds_chosen": 0.07461805641651154,
      "log_odds_ratio": -0.710638701915741,
      "logits/chosen": -2.954481840133667,
      "logits/rejected": -2.947251796722412,
      "logps/chosen": -0.7126928567886353,
      "logps/rejected": -0.7473545074462891,
      "loss": 0.5331,
      "nll_loss": 0.39857620000839233,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.14253857731819153,
      "rewards/margins": 0.006932335905730724,
      "rewards/rejected": -0.14947089552879333,
      "step": 1110
    },
    {
      "epoch": 0.8,
      "grad_norm": 5.59375,
      "learning_rate": 4.74152026131801e-06,
      "log_odds_chosen": 0.06558915972709656,
      "log_odds_ratio": -0.7165695428848267,
      "logits/chosen": -2.9481871128082275,
      "logits/rejected": -2.934065341949463,
      "logps/chosen": -0.7117661237716675,
      "logps/rejected": -0.7454390525817871,
      "loss": 0.5532,
      "nll_loss": 0.41643962264060974,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.14235322177410126,
      "rewards/margins": 0.006734585855156183,
      "rewards/rejected": -0.1490878164768219,
      "step": 1120
    },
    {
      "epoch": 0.81,
      "grad_norm": 5.25,
      "learning_rate": 4.739389719641665e-06,
      "log_odds_chosen": 0.038577549159526825,
      "log_odds_ratio": -0.7274538278579712,
      "logits/chosen": -2.9647698402404785,
      "logits/rejected": -2.953193187713623,
      "logps/chosen": -0.7552378177642822,
      "logps/rejected": -0.767500102519989,
      "loss": 0.5665,
      "nll_loss": 0.42185306549072266,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.15104755759239197,
      "rewards/margins": 0.002452465472742915,
      "rewards/rejected": -0.1535000503063202,
      "step": 1130
    },
    {
      "epoch": 0.81,
      "grad_norm": 5.78125,
      "learning_rate": 4.7372620473819615e-06,
      "log_odds_chosen": 0.14568562805652618,
      "log_odds_ratio": -0.6789036989212036,
      "logits/chosen": -2.9319028854370117,
      "logits/rejected": -2.9229094982147217,
      "logps/chosen": -0.6954749822616577,
      "logps/rejected": -0.7834473848342896,
      "loss": 0.5403,
      "nll_loss": 0.4035823345184326,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.1390949785709381,
      "rewards/margins": 0.01759449765086174,
      "rewards/rejected": -0.15668947994709015,
      "step": 1140
    },
    {
      "epoch": 0.82,
      "grad_norm": 5.09375,
      "learning_rate": 4.735137238103785e-06,
      "log_odds_chosen": -0.04453912377357483,
      "log_odds_ratio": -0.7690579295158386,
      "logits/chosen": -2.9604084491729736,
      "logits/rejected": -2.9683547019958496,
      "logps/chosen": -0.7146620750427246,
      "logps/rejected": -0.6833995580673218,
      "loss": 0.5543,
      "nll_loss": 0.4193571209907532,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.14293241500854492,
      "rewards/margins": -0.006252489052712917,
      "rewards/rejected": -0.13667991757392883,
      "step": 1150
    },
    {
      "epoch": 0.83,
      "grad_norm": 4.59375,
      "learning_rate": 4.7330152853922064e-06,
      "log_odds_chosen": 0.10607735067605972,
      "log_odds_ratio": -0.6870900988578796,
      "logits/chosen": -2.933436870574951,
      "logits/rejected": -2.926907777786255,
      "logps/chosen": -0.707848846912384,
      "logps/rejected": -0.7633363008499146,
      "loss": 0.5204,
      "nll_loss": 0.40916410088539124,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.14156976342201233,
      "rewards/margins": 0.011097497306764126,
      "rewards/rejected": -0.15266726911067963,
      "step": 1160
    },
    {
      "epoch": 0.84,
      "grad_norm": 5.03125,
      "learning_rate": 4.730896182852409e-06,
      "log_odds_chosen": 0.11698383092880249,
      "log_odds_ratio": -0.6935534477233887,
      "logits/chosen": -2.9313931465148926,
      "logits/rejected": -2.911740779876709,
      "logps/chosen": -0.7008348107337952,
      "logps/rejected": -0.7741026282310486,
      "loss": 0.5431,
      "nll_loss": 0.4040239453315735,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.1401669681072235,
      "rewards/margins": 0.014653565362095833,
      "rewards/rejected": -0.1548205316066742,
      "step": 1170
    },
    {
      "epoch": 0.84,
      "grad_norm": 4.28125,
      "learning_rate": 4.72877992410959e-06,
      "log_odds_chosen": 0.10059946775436401,
      "log_odds_ratio": -0.7010030150413513,
      "logits/chosen": -2.946838855743408,
      "logits/rejected": -2.9204533100128174,
      "logps/chosen": -0.6578399538993835,
      "logps/rejected": -0.7220724821090698,
      "loss": 0.5386,
      "nll_loss": 0.3835596740245819,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.13156801462173462,
      "rewards/margins": 0.012846499681472778,
      "rewards/rejected": -0.1444145143032074,
      "step": 1180
    },
    {
      "epoch": 0.85,
      "grad_norm": 4.84375,
      "learning_rate": 4.7266665028088985e-06,
      "log_odds_chosen": 0.08397103101015091,
      "log_odds_ratio": -0.7051702737808228,
      "logits/chosen": -2.93192982673645,
      "logits/rejected": -2.9253973960876465,
      "logps/chosen": -0.6880505681037903,
      "logps/rejected": -0.7314938306808472,
      "loss": 0.5324,
      "nll_loss": 0.3771182596683502,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.13761012256145477,
      "rewards/margins": 0.008688644506037235,
      "rewards/rejected": -0.14629876613616943,
      "step": 1190
    },
    {
      "epoch": 0.86,
      "grad_norm": 5.0,
      "learning_rate": 4.72455591261534e-06,
      "log_odds_chosen": 0.10587283223867416,
      "log_odds_ratio": -0.6958147287368774,
      "logits/chosen": -2.9291391372680664,
      "logits/rejected": -2.931337833404541,
      "logps/chosen": -0.6905062794685364,
      "logps/rejected": -0.7565279603004456,
      "loss": 0.5432,
      "nll_loss": 0.36201146245002747,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.138101264834404,
      "rewards/margins": 0.013204338029026985,
      "rewards/rejected": -0.15130558609962463,
      "step": 1200
    },
    {
      "epoch": 0.86,
      "grad_norm": 5.28125,
      "learning_rate": 4.722448147213712e-06,
      "log_odds_chosen": 0.16628727316856384,
      "log_odds_ratio": -0.6715134382247925,
      "logits/chosen": -2.9251606464385986,
      "logits/rejected": -2.924400806427002,
      "logps/chosen": -0.7159217000007629,
      "logps/rejected": -0.7908354997634888,
      "loss": 0.5327,
      "nll_loss": 0.4329577088356018,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.1431843340396881,
      "rewards/margins": 0.01498276274651289,
      "rewards/rejected": -0.15816709399223328,
      "step": 1210
    },
    {
      "epoch": 0.87,
      "grad_norm": 5.34375,
      "learning_rate": 4.720343200308507e-06,
      "log_odds_chosen": 0.09120135009288788,
      "log_odds_ratio": -0.7072020769119263,
      "logits/chosen": -2.918830394744873,
      "logits/rejected": -2.9098925590515137,
      "logps/chosen": -0.7446082830429077,
      "logps/rejected": -0.7945619821548462,
      "loss": 0.5726,
      "nll_loss": 0.4186829924583435,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.1489216834306717,
      "rewards/margins": 0.009990743361413479,
      "rewards/rejected": -0.15891240537166595,
      "step": 1220
    },
    {
      "epoch": 0.88,
      "grad_norm": 5.15625,
      "learning_rate": 4.7182410656238484e-06,
      "log_odds_chosen": 0.08639942854642868,
      "log_odds_ratio": -0.7023371458053589,
      "logits/chosen": -2.883063793182373,
      "logits/rejected": -2.876861095428467,
      "logps/chosen": -0.7035251259803772,
      "logps/rejected": -0.7428823113441467,
      "loss": 0.5529,
      "nll_loss": 0.3646206855773926,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.14070501923561096,
      "rewards/margins": 0.007871445268392563,
      "rewards/rejected": -0.14857646822929382,
      "step": 1230
    },
    {
      "epoch": 0.89,
      "grad_norm": 5.03125,
      "learning_rate": 4.716141736903407e-06,
      "log_odds_chosen": 0.17533931136131287,
      "log_odds_ratio": -0.6885548830032349,
      "logits/chosen": -2.8661389350891113,
      "logits/rejected": -2.851670980453491,
      "logps/chosen": -0.7199305295944214,
      "logps/rejected": -0.8365727663040161,
      "loss": 0.5332,
      "nll_loss": 0.41772204637527466,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.14398609101772308,
      "rewards/margins": 0.02332843840122223,
      "rewards/rejected": -0.1673145294189453,
      "step": 1240
    },
    {
      "epoch": 0.89,
      "grad_norm": 5.125,
      "learning_rate": 4.714045207910318e-06,
      "log_odds_chosen": 0.13246873021125793,
      "log_odds_ratio": -0.6706294417381287,
      "logits/chosen": -2.8975651264190674,
      "logits/rejected": -2.872434616088867,
      "logps/chosen": -0.6649607419967651,
      "logps/rejected": -0.7357169389724731,
      "loss": 0.5467,
      "nll_loss": 0.41414278745651245,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.13299211859703064,
      "rewards/margins": 0.014151250943541527,
      "rewards/rejected": -0.1471433937549591,
      "step": 1250
    },
    {
      "epoch": 0.9,
      "grad_norm": 5.0625,
      "learning_rate": 4.71195147242711e-06,
      "log_odds_chosen": 0.040602125227451324,
      "log_odds_ratio": -0.7303158640861511,
      "logits/chosen": -2.8778679370880127,
      "logits/rejected": -2.8803482055664062,
      "logps/chosen": -0.7029106020927429,
      "logps/rejected": -0.7276321649551392,
      "loss": 0.559,
      "nll_loss": 0.4467160701751709,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1405821144580841,
      "rewards/margins": 0.004944324027746916,
      "rewards/rejected": -0.1455264538526535,
      "step": 1260
    },
    {
      "epoch": 0.91,
      "grad_norm": 8.0,
      "learning_rate": 4.709860524255622e-06,
      "log_odds_chosen": 0.09002572298049927,
      "log_odds_ratio": -0.7032243609428406,
      "logits/chosen": -2.8805668354034424,
      "logits/rejected": -2.860121965408325,
      "logps/chosen": -0.718974232673645,
      "logps/rejected": -0.7800055742263794,
      "loss": 0.5319,
      "nll_loss": 0.3927960991859436,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.14379484951496124,
      "rewards/margins": 0.012206263840198517,
      "rewards/rejected": -0.15600110590457916,
      "step": 1270
    },
    {
      "epoch": 0.91,
      "grad_norm": 5.125,
      "learning_rate": 4.707772357216934e-06,
      "log_odds_chosen": 0.2139289379119873,
      "log_odds_ratio": -0.6507673859596252,
      "logits/chosen": -2.9283065795898438,
      "logits/rejected": -2.909491777420044,
      "logps/chosen": -0.666816234588623,
      "logps/rejected": -0.7714813947677612,
      "loss": 0.5357,
      "nll_loss": 0.4001480042934418,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.1333632469177246,
      "rewards/margins": 0.020933035761117935,
      "rewards/rejected": -0.15429629385471344,
      "step": 1280
    },
    {
      "epoch": 0.92,
      "grad_norm": 5.21875,
      "learning_rate": 4.705686965151282e-06,
      "log_odds_chosen": 0.08337760716676712,
      "log_odds_ratio": -0.6977395415306091,
      "logits/chosen": -2.866161584854126,
      "logits/rejected": -2.8568389415740967,
      "logps/chosen": -0.6980268955230713,
      "logps/rejected": -0.7350268959999084,
      "loss": 0.5858,
      "nll_loss": 0.44848671555519104,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.13960537314414978,
      "rewards/margins": 0.007400007452815771,
      "rewards/rejected": -0.14700539410114288,
      "step": 1290
    },
    {
      "epoch": 0.93,
      "grad_norm": 5.09375,
      "learning_rate": 4.703604341917987e-06,
      "log_odds_chosen": 0.16698376834392548,
      "log_odds_ratio": -0.6709255576133728,
      "logits/chosen": -2.8881678581237793,
      "logits/rejected": -2.8717174530029297,
      "logps/chosen": -0.6411119699478149,
      "logps/rejected": -0.7108407616615295,
      "loss": 0.5404,
      "nll_loss": 0.39268070459365845,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.12822240591049194,
      "rewards/margins": 0.01394575648009777,
      "rewards/rejected": -0.14216816425323486,
      "step": 1300
    },
    {
      "epoch": 0.94,
      "grad_norm": 5.125,
      "learning_rate": 4.701524481395374e-06,
      "log_odds_chosen": 0.04918034002184868,
      "log_odds_ratio": -0.7179148197174072,
      "logits/chosen": -2.8727264404296875,
      "logits/rejected": -2.858916759490967,
      "logps/chosen": -0.7131275534629822,
      "logps/rejected": -0.7290714979171753,
      "loss": 0.5486,
      "nll_loss": 0.37463003396987915,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.14262551069259644,
      "rewards/margins": 0.0031887758523225784,
      "rewards/rejected": -0.14581426978111267,
      "step": 1310
    },
    {
      "epoch": 0.94,
      "grad_norm": 4.875,
      "learning_rate": 4.699447377480703e-06,
      "log_odds_chosen": 0.1447034478187561,
      "log_odds_ratio": -0.6646226644515991,
      "logits/chosen": -2.8924479484558105,
      "logits/rejected": -2.8799147605895996,
      "logps/chosen": -0.7005911469459534,
      "logps/rejected": -0.773948073387146,
      "loss": 0.5247,
      "nll_loss": 0.38711321353912354,
      "rewards/accuracies": 0.6187499761581421,
      "rewards/chosen": -0.14011821150779724,
      "rewards/margins": 0.01467139832675457,
      "rewards/rejected": -0.15478962659835815,
      "step": 1320
    },
    {
      "epoch": 0.95,
      "grad_norm": 5.125,
      "learning_rate": 4.6973730240900876e-06,
      "log_odds_chosen": 0.18853728473186493,
      "log_odds_ratio": -0.6440411806106567,
      "logits/chosen": -2.9185266494750977,
      "logits/rejected": -2.888824939727783,
      "logps/chosen": -0.6820311546325684,
      "logps/rejected": -0.7798157334327698,
      "loss": 0.5488,
      "nll_loss": 0.4185566008090973,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.13640624284744263,
      "rewards/margins": 0.019556904211640358,
      "rewards/rejected": -0.15596315264701843,
      "step": 1330
    },
    {
      "epoch": 0.96,
      "grad_norm": 5.3125,
      "learning_rate": 4.695301415158426e-06,
      "log_odds_chosen": 0.10568451881408691,
      "log_odds_ratio": -0.7053590416908264,
      "logits/chosen": -2.9057796001434326,
      "logits/rejected": -2.896315813064575,
      "logps/chosen": -0.7402433753013611,
      "logps/rejected": -0.7852991223335266,
      "loss": 0.5757,
      "nll_loss": 0.44732385873794556,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.14804866909980774,
      "rewards/margins": 0.00901114847511053,
      "rewards/rejected": -0.15705981850624084,
      "step": 1340
    },
    {
      "epoch": 0.96,
      "grad_norm": 4.9375,
      "learning_rate": 4.693232544639321e-06,
      "log_odds_chosen": 0.1334969699382782,
      "log_odds_ratio": -0.6749390959739685,
      "logits/chosen": -2.9262282848358154,
      "logits/rejected": -2.9031224250793457,
      "logps/chosen": -0.7033289670944214,
      "logps/rejected": -0.7720553874969482,
      "loss": 0.5335,
      "nll_loss": 0.38161200284957886,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.14066579937934875,
      "rewards/margins": 0.013745295815169811,
      "rewards/rejected": -0.15441109240055084,
      "step": 1350
    },
    {
      "epoch": 0.97,
      "grad_norm": 5.53125,
      "learning_rate": 4.691166406505011e-06,
      "log_odds_chosen": 0.20113444328308105,
      "log_odds_ratio": -0.6593700051307678,
      "logits/chosen": -2.8720855712890625,
      "logits/rejected": -2.865278959274292,
      "logps/chosen": -0.6543204188346863,
      "logps/rejected": -0.7732547521591187,
      "loss": 0.5593,
      "nll_loss": 0.4312739372253418,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.13086409866809845,
      "rewards/margins": 0.023786863312125206,
      "rewards/rejected": -0.1546509563922882,
      "step": 1360
    },
    {
      "epoch": 0.98,
      "grad_norm": 5.5,
      "learning_rate": 4.689102994746289e-06,
      "log_odds_chosen": 0.11934226751327515,
      "log_odds_ratio": -0.6696735620498657,
      "logits/chosen": -2.879538059234619,
      "logits/rejected": -2.8727288246154785,
      "logps/chosen": -0.673319399356842,
      "logps/rejected": -0.7296326756477356,
      "loss": 0.5975,
      "nll_loss": 0.44223251938819885,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.1346638798713684,
      "rewards/margins": 0.011262651532888412,
      "rewards/rejected": -0.14592652022838593,
      "step": 1370
    },
    {
      "epoch": 0.99,
      "grad_norm": 5.5,
      "learning_rate": 4.687042303372439e-06,
      "log_odds_chosen": 0.11940214782953262,
      "log_odds_ratio": -0.6887474060058594,
      "logits/chosen": -2.9131243228912354,
      "logits/rejected": -2.9096319675445557,
      "logps/chosen": -0.6941181421279907,
      "logps/rejected": -0.7482253313064575,
      "loss": 0.5402,
      "nll_loss": 0.40905994176864624,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.13882364332675934,
      "rewards/margins": 0.010821421630680561,
      "rewards/rejected": -0.14964506030082703,
      "step": 1380
    },
    {
      "epoch": 0.99,
      "grad_norm": 4.25,
      "learning_rate": 4.684984326411154e-06,
      "log_odds_chosen": 0.09887897968292236,
      "log_odds_ratio": -0.6886764764785767,
      "logits/chosen": -2.8908066749572754,
      "logits/rejected": -2.8787262439727783,
      "logps/chosen": -0.679020881652832,
      "logps/rejected": -0.7298253774642944,
      "loss": 0.5263,
      "nll_loss": 0.39224857091903687,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.1358041763305664,
      "rewards/margins": 0.010160907171666622,
      "rewards/rejected": -0.1459650695323944,
      "step": 1390
    },
    {
      "epoch": 1.0,
      "grad_norm": 4.65625,
      "learning_rate": 4.68292905790847e-06,
      "log_odds_chosen": 0.2600487768650055,
      "log_odds_ratio": -0.6203187704086304,
      "logits/chosen": -2.9187633991241455,
      "logits/rejected": -2.9031124114990234,
      "logps/chosen": -0.635817289352417,
      "logps/rejected": -0.757480800151825,
      "loss": 0.5639,
      "nll_loss": 0.42110905051231384,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.12716346979141235,
      "rewards/margins": 0.02433270588517189,
      "rewards/rejected": -0.15149617195129395,
      "step": 1400
    },
    {
      "epoch": 1.01,
      "grad_norm": 6.03125,
      "learning_rate": 4.6808764919286885e-06,
      "log_odds_chosen": 0.8848162889480591,
      "log_odds_ratio": -0.37814536690711975,
      "logits/chosen": -2.9017772674560547,
      "logits/rejected": -2.8844223022460938,
      "logps/chosen": -0.5128706097602844,
      "logps/rejected": -0.9588940739631653,
      "loss": 0.4309,
      "nll_loss": 0.3441932797431946,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.10257412493228912,
      "rewards/margins": 0.08920470625162125,
      "rewards/rejected": -0.19177880883216858,
      "step": 1410
    },
    {
      "epoch": 1.01,
      "grad_norm": 4.53125,
      "learning_rate": 4.678826622554307e-06,
      "log_odds_chosen": 0.9211248159408569,
      "log_odds_ratio": -0.3970174491405487,
      "logits/chosen": -2.8897476196289062,
      "logits/rejected": -2.877735137939453,
      "logps/chosen": -0.47735509276390076,
      "logps/rejected": -0.9233940839767456,
      "loss": 0.4206,
      "nll_loss": 0.31756311655044556,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.09547100961208344,
      "rewards/margins": 0.08920779824256897,
      "rewards/rejected": -0.1846788227558136,
      "step": 1420
    },
    {
      "epoch": 1.02,
      "grad_norm": 4.9375,
      "learning_rate": 4.676779443885949e-06,
      "log_odds_chosen": 1.0763452053070068,
      "log_odds_ratio": -0.3526865839958191,
      "logits/chosen": -2.8960213661193848,
      "logits/rejected": -2.8801820278167725,
      "logps/chosen": -0.512055516242981,
      "logps/rejected": -1.0564101934432983,
      "loss": 0.3946,
      "nll_loss": 0.3245953619480133,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.10241111367940903,
      "rewards/margins": 0.1088709607720375,
      "rewards/rejected": -0.21128205955028534,
      "step": 1430
    },
    {
      "epoch": 1.03,
      "grad_norm": 5.4375,
      "learning_rate": 4.674734950042287e-06,
      "log_odds_chosen": 1.138904333114624,
      "log_odds_ratio": -0.33328309655189514,
      "logits/chosen": -2.8903861045837402,
      "logits/rejected": -2.884766101837158,
      "logps/chosen": -0.4885633885860443,
      "logps/rejected": -1.0382764339447021,
      "loss": 0.3688,
      "nll_loss": 0.31521254777908325,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.0977126881480217,
      "rewards/margins": 0.10994259268045425,
      "rewards/rejected": -0.20765526592731476,
      "step": 1440
    },
    {
      "epoch": 1.04,
      "grad_norm": 4.78125,
      "learning_rate": 4.672693135159978e-06,
      "log_odds_chosen": 1.0414109230041504,
      "log_odds_ratio": -0.3578895330429077,
      "logits/chosen": -2.8687214851379395,
      "logits/rejected": -2.853811264038086,
      "logps/chosen": -0.5137967467308044,
      "logps/rejected": -1.0513535737991333,
      "loss": 0.3982,
      "nll_loss": 0.3380189538002014,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10275934636592865,
      "rewards/margins": 0.10751136392354965,
      "rewards/rejected": -0.2102707177400589,
      "step": 1450
    },
    {
      "epoch": 1.04,
      "grad_norm": 6.65625,
      "learning_rate": 4.67065399339359e-06,
      "log_odds_chosen": 1.1234166622161865,
      "log_odds_ratio": -0.35121458768844604,
      "logits/chosen": -2.861860752105713,
      "logits/rejected": -2.8768668174743652,
      "logps/chosen": -0.49000921845436096,
      "logps/rejected": -1.0480070114135742,
      "loss": 0.3922,
      "nll_loss": 0.31293317675590515,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09800183773040771,
      "rewards/margins": 0.11159957945346832,
      "rewards/rejected": -0.20960143208503723,
      "step": 1460
    },
    {
      "epoch": 1.05,
      "grad_norm": 6.96875,
      "learning_rate": 4.668617518915533e-06,
      "log_odds_chosen": 1.04032301902771,
      "log_odds_ratio": -0.35156646370887756,
      "logits/chosen": -2.8752763271331787,
      "logits/rejected": -2.8781304359436035,
      "logps/chosen": -0.5017446279525757,
      "logps/rejected": -1.0297907590866089,
      "loss": 0.3846,
      "nll_loss": 0.31080394983291626,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.10034892708063126,
      "rewards/margins": 0.1056092381477356,
      "rewards/rejected": -0.20595815777778625,
      "step": 1470
    },
    {
      "epoch": 1.06,
      "grad_norm": 5.84375,
      "learning_rate": 4.666583705915985e-06,
      "log_odds_chosen": 1.1882513761520386,
      "log_odds_ratio": -0.32196754217147827,
      "logits/chosen": -2.867017984390259,
      "logits/rejected": -2.8608901500701904,
      "logps/chosen": -0.4866812825202942,
      "logps/rejected": -1.108413577079773,
      "loss": 0.4034,
      "nll_loss": 0.30916067957878113,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.09733626246452332,
      "rewards/margins": 0.12434647977352142,
      "rewards/rejected": -0.22168274223804474,
      "step": 1480
    },
    {
      "epoch": 1.06,
      "grad_norm": 5.90625,
      "learning_rate": 4.664552548602825e-06,
      "log_odds_chosen": 1.2755037546157837,
      "log_odds_ratio": -0.3192000985145569,
      "logits/chosen": -2.8904194831848145,
      "logits/rejected": -2.887146472930908,
      "logps/chosen": -0.4415016174316406,
      "logps/rejected": -1.0573655366897583,
      "loss": 0.375,
      "nll_loss": 0.30936262011528015,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.08830033242702484,
      "rewards/margins": 0.12317276000976562,
      "rewards/rejected": -0.21147307753562927,
      "step": 1490
    },
    {
      "epoch": 1.07,
      "grad_norm": 6.3125,
      "learning_rate": 4.662524041201569e-06,
      "log_odds_chosen": 1.2209784984588623,
      "log_odds_ratio": -0.31655097007751465,
      "logits/chosen": -2.8983540534973145,
      "logits/rejected": -2.876600742340088,
      "logps/chosen": -0.5172096490859985,
      "logps/rejected": -1.1844242811203003,
      "loss": 0.416,
      "nll_loss": 0.3785460293292999,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.10344193130731583,
      "rewards/margins": 0.1334429532289505,
      "rewards/rejected": -0.23688487708568573,
      "step": 1500
    },
    {
      "epoch": 1.08,
      "grad_norm": 6.1875,
      "learning_rate": 4.660498177955291e-06,
      "log_odds_chosen": 1.1757891178131104,
      "log_odds_ratio": -0.316308856010437,
      "logits/chosen": -2.8644537925720215,
      "logits/rejected": -2.8410959243774414,
      "logps/chosen": -0.49577221274375916,
      "logps/rejected": -1.1109702587127686,
      "loss": 0.4093,
      "nll_loss": 0.3233625888824463,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.09915443509817123,
      "rewards/margins": 0.12303964048624039,
      "rewards/rejected": -0.22219407558441162,
      "step": 1510
    },
    {
      "epoch": 1.09,
      "grad_norm": 4.8125,
      "learning_rate": 4.658474953124562e-06,
      "log_odds_chosen": 1.1299588680267334,
      "log_odds_ratio": -0.3389926254749298,
      "logits/chosen": -2.9047207832336426,
      "logits/rejected": -2.9168930053710938,
      "logps/chosen": -0.4910566806793213,
      "logps/rejected": -1.0821782350540161,
      "loss": 0.3979,
      "nll_loss": 0.36464497447013855,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.09821134060621262,
      "rewards/margins": 0.11822430789470673,
      "rewards/rejected": -0.21643562614917755,
      "step": 1520
    },
    {
      "epoch": 1.09,
      "grad_norm": 6.25,
      "learning_rate": 4.656454360987378e-06,
      "log_odds_chosen": 1.2231553792953491,
      "log_odds_ratio": -0.30703750252723694,
      "logits/chosen": -2.8862717151641846,
      "logits/rejected": -2.880195140838623,
      "logps/chosen": -0.45338621735572815,
      "logps/rejected": -1.0659048557281494,
      "loss": 0.3926,
      "nll_loss": 0.32782015204429626,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.09067724645137787,
      "rewards/margins": 0.12250375747680664,
      "rewards/rejected": -0.21318098902702332,
      "step": 1530
    },
    {
      "epoch": 1.1,
      "grad_norm": 5.90625,
      "learning_rate": 4.654436395839094e-06,
      "log_odds_chosen": 1.2722887992858887,
      "log_odds_ratio": -0.3012635111808777,
      "logits/chosen": -2.8365585803985596,
      "logits/rejected": -2.8223185539245605,
      "logps/chosen": -0.498770147562027,
      "logps/rejected": -1.186909556388855,
      "loss": 0.3876,
      "nll_loss": 0.34141403436660767,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.09975402057170868,
      "rewards/margins": 0.13762789964675903,
      "rewards/rejected": -0.23738190531730652,
      "step": 1540
    },
    {
      "epoch": 1.11,
      "grad_norm": 5.25,
      "learning_rate": 4.652421051992354e-06,
      "log_odds_chosen": 1.367235541343689,
      "log_odds_ratio": -0.2841276526451111,
      "logits/chosen": -2.8690426349639893,
      "logits/rejected": -2.860100507736206,
      "logps/chosen": -0.42585864663124084,
      "logps/rejected": -1.0907236337661743,
      "loss": 0.3664,
      "nll_loss": 0.2977783679962158,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.08517173677682877,
      "rewards/margins": 0.13297298550605774,
      "rewards/rejected": -0.2181447297334671,
      "step": 1550
    },
    {
      "epoch": 1.11,
      "grad_norm": 6.9375,
      "learning_rate": 4.650408323777029e-06,
      "log_odds_chosen": 1.1410921812057495,
      "log_odds_ratio": -0.3362571597099304,
      "logits/chosen": -2.853015184402466,
      "logits/rejected": -2.8460183143615723,
      "logps/chosen": -0.5159534215927124,
      "logps/rejected": -1.1035113334655762,
      "loss": 0.392,
      "nll_loss": 0.33157429099082947,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.10319069772958755,
      "rewards/margins": 0.11751158535480499,
      "rewards/rejected": -0.22070229053497314,
      "step": 1560
    },
    {
      "epoch": 1.12,
      "grad_norm": 4.875,
      "learning_rate": 4.6483982055401415e-06,
      "log_odds_chosen": 1.3359148502349854,
      "log_odds_ratio": -0.3214777410030365,
      "logits/chosen": -2.807814359664917,
      "logits/rejected": -2.8036892414093018,
      "logps/chosen": -0.48363837599754333,
      "logps/rejected": -1.1991592645645142,
      "loss": 0.3751,
      "nll_loss": 0.2966926693916321,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.09672766178846359,
      "rewards/margins": 0.1431041657924652,
      "rewards/rejected": -0.23983187973499298,
      "step": 1570
    },
    {
      "epoch": 1.13,
      "grad_norm": 5.65625,
      "learning_rate": 4.646390691645805e-06,
      "log_odds_chosen": 1.219293236732483,
      "log_odds_ratio": -0.34068965911865234,
      "logits/chosen": -2.826866626739502,
      "logits/rejected": -2.8263888359069824,
      "logps/chosen": -0.4953809678554535,
      "logps/rejected": -1.1199510097503662,
      "loss": 0.3645,
      "nll_loss": 0.2939409017562866,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09907619655132294,
      "rewards/margins": 0.1249140128493309,
      "rewards/rejected": -0.22399020195007324,
      "step": 1580
    },
    {
      "epoch": 1.14,
      "grad_norm": 5.875,
      "learning_rate": 4.644385776475159e-06,
      "log_odds_chosen": 1.1653202772140503,
      "log_odds_ratio": -0.3451498746871948,
      "logits/chosen": -2.8366851806640625,
      "logits/rejected": -2.835325002670288,
      "logps/chosen": -0.5062024593353271,
      "logps/rejected": -1.1195242404937744,
      "loss": 0.4054,
      "nll_loss": 0.3258489966392517,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.10124049335718155,
      "rewards/margins": 0.12266434729099274,
      "rewards/rejected": -0.2239048182964325,
      "step": 1590
    },
    {
      "epoch": 1.14,
      "grad_norm": 6.75,
      "learning_rate": 4.642383454426297e-06,
      "log_odds_chosen": 1.174912452697754,
      "log_odds_ratio": -0.3339281678199768,
      "logits/chosen": -2.861581325531006,
      "logits/rejected": -2.8604111671447754,
      "logps/chosen": -0.5135446786880493,
      "logps/rejected": -1.1017265319824219,
      "loss": 0.396,
      "nll_loss": 0.3384125828742981,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.10270893573760986,
      "rewards/margins": 0.11763636022806168,
      "rewards/rejected": -0.22034530341625214,
      "step": 1600
    },
    {
      "epoch": 1.15,
      "grad_norm": 8.125,
      "learning_rate": 4.640383719914205e-06,
      "log_odds_chosen": 1.2484989166259766,
      "log_odds_ratio": -0.30592483282089233,
      "logits/chosen": -2.807584285736084,
      "logits/rejected": -2.813006639480591,
      "logps/chosen": -0.5197979807853699,
      "logps/rejected": -1.181796669960022,
      "loss": 0.4215,
      "nll_loss": 0.3560883402824402,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.10395960509777069,
      "rewards/margins": 0.13239972293376923,
      "rewards/rejected": -0.23635932803153992,
      "step": 1610
    },
    {
      "epoch": 1.16,
      "grad_norm": 6.25,
      "learning_rate": 4.638386567370694e-06,
      "log_odds_chosen": 1.2706489562988281,
      "log_odds_ratio": -0.3202516436576843,
      "logits/chosen": -2.8386385440826416,
      "logits/rejected": -2.835495710372925,
      "logps/chosen": -0.5026634931564331,
      "logps/rejected": -1.1739028692245483,
      "loss": 0.3759,
      "nll_loss": 0.31424498558044434,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.10053269565105438,
      "rewards/margins": 0.13424786925315857,
      "rewards/rejected": -0.23478057980537415,
      "step": 1620
    },
    {
      "epoch": 1.16,
      "grad_norm": 5.3125,
      "learning_rate": 4.636391991244338e-06,
      "log_odds_chosen": 1.2696077823638916,
      "log_odds_ratio": -0.3153325915336609,
      "logits/chosen": -2.858908176422119,
      "logits/rejected": -2.8612444400787354,
      "logps/chosen": -0.48242902755737305,
      "logps/rejected": -1.126427173614502,
      "loss": 0.366,
      "nll_loss": 0.295919805765152,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.09648582339286804,
      "rewards/margins": 0.12879963219165802,
      "rewards/rejected": -0.22528544068336487,
      "step": 1630
    },
    {
      "epoch": 1.17,
      "grad_norm": 6.4375,
      "learning_rate": 4.634399986000405e-06,
      "log_odds_chosen": 1.3514255285263062,
      "log_odds_ratio": -0.2864634096622467,
      "logits/chosen": -2.892820358276367,
      "logits/rejected": -2.8774783611297607,
      "logps/chosen": -0.4613906741142273,
      "logps/rejected": -1.1687535047531128,
      "loss": 0.3837,
      "nll_loss": 0.35217028856277466,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.0922781378030777,
      "rewards/margins": 0.14147259294986725,
      "rewards/rejected": -0.23375073075294495,
      "step": 1640
    },
    {
      "epoch": 1.18,
      "grad_norm": 5.90625,
      "learning_rate": 4.632410546120794e-06,
      "log_odds_chosen": 1.1376657485961914,
      "log_odds_ratio": -0.3578290343284607,
      "logits/chosen": -2.820157527923584,
      "logits/rejected": -2.8282933235168457,
      "logps/chosen": -0.5289143323898315,
      "logps/rejected": -1.1361706256866455,
      "loss": 0.4242,
      "nll_loss": 0.36498746275901794,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.10578285157680511,
      "rewards/margins": 0.12145128101110458,
      "rewards/rejected": -0.2272341251373291,
      "step": 1650
    },
    {
      "epoch": 1.19,
      "grad_norm": 6.5,
      "learning_rate": 4.6304236661039765e-06,
      "log_odds_chosen": 1.2930752038955688,
      "log_odds_ratio": -0.3022990822792053,
      "logits/chosen": -2.8277323246002197,
      "logits/rejected": -2.814790725708008,
      "logps/chosen": -0.471322625875473,
      "logps/rejected": -1.1453874111175537,
      "loss": 0.3746,
      "nll_loss": 0.298636794090271,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.09426452219486237,
      "rewards/margins": 0.13481295108795166,
      "rewards/rejected": -0.2290775030851364,
      "step": 1660
    },
    {
      "epoch": 1.19,
      "grad_norm": 6.875,
      "learning_rate": 4.628439340464919e-06,
      "log_odds_chosen": 1.1731585264205933,
      "log_odds_ratio": -0.33878791332244873,
      "logits/chosen": -2.8189854621887207,
      "logits/rejected": -2.822537660598755,
      "logps/chosen": -0.5024202466011047,
      "logps/rejected": -1.1166616678237915,
      "loss": 0.3923,
      "nll_loss": 0.34138697385787964,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.10048405081033707,
      "rewards/margins": 0.12284828722476959,
      "rewards/rejected": -0.22333233058452606,
      "step": 1670
    },
    {
      "epoch": 1.2,
      "grad_norm": 6.21875,
      "learning_rate": 4.626457563735034e-06,
      "log_odds_chosen": 1.2489228248596191,
      "log_odds_ratio": -0.30081456899642944,
      "logits/chosen": -2.8132882118225098,
      "logits/rejected": -2.8195571899414062,
      "logps/chosen": -0.501319408416748,
      "logps/rejected": -1.1651036739349365,
      "loss": 0.3777,
      "nll_loss": 0.3397257328033447,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.10026389360427856,
      "rewards/margins": 0.13275685906410217,
      "rewards/rejected": -0.23302075266838074,
      "step": 1680
    },
    {
      "epoch": 1.21,
      "grad_norm": 6.5,
      "learning_rate": 4.624478330462108e-06,
      "log_odds_chosen": 1.2629085779190063,
      "log_odds_ratio": -0.3190768361091614,
      "logits/chosen": -2.808572769165039,
      "logits/rejected": -2.8040456771850586,
      "logps/chosen": -0.52824467420578,
      "logps/rejected": -1.2037794589996338,
      "loss": 0.4016,
      "nll_loss": 0.3477191627025604,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.105648934841156,
      "rewards/margins": 0.13510698080062866,
      "rewards/rejected": -0.24075591564178467,
      "step": 1690
    },
    {
      "epoch": 1.21,
      "grad_norm": 5.3125,
      "learning_rate": 4.622501635210244e-06,
      "log_odds_chosen": 1.199979305267334,
      "log_odds_ratio": -0.32586055994033813,
      "logits/chosen": -2.811058759689331,
      "logits/rejected": -2.797663927078247,
      "logps/chosen": -0.5032976269721985,
      "logps/rejected": -1.1406500339508057,
      "loss": 0.412,
      "nll_loss": 0.35429102182388306,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.10065951198339462,
      "rewards/margins": 0.1274704784154892,
      "rewards/rejected": -0.2281300127506256,
      "step": 1700
    },
    {
      "epoch": 1.22,
      "grad_norm": 6.625,
      "learning_rate": 4.62052747255979e-06,
      "log_odds_chosen": 1.2002490758895874,
      "log_odds_ratio": -0.3310433030128479,
      "logits/chosen": -2.8232879638671875,
      "logits/rejected": -2.8157012462615967,
      "logps/chosen": -0.5096195936203003,
      "logps/rejected": -1.1472256183624268,
      "loss": 0.4145,
      "nll_loss": 0.33412426710128784,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.10192392021417618,
      "rewards/margins": 0.12752120196819305,
      "rewards/rejected": -0.22944512963294983,
      "step": 1710
    },
    {
      "epoch": 1.23,
      "grad_norm": 6.25,
      "learning_rate": 4.61855583710729e-06,
      "log_odds_chosen": 1.200430154800415,
      "log_odds_ratio": -0.3242012560367584,
      "logits/chosen": -2.807443141937256,
      "logits/rejected": -2.812464475631714,
      "logps/chosen": -0.4960111975669861,
      "logps/rejected": -1.1150873899459839,
      "loss": 0.3998,
      "nll_loss": 0.3255448639392853,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.0992022305727005,
      "rewards/margins": 0.12381526082754135,
      "rewards/rejected": -0.22301749885082245,
      "step": 1720
    },
    {
      "epoch": 1.24,
      "grad_norm": 6.15625,
      "learning_rate": 4.616586723465408e-06,
      "log_odds_chosen": 1.2508338689804077,
      "log_odds_ratio": -0.31589239835739136,
      "logits/chosen": -2.7934987545013428,
      "logits/rejected": -2.798473358154297,
      "logps/chosen": -0.46048063039779663,
      "logps/rejected": -1.0900535583496094,
      "loss": 0.3718,
      "nll_loss": 0.2973485589027405,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.09209613502025604,
      "rewards/margins": 0.12591460347175598,
      "rewards/rejected": -0.21801073849201202,
      "step": 1730
    },
    {
      "epoch": 1.24,
      "grad_norm": 5.5,
      "learning_rate": 4.614620126262875e-06,
      "log_odds_chosen": 1.2658488750457764,
      "log_odds_ratio": -0.31101447343826294,
      "logits/chosen": -2.7943694591522217,
      "logits/rejected": -2.810157299041748,
      "logps/chosen": -0.5024830102920532,
      "logps/rejected": -1.1770861148834229,
      "loss": 0.3848,
      "nll_loss": 0.3470260202884674,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.10049660503864288,
      "rewards/margins": 0.1349206268787384,
      "rewards/rejected": -0.2354172170162201,
      "step": 1740
    },
    {
      "epoch": 1.25,
      "grad_norm": 5.71875,
      "learning_rate": 4.6126560401444256e-06,
      "log_odds_chosen": 1.2377068996429443,
      "log_odds_ratio": -0.32577404379844666,
      "logits/chosen": -2.8102307319641113,
      "logits/rejected": -2.816279411315918,
      "logps/chosen": -0.4699520170688629,
      "logps/rejected": -1.1359916925430298,
      "loss": 0.3875,
      "nll_loss": 0.3352469503879547,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.09399040788412094,
      "rewards/margins": 0.13320791721343994,
      "rewards/rejected": -0.22719831764698029,
      "step": 1750
    },
    {
      "epoch": 1.26,
      "grad_norm": 5.3125,
      "learning_rate": 4.610694459770736e-06,
      "log_odds_chosen": 1.142558217048645,
      "log_odds_ratio": -0.3530025780200958,
      "logits/chosen": -2.8239128589630127,
      "logits/rejected": -2.8176794052124023,
      "logps/chosen": -0.5302969217300415,
      "logps/rejected": -1.1179604530334473,
      "loss": 0.424,
      "nll_loss": 0.36241117119789124,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.10605937242507935,
      "rewards/margins": 0.11753270775079727,
      "rewards/rejected": -0.22359208762645721,
      "step": 1760
    },
    {
      "epoch": 1.26,
      "grad_norm": 6.71875,
      "learning_rate": 4.6087353798183585e-06,
      "log_odds_chosen": 1.2016445398330688,
      "log_odds_ratio": -0.338972806930542,
      "logits/chosen": -2.8231258392333984,
      "logits/rejected": -2.8218886852264404,
      "logps/chosen": -0.49100199341773987,
      "logps/rejected": -1.0961363315582275,
      "loss": 0.3851,
      "nll_loss": 0.34230172634124756,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.09820040315389633,
      "rewards/margins": 0.1210268959403038,
      "rewards/rejected": -0.21922728419303894,
      "step": 1770
    },
    {
      "epoch": 1.27,
      "grad_norm": 8.0,
      "learning_rate": 4.606778794979673e-06,
      "log_odds_chosen": 1.2381914854049683,
      "log_odds_ratio": -0.32014578580856323,
      "logits/chosen": -2.7523140907287598,
      "logits/rejected": -2.769972801208496,
      "logps/chosen": -0.48253411054611206,
      "logps/rejected": -1.137094259262085,
      "loss": 0.414,
      "nll_loss": 0.3594844937324524,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.09650683403015137,
      "rewards/margins": 0.13091202080249786,
      "rewards/rejected": -0.22741885483264923,
      "step": 1780
    },
    {
      "epoch": 1.28,
      "grad_norm": 5.375,
      "learning_rate": 4.604824699962814e-06,
      "log_odds_chosen": 1.2986476421356201,
      "log_odds_ratio": -0.31768113374710083,
      "logits/chosen": -2.8044631481170654,
      "logits/rejected": -2.8003344535827637,
      "logps/chosen": -0.48466992378234863,
      "logps/rejected": -1.1684885025024414,
      "loss": 0.4,
      "nll_loss": 0.32119065523147583,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09693397581577301,
      "rewards/margins": 0.13676372170448303,
      "rewards/rejected": -0.23369769752025604,
      "step": 1790
    },
    {
      "epoch": 1.29,
      "grad_norm": 5.75,
      "learning_rate": 4.602873089491618e-06,
      "log_odds_chosen": 1.3184231519699097,
      "log_odds_ratio": -0.3061246871948242,
      "logits/chosen": -2.7834739685058594,
      "logits/rejected": -2.7912349700927734,
      "logps/chosen": -0.4672764837741852,
      "logps/rejected": -1.1388607025146484,
      "loss": 0.3609,
      "nll_loss": 0.312313973903656,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.09345529973506927,
      "rewards/margins": 0.1343168318271637,
      "rewards/rejected": -0.22777214646339417,
      "step": 1800
    },
    {
      "epoch": 1.29,
      "grad_norm": 5.40625,
      "learning_rate": 4.600923958305558e-06,
      "log_odds_chosen": 1.221590280532837,
      "log_odds_ratio": -0.325833261013031,
      "logits/chosen": -2.7938928604125977,
      "logits/rejected": -2.787433624267578,
      "logps/chosen": -0.49871402978897095,
      "logps/rejected": -1.1466445922851562,
      "loss": 0.391,
      "nll_loss": 0.3510339856147766,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.09974280744791031,
      "rewards/margins": 0.1295861303806305,
      "rewards/rejected": -0.2293289452791214,
      "step": 1810
    },
    {
      "epoch": 1.3,
      "grad_norm": 6.0,
      "learning_rate": 4.59897730115969e-06,
      "log_odds_chosen": 1.3827050924301147,
      "log_odds_ratio": -0.30398255586624146,
      "logits/chosen": -2.811122417449951,
      "logits/rejected": -2.817436933517456,
      "logps/chosen": -0.4758991599082947,
      "logps/rejected": -1.212922215461731,
      "loss": 0.3903,
      "nll_loss": 0.32494717836380005,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.09517984092235565,
      "rewards/margins": 0.14740462601184845,
      "rewards/rejected": -0.2425844967365265,
      "step": 1820
    },
    {
      "epoch": 1.31,
      "grad_norm": 5.5625,
      "learning_rate": 4.597033112824591e-06,
      "log_odds_chosen": 1.2106988430023193,
      "log_odds_ratio": -0.3271104395389557,
      "logits/chosen": -2.798574924468994,
      "logits/rejected": -2.78853702545166,
      "logps/chosen": -0.5198577642440796,
      "logps/rejected": -1.1642177104949951,
      "loss": 0.3965,
      "nll_loss": 0.3389652669429779,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.10397156327962875,
      "rewards/margins": 0.12887199223041534,
      "rewards/rejected": -0.2328435480594635,
      "step": 1830
    },
    {
      "epoch": 1.31,
      "grad_norm": 6.21875,
      "learning_rate": 4.595091388086298e-06,
      "log_odds_chosen": 1.1952093839645386,
      "log_odds_ratio": -0.3166283071041107,
      "logits/chosen": -2.8241782188415527,
      "logits/rejected": -2.8152718544006348,
      "logps/chosen": -0.5200030207633972,
      "logps/rejected": -1.1538721323013306,
      "loss": 0.3864,
      "nll_loss": 0.30587464570999146,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.10400060564279556,
      "rewards/margins": 0.12677384912967682,
      "rewards/rejected": -0.23077444732189178,
      "step": 1840
    },
    {
      "epoch": 1.32,
      "grad_norm": 6.96875,
      "learning_rate": 4.593152121746254e-06,
      "log_odds_chosen": 1.168222427368164,
      "log_odds_ratio": -0.3400697410106659,
      "logits/chosen": -2.816959857940674,
      "logits/rejected": -2.813157081604004,
      "logps/chosen": -0.4970974326133728,
      "logps/rejected": -1.108543038368225,
      "loss": 0.411,
      "nll_loss": 0.31471529603004456,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.0994194895029068,
      "rewards/margins": 0.12228912115097046,
      "rewards/rejected": -0.22170861065387726,
      "step": 1850
    },
    {
      "epoch": 1.33,
      "grad_norm": 5.59375,
      "learning_rate": 4.591215308621242e-06,
      "log_odds_chosen": 1.2203196287155151,
      "log_odds_ratio": -0.3189007639884949,
      "logits/chosen": -2.8120200634002686,
      "logits/rejected": -2.791141986846924,
      "logps/chosen": -0.4980306029319763,
      "logps/rejected": -1.1655288934707642,
      "loss": 0.3934,
      "nll_loss": 0.31481805443763733,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.09960611909627914,
      "rewards/margins": 0.1334996372461319,
      "rewards/rejected": -0.23310574889183044,
      "step": 1860
    },
    {
      "epoch": 1.34,
      "grad_norm": 6.25,
      "learning_rate": 4.5892809435433355e-06,
      "log_odds_chosen": 1.245577096939087,
      "log_odds_ratio": -0.32619625329971313,
      "logits/chosen": -2.8177731037139893,
      "logits/rejected": -2.8102052211761475,
      "logps/chosen": -0.4943556785583496,
      "logps/rejected": -1.1462085247039795,
      "loss": 0.3925,
      "nll_loss": 0.3532278537750244,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.0988711342215538,
      "rewards/margins": 0.13037055730819702,
      "rewards/rejected": -0.2292417287826538,
      "step": 1870
    },
    {
      "epoch": 1.34,
      "grad_norm": 6.09375,
      "learning_rate": 4.587349021359836e-06,
      "log_odds_chosen": 1.1849457025527954,
      "log_odds_ratio": -0.3331480026245117,
      "logits/chosen": -2.8479719161987305,
      "logits/rejected": -2.8441169261932373,
      "logps/chosen": -0.5096966624259949,
      "logps/rejected": -1.1546456813812256,
      "loss": 0.4052,
      "nll_loss": 0.32981541752815247,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.10193934291601181,
      "rewards/margins": 0.1289898157119751,
      "rewards/rejected": -0.23092913627624512,
      "step": 1880
    },
    {
      "epoch": 1.35,
      "grad_norm": 5.75,
      "learning_rate": 4.585419536933215e-06,
      "log_odds_chosen": 1.235563039779663,
      "log_odds_ratio": -0.3173638880252838,
      "logits/chosen": -2.8725390434265137,
      "logits/rejected": -2.859806776046753,
      "logps/chosen": -0.4742043614387512,
      "logps/rejected": -1.1231261491775513,
      "loss": 0.4115,
      "nll_loss": 0.365268349647522,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.0948408767580986,
      "rewards/margins": 0.12978434562683105,
      "rewards/rejected": -0.22462522983551025,
      "step": 1890
    },
    {
      "epoch": 1.36,
      "grad_norm": 5.4375,
      "learning_rate": 4.583492485141056e-06,
      "log_odds_chosen": 1.3153822422027588,
      "log_odds_ratio": -0.3131297528743744,
      "logits/chosen": -2.783536434173584,
      "logits/rejected": -2.803969144821167,
      "logps/chosen": -0.4793526530265808,
      "logps/rejected": -1.1401029825210571,
      "loss": 0.3573,
      "nll_loss": 0.2991887629032135,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.09587053954601288,
      "rewards/margins": 0.1321500688791275,
      "rewards/rejected": -0.2280205935239792,
      "step": 1900
    },
    {
      "epoch": 1.36,
      "grad_norm": 6.25,
      "learning_rate": 4.581567860876004e-06,
      "log_odds_chosen": 1.2473056316375732,
      "log_odds_ratio": -0.3080199658870697,
      "logits/chosen": -2.8462700843811035,
      "logits/rejected": -2.8370962142944336,
      "logps/chosen": -0.48333874344825745,
      "logps/rejected": -1.148425817489624,
      "loss": 0.4128,
      "nll_loss": 0.3549380302429199,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.09666775166988373,
      "rewards/margins": 0.13301745057106018,
      "rewards/rejected": -0.2296852171421051,
      "step": 1910
    },
    {
      "epoch": 1.37,
      "grad_norm": 6.75,
      "learning_rate": 4.579645659045699e-06,
      "log_odds_chosen": 1.227070689201355,
      "log_odds_ratio": -0.3165205121040344,
      "logits/chosen": -2.8399529457092285,
      "logits/rejected": -2.8465752601623535,
      "logps/chosen": -0.46384549140930176,
      "logps/rejected": -1.0851547718048096,
      "loss": 0.401,
      "nll_loss": 0.3482987880706787,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.09276910126209259,
      "rewards/margins": 0.12426185607910156,
      "rewards/rejected": -0.21703095734119415,
      "step": 1920
    },
    {
      "epoch": 1.38,
      "grad_norm": 4.46875,
      "learning_rate": 4.577725874572724e-06,
      "log_odds_chosen": 1.1611140966415405,
      "log_odds_ratio": -0.34230470657348633,
      "logits/chosen": -2.807835340499878,
      "logits/rejected": -2.817835807800293,
      "logps/chosen": -0.49586695432662964,
      "logps/rejected": -1.093207836151123,
      "loss": 0.3975,
      "nll_loss": 0.31580257415771484,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.0991733968257904,
      "rewards/margins": 0.11946818977594376,
      "rewards/rejected": -0.21864160895347595,
      "step": 1930
    },
    {
      "epoch": 1.39,
      "grad_norm": 6.21875,
      "learning_rate": 4.575808502394551e-06,
      "log_odds_chosen": 1.2263875007629395,
      "log_odds_ratio": -0.3336968421936035,
      "logits/chosen": -2.8074028491973877,
      "logits/rejected": -2.808117628097534,
      "logps/chosen": -0.5058302879333496,
      "logps/rejected": -1.1664202213287354,
      "loss": 0.3892,
      "nll_loss": 0.32398825883865356,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.10116605460643768,
      "rewards/margins": 0.13211797177791595,
      "rewards/rejected": -0.23328404128551483,
      "step": 1940
    },
    {
      "epoch": 1.39,
      "grad_norm": 5.65625,
      "learning_rate": 4.573893537463482e-06,
      "log_odds_chosen": 1.1256129741668701,
      "log_odds_ratio": -0.3365449905395508,
      "logits/chosen": -2.799619197845459,
      "logits/rejected": -2.8067538738250732,
      "logps/chosen": -0.501841127872467,
      "logps/rejected": -1.0864284038543701,
      "loss": 0.3919,
      "nll_loss": 0.3244914412498474,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.10036821663379669,
      "rewards/margins": 0.11691747605800629,
      "rewards/rejected": -0.21728567779064178,
      "step": 1950
    },
    {
      "epoch": 1.4,
      "grad_norm": 7.71875,
      "learning_rate": 4.5719809747465946e-06,
      "log_odds_chosen": 1.2488094568252563,
      "log_odds_ratio": -0.3095954358577728,
      "logits/chosen": -2.80102801322937,
      "logits/rejected": -2.786306381225586,
      "logps/chosen": -0.48401910066604614,
      "logps/rejected": -1.1524279117584229,
      "loss": 0.391,
      "nll_loss": 0.32259485125541687,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.09680382162332535,
      "rewards/margins": 0.1336817443370819,
      "rewards/rejected": -0.23048558831214905,
      "step": 1960
    },
    {
      "epoch": 1.41,
      "grad_norm": 6.65625,
      "learning_rate": 4.570070809225682e-06,
      "log_odds_chosen": 1.185505986213684,
      "log_odds_ratio": -0.3236411213874817,
      "logits/chosen": -2.8083627223968506,
      "logits/rejected": -2.8073079586029053,
      "logps/chosen": -0.497401624917984,
      "logps/rejected": -1.1157503128051758,
      "loss": 0.3984,
      "nll_loss": 0.32068413496017456,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.09948031604290009,
      "rewards/margins": 0.12366974353790283,
      "rewards/rejected": -0.2231500893831253,
      "step": 1970
    },
    {
      "epoch": 1.41,
      "grad_norm": 7.0625,
      "learning_rate": 4.568163035897205e-06,
      "log_odds_chosen": 1.2525742053985596,
      "log_odds_ratio": -0.31062158942222595,
      "logits/chosen": -2.852297782897949,
      "logits/rejected": -2.858063220977783,
      "logps/chosen": -0.5127692222595215,
      "logps/rejected": -1.180938482284546,
      "loss": 0.4053,
      "nll_loss": 0.36488935351371765,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.10255385935306549,
      "rewards/margins": 0.1336338222026825,
      "rewards/rejected": -0.23618769645690918,
      "step": 1980
    },
    {
      "epoch": 1.42,
      "grad_norm": 6.5625,
      "learning_rate": 4.566257649772231e-06,
      "log_odds_chosen": 1.2319920063018799,
      "log_odds_ratio": -0.32411015033721924,
      "logits/chosen": -2.8400864601135254,
      "logits/rejected": -2.835186004638672,
      "logps/chosen": -0.5001600384712219,
      "logps/rejected": -1.1353777647018433,
      "loss": 0.4052,
      "nll_loss": 0.346365749835968,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.10003199428319931,
      "rewards/margins": 0.12704357504844666,
      "rewards/rejected": -0.22707557678222656,
      "step": 1990
    },
    {
      "epoch": 1.43,
      "grad_norm": 5.78125,
      "learning_rate": 4.564354645876385e-06,
      "log_odds_chosen": 1.2825465202331543,
      "log_odds_ratio": -0.31023693084716797,
      "logits/chosen": -2.7589869499206543,
      "logits/rejected": -2.7817113399505615,
      "logps/chosen": -0.48962241411209106,
      "logps/rejected": -1.1758294105529785,
      "loss": 0.3808,
      "nll_loss": 0.3050033152103424,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.09792448580265045,
      "rewards/margins": 0.137241393327713,
      "rewards/rejected": -0.23516587913036346,
      "step": 2000
    },
    {
      "epoch": 1.44,
      "grad_norm": 6.5625,
      "learning_rate": 4.562454019249786e-06,
      "log_odds_chosen": 1.1062853336334229,
      "log_odds_ratio": -0.34708550572395325,
      "logits/chosen": -2.8106868267059326,
      "logits/rejected": -2.802447557449341,
      "logps/chosen": -0.5518588423728943,
      "logps/rejected": -1.1463332176208496,
      "loss": 0.4399,
      "nll_loss": 0.3327658772468567,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.11037178337574005,
      "rewards/margins": 0.11889486014842987,
      "rewards/rejected": -0.22926661372184753,
      "step": 2010
    },
    {
      "epoch": 1.44,
      "grad_norm": 6.65625,
      "learning_rate": 4.560555764947004e-06,
      "log_odds_chosen": 1.2340061664581299,
      "log_odds_ratio": -0.32612934708595276,
      "logits/chosen": -2.822478771209717,
      "logits/rejected": -2.828272819519043,
      "logps/chosen": -0.5015519857406616,
      "logps/rejected": -1.164368987083435,
      "loss": 0.3998,
      "nll_loss": 0.37030792236328125,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.10031040757894516,
      "rewards/margins": 0.13256338238716125,
      "rewards/rejected": -0.2328738272190094,
      "step": 2020
    },
    {
      "epoch": 1.45,
      "grad_norm": 5.5625,
      "learning_rate": 4.5586598780369954e-06,
      "log_odds_chosen": 1.1390058994293213,
      "log_odds_ratio": -0.35169175267219543,
      "logits/chosen": -2.8428843021392822,
      "logits/rejected": -2.846796989440918,
      "logps/chosen": -0.5042189359664917,
      "logps/rejected": -1.092398762702942,
      "loss": 0.3971,
      "nll_loss": 0.3557208478450775,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.10084378719329834,
      "rewards/margins": 0.11763594299554825,
      "rewards/rejected": -0.21847975254058838,
      "step": 2030
    },
    {
      "epoch": 1.46,
      "grad_norm": 6.5625,
      "learning_rate": 4.556766353603058e-06,
      "log_odds_chosen": 1.2142447233200073,
      "log_odds_ratio": -0.3221642076969147,
      "logits/chosen": -2.7607929706573486,
      "logits/rejected": -2.769547939300537,
      "logps/chosen": -0.5120490789413452,
      "logps/rejected": -1.1542648077011108,
      "loss": 0.4132,
      "nll_loss": 0.31557202339172363,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.10240981727838516,
      "rewards/margins": 0.1284431517124176,
      "rewards/rejected": -0.23085296154022217,
      "step": 2040
    },
    {
      "epoch": 1.46,
      "grad_norm": 6.46875,
      "learning_rate": 4.55487518674277e-06,
      "log_odds_chosen": 1.2244555950164795,
      "log_odds_ratio": -0.3199111521244049,
      "logits/chosen": -2.848184108734131,
      "logits/rejected": -2.853646755218506,
      "logps/chosen": -0.4814947545528412,
      "logps/rejected": -1.1112531423568726,
      "loss": 0.4191,
      "nll_loss": 0.33359044790267944,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.0962989553809166,
      "rewards/margins": 0.12595167756080627,
      "rewards/rejected": -0.22225065529346466,
      "step": 2050
    },
    {
      "epoch": 1.47,
      "grad_norm": 5.34375,
      "learning_rate": 4.552986372567943e-06,
      "log_odds_chosen": 1.2204258441925049,
      "log_odds_ratio": -0.3285110592842102,
      "logits/chosen": -2.826977252960205,
      "logits/rejected": -2.8170952796936035,
      "logps/chosen": -0.518255352973938,
      "logps/rejected": -1.1798815727233887,
      "loss": 0.3758,
      "nll_loss": 0.335254967212677,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.10365106910467148,
      "rewards/margins": 0.13232523202896118,
      "rewards/rejected": -0.23597629368305206,
      "step": 2060
    },
    {
      "epoch": 1.48,
      "grad_norm": 5.65625,
      "learning_rate": 4.5510999062045625e-06,
      "log_odds_chosen": 1.268137812614441,
      "log_odds_ratio": -0.3236086964607239,
      "logits/chosen": -2.7644591331481934,
      "logits/rejected": -2.761575222015381,
      "logps/chosen": -0.5131622552871704,
      "logps/rejected": -1.2004690170288086,
      "loss": 0.3893,
      "nll_loss": 0.33567947149276733,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.10263246297836304,
      "rewards/margins": 0.1374613493680954,
      "rewards/rejected": -0.24009379744529724,
      "step": 2070
    },
    {
      "epoch": 1.49,
      "grad_norm": 6.09375,
      "learning_rate": 4.5492157827927435e-06,
      "log_odds_chosen": 1.2079408168792725,
      "log_odds_ratio": -0.33384907245635986,
      "logits/chosen": -2.86165714263916,
      "logits/rejected": -2.8367888927459717,
      "logps/chosen": -0.4977470338344574,
      "logps/rejected": -1.1325632333755493,
      "loss": 0.3736,
      "nll_loss": 0.32109367847442627,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.09954941272735596,
      "rewards/margins": 0.12696322798728943,
      "rewards/rejected": -0.22651264071464539,
      "step": 2080
    },
    {
      "epoch": 1.49,
      "grad_norm": 7.21875,
      "learning_rate": 4.54733399748667e-06,
      "log_odds_chosen": 1.2205212116241455,
      "log_odds_ratio": -0.3305164873600006,
      "logits/chosen": -2.7856006622314453,
      "logits/rejected": -2.7885613441467285,
      "logps/chosen": -0.5106269717216492,
      "logps/rejected": -1.1481513977050781,
      "loss": 0.3855,
      "nll_loss": 0.30324143171310425,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.10212540626525879,
      "rewards/margins": 0.12750491499900818,
      "rewards/rejected": -0.22963032126426697,
      "step": 2090
    },
    {
      "epoch": 1.5,
      "grad_norm": 6.75,
      "learning_rate": 4.5454545454545455e-06,
      "log_odds_chosen": 1.2237420082092285,
      "log_odds_ratio": -0.32826218008995056,
      "logits/chosen": -2.820686101913452,
      "logits/rejected": -2.8283371925354004,
      "logps/chosen": -0.5093476176261902,
      "logps/rejected": -1.168192744255066,
      "loss": 0.3887,
      "nll_loss": 0.3230911195278168,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.10186950862407684,
      "rewards/margins": 0.13176901638507843,
      "rewards/rejected": -0.23363855481147766,
      "step": 2100
    },
    {
      "epoch": 1.51,
      "grad_norm": 6.3125,
      "learning_rate": 4.543577421878542e-06,
      "log_odds_chosen": 1.2703464031219482,
      "log_odds_ratio": -0.31365537643432617,
      "logits/chosen": -2.851209878921509,
      "logits/rejected": -2.8478894233703613,
      "logps/chosen": -0.4876154363155365,
      "logps/rejected": -1.1669528484344482,
      "loss": 0.4003,
      "nll_loss": 0.32528436183929443,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.09752309322357178,
      "rewards/margins": 0.13586744666099548,
      "rewards/rejected": -0.23339056968688965,
      "step": 2110
    },
    {
      "epoch": 1.51,
      "grad_norm": 6.46875,
      "learning_rate": 4.541702621954749e-06,
      "log_odds_chosen": 1.1352488994598389,
      "log_odds_ratio": -0.35285863280296326,
      "logits/chosen": -2.8669912815093994,
      "logits/rejected": -2.872535228729248,
      "logps/chosen": -0.5130313634872437,
      "logps/rejected": -1.1068975925445557,
      "loss": 0.4053,
      "nll_loss": 0.33999350666999817,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.10260625928640366,
      "rewards/margins": 0.11877324432134628,
      "rewards/rejected": -0.22137951850891113,
      "step": 2120
    },
    {
      "epoch": 1.52,
      "grad_norm": 5.71875,
      "learning_rate": 4.539830140893113e-06,
      "log_odds_chosen": 1.4157606363296509,
      "log_odds_ratio": -0.30813926458358765,
      "logits/chosen": -2.8120830059051514,
      "logits/rejected": -2.8225760459899902,
      "logps/chosen": -0.4803242087364197,
      "logps/rejected": -1.2175981998443604,
      "loss": 0.3963,
      "nll_loss": 0.3383215367794037,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.09606485068798065,
      "rewards/margins": 0.14745476841926575,
      "rewards/rejected": -0.2435196191072464,
      "step": 2130
    },
    {
      "epoch": 1.53,
      "grad_norm": 7.625,
      "learning_rate": 4.537959973917404e-06,
      "log_odds_chosen": 1.2107295989990234,
      "log_odds_ratio": -0.3288795053958893,
      "logits/chosen": -2.809863567352295,
      "logits/rejected": -2.8031539916992188,
      "logps/chosen": -0.49784055352211,
      "logps/rejected": -1.1504080295562744,
      "loss": 0.3939,
      "nll_loss": 0.3284316062927246,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.09956810623407364,
      "rewards/margins": 0.1305135190486908,
      "rewards/rejected": -0.23008163273334503,
      "step": 2140
    },
    {
      "epoch": 1.54,
      "grad_norm": 7.46875,
      "learning_rate": 4.536092116265145e-06,
      "log_odds_chosen": 1.1781766414642334,
      "log_odds_ratio": -0.33598729968070984,
      "logits/chosen": -2.8239569664001465,
      "logits/rejected": -2.8153862953186035,
      "logps/chosen": -0.5003775954246521,
      "logps/rejected": -1.1214288473129272,
      "loss": 0.3922,
      "nll_loss": 0.3246839940547943,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.10007552057504654,
      "rewards/margins": 0.12421026080846786,
      "rewards/rejected": -0.2242857962846756,
      "step": 2150
    },
    {
      "epoch": 1.54,
      "grad_norm": 6.46875,
      "learning_rate": 4.534226563187573e-06,
      "log_odds_chosen": 1.1511073112487793,
      "log_odds_ratio": -0.3392142653465271,
      "logits/chosen": -2.8518738746643066,
      "logits/rejected": -2.854708671569824,
      "logps/chosen": -0.5298787355422974,
      "logps/rejected": -1.1405723094940186,
      "loss": 0.3987,
      "nll_loss": 0.37272533774375916,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10597574710845947,
      "rewards/margins": 0.12213869392871857,
      "rewards/rejected": -0.22811445593833923,
      "step": 2160
    },
    {
      "epoch": 1.55,
      "grad_norm": 6.3125,
      "learning_rate": 4.532363309949585e-06,
      "log_odds_chosen": 1.0746400356292725,
      "log_odds_ratio": -0.3582301139831543,
      "logits/chosen": -2.7901675701141357,
      "logits/rejected": -2.7982590198516846,
      "logps/chosen": -0.5267711877822876,
      "logps/rejected": -1.0863219499588013,
      "loss": 0.4085,
      "nll_loss": 0.3383466601371765,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10535424947738647,
      "rewards/margins": 0.1119101420044899,
      "rewards/rejected": -0.21726438403129578,
      "step": 2170
    },
    {
      "epoch": 1.56,
      "grad_norm": 7.375,
      "learning_rate": 4.530502351829687e-06,
      "log_odds_chosen": 1.2173881530761719,
      "log_odds_ratio": -0.33332520723342896,
      "logits/chosen": -2.816040515899658,
      "logits/rejected": -2.8241238594055176,
      "logps/chosen": -0.49707546830177307,
      "logps/rejected": -1.1558698415756226,
      "loss": 0.3933,
      "nll_loss": 0.29985347390174866,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.09941510856151581,
      "rewards/margins": 0.13175883889198303,
      "rewards/rejected": -0.23117394745349884,
      "step": 2180
    },
    {
      "epoch": 1.56,
      "grad_norm": 6.0,
      "learning_rate": 4.528643684119943e-06,
      "log_odds_chosen": 1.2386395931243896,
      "log_odds_ratio": -0.3029495179653168,
      "logits/chosen": -2.780738115310669,
      "logits/rejected": -2.78902530670166,
      "logps/chosen": -0.49545398354530334,
      "logps/rejected": -1.158929705619812,
      "loss": 0.3934,
      "nll_loss": 0.3328419327735901,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.09909079968929291,
      "rewards/margins": 0.13269512355327606,
      "rewards/rejected": -0.23178592324256897,
      "step": 2190
    },
    {
      "epoch": 1.57,
      "grad_norm": 7.34375,
      "learning_rate": 4.526787302125927e-06,
      "log_odds_chosen": 1.2402592897415161,
      "log_odds_ratio": -0.3278099596500397,
      "logits/chosen": -2.8122589588165283,
      "logits/rejected": -2.816236972808838,
      "logps/chosen": -0.5384883284568787,
      "logps/rejected": -1.2339115142822266,
      "loss": 0.4106,
      "nll_loss": 0.3580947518348694,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.10769768059253693,
      "rewards/margins": 0.1390846073627472,
      "rewards/rejected": -0.2467823028564453,
      "step": 2200
    },
    {
      "epoch": 1.58,
      "grad_norm": 6.125,
      "learning_rate": 4.524933201166673e-06,
      "log_odds_chosen": 1.1136146783828735,
      "log_odds_ratio": -0.3713721036911011,
      "logits/chosen": -2.8021061420440674,
      "logits/rejected": -2.813220739364624,
      "logps/chosen": -0.5083444714546204,
      "logps/rejected": -1.091552495956421,
      "loss": 0.4274,
      "nll_loss": 0.36474576592445374,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.10166887938976288,
      "rewards/margins": 0.11664160341024399,
      "rewards/rejected": -0.21831050515174866,
      "step": 2210
    },
    {
      "epoch": 1.59,
      "grad_norm": 8.125,
      "learning_rate": 4.523081376574626e-06,
      "log_odds_chosen": 1.2216575145721436,
      "log_odds_ratio": -0.3291280269622803,
      "logits/chosen": -2.777296304702759,
      "logits/rejected": -2.7747302055358887,
      "logps/chosen": -0.5027607679367065,
      "logps/rejected": -1.1558706760406494,
      "loss": 0.4256,
      "nll_loss": 0.3766383230686188,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.10055215656757355,
      "rewards/margins": 0.1306219846010208,
      "rewards/rejected": -0.23117414116859436,
      "step": 2220
    },
    {
      "epoch": 1.59,
      "grad_norm": 5.875,
      "learning_rate": 4.521231823695586e-06,
      "log_odds_chosen": 1.131630539894104,
      "log_odds_ratio": -0.34842541813850403,
      "logits/chosen": -2.8320116996765137,
      "logits/rejected": -2.8329334259033203,
      "logps/chosen": -0.5016427636146545,
      "logps/rejected": -1.0832736492156982,
      "loss": 0.3947,
      "nll_loss": 0.3312470316886902,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10032854229211807,
      "rewards/margins": 0.11632619798183441,
      "rewards/rejected": -0.21665474772453308,
      "step": 2230
    },
    {
      "epoch": 1.6,
      "grad_norm": 6.4375,
      "learning_rate": 4.519384537888671e-06,
      "log_odds_chosen": 1.2074496746063232,
      "log_odds_ratio": -0.341810017824173,
      "logits/chosen": -2.805079698562622,
      "logits/rejected": -2.7929458618164062,
      "logps/chosen": -0.4954722821712494,
      "logps/rejected": -1.1329338550567627,
      "loss": 0.4051,
      "nll_loss": 0.3333224356174469,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.099094457924366,
      "rewards/margins": 0.12749230861663818,
      "rewards/rejected": -0.2265867441892624,
      "step": 2240
    },
    {
      "epoch": 1.61,
      "grad_norm": 6.59375,
      "learning_rate": 4.517539514526257e-06,
      "log_odds_chosen": 1.3590493202209473,
      "log_odds_ratio": -0.316969096660614,
      "logits/chosen": -2.803619861602783,
      "logits/rejected": -2.77778959274292,
      "logps/chosen": -0.5187050700187683,
      "logps/rejected": -1.2949590682983398,
      "loss": 0.4183,
      "nll_loss": 0.34726181626319885,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.10374102741479874,
      "rewards/margins": 0.15525080263614655,
      "rewards/rejected": -0.2589918076992035,
      "step": 2250
    },
    {
      "epoch": 1.61,
      "grad_norm": 6.375,
      "learning_rate": 4.515696748993935e-06,
      "log_odds_chosen": 1.159447193145752,
      "log_odds_ratio": -0.3347181975841522,
      "logits/chosen": -2.809387683868408,
      "logits/rejected": -2.8114075660705566,
      "logps/chosen": -0.5045318007469177,
      "logps/rejected": -1.105101466178894,
      "loss": 0.4172,
      "nll_loss": 0.3646569848060608,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.1009063571691513,
      "rewards/margins": 0.12011395394802094,
      "rewards/rejected": -0.22102029621601105,
      "step": 2260
    },
    {
      "epoch": 1.62,
      "grad_norm": 6.53125,
      "learning_rate": 4.513856236690462e-06,
      "log_odds_chosen": 1.1499160528182983,
      "log_odds_ratio": -0.34458065032958984,
      "logits/chosen": -2.81234073638916,
      "logits/rejected": -2.8134610652923584,
      "logps/chosen": -0.5100609660148621,
      "logps/rejected": -1.1153619289398193,
      "loss": 0.4097,
      "nll_loss": 0.3220939040184021,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.10201219469308853,
      "rewards/margins": 0.12106022983789444,
      "rewards/rejected": -0.22307243943214417,
      "step": 2270
    },
    {
      "epoch": 1.63,
      "grad_norm": 6.75,
      "learning_rate": 4.51201797302771e-06,
      "log_odds_chosen": 1.307038426399231,
      "log_odds_ratio": -0.32670050859451294,
      "logits/chosen": -2.855238676071167,
      "logits/rejected": -2.830141544342041,
      "logps/chosen": -0.5477741360664368,
      "logps/rejected": -1.310255765914917,
      "loss": 0.4111,
      "nll_loss": 0.3261423110961914,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.10955484211444855,
      "rewards/margins": 0.152496337890625,
      "rewards/rejected": -0.26205116510391235,
      "step": 2280
    },
    {
      "epoch": 1.64,
      "grad_norm": 7.5,
      "learning_rate": 4.510181953430622e-06,
      "log_odds_chosen": 1.170881986618042,
      "log_odds_ratio": -0.3537355363368988,
      "logits/chosen": -2.7898900508880615,
      "logits/rejected": -2.8005497455596924,
      "logps/chosen": -0.5161569714546204,
      "logps/rejected": -1.1166584491729736,
      "loss": 0.4116,
      "nll_loss": 0.33776649832725525,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.10323138535022736,
      "rewards/margins": 0.12010029703378677,
      "rewards/rejected": -0.22333166003227234,
      "step": 2290
    },
    {
      "epoch": 1.64,
      "grad_norm": 6.59375,
      "learning_rate": 4.508348173337162e-06,
      "log_odds_chosen": 1.1670069694519043,
      "log_odds_ratio": -0.3370336890220642,
      "logits/chosen": -2.8468751907348633,
      "logits/rejected": -2.8393032550811768,
      "logps/chosen": -0.5258538722991943,
      "logps/rejected": -1.1647305488586426,
      "loss": 0.4118,
      "nll_loss": 0.3415473699569702,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10517077147960663,
      "rewards/margins": 0.12777534127235413,
      "rewards/rejected": -0.23294611275196075,
      "step": 2300
    },
    {
      "epoch": 1.65,
      "grad_norm": 7.21875,
      "learning_rate": 4.5065166281982656e-06,
      "log_odds_chosen": 1.2587502002716064,
      "log_odds_ratio": -0.31747183203697205,
      "logits/chosen": -2.833258867263794,
      "logits/rejected": -2.828791379928589,
      "logps/chosen": -0.5313466191291809,
      "logps/rejected": -1.2055209875106812,
      "loss": 0.3804,
      "nll_loss": 0.3351854085922241,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.10626931488513947,
      "rewards/margins": 0.1348349153995514,
      "rewards/rejected": -0.24110420048236847,
      "step": 2310
    },
    {
      "epoch": 1.66,
      "grad_norm": 6.375,
      "learning_rate": 4.5046873134777955e-06,
      "log_odds_chosen": 1.1143945455551147,
      "log_odds_ratio": -0.3569260239601135,
      "logits/chosen": -2.8429629802703857,
      "logits/rejected": -2.8426883220672607,
      "logps/chosen": -0.5169599652290344,
      "logps/rejected": -1.0884437561035156,
      "loss": 0.4024,
      "nll_loss": 0.3241182565689087,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10339200496673584,
      "rewards/margins": 0.11429674923419952,
      "rewards/rejected": -0.21768875420093536,
      "step": 2320
    },
    {
      "epoch": 1.66,
      "grad_norm": 6.125,
      "learning_rate": 4.5028602246524934e-06,
      "log_odds_chosen": 1.1783859729766846,
      "log_odds_ratio": -0.3340161442756653,
      "logits/chosen": -2.806804895401001,
      "logits/rejected": -2.7988064289093018,
      "logps/chosen": -0.5388362407684326,
      "logps/rejected": -1.1798697710037231,
      "loss": 0.3924,
      "nll_loss": 0.34538668394088745,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.1077672466635704,
      "rewards/margins": 0.12820670008659363,
      "rewards/rejected": -0.23597395420074463,
      "step": 2330
    },
    {
      "epoch": 1.67,
      "grad_norm": 7.09375,
      "learning_rate": 4.5010353572119344e-06,
      "log_odds_chosen": 1.2434608936309814,
      "log_odds_ratio": -0.3279341459274292,
      "logits/chosen": -2.8133628368377686,
      "logits/rejected": -2.803882360458374,
      "logps/chosen": -0.5466577410697937,
      "logps/rejected": -1.2451903820037842,
      "loss": 0.4016,
      "nll_loss": 0.32733532786369324,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.10933154821395874,
      "rewards/margins": 0.13970652222633362,
      "rewards/rejected": -0.24903810024261475,
      "step": 2340
    },
    {
      "epoch": 1.68,
      "grad_norm": 6.65625,
      "learning_rate": 4.499212706658476e-06,
      "log_odds_chosen": 1.2468292713165283,
      "log_odds_ratio": -0.33085036277770996,
      "logits/chosen": -2.7604432106018066,
      "logits/rejected": -2.767918586730957,
      "logps/chosen": -0.5312229990959167,
      "logps/rejected": -1.232725977897644,
      "loss": 0.3905,
      "nll_loss": 0.28453925251960754,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.10624460875988007,
      "rewards/margins": 0.14030058681964874,
      "rewards/rejected": -0.24654516577720642,
      "step": 2350
    },
    {
      "epoch": 1.69,
      "grad_norm": 6.15625,
      "learning_rate": 4.497392268507216e-06,
      "log_odds_chosen": 1.3811371326446533,
      "log_odds_ratio": -0.3187825083732605,
      "logits/chosen": -2.7945587635040283,
      "logits/rejected": -2.793541431427002,
      "logps/chosen": -0.5095025897026062,
      "logps/rejected": -1.2895538806915283,
      "loss": 0.3888,
      "nll_loss": 0.33172607421875,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.10190053284168243,
      "rewards/margins": 0.15601028501987457,
      "rewards/rejected": -0.2579107880592346,
      "step": 2360
    },
    {
      "epoch": 1.69,
      "grad_norm": 8.1875,
      "learning_rate": 4.495574038285945e-06,
      "log_odds_chosen": 1.2981445789337158,
      "log_odds_ratio": -0.3155536949634552,
      "logits/chosen": -2.8101601600646973,
      "logits/rejected": -2.8108277320861816,
      "logps/chosen": -0.5573509335517883,
      "logps/rejected": -1.3329538106918335,
      "loss": 0.3898,
      "nll_loss": 0.3312874436378479,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.11147018522024155,
      "rewards/margins": 0.1551206111907959,
      "rewards/rejected": -0.26659080386161804,
      "step": 2370
    },
    {
      "epoch": 1.7,
      "grad_norm": 8.5625,
      "learning_rate": 4.493758011535097e-06,
      "log_odds_chosen": 1.2831408977508545,
      "log_odds_ratio": -0.3103974163532257,
      "logits/chosen": -2.823028564453125,
      "logits/rejected": -2.813875436782837,
      "logps/chosen": -0.5119468569755554,
      "logps/rejected": -1.2149946689605713,
      "loss": 0.4083,
      "nll_loss": 0.32668930292129517,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.1023893728852272,
      "rewards/margins": 0.1406095325946808,
      "rewards/rejected": -0.2429989129304886,
      "step": 2380
    },
    {
      "epoch": 1.71,
      "grad_norm": 9.375,
      "learning_rate": 4.491944183807709e-06,
      "log_odds_chosen": 1.1459182500839233,
      "log_odds_ratio": -0.3687919080257416,
      "logits/chosen": -2.850087881088257,
      "logits/rejected": -2.8486053943634033,
      "logps/chosen": -0.5623646974563599,
      "logps/rejected": -1.2140840291976929,
      "loss": 0.3556,
      "nll_loss": 0.285260409116745,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.11247295141220093,
      "rewards/margins": 0.13034388422966003,
      "rewards/rejected": -0.24281683564186096,
      "step": 2390
    },
    {
      "epoch": 1.71,
      "grad_norm": 9.375,
      "learning_rate": 4.490132550669373e-06,
      "log_odds_chosen": 1.240378975868225,
      "log_odds_ratio": -0.3130456507205963,
      "logits/chosen": -2.9485726356506348,
      "logits/rejected": -2.9329330921173096,
      "logps/chosen": -0.5569415092468262,
      "logps/rejected": -1.2546156644821167,
      "loss": 0.4249,
      "nll_loss": 0.3691002428531647,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.11138832569122314,
      "rewards/margins": 0.1395348310470581,
      "rewards/rejected": -0.25092315673828125,
      "step": 2400
    },
    {
      "epoch": 1.72,
      "grad_norm": 7.0625,
      "learning_rate": 4.488323107698186e-06,
      "log_odds_chosen": 1.3868653774261475,
      "log_odds_ratio": -0.2894250750541687,
      "logits/chosen": -2.9048142433166504,
      "logits/rejected": -2.893167018890381,
      "logps/chosen": -0.4791548252105713,
      "logps/rejected": -1.2176661491394043,
      "loss": 0.4143,
      "nll_loss": 0.3569006025791168,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.09583096206188202,
      "rewards/margins": 0.14770226180553436,
      "rewards/rejected": -0.24353322386741638,
      "step": 2410
    },
    {
      "epoch": 1.73,
      "grad_norm": 7.21875,
      "learning_rate": 4.486515850484713e-06,
      "log_odds_chosen": 1.1212716102600098,
      "log_odds_ratio": -0.3596242070198059,
      "logits/chosen": -2.9160571098327637,
      "logits/rejected": -2.9128189086914062,
      "logps/chosen": -0.5376850366592407,
      "logps/rejected": -1.1596351861953735,
      "loss": 0.4265,
      "nll_loss": 0.33928748965263367,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10753700882196426,
      "rewards/margins": 0.12439004331827164,
      "rewards/rejected": -0.2319270670413971,
      "step": 2420
    },
    {
      "epoch": 1.74,
      "grad_norm": 8.5,
      "learning_rate": 4.484710774631934e-06,
      "log_odds_chosen": 1.241854190826416,
      "log_odds_ratio": -0.3230142295360565,
      "logits/chosen": -2.8752593994140625,
      "logits/rejected": -2.8711416721343994,
      "logps/chosen": -0.5400632619857788,
      "logps/rejected": -1.2231850624084473,
      "loss": 0.397,
      "nll_loss": 0.3497290313243866,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.10801263898611069,
      "rewards/margins": 0.13662435114383698,
      "rewards/rejected": -0.24463701248168945,
      "step": 2430
    },
    {
      "epoch": 1.74,
      "grad_norm": 7.28125,
      "learning_rate": 4.482907875755205e-06,
      "log_odds_chosen": 1.3686769008636475,
      "log_odds_ratio": -0.2961091697216034,
      "logits/chosen": -2.8909826278686523,
      "logits/rejected": -2.897916078567505,
      "logps/chosen": -0.5009250640869141,
      "logps/rejected": -1.2302018404006958,
      "loss": 0.4144,
      "nll_loss": 0.36144185066223145,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.10018502175807953,
      "rewards/margins": 0.14585533738136292,
      "rewards/rejected": -0.24604037404060364,
      "step": 2440
    },
    {
      "epoch": 1.75,
      "grad_norm": 7.40625,
      "learning_rate": 4.481107149482208e-06,
      "log_odds_chosen": 1.3370695114135742,
      "log_odds_ratio": -0.30031710863113403,
      "logits/chosen": -2.8956050872802734,
      "logits/rejected": -2.8868050575256348,
      "logps/chosen": -0.5050898194313049,
      "logps/rejected": -1.255013108253479,
      "loss": 0.3929,
      "nll_loss": 0.3568061888217926,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.10101795196533203,
      "rewards/margins": 0.14998464286327362,
      "rewards/rejected": -0.25100260972976685,
      "step": 2450
    },
    {
      "epoch": 1.76,
      "grad_norm": 8.6875,
      "learning_rate": 4.4793085914529136e-06,
      "log_odds_chosen": 1.1334112882614136,
      "log_odds_ratio": -0.3533148169517517,
      "logits/chosen": -2.8731157779693604,
      "logits/rejected": -2.8705520629882812,
      "logps/chosen": -0.5255860090255737,
      "logps/rejected": -1.1402482986450195,
      "loss": 0.3965,
      "nll_loss": 0.34123849868774414,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10511720180511475,
      "rewards/margins": 0.12293247878551483,
      "rewards/rejected": -0.22804968059062958,
      "step": 2460
    },
    {
      "epoch": 1.76,
      "grad_norm": 10.6875,
      "learning_rate": 4.477512197319528e-06,
      "log_odds_chosen": 1.3563247919082642,
      "log_odds_ratio": -0.32014063000679016,
      "logits/chosen": -2.8848843574523926,
      "logits/rejected": -2.879897356033325,
      "logps/chosen": -0.5252963900566101,
      "logps/rejected": -1.3075590133666992,
      "loss": 0.3723,
      "nll_loss": 0.30930250883102417,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10505928099155426,
      "rewards/margins": 0.15645255148410797,
      "rewards/rejected": -0.26151180267333984,
      "step": 2470
    },
    {
      "epoch": 1.77,
      "grad_norm": 19.25,
      "learning_rate": 4.475717962746456e-06,
      "log_odds_chosen": 1.2710864543914795,
      "log_odds_ratio": -0.3212870657444,
      "logits/chosen": -2.875734806060791,
      "logits/rejected": -2.869267702102661,
      "logps/chosen": -0.5439091920852661,
      "logps/rejected": -1.2747299671173096,
      "loss": 0.3726,
      "nll_loss": 0.30996406078338623,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.1087818369269371,
      "rewards/margins": 0.1461641490459442,
      "rewards/rejected": -0.2549459636211395,
      "step": 2480
    },
    {
      "epoch": 1.78,
      "grad_norm": 8.1875,
      "learning_rate": 4.4739258834102515e-06,
      "log_odds_chosen": 1.2608749866485596,
      "log_odds_ratio": -0.346863329410553,
      "logits/chosen": -2.85408878326416,
      "logits/rejected": -2.849069595336914,
      "logps/chosen": -0.5514413118362427,
      "logps/rejected": -1.3013412952423096,
      "loss": 0.3812,
      "nll_loss": 0.33392244577407837,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.11028827726840973,
      "rewards/margins": 0.14997997879981995,
      "rewards/rejected": -0.26026827096939087,
      "step": 2490
    },
    {
      "epoch": 1.79,
      "grad_norm": 7.125,
      "learning_rate": 4.47213595499958e-06,
      "log_odds_chosen": 1.5626336336135864,
      "log_odds_ratio": -0.27938312292099,
      "logits/chosen": -2.928284168243408,
      "logits/rejected": -2.90639328956604,
      "logps/chosen": -0.5721508860588074,
      "logps/rejected": -1.5081751346588135,
      "loss": 0.3923,
      "nll_loss": 0.3527207374572754,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.11443017423152924,
      "rewards/margins": 0.18720485270023346,
      "rewards/rejected": -0.3016350269317627,
      "step": 2500
    },
    {
      "epoch": 1.79,
      "grad_norm": 6.65625,
      "learning_rate": 4.470348173215168e-06,
      "log_odds_chosen": 1.3578099012374878,
      "log_odds_ratio": -0.3152514100074768,
      "logits/chosen": -2.8871371746063232,
      "logits/rejected": -2.874263286590576,
      "logps/chosen": -0.5261938571929932,
      "logps/rejected": -1.3208293914794922,
      "loss": 0.3849,
      "nll_loss": 0.33318930864334106,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.10523878037929535,
      "rewards/margins": 0.15892712771892548,
      "rewards/rejected": -0.26416587829589844,
      "step": 2510
    },
    {
      "epoch": 1.8,
      "grad_norm": 12.5625,
      "learning_rate": 4.468562533769766e-06,
      "log_odds_chosen": 1.367193579673767,
      "log_odds_ratio": -0.3165552616119385,
      "logits/chosen": -2.8899295330047607,
      "logits/rejected": -2.890939950942993,
      "logps/chosen": -0.5429189205169678,
      "logps/rejected": -1.3376638889312744,
      "loss": 0.4111,
      "nll_loss": 0.33587661385536194,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.10858378559350967,
      "rewards/margins": 0.15894900262355804,
      "rewards/rejected": -0.2675327956676483,
      "step": 2520
    },
    {
      "epoch": 1.81,
      "grad_norm": 12.8125,
      "learning_rate": 4.4667790323881e-06,
      "log_odds_chosen": 1.2805125713348389,
      "log_odds_ratio": -0.33233442902565,
      "logits/chosen": -2.871812105178833,
      "logits/rejected": -2.852710723876953,
      "logps/chosen": -0.6220626831054688,
      "logps/rejected": -1.3669788837432861,
      "loss": 0.4336,
      "nll_loss": 0.4037063717842102,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.12441255897283554,
      "rewards/margins": 0.14898324012756348,
      "rewards/rejected": -0.2733957767486572,
      "step": 2530
    },
    {
      "epoch": 1.81,
      "grad_norm": 7.0625,
      "learning_rate": 4.464997664806832e-06,
      "log_odds_chosen": 1.1691948175430298,
      "log_odds_ratio": -0.35392698645591736,
      "logits/chosen": -2.857517719268799,
      "logits/rejected": -2.862380266189575,
      "logps/chosen": -0.5593948364257812,
      "logps/rejected": -1.201662302017212,
      "loss": 0.3871,
      "nll_loss": 0.31616219878196716,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.11187896877527237,
      "rewards/margins": 0.12845350801944733,
      "rewards/rejected": -0.2403324842453003,
      "step": 2540
    },
    {
      "epoch": 1.82,
      "grad_norm": 6.4375,
      "learning_rate": 4.463218426774518e-06,
      "log_odds_chosen": 1.4263107776641846,
      "log_odds_ratio": -0.3046194911003113,
      "logits/chosen": -2.891172170639038,
      "logits/rejected": -2.8799169063568115,
      "logps/chosen": -0.5329869389533997,
      "logps/rejected": -1.3692753314971924,
      "loss": 0.3873,
      "nll_loss": 0.3430352509021759,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.10659738630056381,
      "rewards/margins": 0.16725768148899078,
      "rewards/rejected": -0.273855060338974,
      "step": 2550
    },
    {
      "epoch": 1.83,
      "grad_norm": 6.40625,
      "learning_rate": 4.461441314051561e-06,
      "log_odds_chosen": 1.4294536113739014,
      "log_odds_ratio": -0.2968199849128723,
      "logits/chosen": -2.9138002395629883,
      "logits/rejected": -2.8921501636505127,
      "logps/chosen": -0.4996632933616638,
      "logps/rejected": -1.3187425136566162,
      "loss": 0.3745,
      "nll_loss": 0.3486596941947937,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.09993267059326172,
      "rewards/margins": 0.16381582617759705,
      "rewards/rejected": -0.2637484669685364,
      "step": 2560
    },
    {
      "epoch": 1.84,
      "grad_norm": 7.59375,
      "learning_rate": 4.459666322410172e-06,
      "log_odds_chosen": 1.3704941272735596,
      "log_odds_ratio": -0.3186499774456024,
      "logits/chosen": -2.883667469024658,
      "logits/rejected": -2.8775885105133057,
      "logps/chosen": -0.5285792350769043,
      "logps/rejected": -1.3211987018585205,
      "loss": 0.4048,
      "nll_loss": 0.3316243290901184,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.10571584850549698,
      "rewards/margins": 0.15852385759353638,
      "rewards/rejected": -0.26423972845077515,
      "step": 2570
    },
    {
      "epoch": 1.84,
      "grad_norm": 9.1875,
      "learning_rate": 4.457893447634326e-06,
      "log_odds_chosen": 1.2892141342163086,
      "log_odds_ratio": -0.33191436529159546,
      "logits/chosen": -2.9250831604003906,
      "logits/rejected": -2.902353286743164,
      "logps/chosen": -0.580119788646698,
      "logps/rejected": -1.3289637565612793,
      "loss": 0.4149,
      "nll_loss": 0.35767120122909546,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.1160239577293396,
      "rewards/margins": 0.14976876974105835,
      "rewards/rejected": -0.26579275727272034,
      "step": 2580
    },
    {
      "epoch": 1.85,
      "grad_norm": 10.5625,
      "learning_rate": 4.456122685519721e-06,
      "log_odds_chosen": 1.394887089729309,
      "log_odds_ratio": -0.3056487441062927,
      "logits/chosen": -2.8950932025909424,
      "logits/rejected": -2.873574733734131,
      "logps/chosen": -0.5190961956977844,
      "logps/rejected": -1.3081409931182861,
      "loss": 0.4005,
      "nll_loss": 0.31329116225242615,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.10381922870874405,
      "rewards/margins": 0.15780898928642273,
      "rewards/rejected": -0.2616282105445862,
      "step": 2590
    },
    {
      "epoch": 1.86,
      "grad_norm": 7.09375,
      "learning_rate": 4.45435403187374e-06,
      "log_odds_chosen": 1.507157325744629,
      "log_odds_ratio": -0.2677215039730072,
      "logits/chosen": -2.8969600200653076,
      "logits/rejected": -2.88423752784729,
      "logps/chosen": -0.4688204824924469,
      "logps/rejected": -1.299645185470581,
      "loss": 0.3953,
      "nll_loss": 0.32218068838119507,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.09376410394906998,
      "rewards/margins": 0.16616491973400116,
      "rewards/rejected": -0.25992903113365173,
      "step": 2600
    },
    {
      "epoch": 1.86,
      "grad_norm": 9.4375,
      "learning_rate": 4.452587482515399e-06,
      "log_odds_chosen": 1.3385193347930908,
      "log_odds_ratio": -0.31111469864845276,
      "logits/chosen": -2.907179355621338,
      "logits/rejected": -2.8795833587646484,
      "logps/chosen": -0.5588836669921875,
      "logps/rejected": -1.3447484970092773,
      "loss": 0.398,
      "nll_loss": 0.3411378264427185,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.11177675426006317,
      "rewards/margins": 0.15717296302318573,
      "rewards/rejected": -0.2689497172832489,
      "step": 2610
    },
    {
      "epoch": 1.87,
      "grad_norm": 9.5625,
      "learning_rate": 4.450823033275315e-06,
      "log_odds_chosen": 1.338440179824829,
      "log_odds_ratio": -0.3163678050041199,
      "logits/chosen": -2.9387764930725098,
      "logits/rejected": -2.9056990146636963,
      "logps/chosen": -0.5297420024871826,
      "logps/rejected": -1.322826862335205,
      "loss": 0.422,
      "nll_loss": 0.3693048357963562,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.10594840347766876,
      "rewards/margins": 0.15861697494983673,
      "rewards/rejected": -0.2645653784275055,
      "step": 2620
    },
    {
      "epoch": 1.88,
      "grad_norm": 7.65625,
      "learning_rate": 4.4490606799956615e-06,
      "log_odds_chosen": 1.4915597438812256,
      "log_odds_ratio": -0.28770941495895386,
      "logits/chosen": -2.8788347244262695,
      "logits/rejected": -2.8595736026763916,
      "logps/chosen": -0.5416139364242554,
      "logps/rejected": -1.4380385875701904,
      "loss": 0.4005,
      "nll_loss": 0.34102195501327515,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.10832278430461884,
      "rewards/margins": 0.1792849451303482,
      "rewards/rejected": -0.28760772943496704,
      "step": 2630
    },
    {
      "epoch": 1.89,
      "grad_norm": 9.5625,
      "learning_rate": 4.447300418530126e-06,
      "log_odds_chosen": 1.3638993501663208,
      "log_odds_ratio": -0.32332590222358704,
      "logits/chosen": -2.916489601135254,
      "logits/rejected": -2.8951425552368164,
      "logps/chosen": -0.5407701730728149,
      "logps/rejected": -1.3687996864318848,
      "loss": 0.4186,
      "nll_loss": 0.350581556558609,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.10815402120351791,
      "rewards/margins": 0.16560593247413635,
      "rewards/rejected": -0.27375996112823486,
      "step": 2640
    },
    {
      "epoch": 1.89,
      "grad_norm": 8.4375,
      "learning_rate": 4.4455422447438715e-06,
      "log_odds_chosen": 1.3353352546691895,
      "log_odds_ratio": -0.3252139091491699,
      "logits/chosen": -2.880558729171753,
      "logits/rejected": -2.8813929557800293,
      "logps/chosen": -0.5225256681442261,
      "logps/rejected": -1.2812443971633911,
      "loss": 0.4082,
      "nll_loss": 0.32984524965286255,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.10450513660907745,
      "rewards/margins": 0.15174375474452972,
      "rewards/rejected": -0.25624892115592957,
      "step": 2650
    },
    {
      "epoch": 1.9,
      "grad_norm": 9.0625,
      "learning_rate": 4.443786154513493e-06,
      "log_odds_chosen": 1.541635513305664,
      "log_odds_ratio": -0.2674759328365326,
      "logits/chosen": -2.883507490158081,
      "logits/rejected": -2.8745486736297607,
      "logps/chosen": -0.5493952035903931,
      "logps/rejected": -1.4771674871444702,
      "loss": 0.3756,
      "nll_loss": 0.2988761067390442,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.1098790392279625,
      "rewards/margins": 0.18555445969104767,
      "rewards/rejected": -0.2954334616661072,
      "step": 2660
    },
    {
      "epoch": 1.91,
      "grad_norm": 12.9375,
      "learning_rate": 4.442032143726981e-06,
      "log_odds_chosen": 1.3634321689605713,
      "log_odds_ratio": -0.30965596437454224,
      "logits/chosen": -2.8866159915924072,
      "logits/rejected": -2.8669233322143555,
      "logps/chosen": -0.602931022644043,
      "logps/rejected": -1.4159696102142334,
      "loss": 0.4085,
      "nll_loss": 0.362027645111084,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.12058620154857635,
      "rewards/margins": 0.16260775923728943,
      "rewards/rejected": -0.283193975687027,
      "step": 2670
    },
    {
      "epoch": 1.91,
      "grad_norm": 13.9375,
      "learning_rate": 4.440280208283675e-06,
      "log_odds_chosen": 1.3336797952651978,
      "log_odds_ratio": -0.32034236192703247,
      "logits/chosen": -2.9205827713012695,
      "logits/rejected": -2.8959076404571533,
      "logps/chosen": -0.5290694236755371,
      "logps/rejected": -1.265318512916565,
      "loss": 0.3972,
      "nll_loss": 0.3247922360897064,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.1058138832449913,
      "rewards/margins": 0.14724981784820557,
      "rewards/rejected": -0.25306370854377747,
      "step": 2680
    },
    {
      "epoch": 1.92,
      "grad_norm": 12.0625,
      "learning_rate": 4.43853034409423e-06,
      "log_odds_chosen": 1.4389830827713013,
      "log_odds_ratio": -0.308421790599823,
      "logits/chosen": -2.900420904159546,
      "logits/rejected": -2.879181385040283,
      "logps/chosen": -0.5513266324996948,
      "logps/rejected": -1.4156787395477295,
      "loss": 0.3989,
      "nll_loss": 0.3391249179840088,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.11026531457901001,
      "rewards/margins": 0.1728704273700714,
      "rewards/rejected": -0.2831357419490814,
      "step": 2690
    },
    {
      "epoch": 1.93,
      "grad_norm": 7.46875,
      "learning_rate": 4.43678254708057e-06,
      "log_odds_chosen": 1.2590786218643188,
      "log_odds_ratio": -0.3339710831642151,
      "logits/chosen": -2.8714280128479004,
      "logits/rejected": -2.8531947135925293,
      "logps/chosen": -0.5529037117958069,
      "logps/rejected": -1.291151762008667,
      "loss": 0.4083,
      "nll_loss": 0.34685009717941284,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.11058074235916138,
      "rewards/margins": 0.1476496458053589,
      "rewards/rejected": -0.25823038816452026,
      "step": 2700
    },
    {
      "epoch": 1.94,
      "grad_norm": 10.8125,
      "learning_rate": 4.435036813175853e-06,
      "log_odds_chosen": 1.296800971031189,
      "log_odds_ratio": -0.3468873202800751,
      "logits/chosen": -2.9166648387908936,
      "logits/rejected": -2.905029773712158,
      "logps/chosen": -0.5468270778656006,
      "logps/rejected": -1.318878412246704,
      "loss": 0.4244,
      "nll_loss": 0.3611888289451599,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10936541855335236,
      "rewards/margins": 0.15441028773784637,
      "rewards/rejected": -0.26377567648887634,
      "step": 2710
    },
    {
      "epoch": 1.94,
      "grad_norm": 7.59375,
      "learning_rate": 4.4332931383244296e-06,
      "log_odds_chosen": 1.5938103199005127,
      "log_odds_ratio": -0.2786775231361389,
      "logits/chosen": -2.84563946723938,
      "logits/rejected": -2.81329345703125,
      "logps/chosen": -0.5737396478652954,
      "logps/rejected": -1.5809630155563354,
      "loss": 0.4009,
      "nll_loss": 0.3476629853248596,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.11474792659282684,
      "rewards/margins": 0.20144470036029816,
      "rewards/rejected": -0.316192626953125,
      "step": 2720
    },
    {
      "epoch": 1.95,
      "grad_norm": 9.125,
      "learning_rate": 4.431551518481802e-06,
      "log_odds_chosen": 1.4222679138183594,
      "log_odds_ratio": -0.30235162377357483,
      "logits/chosen": -2.859123468399048,
      "logits/rejected": -2.8459434509277344,
      "logps/chosen": -0.5479883551597595,
      "logps/rejected": -1.3775622844696045,
      "loss": 0.3895,
      "nll_loss": 0.31317585706710815,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.10959766805171967,
      "rewards/margins": 0.16591483354568481,
      "rewards/rejected": -0.2755124866962433,
      "step": 2730
    },
    {
      "epoch": 1.96,
      "grad_norm": 8.1875,
      "learning_rate": 4.429811949614588e-06,
      "log_odds_chosen": 1.3053306341171265,
      "log_odds_ratio": -0.3146135210990906,
      "logits/chosen": -2.9177818298339844,
      "logits/rejected": -2.8973758220672607,
      "logps/chosen": -0.5131221413612366,
      "logps/rejected": -1.2459287643432617,
      "loss": 0.4246,
      "nll_loss": 0.37119024991989136,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.10262443125247955,
      "rewards/margins": 0.14656129479408264,
      "rewards/rejected": -0.2491857260465622,
      "step": 2740
    },
    {
      "epoch": 1.96,
      "grad_norm": 6.4375,
      "learning_rate": 4.428074427700477e-06,
      "log_odds_chosen": 1.3217928409576416,
      "log_odds_ratio": -0.30279672145843506,
      "logits/chosen": -2.877779245376587,
      "logits/rejected": -2.860002279281616,
      "logps/chosen": -0.5184148550033569,
      "logps/rejected": -1.2421989440917969,
      "loss": 0.3943,
      "nll_loss": 0.32501065731048584,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.1036829724907875,
      "rewards/margins": 0.14475682377815247,
      "rewards/rejected": -0.24843978881835938,
      "step": 2750
    },
    {
      "epoch": 1.97,
      "grad_norm": 16.125,
      "learning_rate": 4.426338948728195e-06,
      "log_odds_chosen": 1.3941218852996826,
      "log_odds_ratio": -0.3162110447883606,
      "logits/chosen": -2.8602135181427,
      "logits/rejected": -2.820308208465576,
      "logps/chosen": -0.5790046453475952,
      "logps/rejected": -1.4153722524642944,
      "loss": 0.4456,
      "nll_loss": 0.3927566409111023,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.11580093950033188,
      "rewards/margins": 0.16727350652217865,
      "rewards/rejected": -0.28307443857192993,
      "step": 2760
    },
    {
      "epoch": 1.98,
      "grad_norm": 7.15625,
      "learning_rate": 4.424605508697463e-06,
      "log_odds_chosen": 1.4302750825881958,
      "log_odds_ratio": -0.3106474280357361,
      "logits/chosen": -2.8623061180114746,
      "logits/rejected": -2.825331449508667,
      "logps/chosen": -0.5542712211608887,
      "logps/rejected": -1.4113256931304932,
      "loss": 0.3867,
      "nll_loss": 0.34051352739334106,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.11085423082113266,
      "rewards/margins": 0.17141090333461761,
      "rewards/rejected": -0.2822651267051697,
      "step": 2770
    },
    {
      "epoch": 1.99,
      "grad_norm": 11.4375,
      "learning_rate": 4.42287410361896e-06,
      "log_odds_chosen": 1.4414396286010742,
      "log_odds_ratio": -0.32269230484962463,
      "logits/chosen": -2.8639965057373047,
      "logits/rejected": -2.8246994018554688,
      "logps/chosen": -0.5694876313209534,
      "logps/rejected": -1.4324452877044678,
      "loss": 0.3603,
      "nll_loss": 0.307388037443161,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.11389752477407455,
      "rewards/margins": 0.17259155213832855,
      "rewards/rejected": -0.2864890694618225,
      "step": 2780
    },
    {
      "epoch": 1.99,
      "grad_norm": 12.0625,
      "learning_rate": 4.421144729514289e-06,
      "log_odds_chosen": 1.4677602052688599,
      "log_odds_ratio": -0.3058595061302185,
      "logits/chosen": -2.8635823726654053,
      "logits/rejected": -2.80998158454895,
      "logps/chosen": -0.5519368052482605,
      "logps/rejected": -1.4589126110076904,
      "loss": 0.4043,
      "nll_loss": 0.3577142357826233,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.11038736253976822,
      "rewards/margins": 0.18139515817165375,
      "rewards/rejected": -0.2917824983596802,
      "step": 2790
    },
    {
      "epoch": 2.0,
      "grad_norm": 12.75,
      "learning_rate": 4.419417382415923e-06,
      "log_odds_chosen": 1.4039043188095093,
      "log_odds_ratio": -0.313946932554245,
      "logits/chosen": -2.8399224281311035,
      "logits/rejected": -2.791377544403076,
      "logps/chosen": -0.5001746416091919,
      "logps/rejected": -1.3304522037506104,
      "loss": 0.3924,
      "nll_loss": 0.3202509880065918,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.1000349372625351,
      "rewards/margins": 0.16605551540851593,
      "rewards/rejected": -0.26609042286872864,
      "step": 2800
    },
    {
      "epoch": 2.01,
      "grad_norm": 5.3125,
      "learning_rate": 4.417692058367186e-06,
      "log_odds_chosen": 2.910543441772461,
      "log_odds_ratio": -0.10447581857442856,
      "logits/chosen": -2.8468985557556152,
      "logits/rejected": -2.7128536701202393,
      "logps/chosen": -0.390835702419281,
      "logps/rejected": -2.187709331512451,
      "loss": 0.2834,
      "nll_loss": 0.2709491550922394,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.0781671404838562,
      "rewards/margins": 0.35937467217445374,
      "rewards/rejected": -0.43754181265830994,
      "step": 2810
    },
    {
      "epoch": 2.01,
      "grad_norm": 9.5625,
      "learning_rate": 4.415968753422204e-06,
      "log_odds_chosen": 3.4466099739074707,
      "log_odds_ratio": -0.07032948732376099,
      "logits/chosen": -2.8091824054718018,
      "logits/rejected": -2.6169466972351074,
      "logps/chosen": -0.31530073285102844,
      "logps/rejected": -2.4634060859680176,
      "loss": 0.2586,
      "nll_loss": 0.2396393120288849,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06306014955043793,
      "rewards/margins": 0.4296211302280426,
      "rewards/rejected": -0.4926813244819641,
      "step": 2820
    },
    {
      "epoch": 2.02,
      "grad_norm": 4.65625,
      "learning_rate": 4.414247463645868e-06,
      "log_odds_chosen": 3.7165610790252686,
      "log_odds_ratio": -0.06657154858112335,
      "logits/chosen": -2.7692294120788574,
      "logits/rejected": -2.4765193462371826,
      "logps/chosen": -0.37618759274482727,
      "logps/rejected": -2.8358898162841797,
      "loss": 0.2718,
      "nll_loss": 0.2537682354450226,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07523752003908157,
      "rewards/margins": 0.491940438747406,
      "rewards/rejected": -0.5671780109405518,
      "step": 2830
    },
    {
      "epoch": 2.03,
      "grad_norm": 6.1875,
      "learning_rate": 4.4125281851137995e-06,
      "log_odds_chosen": 5.014394283294678,
      "log_odds_ratio": -0.07537925243377686,
      "logits/chosen": -2.721604108810425,
      "logits/rejected": -2.253276824951172,
      "logps/chosen": -0.3364471197128296,
      "logps/rejected": -4.104693412780762,
      "loss": 0.2853,
      "nll_loss": 0.24688100814819336,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06728943437337875,
      "rewards/margins": 0.7536492943763733,
      "rewards/rejected": -0.820938766002655,
      "step": 2840
    },
    {
      "epoch": 2.03,
      "grad_norm": 7.8125,
      "learning_rate": 4.41081091391231e-06,
      "log_odds_chosen": 3.7566096782684326,
      "log_odds_ratio": -0.06575653702020645,
      "logits/chosen": -2.7676377296447754,
      "logits/rejected": -2.4655532836914062,
      "logps/chosen": -0.3585343360900879,
      "logps/rejected": -2.8366215229034424,
      "loss": 0.2814,
      "nll_loss": 0.27807971835136414,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.0717068687081337,
      "rewards/margins": 0.49561747908592224,
      "rewards/rejected": -0.5673243403434753,
      "step": 2850
    },
    {
      "epoch": 2.04,
      "grad_norm": 13.875,
      "learning_rate": 4.409095646138363e-06,
      "log_odds_chosen": 3.4027340412139893,
      "log_odds_ratio": -0.08646047860383987,
      "logits/chosen": -2.78098726272583,
      "logits/rejected": -2.575989246368408,
      "logps/chosen": -0.3481005132198334,
      "logps/rejected": -2.515467405319214,
      "loss": 0.2687,
      "nll_loss": 0.2460239678621292,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.06962011754512787,
      "rewards/margins": 0.4334734380245209,
      "rewards/rejected": -0.5030934810638428,
      "step": 2860
    },
    {
      "epoch": 2.05,
      "grad_norm": 5.5625,
      "learning_rate": 4.4073823778995425e-06,
      "log_odds_chosen": 3.673715114593506,
      "log_odds_ratio": -0.06583552062511444,
      "logits/chosen": -2.7850005626678467,
      "logits/rejected": -2.458684206008911,
      "logps/chosen": -0.3202974200248718,
      "logps/rejected": -2.65244197845459,
      "loss": 0.2405,
      "nll_loss": 0.2267676144838333,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06405948847532272,
      "rewards/margins": 0.4664289355278015,
      "rewards/rejected": -0.5304883718490601,
      "step": 2870
    },
    {
      "epoch": 2.06,
      "grad_norm": 12.8125,
      "learning_rate": 4.405671105314009e-06,
      "log_odds_chosen": 4.673083782196045,
      "log_odds_ratio": -0.05653030425310135,
      "logits/chosen": -2.6835262775421143,
      "logits/rejected": -2.0964083671569824,
      "logps/chosen": -0.336531400680542,
      "logps/rejected": -3.6344618797302246,
      "loss": 0.2548,
      "nll_loss": 0.23812773823738098,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.0673062726855278,
      "rewards/margins": 0.6595860719680786,
      "rewards/rejected": -0.726892352104187,
      "step": 2880
    },
    {
      "epoch": 2.06,
      "grad_norm": 5.71875,
      "learning_rate": 4.4039618245104645e-06,
      "log_odds_chosen": 5.218452453613281,
      "log_odds_ratio": -0.06279017776250839,
      "logits/chosen": -2.765897274017334,
      "logits/rejected": -2.0521953105926514,
      "logps/chosen": -0.3615598678588867,
      "logps/rejected": -4.352395534515381,
      "loss": 0.2837,
      "nll_loss": 0.2878434956073761,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.07231196761131287,
      "rewards/margins": 0.7981672286987305,
      "rewards/rejected": -0.8704792261123657,
      "step": 2890
    },
    {
      "epoch": 2.07,
      "grad_norm": 9.125,
      "learning_rate": 4.4022545316281195e-06,
      "log_odds_chosen": 5.200108051300049,
      "log_odds_ratio": -0.05653291940689087,
      "logits/chosen": -2.7568302154541016,
      "logits/rejected": -2.0886034965515137,
      "logps/chosen": -0.34464651346206665,
      "logps/rejected": -4.230252742767334,
      "loss": 0.2593,
      "nll_loss": 0.26374679803848267,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06892929971218109,
      "rewards/margins": 0.7771213054656982,
      "rewards/rejected": -0.8460506200790405,
      "step": 2900
    },
    {
      "epoch": 2.08,
      "grad_norm": 9.375,
      "learning_rate": 4.40054922281665e-06,
      "log_odds_chosen": 6.3368120193481445,
      "log_odds_ratio": -0.04714951664209366,
      "logits/chosen": -2.7386841773986816,
      "logits/rejected": -2.0858142375946045,
      "logps/chosen": -0.342684805393219,
      "logps/rejected": -5.299139499664307,
      "loss": 0.2966,
      "nll_loss": 0.28953665494918823,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06853695958852768,
      "rewards/margins": 0.9912910461425781,
      "rewards/rejected": -1.0598279237747192,
      "step": 2910
    },
    {
      "epoch": 2.08,
      "grad_norm": 10.125,
      "learning_rate": 4.398845894236168e-06,
      "log_odds_chosen": 6.120891094207764,
      "log_odds_ratio": -0.045149557292461395,
      "logits/chosen": -2.792410135269165,
      "logits/rejected": -2.072402000427246,
      "logps/chosen": -0.3363955020904541,
      "logps/rejected": -5.071963787078857,
      "loss": 0.2825,
      "nll_loss": 0.25938451290130615,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06727909296751022,
      "rewards/margins": 0.9471136927604675,
      "rewards/rejected": -1.014392614364624,
      "step": 2920
    },
    {
      "epoch": 2.09,
      "grad_norm": 8.875,
      "learning_rate": 4.397144542057179e-06,
      "log_odds_chosen": 4.736415863037109,
      "log_odds_ratio": -0.06085144728422165,
      "logits/chosen": -2.801853656768799,
      "logits/rejected": -2.182584047317505,
      "logps/chosen": -0.36493486166000366,
      "logps/rejected": -3.7831199169158936,
      "loss": 0.2756,
      "nll_loss": 0.2436126172542572,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.07298697531223297,
      "rewards/margins": 0.6836371421813965,
      "rewards/rejected": -0.7566241025924683,
      "step": 2930
    },
    {
      "epoch": 2.1,
      "grad_norm": 7.75,
      "learning_rate": 4.395445162460548e-06,
      "log_odds_chosen": 6.897581577301025,
      "log_odds_ratio": -0.03966455161571503,
      "logits/chosen": -2.73842191696167,
      "logits/rejected": -1.9335638284683228,
      "logps/chosen": -0.3176586329936981,
      "logps/rejected": -5.78334903717041,
      "loss": 0.251,
      "nll_loss": 0.23550419509410858,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.06353173404932022,
      "rewards/margins": 1.0931380987167358,
      "rewards/rejected": -1.1566698551177979,
      "step": 2940
    },
    {
      "epoch": 2.11,
      "grad_norm": 5.59375,
      "learning_rate": 4.393747751637469e-06,
      "log_odds_chosen": 10.120447158813477,
      "log_odds_ratio": -0.03257293254137039,
      "logits/chosen": -2.7335376739501953,
      "logits/rejected": -1.6728357076644897,
      "logps/chosen": -0.32205477356910706,
      "logps/rejected": -8.999835968017578,
      "loss": 0.2672,
      "nll_loss": 0.2610648274421692,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06441095471382141,
      "rewards/margins": 1.7355563640594482,
      "rewards/rejected": -1.7999674081802368,
      "step": 2950
    },
    {
      "epoch": 2.11,
      "grad_norm": 14.3125,
      "learning_rate": 4.392052305789416e-06,
      "log_odds_chosen": 9.041637420654297,
      "log_odds_ratio": -0.039078570902347565,
      "logits/chosen": -2.7224085330963135,
      "logits/rejected": -1.6541340351104736,
      "logps/chosen": -0.3390257954597473,
      "logps/rejected": -7.973293304443359,
      "loss": 0.2984,
      "nll_loss": 0.2727344036102295,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06780517101287842,
      "rewards/margins": 1.5268534421920776,
      "rewards/rejected": -1.594658613204956,
      "step": 2960
    },
    {
      "epoch": 2.12,
      "grad_norm": 10.8125,
      "learning_rate": 4.390358821128123e-06,
      "log_odds_chosen": 9.127324104309082,
      "log_odds_ratio": -0.04081900417804718,
      "logits/chosen": -2.6818902492523193,
      "logits/rejected": -1.3260700702667236,
      "logps/chosen": -0.3540460765361786,
      "logps/rejected": -8.071810722351074,
      "loss": 0.2843,
      "nll_loss": 0.28363126516342163,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.07080921530723572,
      "rewards/margins": 1.5435529947280884,
      "rewards/rejected": -1.6143620014190674,
      "step": 2970
    },
    {
      "epoch": 2.13,
      "grad_norm": 18.0,
      "learning_rate": 4.388667293875536e-06,
      "log_odds_chosen": 5.46827507019043,
      "log_odds_ratio": -0.13300354778766632,
      "logits/chosen": -2.7808876037597656,
      "logits/rejected": -2.0889151096343994,
      "logps/chosen": -0.4834941327571869,
      "logps/rejected": -4.695062160491943,
      "loss": 0.275,
      "nll_loss": 0.26671329140663147,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.0966988280415535,
      "rewards/margins": 0.8423136472702026,
      "rewards/rejected": -0.9390126466751099,
      "step": 2980
    },
    {
      "epoch": 2.13,
      "grad_norm": 169.0,
      "learning_rate": 4.386977720263786e-06,
      "log_odds_chosen": 6.041357040405273,
      "log_odds_ratio": -0.05618007108569145,
      "logits/chosen": -2.7575278282165527,
      "logits/rejected": -1.7493776082992554,
      "logps/chosen": -0.3787495195865631,
      "logps/rejected": -5.060299873352051,
      "loss": 0.2629,
      "nll_loss": 0.26606082916259766,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.07574990391731262,
      "rewards/margins": 0.9363101124763489,
      "rewards/rejected": -1.0120599269866943,
      "step": 2990
    },
    {
      "epoch": 2.14,
      "grad_norm": 5.9375,
      "learning_rate": 4.385290096535147e-06,
      "log_odds_chosen": 4.69273567199707,
      "log_odds_ratio": -0.05490034073591232,
      "logits/chosen": -2.8089425563812256,
      "logits/rejected": -2.2398369312286377,
      "logps/chosen": -0.30732420086860657,
      "logps/rejected": -3.626305103302002,
      "loss": 0.2486,
      "nll_loss": 0.2282293289899826,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.061464838683605194,
      "rewards/margins": 0.663796067237854,
      "rewards/rejected": -0.7252610325813293,
      "step": 3000
    },
    {
      "epoch": 2.15,
      "grad_norm": 12.6875,
      "learning_rate": 4.383604418942005e-06,
      "log_odds_chosen": 6.116213321685791,
      "log_odds_ratio": -0.03878642991185188,
      "logits/chosen": -2.6938247680664062,
      "logits/rejected": -1.8152358531951904,
      "logps/chosen": -0.35414764285087585,
      "logps/rejected": -5.104779243469238,
      "loss": 0.2722,
      "nll_loss": 0.252914160490036,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.07082952558994293,
      "rewards/margins": 0.9501264691352844,
      "rewards/rejected": -1.0209559202194214,
      "step": 3010
    },
    {
      "epoch": 2.16,
      "grad_norm": 6.125,
      "learning_rate": 4.381920683746824e-06,
      "log_odds_chosen": 5.905968189239502,
      "log_odds_ratio": -0.0426829531788826,
      "logits/chosen": -2.756247043609619,
      "logits/rejected": -2.1121325492858887,
      "logps/chosen": -0.3639851212501526,
      "logps/rejected": -4.953127384185791,
      "loss": 0.2756,
      "nll_loss": 0.27040037512779236,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0727970153093338,
      "rewards/margins": 0.9178284406661987,
      "rewards/rejected": -0.9906253814697266,
      "step": 3020
    },
    {
      "epoch": 2.16,
      "grad_norm": 5.8125,
      "learning_rate": 4.380238887222108e-06,
      "log_odds_chosen": 6.093326568603516,
      "log_odds_ratio": -0.04192028567194939,
      "logits/chosen": -2.7782235145568848,
      "logits/rejected": -1.9548759460449219,
      "logps/chosen": -0.31508156657218933,
      "logps/rejected": -4.972496032714844,
      "loss": 0.2734,
      "nll_loss": 0.2683493494987488,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.06301631033420563,
      "rewards/margins": 0.9314829707145691,
      "rewards/rejected": -0.9944992065429688,
      "step": 3030
    },
    {
      "epoch": 2.17,
      "grad_norm": 6.03125,
      "learning_rate": 4.378559025650368e-06,
      "log_odds_chosen": 7.272418022155762,
      "log_odds_ratio": -0.056223928928375244,
      "logits/chosen": -2.7609264850616455,
      "logits/rejected": -1.9744220972061157,
      "logps/chosen": -0.3785443902015686,
      "logps/rejected": -6.300936698913574,
      "loss": 0.2611,
      "nll_loss": 0.26268187165260315,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.07570887356996536,
      "rewards/margins": 1.184478521347046,
      "rewards/rejected": -1.2601875066757202,
      "step": 3040
    },
    {
      "epoch": 2.18,
      "grad_norm": 5.21875,
      "learning_rate": 4.376881095324086e-06,
      "log_odds_chosen": 8.435240745544434,
      "log_odds_ratio": -0.032564710825681686,
      "logits/chosen": -2.803131580352783,
      "logits/rejected": -1.676296591758728,
      "logps/chosen": -0.3182581067085266,
      "logps/rejected": -7.271533012390137,
      "loss": 0.2522,
      "nll_loss": 0.23994970321655273,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.06365162134170532,
      "rewards/margins": 1.3906551599502563,
      "rewards/rejected": -1.454306721687317,
      "step": 3050
    },
    {
      "epoch": 2.18,
      "grad_norm": 7.5625,
      "learning_rate": 4.375205092545683e-06,
      "log_odds_chosen": 8.230413436889648,
      "log_odds_ratio": -0.05366669222712517,
      "logits/chosen": -2.786719560623169,
      "logits/rejected": -1.8391005992889404,
      "logps/chosen": -0.3586600422859192,
      "logps/rejected": -7.209918022155762,
      "loss": 0.2779,
      "nll_loss": 0.2507212460041046,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.07173201441764832,
      "rewards/margins": 1.3702516555786133,
      "rewards/rejected": -1.4419835805892944,
      "step": 3060
    },
    {
      "epoch": 2.19,
      "grad_norm": 5.125,
      "learning_rate": 4.373531013627483e-06,
      "log_odds_chosen": 8.445428848266602,
      "log_odds_ratio": -0.03792678564786911,
      "logits/chosen": -2.7394258975982666,
      "logits/rejected": -1.4675519466400146,
      "logps/chosen": -0.3647417426109314,
      "logps/rejected": -7.469712734222412,
      "loss": 0.2809,
      "nll_loss": 0.26669415831565857,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.07294835150241852,
      "rewards/margins": 1.4209941625595093,
      "rewards/rejected": -1.4939426183700562,
      "step": 3070
    },
    {
      "epoch": 2.2,
      "grad_norm": 170.0,
      "learning_rate": 4.371858854891681e-06,
      "log_odds_chosen": 6.993101596832275,
      "log_odds_ratio": -0.06284736096858978,
      "logits/chosen": -2.7770023345947266,
      "logits/rejected": -1.9066228866577148,
      "logps/chosen": -0.36158767342567444,
      "logps/rejected": -6.016913414001465,
      "loss": 0.2731,
      "nll_loss": 0.25095489621162415,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.07231752574443817,
      "rewards/margins": 1.1310651302337646,
      "rewards/rejected": -1.2033826112747192,
      "step": 3080
    },
    {
      "epoch": 2.21,
      "grad_norm": 20.375,
      "learning_rate": 4.370188612670307e-06,
      "log_odds_chosen": 6.399539947509766,
      "log_odds_ratio": -0.05658464506268501,
      "logits/chosen": -2.7655324935913086,
      "logits/rejected": -1.907060980796814,
      "logps/chosen": -0.3928866684436798,
      "logps/rejected": -5.48469877243042,
      "loss": 0.2814,
      "nll_loss": 0.2612989544868469,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.07857732474803925,
      "rewards/margins": 1.0183625221252441,
      "rewards/rejected": -1.0969398021697998,
      "step": 3090
    },
    {
      "epoch": 2.21,
      "grad_norm": 10.8125,
      "learning_rate": 4.36852028330519e-06,
      "log_odds_chosen": 6.503146171569824,
      "log_odds_ratio": -0.04645160958170891,
      "logits/chosen": -2.803072929382324,
      "logits/rejected": -1.9909429550170898,
      "logps/chosen": -0.38660162687301636,
      "logps/rejected": -5.571995735168457,
      "loss": 0.2914,
      "nll_loss": 0.2930335998535156,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.07732032984495163,
      "rewards/margins": 1.0370789766311646,
      "rewards/rejected": -1.1143993139266968,
      "step": 3100
    },
    {
      "epoch": 2.22,
      "grad_norm": 7.46875,
      "learning_rate": 4.3668538631479314e-06,
      "log_odds_chosen": 6.382928848266602,
      "log_odds_ratio": -0.04722817987203598,
      "logits/chosen": -2.793405055999756,
      "logits/rejected": -1.897592544555664,
      "logps/chosen": -0.36716800928115845,
      "logps/rejected": -5.360346794128418,
      "loss": 0.2719,
      "nll_loss": 0.2766880989074707,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.07343360781669617,
      "rewards/margins": 0.9986356496810913,
      "rewards/rejected": -1.0720694065093994,
      "step": 3110
    },
    {
      "epoch": 2.23,
      "grad_norm": 7.53125,
      "learning_rate": 4.365189348559864e-06,
      "log_odds_chosen": 7.294475555419922,
      "log_odds_ratio": -0.04280845448374748,
      "logits/chosen": -2.794233560562134,
      "logits/rejected": -1.9600608348846436,
      "logps/chosen": -0.3509625494480133,
      "logps/rejected": -6.329296112060547,
      "loss": 0.26,
      "nll_loss": 0.24876642227172852,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.07019250839948654,
      "rewards/margins": 1.1956669092178345,
      "rewards/rejected": -1.2658593654632568,
      "step": 3120
    },
    {
      "epoch": 2.23,
      "grad_norm": 105.0,
      "learning_rate": 4.363526735912025e-06,
      "log_odds_chosen": 8.40985107421875,
      "log_odds_ratio": -0.03858170285820961,
      "logits/chosen": -2.7838852405548096,
      "logits/rejected": -1.66619873046875,
      "logps/chosen": -0.39186400175094604,
      "logps/rejected": -7.510270118713379,
      "loss": 0.2711,
      "nll_loss": 0.27240419387817383,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07837279886007309,
      "rewards/margins": 1.423681378364563,
      "rewards/rejected": -1.5020540952682495,
      "step": 3130
    },
    {
      "epoch": 2.24,
      "grad_norm": 48.25,
      "learning_rate": 4.361866021585114e-06,
      "log_odds_chosen": 8.032949447631836,
      "log_odds_ratio": -0.03759379684925079,
      "logits/chosen": -2.8125336170196533,
      "logits/rejected": -1.9667034149169922,
      "logps/chosen": -0.3394584059715271,
      "logps/rejected": -6.949484348297119,
      "loss": 0.277,
      "nll_loss": 0.2678987979888916,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.0678916722536087,
      "rewards/margins": 1.322005271911621,
      "rewards/rejected": -1.3898969888687134,
      "step": 3140
    },
    {
      "epoch": 2.25,
      "grad_norm": 9.625,
      "learning_rate": 4.360207201969474e-06,
      "log_odds_chosen": 6.880602836608887,
      "log_odds_ratio": -0.040961816906929016,
      "logits/chosen": -2.7715115547180176,
      "logits/rejected": -2.049297571182251,
      "logps/chosen": -0.34252291917800903,
      "logps/rejected": -5.822251319885254,
      "loss": 0.2519,
      "nll_loss": 0.23448018729686737,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06850457936525345,
      "rewards/margins": 1.0959457159042358,
      "rewards/rejected": -1.1644504070281982,
      "step": 3150
    },
    {
      "epoch": 2.26,
      "grad_norm": 120.0,
      "learning_rate": 4.358550273465042e-06,
      "log_odds_chosen": 7.489804267883301,
      "log_odds_ratio": -0.030518781393766403,
      "logits/chosen": -2.8056952953338623,
      "logits/rejected": -1.9100799560546875,
      "logps/chosen": -0.3443801999092102,
      "logps/rejected": -6.379244804382324,
      "loss": 0.267,
      "nll_loss": 0.25786659121513367,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06887603551149368,
      "rewards/margins": 1.2069729566574097,
      "rewards/rejected": -1.2758489847183228,
      "step": 3160
    },
    {
      "epoch": 2.26,
      "grad_norm": 6.875,
      "learning_rate": 4.356895232481328e-06,
      "log_odds_chosen": 8.904826164245605,
      "log_odds_ratio": -0.02953409031033516,
      "logits/chosen": -2.8109521865844727,
      "logits/rejected": -1.7148170471191406,
      "logps/chosen": -0.32581400871276855,
      "logps/rejected": -7.794666290283203,
      "loss": 0.2778,
      "nll_loss": 0.2546003758907318,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.065162792801857,
      "rewards/margins": 1.4937704801559448,
      "rewards/rejected": -1.5589332580566406,
      "step": 3170
    },
    {
      "epoch": 2.27,
      "grad_norm": 64.5,
      "learning_rate": 4.3552420754373795e-06,
      "log_odds_chosen": 9.826112747192383,
      "log_odds_ratio": -0.03542652726173401,
      "logits/chosen": -2.755399465560913,
      "logits/rejected": -1.6918747425079346,
      "logps/chosen": -0.37103694677352905,
      "logps/rejected": -8.79539680480957,
      "loss": 0.2858,
      "nll_loss": 0.27970394492149353,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.07420738786458969,
      "rewards/margins": 1.6848719120025635,
      "rewards/rejected": -1.7590793371200562,
      "step": 3180
    },
    {
      "epoch": 2.28,
      "grad_norm": 71.0,
      "learning_rate": 4.353590798761745e-06,
      "log_odds_chosen": 7.645397186279297,
      "log_odds_ratio": -0.041773177683353424,
      "logits/chosen": -2.7663168907165527,
      "logits/rejected": -1.89877188205719,
      "logps/chosen": -0.3466086685657501,
      "logps/rejected": -6.663360595703125,
      "loss": 0.2852,
      "nll_loss": 0.2754463255405426,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.06932173669338226,
      "rewards/margins": 1.263350248336792,
      "rewards/rejected": -1.3326722383499146,
      "step": 3190
    },
    {
      "epoch": 2.28,
      "grad_norm": 8.0,
      "learning_rate": 4.351941398892446e-06,
      "log_odds_chosen": 10.17717170715332,
      "log_odds_ratio": -0.02503364160656929,
      "logits/chosen": -2.7787466049194336,
      "logits/rejected": -1.7833493947982788,
      "logps/chosen": -0.38776397705078125,
      "logps/rejected": -9.250219345092773,
      "loss": 0.291,
      "nll_loss": 0.3181122839450836,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07755279541015625,
      "rewards/margins": 1.7724910974502563,
      "rewards/rejected": -1.8500440120697021,
      "step": 3200
    },
    {
      "epoch": 2.29,
      "grad_norm": 4.6875,
      "learning_rate": 4.350293872276944e-06,
      "log_odds_chosen": 8.712306022644043,
      "log_odds_ratio": -0.028896396979689598,
      "logits/chosen": -2.7661192417144775,
      "logits/rejected": -1.6439266204833984,
      "logps/chosen": -0.3586076498031616,
      "logps/rejected": -7.733296871185303,
      "loss": 0.2939,
      "nll_loss": 0.28883105516433716,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07172153890132904,
      "rewards/margins": 1.474937915802002,
      "rewards/rejected": -1.5466594696044922,
      "step": 3210
    },
    {
      "epoch": 2.3,
      "grad_norm": 14.3125,
      "learning_rate": 4.348648215372106e-06,
      "log_odds_chosen": 9.273715019226074,
      "log_odds_ratio": -0.025510499253869057,
      "logits/chosen": -2.7492473125457764,
      "logits/rejected": -1.7804412841796875,
      "logps/chosen": -0.3289060592651367,
      "logps/rejected": -8.170428276062012,
      "loss": 0.287,
      "nll_loss": 0.2702774405479431,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06578120589256287,
      "rewards/margins": 1.568304419517517,
      "rewards/rejected": -1.6340856552124023,
      "step": 3220
    },
    {
      "epoch": 2.31,
      "grad_norm": 5.53125,
      "learning_rate": 4.347004424644176e-06,
      "log_odds_chosen": 9.255958557128906,
      "log_odds_ratio": -0.04089302942156792,
      "logits/chosen": -2.817354202270508,
      "logits/rejected": -1.8950843811035156,
      "logps/chosen": -0.32880938053131104,
      "logps/rejected": -8.185022354125977,
      "loss": 0.2918,
      "nll_loss": 0.2752782106399536,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.06576187163591385,
      "rewards/margins": 1.5712426900863647,
      "rewards/rejected": -1.6370046138763428,
      "step": 3230
    },
    {
      "epoch": 2.31,
      "grad_norm": 6.40625,
      "learning_rate": 4.34536249656874e-06,
      "log_odds_chosen": 8.917261123657227,
      "log_odds_ratio": -0.04923417791724205,
      "logits/chosen": -2.776374101638794,
      "logits/rejected": -1.8802967071533203,
      "logps/chosen": -0.42338424921035767,
      "logps/rejected": -7.919134616851807,
      "loss": 0.2844,
      "nll_loss": 0.29589223861694336,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.0846768468618393,
      "rewards/margins": 1.499150037765503,
      "rewards/rejected": -1.583827018737793,
      "step": 3240
    },
    {
      "epoch": 2.32,
      "grad_norm": 6.5,
      "learning_rate": 4.3437224276306945e-06,
      "log_odds_chosen": 7.567285060882568,
      "log_odds_ratio": -0.028490770608186722,
      "logits/chosen": -2.786663770675659,
      "logits/rejected": -1.9782333374023438,
      "logps/chosen": -0.33940792083740234,
      "logps/rejected": -6.447556495666504,
      "loss": 0.3091,
      "nll_loss": 0.3036688566207886,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.06788158416748047,
      "rewards/margins": 1.2216296195983887,
      "rewards/rejected": -1.2895113229751587,
      "step": 3250
    },
    {
      "epoch": 2.33,
      "grad_norm": 17.875,
      "learning_rate": 4.342084214324218e-06,
      "log_odds_chosen": 7.1914873123168945,
      "log_odds_ratio": -0.0445329025387764,
      "logits/chosen": -2.756364345550537,
      "logits/rejected": -2.037761926651001,
      "logps/chosen": -0.3597845435142517,
      "logps/rejected": -6.1982855796813965,
      "loss": 0.3067,
      "nll_loss": 0.2898246645927429,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07195691019296646,
      "rewards/margins": 1.1677000522613525,
      "rewards/rejected": -1.2396571636199951,
      "step": 3260
    },
    {
      "epoch": 2.33,
      "grad_norm": 13.0625,
      "learning_rate": 4.340447853152738e-06,
      "log_odds_chosen": 8.178131103515625,
      "log_odds_ratio": -0.05221225693821907,
      "logits/chosen": -2.783968925476074,
      "logits/rejected": -1.8930259943008423,
      "logps/chosen": -0.36348551511764526,
      "logps/rejected": -7.173810005187988,
      "loss": 0.2735,
      "nll_loss": 0.258183091878891,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.07269710302352905,
      "rewards/margins": 1.3620648384094238,
      "rewards/rejected": -1.434761881828308,
      "step": 3270
    },
    {
      "epoch": 2.34,
      "grad_norm": 5.71875,
      "learning_rate": 4.338813340628896e-06,
      "log_odds_chosen": 9.553061485290527,
      "log_odds_ratio": -0.037361737340688705,
      "logits/chosen": -2.797703266143799,
      "logits/rejected": -1.5537981986999512,
      "logps/chosen": -0.33459705114364624,
      "logps/rejected": -8.510927200317383,
      "loss": 0.2714,
      "nll_loss": 0.27574431896209717,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.06691941618919373,
      "rewards/margins": 1.6352659463882446,
      "rewards/rejected": -1.7021853923797607,
      "step": 3280
    },
    {
      "epoch": 2.35,
      "grad_norm": 12.9375,
      "learning_rate": 4.337180673274523e-06,
      "log_odds_chosen": 8.518575668334961,
      "log_odds_ratio": -0.04755989462137222,
      "logits/chosen": -2.8275094032287598,
      "logits/rejected": -1.775490403175354,
      "logps/chosen": -0.33448418974876404,
      "logps/rejected": -7.48303747177124,
      "loss": 0.2834,
      "nll_loss": 0.27971869707107544,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.06689684092998505,
      "rewards/margins": 1.4297107458114624,
      "rewards/rejected": -1.4966075420379639,
      "step": 3290
    },
    {
      "epoch": 2.36,
      "grad_norm": 18.625,
      "learning_rate": 4.3355498476206e-06,
      "log_odds_chosen": 9.549077033996582,
      "log_odds_ratio": -0.0418349914252758,
      "logits/chosen": -2.8091912269592285,
      "logits/rejected": -1.7388540506362915,
      "logps/chosen": -0.3228936493396759,
      "logps/rejected": -8.422323226928711,
      "loss": 0.2922,
      "nll_loss": 0.28200000524520874,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.06457872688770294,
      "rewards/margins": 1.6198861598968506,
      "rewards/rejected": -1.6844650506973267,
      "step": 3300
    },
    {
      "epoch": 2.36,
      "grad_norm": 5.375,
      "learning_rate": 4.333920860207238e-06,
      "log_odds_chosen": 9.765109062194824,
      "log_odds_ratio": -0.03472736105322838,
      "logits/chosen": -2.820134162902832,
      "logits/rejected": -1.793215036392212,
      "logps/chosen": -0.4387420117855072,
      "logps/rejected": -8.810163497924805,
      "loss": 0.2826,
      "nll_loss": 0.2727591395378113,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.08774841576814651,
      "rewards/margins": 1.6742843389511108,
      "rewards/rejected": -1.7620328664779663,
      "step": 3310
    },
    {
      "epoch": 2.37,
      "grad_norm": 9.0625,
      "learning_rate": 4.332293707583636e-06,
      "log_odds_chosen": 9.640665054321289,
      "log_odds_ratio": -0.03734064847230911,
      "logits/chosen": -2.774531841278076,
      "logits/rejected": -1.7828537225723267,
      "logps/chosen": -0.41496315598487854,
      "logps/rejected": -8.623726844787598,
      "loss": 0.3401,
      "nll_loss": 0.3533519506454468,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.08299262821674347,
      "rewards/margins": 1.6417526006698608,
      "rewards/rejected": -1.7247453927993774,
      "step": 3320
    },
    {
      "epoch": 2.38,
      "grad_norm": 48.75,
      "learning_rate": 4.330668386308059e-06,
      "log_odds_chosen": 8.390340805053711,
      "log_odds_ratio": -0.03434724360704422,
      "logits/chosen": -2.824404716491699,
      "logits/rejected": -1.6728851795196533,
      "logps/chosen": -0.36389368772506714,
      "logps/rejected": -7.420858860015869,
      "loss": 0.2978,
      "nll_loss": 0.28047677874565125,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07277874648571014,
      "rewards/margins": 1.4113929271697998,
      "rewards/rejected": -1.4841716289520264,
      "step": 3330
    },
    {
      "epoch": 2.38,
      "grad_norm": 27.25,
      "learning_rate": 4.329044892947799e-06,
      "log_odds_chosen": 7.926604270935059,
      "log_odds_ratio": -0.04124735668301582,
      "logits/chosen": -2.8356728553771973,
      "logits/rejected": -1.8978474140167236,
      "logps/chosen": -0.3497302830219269,
      "logps/rejected": -6.923464775085449,
      "loss": 0.258,
      "nll_loss": 0.24766549468040466,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.06994606554508209,
      "rewards/margins": 1.3147468566894531,
      "rewards/rejected": -1.384692907333374,
      "step": 3340
    },
    {
      "epoch": 2.39,
      "grad_norm": 56.0,
      "learning_rate": 4.327423224079155e-06,
      "log_odds_chosen": 6.804099082946777,
      "log_odds_ratio": -0.05955817550420761,
      "logits/chosen": -2.8271944522857666,
      "logits/rejected": -2.0780460834503174,
      "logps/chosen": -0.3145996928215027,
      "logps/rejected": -5.751158237457275,
      "loss": 0.2661,
      "nll_loss": 0.25528645515441895,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.06291993707418442,
      "rewards/margins": 1.087311863899231,
      "rewards/rejected": -1.1502315998077393,
      "step": 3350
    },
    {
      "epoch": 2.4,
      "grad_norm": 21.5,
      "learning_rate": 4.325803376287392e-06,
      "log_odds_chosen": 8.253396987915039,
      "log_odds_ratio": -0.0472552627325058,
      "logits/chosen": -2.797362804412842,
      "logits/rejected": -1.7111046314239502,
      "logps/chosen": -0.35209912061691284,
      "logps/rejected": -7.188366889953613,
      "loss": 0.2921,
      "nll_loss": 0.3038533329963684,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.07041982561349869,
      "rewards/margins": 1.3672535419464111,
      "rewards/rejected": -1.4376734495162964,
      "step": 3360
    },
    {
      "epoch": 2.41,
      "grad_norm": 8.6875,
      "learning_rate": 4.32418534616672e-06,
      "log_odds_chosen": 10.038507461547852,
      "log_odds_ratio": -0.025366192683577538,
      "logits/chosen": -2.804220676422119,
      "logits/rejected": -1.6713545322418213,
      "logps/chosen": -0.35934942960739136,
      "logps/rejected": -9.00309944152832,
      "loss": 0.2658,
      "nll_loss": 0.26834237575531006,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.0718698799610138,
      "rewards/margins": 1.728750228881836,
      "rewards/rejected": -1.8006200790405273,
      "step": 3370
    },
    {
      "epoch": 2.41,
      "grad_norm": 6.0,
      "learning_rate": 4.322569130320256e-06,
      "log_odds_chosen": 9.466753005981445,
      "log_odds_ratio": -0.03348467871546745,
      "logits/chosen": -2.8029136657714844,
      "logits/rejected": -1.865025520324707,
      "logps/chosen": -0.38565486669540405,
      "logps/rejected": -8.445741653442383,
      "loss": 0.2768,
      "nll_loss": 0.2976570129394531,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.07713097333908081,
      "rewards/margins": 1.6120173931121826,
      "rewards/rejected": -1.6891483068466187,
      "step": 3380
    },
    {
      "epoch": 2.42,
      "grad_norm": 13.5,
      "learning_rate": 4.320954725359999e-06,
      "log_odds_chosen": 9.79299259185791,
      "log_odds_ratio": -0.027992457151412964,
      "logits/chosen": -2.841531753540039,
      "logits/rejected": -1.9643710851669312,
      "logps/chosen": -0.35286644101142883,
      "logps/rejected": -8.675617218017578,
      "loss": 0.2939,
      "nll_loss": 0.3023623824119568,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07057328522205353,
      "rewards/margins": 1.6645504236221313,
      "rewards/rejected": -1.735123634338379,
      "step": 3390
    },
    {
      "epoch": 2.43,
      "grad_norm": 41.25,
      "learning_rate": 4.319342127906801e-06,
      "log_odds_chosen": 9.294361114501953,
      "log_odds_ratio": -0.045621536672115326,
      "logits/chosen": -2.8277316093444824,
      "logits/rejected": -1.8078911304473877,
      "logps/chosen": -0.36391180753707886,
      "logps/rejected": -8.31618595123291,
      "loss": 0.2618,
      "nll_loss": 0.25773972272872925,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.07278236001729965,
      "rewards/margins": 1.5904548168182373,
      "rewards/rejected": -1.66323721408844,
      "step": 3400
    },
    {
      "epoch": 2.43,
      "grad_norm": 5.65625,
      "learning_rate": 4.317731334590332e-06,
      "log_odds_chosen": 9.953015327453613,
      "log_odds_ratio": -0.04586916044354439,
      "logits/chosen": -2.7547903060913086,
      "logits/rejected": -1.697317361831665,
      "logps/chosen": -0.3400295674800873,
      "logps/rejected": -8.843751907348633,
      "loss": 0.2692,
      "nll_loss": 0.24716739356517792,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.06800590455532074,
      "rewards/margins": 1.700744390487671,
      "rewards/rejected": -1.7687504291534424,
      "step": 3410
    },
    {
      "epoch": 2.44,
      "grad_norm": 6.78125,
      "learning_rate": 4.316122342049056e-06,
      "log_odds_chosen": 11.804905891418457,
      "log_odds_ratio": -0.023801427334547043,
      "logits/chosen": -2.755643844604492,
      "logits/rejected": -1.4954235553741455,
      "logps/chosen": -0.4441162943840027,
      "logps/rejected": -10.793535232543945,
      "loss": 0.2901,
      "nll_loss": 0.3134029507637024,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08882326632738113,
      "rewards/margins": 2.0698838233947754,
      "rewards/rejected": -2.1587071418762207,
      "step": 3420
    },
    {
      "epoch": 2.45,
      "grad_norm": 7.03125,
      "learning_rate": 4.314515146930197e-06,
      "log_odds_chosen": 9.51766586303711,
      "log_odds_ratio": -0.029458215460181236,
      "logits/chosen": -2.814960241317749,
      "logits/rejected": -1.9240309000015259,
      "logps/chosen": -0.41664624214172363,
      "logps/rejected": -8.569226264953613,
      "loss": 0.2687,
      "nll_loss": 0.25920039415359497,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.08332924544811249,
      "rewards/margins": 1.6305160522460938,
      "rewards/rejected": -1.7138452529907227,
      "step": 3430
    },
    {
      "epoch": 2.46,
      "grad_norm": 5.875,
      "learning_rate": 4.312909745889715e-06,
      "log_odds_chosen": 7.934914588928223,
      "log_odds_ratio": -0.033872656524181366,
      "logits/chosen": -2.7913076877593994,
      "logits/rejected": -1.9354534149169922,
      "logps/chosen": -0.411059707403183,
      "logps/rejected": -7.0791335105896,
      "loss": 0.2602,
      "nll_loss": 0.24955859780311584,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.082211934030056,
      "rewards/margins": 1.3336145877838135,
      "rewards/rejected": -1.4158265590667725,
      "step": 3440
    },
    {
      "epoch": 2.46,
      "grad_norm": 37.75,
      "learning_rate": 4.311306135592269e-06,
      "log_odds_chosen": 8.438997268676758,
      "log_odds_ratio": -0.04086356610059738,
      "logits/chosen": -2.8126413822174072,
      "logits/rejected": -1.8445075750350952,
      "logps/chosen": -0.3259912133216858,
      "logps/rejected": -7.326315879821777,
      "loss": 0.2547,
      "nll_loss": 0.24355188012123108,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.06519824266433716,
      "rewards/margins": 1.4000650644302368,
      "rewards/rejected": -1.4652631282806396,
      "step": 3450
    },
    {
      "epoch": 2.47,
      "grad_norm": 11.5625,
      "learning_rate": 4.309704312711197e-06,
      "log_odds_chosen": 7.0475311279296875,
      "log_odds_ratio": -0.04972859099507332,
      "logits/chosen": -2.7898261547088623,
      "logits/rejected": -2.0793161392211914,
      "logps/chosen": -0.3466077148914337,
      "logps/rejected": -5.998042106628418,
      "loss": 0.2998,
      "nll_loss": 0.27084097266197205,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.06932154297828674,
      "rewards/margins": 1.1302868127822876,
      "rewards/rejected": -1.199608564376831,
      "step": 3460
    },
    {
      "epoch": 2.48,
      "grad_norm": 25.0,
      "learning_rate": 4.3081042739284794e-06,
      "log_odds_chosen": 9.243393898010254,
      "log_odds_ratio": -0.03272037208080292,
      "logits/chosen": -2.832883834838867,
      "logits/rejected": -1.8257993459701538,
      "logps/chosen": -0.3495597839355469,
      "logps/rejected": -8.209525108337402,
      "loss": 0.264,
      "nll_loss": 0.24593329429626465,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.06991195678710938,
      "rewards/margins": 1.5719932317733765,
      "rewards/rejected": -1.6419051885604858,
      "step": 3470
    },
    {
      "epoch": 2.48,
      "grad_norm": 11.125,
      "learning_rate": 4.306506015934716e-06,
      "log_odds_chosen": 8.312199592590332,
      "log_odds_ratio": -0.024369016289711,
      "logits/chosen": -2.8706321716308594,
      "logits/rejected": -1.9261465072631836,
      "logps/chosen": -0.3649846911430359,
      "logps/rejected": -7.237170219421387,
      "loss": 0.2681,
      "nll_loss": 0.25165751576423645,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07299693673849106,
      "rewards/margins": 1.3744370937347412,
      "rewards/rejected": -1.4474341869354248,
      "step": 3480
    },
    {
      "epoch": 2.49,
      "grad_norm": 58.5,
      "learning_rate": 4.304909535429091e-06,
      "log_odds_chosen": 10.501726150512695,
      "log_odds_ratio": -0.029145091772079468,
      "logits/chosen": -2.8260130882263184,
      "logits/rejected": -1.6204798221588135,
      "logps/chosen": -0.3936752378940582,
      "logps/rejected": -9.544812202453613,
      "loss": 0.2738,
      "nll_loss": 0.2682205140590668,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.07873504608869553,
      "rewards/margins": 1.8302274942398071,
      "rewards/rejected": -1.9089622497558594,
      "step": 3490
    },
    {
      "epoch": 2.5,
      "grad_norm": 15.9375,
      "learning_rate": 4.303314829119352e-06,
      "log_odds_chosen": 10.030715942382812,
      "log_odds_ratio": -0.04658619314432144,
      "logits/chosen": -2.796156406402588,
      "logits/rejected": -1.8100440502166748,
      "logps/chosen": -0.3424500823020935,
      "logps/rejected": -8.992310523986816,
      "loss": 0.2764,
      "nll_loss": 0.24179503321647644,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.06849001348018646,
      "rewards/margins": 1.7299721240997314,
      "rewards/rejected": -1.798462152481079,
      "step": 3500
    },
    {
      "epoch": 2.51,
      "grad_norm": 122.5,
      "learning_rate": 4.301721893721773e-06,
      "log_odds_chosen": 10.208802223205566,
      "log_odds_ratio": -0.04213592782616615,
      "logits/chosen": -2.797102212905884,
      "logits/rejected": -1.4663350582122803,
      "logps/chosen": -0.3950093686580658,
      "logps/rejected": -9.20128345489502,
      "loss": 0.301,
      "nll_loss": 0.3496857285499573,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.07900188118219376,
      "rewards/margins": 1.7612546682357788,
      "rewards/rejected": -1.840256690979004,
      "step": 3510
    },
    {
      "epoch": 2.51,
      "grad_norm": 10.25,
      "learning_rate": 4.300130725961134e-06,
      "log_odds_chosen": 6.270288467407227,
      "log_odds_ratio": -0.05212852358818054,
      "logits/chosen": -2.8619461059570312,
      "logits/rejected": -2.255450487136841,
      "logps/chosen": -0.36604180932044983,
      "logps/rejected": -5.2709221839904785,
      "loss": 0.265,
      "nll_loss": 0.26578956842422485,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.07320836186408997,
      "rewards/margins": 0.9809761047363281,
      "rewards/rejected": -1.0541845560073853,
      "step": 3520
    },
    {
      "epoch": 2.52,
      "grad_norm": 10.125,
      "learning_rate": 4.298541322570686e-06,
      "log_odds_chosen": 6.3298749923706055,
      "log_odds_ratio": -0.03688116371631622,
      "logits/chosen": -2.834110736846924,
      "logits/rejected": -2.0629196166992188,
      "logps/chosen": -0.3773978352546692,
      "logps/rejected": -5.346269130706787,
      "loss": 0.247,
      "nll_loss": 0.24589797854423523,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.07547955214977264,
      "rewards/margins": 0.9937742948532104,
      "rewards/rejected": -1.0692538022994995,
      "step": 3530
    },
    {
      "epoch": 2.53,
      "grad_norm": 136.0,
      "learning_rate": 4.296953680292129e-06,
      "log_odds_chosen": 7.077627658843994,
      "log_odds_ratio": -0.06007321551442146,
      "logits/chosen": -2.754781484603882,
      "logits/rejected": -1.8707349300384521,
      "logps/chosen": -0.41770386695861816,
      "logps/rejected": -6.089178085327148,
      "loss": 0.3063,
      "nll_loss": 0.3370429575443268,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.08354077488183975,
      "rewards/margins": 1.134294867515564,
      "rewards/rejected": -1.2178356647491455,
      "step": 3540
    },
    {
      "epoch": 2.53,
      "grad_norm": 7.1875,
      "learning_rate": 4.295367795875578e-06,
      "log_odds_chosen": 4.167150974273682,
      "log_odds_ratio": -0.06860674917697906,
      "logits/chosen": -2.8772263526916504,
      "logits/rejected": -2.3703644275665283,
      "logps/chosen": -0.3609137237071991,
      "logps/rejected": -3.2382397651672363,
      "loss": 0.2878,
      "nll_loss": 0.3157772421836853,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.07218273729085922,
      "rewards/margins": 0.575465202331543,
      "rewards/rejected": -0.6476479172706604,
      "step": 3550
    },
    {
      "epoch": 2.54,
      "grad_norm": 7.9375,
      "learning_rate": 4.293783666079539e-06,
      "log_odds_chosen": 6.502009391784668,
      "log_odds_ratio": -0.03050677478313446,
      "logits/chosen": -2.6957783699035645,
      "logits/rejected": -1.993639349937439,
      "logps/chosen": -0.31069567799568176,
      "logps/rejected": -5.301680088043213,
      "loss": 0.2675,
      "nll_loss": 0.2550848424434662,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06213913485407829,
      "rewards/margins": 0.9981969594955444,
      "rewards/rejected": -1.0603359937667847,
      "step": 3560
    },
    {
      "epoch": 2.55,
      "grad_norm": 8.875,
      "learning_rate": 4.292201287670881e-06,
      "log_odds_chosen": 7.6693291664123535,
      "log_odds_ratio": -0.04327741265296936,
      "logits/chosen": -2.78936505317688,
      "logits/rejected": -1.8133246898651123,
      "logps/chosen": -0.42709359526634216,
      "logps/rejected": -6.817948818206787,
      "loss": 0.3028,
      "nll_loss": 0.2974828779697418,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.0854187160730362,
      "rewards/margins": 1.2781710624694824,
      "rewards/rejected": -1.363589882850647,
      "step": 3570
    },
    {
      "epoch": 2.56,
      "grad_norm": 7.90625,
      "learning_rate": 4.2906206574248056e-06,
      "log_odds_chosen": 7.7335991859436035,
      "log_odds_ratio": -0.03188318759202957,
      "logits/chosen": -2.823167085647583,
      "logits/rejected": -1.8172123432159424,
      "logps/chosen": -0.39690691232681274,
      "logps/rejected": -6.871090888977051,
      "loss": 0.2829,
      "nll_loss": 0.26852673292160034,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.07938139140605927,
      "rewards/margins": 1.2948367595672607,
      "rewards/rejected": -1.3742179870605469,
      "step": 3580
    },
    {
      "epoch": 2.56,
      "grad_norm": 8.375,
      "learning_rate": 4.289041772124823e-06,
      "log_odds_chosen": 7.018429756164551,
      "log_odds_ratio": -0.04488766938447952,
      "logits/chosen": -2.8090367317199707,
      "logits/rejected": -2.016038417816162,
      "logps/chosen": -0.4279165267944336,
      "logps/rejected": -6.177850246429443,
      "loss": 0.2934,
      "nll_loss": 0.27853289246559143,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.08558329194784164,
      "rewards/margins": 1.1499868631362915,
      "rewards/rejected": -1.235569953918457,
      "step": 3590
    },
    {
      "epoch": 2.57,
      "grad_norm": 7.84375,
      "learning_rate": 4.2874646285627205e-06,
      "log_odds_chosen": 7.609469413757324,
      "log_odds_ratio": -0.029052983969449997,
      "logits/chosen": -2.792881727218628,
      "logits/rejected": -1.8695917129516602,
      "logps/chosen": -0.35575172305107117,
      "logps/rejected": -6.599902153015137,
      "loss": 0.2679,
      "nll_loss": 0.26382261514663696,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07115034759044647,
      "rewards/margins": 1.2488303184509277,
      "rewards/rejected": -1.3199807405471802,
      "step": 3600
    },
    {
      "epoch": 2.58,
      "grad_norm": 11.0625,
      "learning_rate": 4.2858892235385405e-06,
      "log_odds_chosen": 9.122172355651855,
      "log_odds_ratio": -0.02934536337852478,
      "logits/chosen": -2.7800889015197754,
      "logits/rejected": -1.775040626525879,
      "logps/chosen": -0.40309590101242065,
      "logps/rejected": -8.151556968688965,
      "loss": 0.2977,
      "nll_loss": 0.28484228253364563,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08061918616294861,
      "rewards/margins": 1.5496922731399536,
      "rewards/rejected": -1.630311369895935,
      "step": 3610
    },
    {
      "epoch": 2.58,
      "grad_norm": 5.03125,
      "learning_rate": 4.2843155538605454e-06,
      "log_odds_chosen": 9.74396800994873,
      "log_odds_ratio": -0.03267233446240425,
      "logits/chosen": -2.8342878818511963,
      "logits/rejected": -1.560414433479309,
      "logps/chosen": -0.358869731426239,
      "logps/rejected": -8.761539459228516,
      "loss": 0.286,
      "nll_loss": 0.2713310122489929,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.071773961186409,
      "rewards/margins": 1.680533766746521,
      "rewards/rejected": -1.7523078918457031,
      "step": 3620
    },
    {
      "epoch": 2.59,
      "grad_norm": 8.3125,
      "learning_rate": 4.2827436163452e-06,
      "log_odds_chosen": 8.96192741394043,
      "log_odds_ratio": -0.025754928588867188,
      "logits/chosen": -2.807020664215088,
      "logits/rejected": -1.7215229272842407,
      "logps/chosen": -0.37024766206741333,
      "logps/rejected": -7.972418785095215,
      "loss": 0.2918,
      "nll_loss": 0.2985715866088867,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07404953241348267,
      "rewards/margins": 1.5204341411590576,
      "rewards/rejected": -1.5944838523864746,
      "step": 3630
    },
    {
      "epoch": 2.6,
      "grad_norm": 27.375,
      "learning_rate": 4.2811734078171365e-06,
      "log_odds_chosen": 9.322843551635742,
      "log_odds_ratio": -0.02989858388900757,
      "logits/chosen": -2.800328016281128,
      "logits/rejected": -1.6582973003387451,
      "logps/chosen": -0.33669382333755493,
      "logps/rejected": -8.228538513183594,
      "loss": 0.2924,
      "nll_loss": 0.294689804315567,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.06733877211809158,
      "rewards/margins": 1.578368902206421,
      "rewards/rejected": -1.6457077264785767,
      "step": 3640
    },
    {
      "epoch": 2.61,
      "grad_norm": 83.5,
      "learning_rate": 4.27960492510913e-06,
      "log_odds_chosen": 10.60124397277832,
      "log_odds_ratio": -0.03527144342660904,
      "logits/chosen": -2.7842440605163574,
      "logits/rejected": -1.780975103378296,
      "logps/chosen": -0.35910865664482117,
      "logps/rejected": -9.61752986907959,
      "loss": 0.2886,
      "nll_loss": 0.26610469818115234,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07182172685861588,
      "rewards/margins": 1.851684331893921,
      "rewards/rejected": -1.9235061407089233,
      "step": 3650
    },
    {
      "epoch": 2.61,
      "grad_norm": 11.6875,
      "learning_rate": 4.278038165062074e-06,
      "log_odds_chosen": 9.46485424041748,
      "log_odds_ratio": -0.030444592237472534,
      "logits/chosen": -2.7923178672790527,
      "logits/rejected": -1.7434918880462646,
      "logps/chosen": -0.36185431480407715,
      "logps/rejected": -8.415678977966309,
      "loss": 0.2739,
      "nll_loss": 0.2679918110370636,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.07237086445093155,
      "rewards/margins": 1.6107648611068726,
      "rewards/rejected": -1.6831356287002563,
      "step": 3660
    },
    {
      "epoch": 2.62,
      "grad_norm": 11.875,
      "learning_rate": 4.276473124524951e-06,
      "log_odds_chosen": 10.846599578857422,
      "log_odds_ratio": -0.0410008430480957,
      "logits/chosen": -2.814073324203491,
      "logits/rejected": -1.7574056386947632,
      "logps/chosen": -0.3752827048301697,
      "logps/rejected": -9.809711456298828,
      "loss": 0.3038,
      "nll_loss": 0.2877393960952759,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.0750565379858017,
      "rewards/margins": 1.886885643005371,
      "rewards/rejected": -1.9619423151016235,
      "step": 3670
    },
    {
      "epoch": 2.63,
      "grad_norm": 80.5,
      "learning_rate": 4.274909800354809e-06,
      "log_odds_chosen": 8.6845703125,
      "log_odds_ratio": -0.04010792821645737,
      "logits/chosen": -2.8353562355041504,
      "logits/rejected": -1.9117534160614014,
      "logps/chosen": -0.36866775155067444,
      "logps/rejected": -7.723712921142578,
      "loss": 0.2716,
      "nll_loss": 0.28985413908958435,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07373355329036713,
      "rewards/margins": 1.4710088968276978,
      "rewards/rejected": -1.5447423458099365,
      "step": 3680
    },
    {
      "epoch": 2.63,
      "grad_norm": 25.5,
      "learning_rate": 4.27334818941673e-06,
      "log_odds_chosen": 11.139304161071777,
      "log_odds_ratio": -0.02806129679083824,
      "logits/chosen": -2.829277515411377,
      "logits/rejected": -1.7117398977279663,
      "logps/chosen": -0.43500715494155884,
      "logps/rejected": -10.296022415161133,
      "loss": 0.298,
      "nll_loss": 0.30640918016433716,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08700142055749893,
      "rewards/margins": 1.972203254699707,
      "rewards/rejected": -2.0592048168182373,
      "step": 3690
    },
    {
      "epoch": 2.64,
      "grad_norm": 14.0625,
      "learning_rate": 4.271788288583805e-06,
      "log_odds_chosen": 9.920567512512207,
      "log_odds_ratio": -0.0430777445435524,
      "logits/chosen": -2.800262928009033,
      "logits/rejected": -1.6749203205108643,
      "logps/chosen": -0.3537217676639557,
      "logps/rejected": -8.953767776489258,
      "loss": 0.292,
      "nll_loss": 0.28976768255233765,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.0707443580031395,
      "rewards/margins": 1.720009207725525,
      "rewards/rejected": -1.7907536029815674,
      "step": 3700
    },
    {
      "epoch": 2.65,
      "grad_norm": 71.0,
      "learning_rate": 4.270230094737115e-06,
      "log_odds_chosen": 10.269146919250488,
      "log_odds_ratio": -0.03263562172651291,
      "logits/chosen": -2.8407130241394043,
      "logits/rejected": -1.8693710565567017,
      "logps/chosen": -0.3560786247253418,
      "logps/rejected": -9.248159408569336,
      "loss": 0.2528,
      "nll_loss": 0.24508798122406006,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.07121572643518448,
      "rewards/margins": 1.7784162759780884,
      "rewards/rejected": -1.8496320247650146,
      "step": 3710
    },
    {
      "epoch": 2.66,
      "grad_norm": 163.0,
      "learning_rate": 4.268673604765692e-06,
      "log_odds_chosen": 8.313056945800781,
      "log_odds_ratio": -0.04137984663248062,
      "logits/chosen": -2.8410933017730713,
      "logits/rejected": -1.973070502281189,
      "logps/chosen": -0.3569386601448059,
      "logps/rejected": -7.325540065765381,
      "loss": 0.2936,
      "nll_loss": 0.3143910765647888,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.07138773798942566,
      "rewards/margins": 1.3937203884124756,
      "rewards/rejected": -1.4651081562042236,
      "step": 3720
    },
    {
      "epoch": 2.66,
      "grad_norm": 69.0,
      "learning_rate": 4.267118815566505e-06,
      "log_odds_chosen": 11.062416076660156,
      "log_odds_ratio": -0.02929757535457611,
      "logits/chosen": -2.8256027698516846,
      "logits/rejected": -1.8267700672149658,
      "logps/chosen": -0.34090739488601685,
      "logps/rejected": -10.022697448730469,
      "loss": 0.2669,
      "nll_loss": 0.2610389292240143,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06818147003650665,
      "rewards/margins": 1.936357855796814,
      "rewards/rejected": -2.0045394897460938,
      "step": 3730
    },
    {
      "epoch": 2.67,
      "grad_norm": 7.21875,
      "learning_rate": 4.265565724044426e-06,
      "log_odds_chosen": 9.198521614074707,
      "log_odds_ratio": -0.03330180048942566,
      "logits/chosen": -2.8640224933624268,
      "logits/rejected": -1.98549485206604,
      "logps/chosen": -0.41554588079452515,
      "logps/rejected": -8.330385208129883,
      "loss": 0.2867,
      "nll_loss": 0.29421791434288025,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.08310917764902115,
      "rewards/margins": 1.58296799659729,
      "rewards/rejected": -1.6660770177841187,
      "step": 3740
    },
    {
      "epoch": 2.68,
      "grad_norm": 26.125,
      "learning_rate": 4.264014327112208e-06,
      "log_odds_chosen": 8.376938819885254,
      "log_odds_ratio": -0.03763898089528084,
      "logits/chosen": -2.8337912559509277,
      "logits/rejected": -1.9237620830535889,
      "logps/chosen": -0.33510148525238037,
      "logps/rejected": -7.281014919281006,
      "loss": 0.2649,
      "nll_loss": 0.23882155120372772,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.06702030450105667,
      "rewards/margins": 1.389182686805725,
      "rewards/rejected": -1.4562032222747803,
      "step": 3750
    },
    {
      "epoch": 2.68,
      "grad_norm": 6.375,
      "learning_rate": 4.26246462169046e-06,
      "log_odds_chosen": 7.407693386077881,
      "log_odds_ratio": -0.0290813185274601,
      "logits/chosen": -2.852565050125122,
      "logits/rejected": -2.0294179916381836,
      "logps/chosen": -0.3306184411048889,
      "logps/rejected": -6.291057586669922,
      "loss": 0.2842,
      "nll_loss": 0.28192299604415894,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06612369418144226,
      "rewards/margins": 1.1920878887176514,
      "rewards/rejected": -1.2582114934921265,
      "step": 3760
    },
    {
      "epoch": 2.69,
      "grad_norm": 27.5,
      "learning_rate": 4.260916604707614e-06,
      "log_odds_chosen": 8.492695808410645,
      "log_odds_ratio": -0.040566060692071915,
      "logits/chosen": -2.8278088569641113,
      "logits/rejected": -1.880413293838501,
      "logps/chosen": -0.40683627128601074,
      "logps/rejected": -7.554902076721191,
      "loss": 0.269,
      "nll_loss": 0.2897428870201111,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.08136724680662155,
      "rewards/margins": 1.4296132326126099,
      "rewards/rejected": -1.5109803676605225,
      "step": 3770
    },
    {
      "epoch": 2.7,
      "grad_norm": 5.75,
      "learning_rate": 4.25937027309991e-06,
      "log_odds_chosen": 7.113955497741699,
      "log_odds_ratio": -0.034956224262714386,
      "logits/chosen": -2.863619089126587,
      "logits/rejected": -2.0861170291900635,
      "logps/chosen": -0.3817751407623291,
      "logps/rejected": -6.190094947814941,
      "loss": 0.3086,
      "nll_loss": 0.32031726837158203,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07635502517223358,
      "rewards/margins": 1.1616640090942383,
      "rewards/rejected": -1.2380189895629883,
      "step": 3780
    },
    {
      "epoch": 2.71,
      "grad_norm": 11.5625,
      "learning_rate": 4.257825623811364e-06,
      "log_odds_chosen": 6.6032867431640625,
      "log_odds_ratio": -0.04721903055906296,
      "logits/chosen": -2.870551824569702,
      "logits/rejected": -2.1494574546813965,
      "logps/chosen": -0.3325832486152649,
      "logps/rejected": -5.56205940246582,
      "loss": 0.2733,
      "nll_loss": 0.2754889130592346,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.06651665270328522,
      "rewards/margins": 1.0458953380584717,
      "rewards/rejected": -1.1124117374420166,
      "step": 3790
    },
    {
      "epoch": 2.71,
      "grad_norm": 13.625,
      "learning_rate": 4.256282653793743e-06,
      "log_odds_chosen": 9.226226806640625,
      "log_odds_ratio": -0.04071826487779617,
      "logits/chosen": -2.8239336013793945,
      "logits/rejected": -1.799629807472229,
      "logps/chosen": -0.37041693925857544,
      "logps/rejected": -8.24487018585205,
      "loss": 0.2801,
      "nll_loss": 0.2625892162322998,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.07408339530229568,
      "rewards/margins": 1.5748907327651978,
      "rewards/rejected": -1.648974061012268,
      "step": 3800
    },
    {
      "epoch": 2.72,
      "grad_norm": 43.0,
      "learning_rate": 4.254741360006543e-06,
      "log_odds_chosen": 7.27142858505249,
      "log_odds_ratio": -0.04866673797369003,
      "logits/chosen": -2.8658313751220703,
      "logits/rejected": -2.249227523803711,
      "logps/chosen": -0.3399750590324402,
      "logps/rejected": -6.300660133361816,
      "loss": 0.2534,
      "nll_loss": 0.22867396473884583,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.06799499690532684,
      "rewards/margins": 1.1921371221542358,
      "rewards/rejected": -1.2601318359375,
      "step": 3810
    },
    {
      "epoch": 2.73,
      "grad_norm": 7.5625,
      "learning_rate": 4.25320173941696e-06,
      "log_odds_chosen": 8.403349876403809,
      "log_odds_ratio": -0.03414331004023552,
      "logits/chosen": -2.849536657333374,
      "logits/rejected": -2.041189193725586,
      "logps/chosen": -0.3983810842037201,
      "logps/rejected": -7.541253566741943,
      "loss": 0.277,
      "nll_loss": 0.2703934609889984,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07967622578144073,
      "rewards/margins": 1.428574562072754,
      "rewards/rejected": -1.5082508325576782,
      "step": 3820
    },
    {
      "epoch": 2.73,
      "grad_norm": 11.6875,
      "learning_rate": 4.251663788999866e-06,
      "log_odds_chosen": 8.267216682434082,
      "log_odds_ratio": -0.03860500827431679,
      "logits/chosen": -2.8517775535583496,
      "logits/rejected": -2.037395477294922,
      "logps/chosen": -0.35646042227745056,
      "logps/rejected": -7.292947292327881,
      "loss": 0.2727,
      "nll_loss": 0.262928307056427,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.07129208743572235,
      "rewards/margins": 1.3872973918914795,
      "rewards/rejected": -1.4585894346237183,
      "step": 3830
    },
    {
      "epoch": 2.74,
      "grad_norm": 24.625,
      "learning_rate": 4.250127505737787e-06,
      "log_odds_chosen": 9.158185958862305,
      "log_odds_ratio": -0.02719098888337612,
      "logits/chosen": -2.81072735786438,
      "logits/rejected": -1.8471095561981201,
      "logps/chosen": -0.3558461368083954,
      "logps/rejected": -8.152399063110352,
      "loss": 0.2867,
      "nll_loss": 0.2748354375362396,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.07116921991109848,
      "rewards/margins": 1.5593106746673584,
      "rewards/rejected": -1.6304798126220703,
      "step": 3840
    },
    {
      "epoch": 2.75,
      "grad_norm": 6.21875,
      "learning_rate": 4.2485928866208736e-06,
      "log_odds_chosen": 5.923229217529297,
      "log_odds_ratio": -0.0449279360473156,
      "logits/chosen": -2.8480160236358643,
      "logits/rejected": -2.3773159980773926,
      "logps/chosen": -0.3430297374725342,
      "logps/rejected": -4.906435012817383,
      "loss": 0.2786,
      "nll_loss": 0.26425907015800476,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0686059445142746,
      "rewards/margins": 0.9126811027526855,
      "rewards/rejected": -0.981286883354187,
      "step": 3850
    },
    {
      "epoch": 2.76,
      "grad_norm": 11.9375,
      "learning_rate": 4.247059928646881e-06,
      "log_odds_chosen": 9.403549194335938,
      "log_odds_ratio": -0.04165271669626236,
      "logits/chosen": -2.783756971359253,
      "logits/rejected": -1.7871586084365845,
      "logps/chosen": -0.44730639457702637,
      "logps/rejected": -8.516668319702148,
      "loss": 0.3078,
      "nll_loss": 0.2746645212173462,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.08946128189563751,
      "rewards/margins": 1.6138725280761719,
      "rewards/rejected": -1.7033336162567139,
      "step": 3860
    },
    {
      "epoch": 2.76,
      "grad_norm": 39.25,
      "learning_rate": 4.245528628821135e-06,
      "log_odds_chosen": 8.89954662322998,
      "log_odds_ratio": -0.02781747281551361,
      "logits/chosen": -2.8360252380371094,
      "logits/rejected": -1.8986726999282837,
      "logps/chosen": -0.3417971730232239,
      "logps/rejected": -7.838662147521973,
      "loss": 0.2597,
      "nll_loss": 0.28470245003700256,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06835943460464478,
      "rewards/margins": 1.4993728399276733,
      "rewards/rejected": -1.5677324533462524,
      "step": 3870
    },
    {
      "epoch": 2.77,
      "grad_norm": 203.0,
      "learning_rate": 4.243998984156526e-06,
      "log_odds_chosen": 9.094325065612793,
      "log_odds_ratio": -0.033297426998615265,
      "logits/chosen": -2.8292903900146484,
      "logits/rejected": -2.0152010917663574,
      "logps/chosen": -0.35362544655799866,
      "logps/rejected": -8.119251251220703,
      "loss": 0.2861,
      "nll_loss": 0.26851797103881836,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07072508335113525,
      "rewards/margins": 1.5531253814697266,
      "rewards/rejected": -1.6238505840301514,
      "step": 3880
    },
    {
      "epoch": 2.78,
      "grad_norm": 7.0,
      "learning_rate": 4.242470991673459e-06,
      "log_odds_chosen": 8.757810592651367,
      "log_odds_ratio": -0.02989915944635868,
      "logits/chosen": -2.8213298320770264,
      "logits/rejected": -2.0070536136627197,
      "logps/chosen": -0.4033467173576355,
      "logps/rejected": -7.789436340332031,
      "loss": 0.2885,
      "nll_loss": 0.27011099457740784,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0806693509221077,
      "rewards/margins": 1.4772179126739502,
      "rewards/rejected": -1.5578871965408325,
      "step": 3890
    },
    {
      "epoch": 2.78,
      "grad_norm": 247.0,
      "learning_rate": 4.240944648399854e-06,
      "log_odds_chosen": 9.707173347473145,
      "log_odds_ratio": -0.030115026980638504,
      "logits/chosen": -2.812969923019409,
      "logits/rejected": -1.7806400060653687,
      "logps/chosen": -0.3756954073905945,
      "logps/rejected": -8.687716484069824,
      "loss": 0.2924,
      "nll_loss": 0.2944888770580292,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.07513907551765442,
      "rewards/margins": 1.6624042987823486,
      "rewards/rejected": -1.7375433444976807,
      "step": 3900
    },
    {
      "epoch": 2.79,
      "grad_norm": 7.3125,
      "learning_rate": 4.239419951371107e-06,
      "log_odds_chosen": 7.855190277099609,
      "log_odds_ratio": -0.049606241285800934,
      "logits/chosen": -2.8382956981658936,
      "logits/rejected": -1.9690237045288086,
      "logps/chosen": -0.3678521513938904,
      "logps/rejected": -6.8809919357299805,
      "loss": 0.246,
      "nll_loss": 0.24365882575511932,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.07357043772935867,
      "rewards/margins": 1.3026279211044312,
      "rewards/rejected": -1.3761985301971436,
      "step": 3910
    },
    {
      "epoch": 2.8,
      "grad_norm": 8.0625,
      "learning_rate": 4.237896897630065e-06,
      "log_odds_chosen": 9.103668212890625,
      "log_odds_ratio": -0.01939046010375023,
      "logits/chosen": -2.8209850788116455,
      "logits/rejected": -1.7593700885772705,
      "logps/chosen": -0.3552018702030182,
      "logps/rejected": -8.111700057983398,
      "loss": 0.2691,
      "nll_loss": 0.26674580574035645,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07104037702083588,
      "rewards/margins": 1.5512996912002563,
      "rewards/rejected": -1.6223399639129639,
      "step": 3920
    },
    {
      "epoch": 2.81,
      "grad_norm": 6.625,
      "learning_rate": 4.2363754842270135e-06,
      "log_odds_chosen": 8.898262023925781,
      "log_odds_ratio": -0.03263109177350998,
      "logits/chosen": -2.8059921264648438,
      "logits/rejected": -1.9720518589019775,
      "logps/chosen": -0.3654606342315674,
      "logps/rejected": -7.9605865478515625,
      "loss": 0.2722,
      "nll_loss": 0.24651777744293213,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.07309212535619736,
      "rewards/margins": 1.5190250873565674,
      "rewards/rejected": -1.592117190361023,
      "step": 3930
    },
    {
      "epoch": 2.81,
      "grad_norm": 10.9375,
      "learning_rate": 4.23485570821964e-06,
      "log_odds_chosen": 10.18175220489502,
      "log_odds_ratio": -0.029537459835410118,
      "logits/chosen": -2.805284023284912,
      "logits/rejected": -1.7588096857070923,
      "logps/chosen": -0.3637697100639343,
      "logps/rejected": -9.15390682220459,
      "loss": 0.2921,
      "nll_loss": 0.28845328092575073,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07275393605232239,
      "rewards/margins": 1.7580273151397705,
      "rewards/rejected": -1.8307812213897705,
      "step": 3940
    },
    {
      "epoch": 2.82,
      "grad_norm": 276.0,
      "learning_rate": 4.233337566673017e-06,
      "log_odds_chosen": 10.128081321716309,
      "log_odds_ratio": -0.05651165917515755,
      "logits/chosen": -2.7642838954925537,
      "logits/rejected": -1.8633878231048584,
      "logps/chosen": -0.4686155915260315,
      "logps/rejected": -9.309432029724121,
      "loss": 0.299,
      "nll_loss": 0.2807057797908783,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.0937231183052063,
      "rewards/margins": 1.7681634426116943,
      "rewards/rejected": -1.8618863821029663,
      "step": 3950
    },
    {
      "epoch": 2.83,
      "grad_norm": 9.0625,
      "learning_rate": 4.2318210566595795e-06,
      "log_odds_chosen": 10.621389389038086,
      "log_odds_ratio": -0.01841096207499504,
      "logits/chosen": -2.835549831390381,
      "logits/rejected": -1.8261913061141968,
      "logps/chosen": -0.3856840133666992,
      "logps/rejected": -9.641094207763672,
      "loss": 0.2804,
      "nll_loss": 0.28866028785705566,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.0771368071436882,
      "rewards/margins": 1.851082444190979,
      "rewards/rejected": -1.928219199180603,
      "step": 3960
    },
    {
      "epoch": 2.83,
      "grad_norm": 50.5,
      "learning_rate": 4.230306175259094e-06,
      "log_odds_chosen": 8.465916633605957,
      "log_odds_ratio": -0.029557716101408005,
      "logits/chosen": -2.8356223106384277,
      "logits/rejected": -2.1212961673736572,
      "logps/chosen": -0.3706379532814026,
      "logps/rejected": -7.479179382324219,
      "loss": 0.2831,
      "nll_loss": 0.28696635365486145,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07412759214639664,
      "rewards/margins": 1.421708345413208,
      "rewards/rejected": -1.495835781097412,
      "step": 3970
    },
    {
      "epoch": 2.84,
      "grad_norm": 9.375,
      "learning_rate": 4.228792919558642e-06,
      "log_odds_chosen": 9.524942398071289,
      "log_odds_ratio": -0.03312790021300316,
      "logits/chosen": -2.817082405090332,
      "logits/rejected": -1.7293701171875,
      "logps/chosen": -0.35598936676979065,
      "logps/rejected": -8.479887008666992,
      "loss": 0.2787,
      "nll_loss": 0.28613555431365967,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.07119788229465485,
      "rewards/margins": 1.6247799396514893,
      "rewards/rejected": -1.6959775686264038,
      "step": 3980
    },
    {
      "epoch": 2.85,
      "grad_norm": 8.625,
      "learning_rate": 4.227281286652593e-06,
      "log_odds_chosen": 9.673505783081055,
      "log_odds_ratio": -0.027351032942533493,
      "logits/chosen": -2.8485827445983887,
      "logits/rejected": -1.9277604818344116,
      "logps/chosen": -0.36449387669563293,
      "logps/rejected": -8.6371431350708,
      "loss": 0.2689,
      "nll_loss": 0.25819873809814453,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.07289878278970718,
      "rewards/margins": 1.6545298099517822,
      "rewards/rejected": -1.727428674697876,
      "step": 3990
    },
    {
      "epoch": 2.86,
      "grad_norm": 11.875,
      "learning_rate": 4.2257712736425835e-06,
      "log_odds_chosen": 10.454000473022461,
      "log_odds_ratio": -0.021729234606027603,
      "logits/chosen": -2.8186697959899902,
      "logits/rejected": -1.75783371925354,
      "logps/chosen": -0.4553149342536926,
      "logps/rejected": -9.653711318969727,
      "loss": 0.2844,
      "nll_loss": 0.27239570021629333,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0910629853606224,
      "rewards/margins": 1.8396793603897095,
      "rewards/rejected": -1.9307425022125244,
      "step": 4000
    },
    {
      "epoch": 2.86,
      "grad_norm": 4.3125,
      "learning_rate": 4.224262877637488e-06,
      "log_odds_chosen": 9.088384628295898,
      "log_odds_ratio": -0.044943638145923615,
      "logits/chosen": -2.779998302459717,
      "logits/rejected": -1.8532028198242188,
      "logps/chosen": -0.4339783191680908,
      "logps/rejected": -8.246506690979004,
      "loss": 0.2768,
      "nll_loss": 0.27718260884284973,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.08679567277431488,
      "rewards/margins": 1.5625057220458984,
      "rewards/rejected": -1.649301290512085,
      "step": 4010
    },
    {
      "epoch": 2.87,
      "grad_norm": 8.125,
      "learning_rate": 4.2227560957534054e-06,
      "log_odds_chosen": 12.664664268493652,
      "log_odds_ratio": -0.027644852176308632,
      "logits/chosen": -2.8048336505889893,
      "logits/rejected": -1.6057932376861572,
      "logps/chosen": -0.3959696888923645,
      "logps/rejected": -11.737017631530762,
      "loss": 0.3086,
      "nll_loss": 0.2660156190395355,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07919393479824066,
      "rewards/margins": 2.26820969581604,
      "rewards/rejected": -2.3474037647247314,
      "step": 4020
    },
    {
      "epoch": 2.88,
      "grad_norm": 109.0,
      "learning_rate": 4.221250925113625e-06,
      "log_odds_chosen": 9.165689468383789,
      "log_odds_ratio": -0.07480543851852417,
      "logits/chosen": -2.8110146522521973,
      "logits/rejected": -1.914781928062439,
      "logps/chosen": -0.40431445837020874,
      "logps/rejected": -8.292974472045898,
      "loss": 0.2893,
      "nll_loss": 0.2831747233867645,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.08086288720369339,
      "rewards/margins": 1.5777318477630615,
      "rewards/rejected": -1.6585948467254639,
      "step": 4030
    },
    {
      "epoch": 2.88,
      "grad_norm": 7.65625,
      "learning_rate": 4.219747362848612e-06,
      "log_odds_chosen": 11.30323314666748,
      "log_odds_ratio": -0.03717976063489914,
      "logits/chosen": -2.7821929454803467,
      "logits/rejected": -1.768080472946167,
      "logps/chosen": -0.3442782461643219,
      "logps/rejected": -10.223400115966797,
      "loss": 0.2818,
      "nll_loss": 0.2807686924934387,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.0688556432723999,
      "rewards/margins": 1.9758243560791016,
      "rewards/rejected": -2.044680118560791,
      "step": 4040
    },
    {
      "epoch": 2.89,
      "grad_norm": 9.0,
      "learning_rate": 4.2182454060959784e-06,
      "log_odds_chosen": 11.626094818115234,
      "log_odds_ratio": -0.02243855968117714,
      "logits/chosen": -2.83784818649292,
      "logits/rejected": -1.7854284048080444,
      "logps/chosen": -0.375316321849823,
      "logps/rejected": -10.655095100402832,
      "loss": 0.2895,
      "nll_loss": 0.29837626218795776,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07506327331066132,
      "rewards/margins": 2.055955648422241,
      "rewards/rejected": -2.131019115447998,
      "step": 4050
    },
    {
      "epoch": 2.9,
      "grad_norm": 6.25,
      "learning_rate": 4.216745052000467e-06,
      "log_odds_chosen": 9.441912651062012,
      "log_odds_ratio": -0.07335133850574493,
      "logits/chosen": -2.814119815826416,
      "logits/rejected": -2.0087761878967285,
      "logps/chosen": -0.40029245615005493,
      "logps/rejected": -8.542582511901855,
      "loss": 0.253,
      "nll_loss": 0.24284212291240692,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.0800584927201271,
      "rewards/margins": 1.6284582614898682,
      "rewards/rejected": -1.7085163593292236,
      "step": 4060
    },
    {
      "epoch": 2.91,
      "grad_norm": 8.5,
      "learning_rate": 4.21524629771392e-06,
      "log_odds_chosen": 11.409785270690918,
      "log_odds_ratio": -0.032930441200733185,
      "logits/chosen": -2.830761194229126,
      "logits/rejected": -1.863943338394165,
      "logps/chosen": -0.3692954480648041,
      "logps/rejected": -10.434585571289062,
      "loss": 0.288,
      "nll_loss": 0.2768712639808655,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.07385909557342529,
      "rewards/margins": 2.0130581855773926,
      "rewards/rejected": -2.0869174003601074,
      "step": 4070
    },
    {
      "epoch": 2.91,
      "grad_norm": 5.15625,
      "learning_rate": 4.213749140395264e-06,
      "log_odds_chosen": 10.989635467529297,
      "log_odds_ratio": -0.0375329926609993,
      "logits/chosen": -2.829820394515991,
      "logits/rejected": -2.0580878257751465,
      "logps/chosen": -0.35255834460258484,
      "logps/rejected": -9.971551895141602,
      "loss": 0.2844,
      "nll_loss": 0.26232025027275085,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.07051166892051697,
      "rewards/margins": 1.9237985610961914,
      "rewards/rejected": -1.9943103790283203,
      "step": 4080
    },
    {
      "epoch": 2.92,
      "grad_norm": 72.5,
      "learning_rate": 4.2122535772104825e-06,
      "log_odds_chosen": 8.69615364074707,
      "log_odds_ratio": -0.056691087782382965,
      "logits/chosen": -2.840405225753784,
      "logits/rejected": -1.9989845752716064,
      "logps/chosen": -0.3394516110420227,
      "logps/rejected": -7.6478590965271,
      "loss": 0.2743,
      "nll_loss": 0.2390586882829666,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.06789031624794006,
      "rewards/margins": 1.4616813659667969,
      "rewards/rejected": -1.529571771621704,
      "step": 4090
    },
    {
      "epoch": 2.93,
      "grad_norm": 6.90625,
      "learning_rate": 4.2107596053325946e-06,
      "log_odds_chosen": 8.241374969482422,
      "log_odds_ratio": -0.04560881108045578,
      "logits/chosen": -2.870729446411133,
      "logits/rejected": -2.089569568634033,
      "logps/chosen": -0.3773348033428192,
      "logps/rejected": -7.359114170074463,
      "loss": 0.2822,
      "nll_loss": 0.29517418146133423,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.07546695321798325,
      "rewards/margins": 1.3963558673858643,
      "rewards/rejected": -1.471822738647461,
      "step": 4100
    },
    {
      "epoch": 2.93,
      "grad_norm": 5.8125,
      "learning_rate": 4.209267221941637e-06,
      "log_odds_chosen": 8.46093463897705,
      "log_odds_ratio": -0.050915759056806564,
      "logits/chosen": -2.8458335399627686,
      "logits/rejected": -1.9423186779022217,
      "logps/chosen": -0.3869299292564392,
      "logps/rejected": -7.5012969970703125,
      "loss": 0.2991,
      "nll_loss": 0.2835760712623596,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.07738598436117172,
      "rewards/margins": 1.4228734970092773,
      "rewards/rejected": -1.5002593994140625,
      "step": 4110
    },
    {
      "epoch": 2.94,
      "grad_norm": 22.25,
      "learning_rate": 4.207776424224631e-06,
      "log_odds_chosen": 10.923059463500977,
      "log_odds_ratio": -0.023947065696120262,
      "logits/chosen": -2.816763401031494,
      "logits/rejected": -1.7249367237091064,
      "logps/chosen": -0.3471135199069977,
      "logps/rejected": -9.89926815032959,
      "loss": 0.2673,
      "nll_loss": 0.25374776124954224,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06942270696163177,
      "rewards/margins": 1.910431146621704,
      "rewards/rejected": -1.979853868484497,
      "step": 4120
    },
    {
      "epoch": 2.95,
      "grad_norm": 7.0625,
      "learning_rate": 4.206287209375573e-06,
      "log_odds_chosen": 9.080406188964844,
      "log_odds_ratio": -0.05416526645421982,
      "logits/chosen": -2.8165950775146484,
      "logits/rejected": -1.8180831670761108,
      "logps/chosen": -0.363316148519516,
      "logps/rejected": -8.077341079711914,
      "loss": 0.2679,
      "nll_loss": 0.26229244470596313,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.07266323268413544,
      "rewards/margins": 1.5428051948547363,
      "rewards/rejected": -1.6154683828353882,
      "step": 4130
    },
    {
      "epoch": 2.96,
      "grad_norm": 6.53125,
      "learning_rate": 4.204799574595403e-06,
      "log_odds_chosen": 6.818055629730225,
      "log_odds_ratio": -0.05800958722829819,
      "logits/chosen": -2.8333072662353516,
      "logits/rejected": -2.186479330062866,
      "logps/chosen": -0.3609169125556946,
      "logps/rejected": -5.922616958618164,
      "loss": 0.2654,
      "nll_loss": 0.2335742712020874,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07218338549137115,
      "rewards/margins": 1.112339973449707,
      "rewards/rejected": -1.184523344039917,
      "step": 4140
    },
    {
      "epoch": 2.96,
      "grad_norm": 15.625,
      "learning_rate": 4.203313517091987e-06,
      "log_odds_chosen": 6.995954990386963,
      "log_odds_ratio": -0.04991995543241501,
      "logits/chosen": -2.8100647926330566,
      "logits/rejected": -2.1644890308380127,
      "logps/chosen": -0.3308708071708679,
      "logps/rejected": -5.982227325439453,
      "loss": 0.2739,
      "nll_loss": 0.23660406470298767,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.06617416441440582,
      "rewards/margins": 1.130271553993225,
      "rewards/rejected": -1.1964454650878906,
      "step": 4150
    },
    {
      "epoch": 2.97,
      "grad_norm": 24.5,
      "learning_rate": 4.201829034080091e-06,
      "log_odds_chosen": 9.040159225463867,
      "log_odds_ratio": -0.048347409814596176,
      "logits/chosen": -2.8145790100097656,
      "logits/rejected": -1.903846025466919,
      "logps/chosen": -0.40995389223098755,
      "logps/rejected": -8.149190902709961,
      "loss": 0.2742,
      "nll_loss": 0.28532686829566956,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.08199077844619751,
      "rewards/margins": 1.5478473901748657,
      "rewards/rejected": -1.629838228225708,
      "step": 4160
    },
    {
      "epoch": 2.98,
      "grad_norm": 9.0625,
      "learning_rate": 4.200346122781363e-06,
      "log_odds_chosen": 6.394429683685303,
      "log_odds_ratio": -0.048393916338682175,
      "logits/chosen": -2.858705997467041,
      "logits/rejected": -2.1757659912109375,
      "logps/chosen": -0.36599108576774597,
      "logps/rejected": -5.35428524017334,
      "loss": 0.2785,
      "nll_loss": 0.2647210955619812,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -0.07319821417331696,
      "rewards/margins": 0.9976588487625122,
      "rewards/rejected": -1.0708571672439575,
      "step": 4170
    },
    {
      "epoch": 2.98,
      "grad_norm": 8.0625,
      "learning_rate": 4.1988647804243155e-06,
      "log_odds_chosen": 7.234747409820557,
      "log_odds_ratio": -0.03427041321992874,
      "logits/chosen": -2.8021435737609863,
      "logits/rejected": -2.0722737312316895,
      "logps/chosen": -0.4443022310733795,
      "logps/rejected": -6.377936840057373,
      "loss": 0.2923,
      "nll_loss": 0.2961937487125397,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.08886045217514038,
      "rewards/margins": 1.1867269277572632,
      "rewards/rejected": -1.2755874395370483,
      "step": 4180
    },
    {
      "epoch": 2.99,
      "grad_norm": 15.5,
      "learning_rate": 4.197385004244289e-06,
      "log_odds_chosen": 8.025124549865723,
      "log_odds_ratio": -0.023739898577332497,
      "logits/chosen": -2.865403413772583,
      "logits/rejected": -2.047375202178955,
      "logps/chosen": -0.3514173626899719,
      "logps/rejected": -6.971765995025635,
      "loss": 0.2778,
      "nll_loss": 0.2846386432647705,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07028347253799438,
      "rewards/margins": 1.3240697383880615,
      "rewards/rejected": -1.3943531513214111,
      "step": 4190
    },
    {
      "epoch": 3.0,
      "grad_norm": 6.8125,
      "learning_rate": 4.195906791483446e-06,
      "log_odds_chosen": 9.17123031616211,
      "log_odds_ratio": -0.03721408173441887,
      "logits/chosen": -2.8432440757751465,
      "logits/rejected": -1.8437057733535767,
      "logps/chosen": -0.40266337990760803,
      "logps/rejected": -8.306893348693848,
      "loss": 0.2994,
      "nll_loss": 0.289571076631546,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -0.08053267002105713,
      "rewards/margins": 1.580845832824707,
      "rewards/rejected": -1.6613785028457642,
      "step": 4200
    },
    {
      "epoch": 3.0,
      "step": 4200,
      "total_flos": 0.0,
      "train_loss": 0.4132561104070573,
      "train_runtime": 52660.5661,
      "train_samples_per_second": 2.553,
      "train_steps_per_second": 0.08
    }
  ],
  "logging_steps": 10,
  "max_steps": 4200,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}