{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 6793,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.00014721036360959813,
      "grad_norm": 59.3047116816907,
      "learning_rate": 7.352941176470588e-10,
      "logits/chosen": -2.9195547103881836,
      "logits/rejected": -2.4565553665161133,
      "logps/chosen": -421.782470703125,
      "logps/rejected": -89.33955383300781,
      "loss": -0.5,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.0014721036360959812,
      "grad_norm": 62.717713562215444,
      "learning_rate": 7.352941176470588e-09,
      "logits/chosen": -2.61781644821167,
      "logits/rejected": -2.573765516281128,
      "logps/chosen": -334.68304443359375,
      "logps/rejected": -234.80751037597656,
      "loss": -0.5006,
      "rewards/accuracies": 0.5925925970077515,
      "rewards/chosen": 0.042253851890563965,
      "rewards/margins": 0.13390761613845825,
      "rewards/rejected": -0.0916537418961525,
      "step": 10
    },
    {
      "epoch": 0.0029442072721919624,
      "grad_norm": 56.14890631538051,
      "learning_rate": 1.4705882352941176e-08,
      "logits/chosen": -2.6124911308288574,
      "logits/rejected": -2.6781249046325684,
      "logps/chosen": -264.3174743652344,
      "logps/rejected": -236.8070526123047,
      "loss": -0.4998,
      "rewards/accuracies": 0.46666669845581055,
      "rewards/chosen": 0.062340669333934784,
      "rewards/margins": 0.014057544060051441,
      "rewards/rejected": 0.048283133655786514,
      "step": 20
    },
    {
      "epoch": 0.004416310908287944,
      "grad_norm": 51.27665371287587,
      "learning_rate": 2.2058823529411764e-08,
      "logits/chosen": -2.7284631729125977,
      "logits/rejected": -2.6887402534484863,
      "logps/chosen": -205.87094116210938,
      "logps/rejected": -213.82528686523438,
      "loss": -0.5006,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": 0.11215279996395111,
      "rewards/margins": 0.10462869703769684,
      "rewards/rejected": 0.0075241089798510075,
      "step": 30
    },
    {
      "epoch": 0.005888414544383925,
      "grad_norm": 51.726238559255876,
      "learning_rate": 2.941176470588235e-08,
      "logits/chosen": -2.729477643966675,
      "logits/rejected": -2.6255459785461426,
      "logps/chosen": -251.6602325439453,
      "logps/rejected": -393.7332458496094,
      "loss": -0.4999,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": 0.1325322389602661,
      "rewards/margins": 0.05858485773205757,
      "rewards/rejected": 0.07394739240407944,
      "step": 40
    },
    {
      "epoch": 0.007360518180479906,
      "grad_norm": 68.45327109861037,
      "learning_rate": 3.676470588235294e-08,
      "logits/chosen": -2.646000862121582,
      "logits/rejected": -2.6366517543792725,
      "logps/chosen": -290.57415771484375,
      "logps/rejected": -222.02023315429688,
      "loss": -0.5006,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.11794261634349823,
      "rewards/margins": 0.197869211435318,
      "rewards/rejected": -0.07992658764123917,
      "step": 50
    },
    {
      "epoch": 0.008832621816575887,
      "grad_norm": 63.943666716690366,
      "learning_rate": 4.411764705882353e-08,
      "logits/chosen": -2.7703545093536377,
      "logits/rejected": -2.74164080619812,
      "logps/chosen": -327.62249755859375,
      "logps/rejected": -330.29437255859375,
      "loss": -0.5,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.1268714964389801,
      "rewards/margins": 0.04060954973101616,
      "rewards/rejected": 0.08626194298267365,
      "step": 60
    },
    {
      "epoch": 0.010304725452671868,
      "grad_norm": 59.11643477749268,
      "learning_rate": 5.147058823529411e-08,
      "logits/chosen": -2.567875623703003,
      "logits/rejected": -2.4987878799438477,
      "logps/chosen": -262.30279541015625,
      "logps/rejected": -311.67938232421875,
      "loss": -0.5002,
      "rewards/accuracies": 0.5666666626930237,
      "rewards/chosen": 0.10554748773574829,
      "rewards/margins": 0.057436250150203705,
      "rewards/rejected": 0.04811124876141548,
      "step": 70
    },
    {
      "epoch": 0.01177682908876785,
      "grad_norm": 56.721775913846564,
      "learning_rate": 5.88235294117647e-08,
      "logits/chosen": -2.796175479888916,
      "logits/rejected": -2.68497371673584,
      "logps/chosen": -260.50946044921875,
      "logps/rejected": -233.54541015625,
      "loss": -0.501,
      "rewards/accuracies": 0.5666666626930237,
      "rewards/chosen": 0.15474791824817657,
      "rewards/margins": 0.0889195054769516,
      "rewards/rejected": 0.06582838296890259,
      "step": 80
    },
    {
      "epoch": 0.01324893272486383,
      "grad_norm": 53.35885815375979,
      "learning_rate": 6.617647058823529e-08,
      "logits/chosen": -2.803354024887085,
      "logits/rejected": -2.7273952960968018,
      "logps/chosen": -303.0828857421875,
      "logps/rejected": -232.6864776611328,
      "loss": -0.5007,
      "rewards/accuracies": 0.46666663885116577,
      "rewards/chosen": 0.14413569867610931,
      "rewards/margins": -0.022162145003676414,
      "rewards/rejected": 0.16629785299301147,
      "step": 90
    },
    {
      "epoch": 0.014721036360959812,
      "grad_norm": 57.564270736371775,
      "learning_rate": 7.352941176470588e-08,
      "logits/chosen": -2.7901253700256348,
      "logits/rejected": -2.6695311069488525,
      "logps/chosen": -263.35443115234375,
      "logps/rejected": -217.3997344970703,
      "loss": -0.5016,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": 0.42978033423423767,
      "rewards/margins": 0.30862101912498474,
      "rewards/rejected": 0.12115930020809174,
      "step": 100
    },
    {
      "epoch": 0.016193139997055794,
      "grad_norm": 54.13898822345304,
      "learning_rate": 8.088235294117647e-08,
      "logits/chosen": -2.8039963245391846,
      "logits/rejected": -2.7367870807647705,
      "logps/chosen": -367.28369140625,
      "logps/rejected": -291.3222961425781,
      "loss": -0.5022,
      "rewards/accuracies": 0.533333420753479,
      "rewards/chosen": 0.5604602098464966,
      "rewards/margins": 0.22085995972156525,
      "rewards/rejected": 0.33960023522377014,
      "step": 110
    },
    {
      "epoch": 0.017665243633151775,
      "grad_norm": 69.48648735646931,
      "learning_rate": 8.823529411764706e-08,
      "logits/chosen": -2.6727895736694336,
      "logits/rejected": -2.6338202953338623,
      "logps/chosen": -282.6793212890625,
      "logps/rejected": -291.5563049316406,
      "loss": -0.5036,
      "rewards/accuracies": 0.6999999284744263,
      "rewards/chosen": 0.3824855387210846,
      "rewards/margins": 0.35269397497177124,
      "rewards/rejected": 0.0297915730625391,
      "step": 120
    },
    {
      "epoch": 0.019137347269247755,
      "grad_norm": 63.63485998378907,
      "learning_rate": 9.558823529411763e-08,
      "logits/chosen": -2.766634702682495,
      "logits/rejected": -2.6744561195373535,
      "logps/chosen": -303.455078125,
      "logps/rejected": -222.0045166015625,
      "loss": -0.5034,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": 0.838738739490509,
      "rewards/margins": 0.7629243731498718,
      "rewards/rejected": 0.0758143812417984,
      "step": 130
    },
    {
      "epoch": 0.020609450905343735,
      "grad_norm": 64.89277754582618,
      "learning_rate": 1.0294117647058822e-07,
      "logits/chosen": -2.7446699142456055,
      "logits/rejected": -2.6963133811950684,
      "logps/chosen": -200.55355834960938,
      "logps/rejected": -195.1848907470703,
      "loss": -0.5031,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": 0.5240464210510254,
      "rewards/margins": 0.5540199279785156,
      "rewards/rejected": -0.029973510652780533,
      "step": 140
    },
    {
      "epoch": 0.022081554541439716,
      "grad_norm": 64.49295539864423,
      "learning_rate": 1.1029411764705881e-07,
      "logits/chosen": -2.773139715194702,
      "logits/rejected": -2.6096127033233643,
      "logps/chosen": -267.8309326171875,
      "logps/rejected": -229.3339385986328,
      "loss": -0.5073,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 1.0566240549087524,
      "rewards/margins": 0.8906691670417786,
      "rewards/rejected": 0.1659548431634903,
      "step": 150
    },
    {
      "epoch": 0.0235536581775357,
      "grad_norm": 49.4147047820236,
      "learning_rate": 1.176470588235294e-07,
      "logits/chosen": -2.7843563556671143,
      "logits/rejected": -2.671734094619751,
      "logps/chosen": -232.1531982421875,
      "logps/rejected": -218.7380828857422,
      "loss": -0.5083,
      "rewards/accuracies": 0.5666666626930237,
      "rewards/chosen": 0.4854610860347748,
      "rewards/margins": 0.3678571581840515,
      "rewards/rejected": 0.11760394275188446,
      "step": 160
    },
    {
      "epoch": 0.02502576181363168,
      "grad_norm": 151.81301853213876,
      "learning_rate": 1.25e-07,
      "logits/chosen": -2.7112841606140137,
      "logits/rejected": -2.697054147720337,
      "logps/chosen": -295.9041442871094,
      "logps/rejected": -252.22592163085938,
      "loss": -0.5111,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 1.8192431926727295,
      "rewards/margins": 1.2015241384506226,
      "rewards/rejected": 0.6177191138267517,
      "step": 170
    },
    {
      "epoch": 0.02649786544972766,
      "grad_norm": 61.50718590464541,
      "learning_rate": 1.3235294117647057e-07,
      "logits/chosen": -2.777191638946533,
      "logits/rejected": -2.740993022918701,
      "logps/chosen": -205.4273223876953,
      "logps/rejected": -221.2201385498047,
      "loss": -0.5126,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": 1.955256700515747,
      "rewards/margins": 1.3177016973495483,
      "rewards/rejected": 0.6375550031661987,
      "step": 180
    },
    {
      "epoch": 0.02796996908582364,
      "grad_norm": 51.44749302054236,
      "learning_rate": 1.3970588235294117e-07,
      "logits/chosen": -2.7335312366485596,
      "logits/rejected": -2.7274670600891113,
      "logps/chosen": -244.8299560546875,
      "logps/rejected": -260.83514404296875,
      "loss": -0.5153,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": 1.4504351615905762,
      "rewards/margins": 0.6974285840988159,
      "rewards/rejected": 0.7530065178871155,
      "step": 190
    },
    {
      "epoch": 0.029442072721919624,
      "grad_norm": 59.56243081950517,
      "learning_rate": 1.4705882352941175e-07,
      "logits/chosen": -2.7603297233581543,
      "logits/rejected": -2.6506123542785645,
      "logps/chosen": -332.4497375488281,
      "logps/rejected": -255.1281280517578,
      "loss": -0.5174,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": 1.7580398321151733,
      "rewards/margins": 1.945922613143921,
      "rewards/rejected": -0.18788254261016846,
      "step": 200
    },
    {
      "epoch": 0.030914176358015605,
      "grad_norm": 63.17962906853604,
      "learning_rate": 1.5441176470588236e-07,
      "logits/chosen": -2.6521596908569336,
      "logits/rejected": -2.6083004474639893,
      "logps/chosen": -259.6528015136719,
      "logps/rejected": -211.9808807373047,
      "loss": -0.5343,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": 1.2981822490692139,
      "rewards/margins": 2.8318989276885986,
      "rewards/rejected": -1.5337167978286743,
      "step": 210
    },
    {
      "epoch": 0.03238627999411159,
      "grad_norm": 82.06637072200158,
      "learning_rate": 1.6176470588235293e-07,
      "logits/chosen": -2.582864761352539,
      "logits/rejected": -2.5215964317321777,
      "logps/chosen": -258.61572265625,
      "logps/rejected": -197.9263916015625,
      "loss": -0.5429,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": 0.46621522307395935,
      "rewards/margins": 4.356805801391602,
      "rewards/rejected": -3.8905906677246094,
      "step": 220
    },
    {
      "epoch": 0.033858383630207566,
      "grad_norm": 75.03772013756237,
      "learning_rate": 1.6911764705882354e-07,
      "logits/chosen": -2.720737934112549,
      "logits/rejected": -2.587012767791748,
      "logps/chosen": -357.04229736328125,
      "logps/rejected": -262.19732666015625,
      "loss": -0.5307,
      "rewards/accuracies": 0.5999999642372131,
      "rewards/chosen": 3.6453399658203125,
      "rewards/margins": 3.965730667114258,
      "rewards/rejected": -0.3203909397125244,
      "step": 230
    },
    {
      "epoch": 0.03533048726630355,
      "grad_norm": 50.61131610619733,
      "learning_rate": 1.764705882352941e-07,
      "logits/chosen": -2.7071075439453125,
      "logits/rejected": -2.591439723968506,
      "logps/chosen": -268.3920593261719,
      "logps/rejected": -265.4142761230469,
      "loss": -0.5248,
      "rewards/accuracies": 0.6999999284744263,
      "rewards/chosen": 1.2308013439178467,
      "rewards/margins": 1.1662706136703491,
      "rewards/rejected": 0.06453065574169159,
      "step": 240
    },
    {
      "epoch": 0.036802590902399526,
      "grad_norm": 58.15393506830767,
      "learning_rate": 1.8382352941176472e-07,
      "logits/chosen": -2.574489116668701,
      "logits/rejected": -2.606422185897827,
      "logps/chosen": -242.9832000732422,
      "logps/rejected": -248.2828369140625,
      "loss": -0.5286,
      "rewards/accuracies": 0.5333333611488342,
      "rewards/chosen": 2.7258193492889404,
      "rewards/margins": 3.5358548164367676,
      "rewards/rejected": -0.8100353479385376,
      "step": 250
    },
    {
      "epoch": 0.03827469453849551,
      "grad_norm": 61.41411180596185,
      "learning_rate": 1.9117647058823527e-07,
      "logits/chosen": -2.748314619064331,
      "logits/rejected": -2.5809173583984375,
      "logps/chosen": -272.1921081542969,
      "logps/rejected": -226.16494750976562,
      "loss": -0.5357,
      "rewards/accuracies": 0.5666667222976685,
      "rewards/chosen": 0.6965977549552917,
      "rewards/margins": 2.2068471908569336,
      "rewards/rejected": -1.510249137878418,
      "step": 260
    },
    {
      "epoch": 0.039746798174591494,
      "grad_norm": 53.2742305166138,
      "learning_rate": 1.9852941176470587e-07,
      "logits/chosen": -2.637295722961426,
      "logits/rejected": -2.6960065364837646,
      "logps/chosen": -293.53662109375,
      "logps/rejected": -284.0157775878906,
      "loss": -0.566,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": 4.886682033538818,
      "rewards/margins": 3.699345111846924,
      "rewards/rejected": 1.187337040901184,
      "step": 270
    },
    {
      "epoch": 0.04121890181068747,
      "grad_norm": 67.44690377969081,
      "learning_rate": 2.0588235294117645e-07,
      "logits/chosen": -2.7481675148010254,
      "logits/rejected": -2.6167244911193848,
      "logps/chosen": -319.9439392089844,
      "logps/rejected": -290.9716796875,
      "loss": -0.5819,
      "rewards/accuracies": 0.7999999523162842,
      "rewards/chosen": 7.5188422203063965,
      "rewards/margins": 9.251741409301758,
      "rewards/rejected": -1.7328999042510986,
      "step": 280
    },
    {
      "epoch": 0.042691005446783455,
      "grad_norm": 87.91638388851798,
      "learning_rate": 2.1323529411764705e-07,
      "logits/chosen": -2.6716346740722656,
      "logits/rejected": -2.647164821624756,
      "logps/chosen": -273.6284484863281,
      "logps/rejected": -228.8042755126953,
      "loss": -0.5634,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": 4.426143169403076,
      "rewards/margins": 7.126317501068115,
      "rewards/rejected": -2.700174331665039,
      "step": 290
    },
    {
      "epoch": 0.04416310908287943,
      "grad_norm": 61.82588905177701,
      "learning_rate": 2.2058823529411763e-07,
      "logits/chosen": -2.6839542388916016,
      "logits/rejected": -2.577425003051758,
      "logps/chosen": -277.6145324707031,
      "logps/rejected": -250.5657958984375,
      "loss": -0.5655,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -2.387023448944092,
      "rewards/margins": 6.0797624588012695,
      "rewards/rejected": -8.46678638458252,
      "step": 300
    },
    {
      "epoch": 0.045635212718975415,
      "grad_norm": 90.58261503598163,
      "learning_rate": 2.2794117647058823e-07,
      "logits/chosen": -2.7629926204681396,
      "logits/rejected": -2.6530611515045166,
      "logps/chosen": -311.4811706542969,
      "logps/rejected": -273.9911804199219,
      "loss": -0.5693,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": 3.209371566772461,
      "rewards/margins": 14.616223335266113,
      "rewards/rejected": -11.406851768493652,
      "step": 310
    },
    {
      "epoch": 0.0471073163550714,
      "grad_norm": 67.08713113186985,
      "learning_rate": 2.352941176470588e-07,
      "logits/chosen": -2.5868704319000244,
      "logits/rejected": -2.5623393058776855,
      "logps/chosen": -254.93618774414062,
      "logps/rejected": -257.2383117675781,
      "loss": -0.5679,
      "rewards/accuracies": 0.6999999284744263,
      "rewards/chosen": 6.649624824523926,
      "rewards/margins": 8.101293563842773,
      "rewards/rejected": -1.4516680240631104,
      "step": 320
    },
    {
      "epoch": 0.048579419991167376,
      "grad_norm": 79.0714158762543,
      "learning_rate": 2.426470588235294e-07,
      "logits/chosen": -2.6178786754608154,
      "logits/rejected": -2.525106906890869,
      "logps/chosen": -228.4622802734375,
      "logps/rejected": -203.71871948242188,
      "loss": -0.5887,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 8.58758544921875,
      "rewards/margins": 12.349942207336426,
      "rewards/rejected": -3.7623565196990967,
      "step": 330
    },
    {
      "epoch": 0.05005152362726336,
      "grad_norm": 60.82826755399802,
      "learning_rate": 2.5e-07,
      "logits/chosen": -2.7113709449768066,
      "logits/rejected": -2.6892166137695312,
      "logps/chosen": -298.3251647949219,
      "logps/rejected": -354.0218200683594,
      "loss": -0.5874,
      "rewards/accuracies": 0.5333333015441895,
      "rewards/chosen": 1.769512414932251,
      "rewards/margins": 12.34203052520752,
      "rewards/rejected": -10.572517395019531,
      "step": 340
    },
    {
      "epoch": 0.051523627263359344,
      "grad_norm": 78.11188348388266,
      "learning_rate": 2.5735294117647057e-07,
      "logits/chosen": -2.648749589920044,
      "logits/rejected": -2.5208377838134766,
      "logps/chosen": -316.731689453125,
      "logps/rejected": -266.75848388671875,
      "loss": -0.5691,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -3.76812481880188,
      "rewards/margins": 15.556530952453613,
      "rewards/rejected": -19.324655532836914,
      "step": 350
    },
    {
      "epoch": 0.05299573089945532,
      "grad_norm": 83.28442522152687,
      "learning_rate": 2.6470588235294114e-07,
      "logits/chosen": -2.7165980339050293,
      "logits/rejected": -2.6670279502868652,
      "logps/chosen": -244.55905151367188,
      "logps/rejected": -235.98751831054688,
      "loss": -0.5645,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -2.439885139465332,
      "rewards/margins": 9.439864158630371,
      "rewards/rejected": -11.87975025177002,
      "step": 360
    },
    {
      "epoch": 0.054467834535551304,
      "grad_norm": 61.35097647888104,
      "learning_rate": 2.720588235294117e-07,
      "logits/chosen": -2.7003684043884277,
      "logits/rejected": -2.740440607070923,
      "logps/chosen": -260.4454040527344,
      "logps/rejected": -264.84832763671875,
      "loss": -0.6254,
      "rewards/accuracies": 0.6999999284744263,
      "rewards/chosen": 7.243973731994629,
      "rewards/margins": 8.225504875183105,
      "rewards/rejected": -0.981531023979187,
      "step": 370
    },
    {
      "epoch": 0.05593993817164728,
      "grad_norm": 64.73711456488456,
      "learning_rate": 2.7941176470588235e-07,
      "logits/chosen": -2.725066661834717,
      "logits/rejected": -2.6576786041259766,
      "logps/chosen": -268.77606201171875,
      "logps/rejected": -228.33663940429688,
      "loss": -0.5942,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": 3.2542526721954346,
      "rewards/margins": 10.755142211914062,
      "rewards/rejected": -7.500891208648682,
      "step": 380
    },
    {
      "epoch": 0.057412041807743265,
      "grad_norm": 93.61699405355341,
      "learning_rate": 2.8676470588235293e-07,
      "logits/chosen": -2.70733380317688,
      "logits/rejected": -2.7066421508789062,
      "logps/chosen": -267.53656005859375,
      "logps/rejected": -265.4286193847656,
      "loss": -0.5646,
      "rewards/accuracies": 0.5333333611488342,
      "rewards/chosen": 0.5764063000679016,
      "rewards/margins": 3.5772106647491455,
      "rewards/rejected": -3.0008046627044678,
      "step": 390
    },
    {
      "epoch": 0.05888414544383925,
      "grad_norm": 95.8288489851453,
      "learning_rate": 2.941176470588235e-07,
      "logits/chosen": -2.7099833488464355,
      "logits/rejected": -2.6187198162078857,
      "logps/chosen": -212.15188598632812,
      "logps/rejected": -228.40219116210938,
      "loss": -0.6496,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": 5.638246059417725,
      "rewards/margins": 12.200549125671387,
      "rewards/rejected": -6.5623040199279785,
      "step": 400
    },
    {
      "epoch": 0.060356249079935226,
      "grad_norm": 89.98299190290189,
      "learning_rate": 3.014705882352941e-07,
      "logits/chosen": -2.6361050605773926,
      "logits/rejected": -2.517089366912842,
      "logps/chosen": -311.2821960449219,
      "logps/rejected": -305.902587890625,
      "loss": -0.6187,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": 3.9842586517333984,
      "rewards/margins": 14.787178039550781,
      "rewards/rejected": -10.8029203414917,
      "step": 410
    },
    {
      "epoch": 0.06182835271603121,
      "grad_norm": 71.91134475829398,
      "learning_rate": 3.088235294117647e-07,
      "logits/chosen": -2.779949188232422,
      "logits/rejected": -2.657057046890259,
      "logps/chosen": -327.05633544921875,
      "logps/rejected": -310.15838623046875,
      "loss": -0.572,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": 0.5427759289741516,
      "rewards/margins": 14.023786544799805,
      "rewards/rejected": -13.481012344360352,
      "step": 420
    },
    {
      "epoch": 0.0633004563521272,
      "grad_norm": 61.06132374410608,
      "learning_rate": 3.161764705882353e-07,
      "logits/chosen": -2.671093463897705,
      "logits/rejected": -2.6272449493408203,
      "logps/chosen": -178.1507568359375,
      "logps/rejected": -203.5655517578125,
      "loss": -0.5921,
      "rewards/accuracies": 0.6999999284744263,
      "rewards/chosen": -1.2086782455444336,
      "rewards/margins": 14.08912467956543,
      "rewards/rejected": -15.297802925109863,
      "step": 430
    },
    {
      "epoch": 0.06477255998822318,
      "grad_norm": 57.80411070307135,
      "learning_rate": 3.2352941176470586e-07,
      "logits/chosen": -2.717437505722046,
      "logits/rejected": -2.641340732574463,
      "logps/chosen": -225.2947235107422,
      "logps/rejected": -227.05111694335938,
      "loss": -0.6533,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": -6.093857765197754,
      "rewards/margins": 29.759984970092773,
      "rewards/rejected": -35.853843688964844,
      "step": 440
    },
    {
      "epoch": 0.06624466362431915,
      "grad_norm": 97.41923218617036,
      "learning_rate": 3.3088235294117644e-07,
      "logits/chosen": -2.73636531829834,
      "logits/rejected": -2.663825750350952,
      "logps/chosen": -379.1432189941406,
      "logps/rejected": -351.1720275878906,
      "loss": -0.5792,
      "rewards/accuracies": 0.5666667222976685,
      "rewards/chosen": -17.862075805664062,
      "rewards/margins": 4.691006183624268,
      "rewards/rejected": -22.553081512451172,
      "step": 450
    },
    {
      "epoch": 0.06771676726041513,
      "grad_norm": 69.79108612357955,
      "learning_rate": 3.3823529411764707e-07,
      "logits/chosen": -2.884448528289795,
      "logits/rejected": -2.717114210128784,
      "logps/chosen": -280.7251892089844,
      "logps/rejected": -238.0408935546875,
      "loss": -0.6147,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": 7.1150994300842285,
      "rewards/margins": 28.3294734954834,
      "rewards/rejected": -21.214374542236328,
      "step": 460
    },
    {
      "epoch": 0.06918887089651111,
      "grad_norm": 81.40799402287868,
      "learning_rate": 3.4558823529411765e-07,
      "logits/chosen": -2.830606698989868,
      "logits/rejected": -2.7466413974761963,
      "logps/chosen": -212.4346160888672,
      "logps/rejected": -210.50015258789062,
      "loss": -0.6214,
      "rewards/accuracies": 0.6999999284744263,
      "rewards/chosen": 3.537209987640381,
      "rewards/margins": 13.874101638793945,
      "rewards/rejected": -10.336891174316406,
      "step": 470
    },
    {
      "epoch": 0.0706609745326071,
      "grad_norm": 60.93723069174302,
      "learning_rate": 3.529411764705882e-07,
      "logits/chosen": -2.7997937202453613,
      "logits/rejected": -2.698042154312134,
      "logps/chosen": -280.5040588378906,
      "logps/rejected": -266.04449462890625,
      "loss": -0.6656,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 7.506966590881348,
      "rewards/margins": 28.281103134155273,
      "rewards/rejected": -20.774137496948242,
      "step": 480
    },
    {
      "epoch": 0.07213307816870308,
      "grad_norm": 53.699235220438965,
      "learning_rate": 3.602941176470588e-07,
      "logits/chosen": -2.858339548110962,
      "logits/rejected": -2.7603535652160645,
      "logps/chosen": -323.2521057128906,
      "logps/rejected": -299.0159912109375,
      "loss": -0.6162,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.6800538897514343,
      "rewards/margins": 15.3016939163208,
      "rewards/rejected": -15.981745719909668,
      "step": 490
    },
    {
      "epoch": 0.07360518180479905,
      "grad_norm": 65.19983075575855,
      "learning_rate": 3.6764705882352943e-07,
      "logits/chosen": -2.7707672119140625,
      "logits/rejected": -2.8162882328033447,
      "logps/chosen": -215.8936309814453,
      "logps/rejected": -260.87811279296875,
      "loss": -0.644,
      "rewards/accuracies": 0.6999999284744263,
      "rewards/chosen": -8.859192848205566,
      "rewards/margins": 21.8640079498291,
      "rewards/rejected": -30.723196029663086,
      "step": 500
    },
    {
      "epoch": 0.07507728544089504,
      "grad_norm": 95.71207034761002,
      "learning_rate": 3.75e-07,
      "logits/chosen": -2.74477219581604,
      "logits/rejected": -2.638737440109253,
      "logps/chosen": -317.5669860839844,
      "logps/rejected": -341.0974426269531,
      "loss": -0.6321,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": 3.8060078620910645,
      "rewards/margins": 41.86675262451172,
      "rewards/rejected": -38.06074905395508,
      "step": 510
    },
    {
      "epoch": 0.07654938907699102,
      "grad_norm": 68.94127927971576,
      "learning_rate": 3.8235294117647053e-07,
      "logits/chosen": -2.5374088287353516,
      "logits/rejected": -2.4168717861175537,
      "logps/chosen": -227.83541870117188,
      "logps/rejected": -261.05181884765625,
      "loss": -0.6334,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": -5.480355262756348,
      "rewards/margins": 24.557945251464844,
      "rewards/rejected": -30.038299560546875,
      "step": 520
    },
    {
      "epoch": 0.078021492713087,
      "grad_norm": 87.64735851894017,
      "learning_rate": 3.8970588235294116e-07,
      "logits/chosen": -2.7724156379699707,
      "logits/rejected": -2.6891961097717285,
      "logps/chosen": -287.04461669921875,
      "logps/rejected": -286.272216796875,
      "loss": -0.5904,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -2.3728575706481934,
      "rewards/margins": 13.43122673034668,
      "rewards/rejected": -15.804086685180664,
      "step": 530
    },
    {
      "epoch": 0.07949359634918299,
      "grad_norm": 78.90025977455906,
      "learning_rate": 3.9705882352941174e-07,
      "logits/chosen": -2.8418171405792236,
      "logits/rejected": -2.637730360031128,
      "logps/chosen": -313.7364807128906,
      "logps/rejected": -272.36383056640625,
      "loss": -0.5988,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 2.1098432540893555,
      "rewards/margins": 11.420353889465332,
      "rewards/rejected": -9.310510635375977,
      "step": 540
    },
    {
      "epoch": 0.08096569998527896,
      "grad_norm": 113.38409504295495,
      "learning_rate": 4.044117647058823e-07,
      "logits/chosen": -2.5933775901794434,
      "logits/rejected": -2.596010208129883,
      "logps/chosen": -326.6166076660156,
      "logps/rejected": -320.9649658203125,
      "loss": -0.6969,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": 9.6653413772583,
      "rewards/margins": 33.90729904174805,
      "rewards/rejected": -24.241954803466797,
      "step": 550
    },
    {
      "epoch": 0.08243780362137494,
      "grad_norm": 94.65417572351423,
      "learning_rate": 4.117647058823529e-07,
      "logits/chosen": -2.748504161834717,
      "logits/rejected": -2.6171000003814697,
      "logps/chosen": -245.5800018310547,
      "logps/rejected": -253.0184326171875,
      "loss": -0.6267,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 1.0452303886413574,
      "rewards/margins": 17.541004180908203,
      "rewards/rejected": -16.495773315429688,
      "step": 560
    },
    {
      "epoch": 0.08390990725747093,
      "grad_norm": 64.61636556221126,
      "learning_rate": 4.191176470588235e-07,
      "logits/chosen": -2.741159439086914,
      "logits/rejected": -2.4894185066223145,
      "logps/chosen": -299.0730285644531,
      "logps/rejected": -273.1308898925781,
      "loss": -0.6281,
      "rewards/accuracies": 0.9666666984558105,
      "rewards/chosen": 4.198310375213623,
      "rewards/margins": 45.12836456298828,
      "rewards/rejected": -40.9300537109375,
      "step": 570
    },
    {
      "epoch": 0.08538201089356691,
      "grad_norm": 72.8371639729045,
      "learning_rate": 4.264705882352941e-07,
      "logits/chosen": -2.5701775550842285,
      "logits/rejected": -2.5074493885040283,
      "logps/chosen": -341.37701416015625,
      "logps/rejected": -259.933837890625,
      "loss": -0.6109,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -20.600797653198242,
      "rewards/margins": 9.473933219909668,
      "rewards/rejected": -30.07473373413086,
      "step": 580
    },
    {
      "epoch": 0.08685411452966289,
      "grad_norm": 105.51940135002695,
      "learning_rate": 4.338235294117647e-07,
      "logits/chosen": -2.587874174118042,
      "logits/rejected": -2.526446580886841,
      "logps/chosen": -289.45635986328125,
      "logps/rejected": -295.77532958984375,
      "loss": -0.6777,
      "rewards/accuracies": 0.7999999523162842,
      "rewards/chosen": -2.739182233810425,
      "rewards/margins": 30.27170181274414,
      "rewards/rejected": -33.01088333129883,
      "step": 590
    },
    {
      "epoch": 0.08832621816575886,
      "grad_norm": 81.73213444001422,
      "learning_rate": 4.4117647058823526e-07,
      "logits/chosen": -2.439671516418457,
      "logits/rejected": -2.507843017578125,
      "logps/chosen": -284.27679443359375,
      "logps/rejected": -326.23211669921875,
      "loss": -0.6394,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -9.412282943725586,
      "rewards/margins": 33.25923538208008,
      "rewards/rejected": -42.67152404785156,
      "step": 600
    },
    {
      "epoch": 0.08979832180185485,
      "grad_norm": 82.34602240018579,
      "learning_rate": 4.485294117647059e-07,
      "logits/chosen": -2.5409388542175293,
      "logits/rejected": -2.405757427215576,
      "logps/chosen": -265.4460144042969,
      "logps/rejected": -282.75128173828125,
      "loss": -0.6557,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -1.4953162670135498,
      "rewards/margins": 26.386011123657227,
      "rewards/rejected": -27.88132667541504,
      "step": 610
    },
    {
      "epoch": 0.09127042543795083,
      "grad_norm": 58.39554095838379,
      "learning_rate": 4.5588235294117646e-07,
      "logits/chosen": -2.6100566387176514,
      "logits/rejected": -2.4268908500671387,
      "logps/chosen": -245.78353881835938,
      "logps/rejected": -221.22189331054688,
      "loss": -0.6245,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.006898021791130304,
      "rewards/margins": 22.50055694580078,
      "rewards/rejected": -22.493661880493164,
      "step": 620
    },
    {
      "epoch": 0.09274252907404681,
      "grad_norm": 106.62943229277286,
      "learning_rate": 4.6323529411764704e-07,
      "logits/chosen": -2.6467843055725098,
      "logits/rejected": -2.4030632972717285,
      "logps/chosen": -310.4133605957031,
      "logps/rejected": -281.5668029785156,
      "loss": -0.7354,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": 10.747413635253906,
      "rewards/margins": 37.000389099121094,
      "rewards/rejected": -26.252975463867188,
      "step": 630
    },
    {
      "epoch": 0.0942146327101428,
      "grad_norm": 82.29085053499658,
      "learning_rate": 4.705882352941176e-07,
      "logits/chosen": -2.317124843597412,
      "logits/rejected": -2.0745816230773926,
      "logps/chosen": -296.77081298828125,
      "logps/rejected": -311.05267333984375,
      "loss": -0.6343,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -11.587621688842773,
      "rewards/margins": 29.84322738647461,
      "rewards/rejected": -41.430850982666016,
      "step": 640
    },
    {
      "epoch": 0.09568673634623878,
      "grad_norm": 84.94260930526677,
      "learning_rate": 4.779411764705882e-07,
      "logits/chosen": -2.380723476409912,
      "logits/rejected": -2.109708070755005,
      "logps/chosen": -297.1458435058594,
      "logps/rejected": -315.0326232910156,
      "loss": -0.6994,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 8.184403419494629,
      "rewards/margins": 30.700927734375,
      "rewards/rejected": -22.516525268554688,
      "step": 650
    },
    {
      "epoch": 0.09715883998233475,
      "grad_norm": 102.29518762051372,
      "learning_rate": 4.852941176470588e-07,
      "logits/chosen": -2.258287191390991,
      "logits/rejected": -1.9773895740509033,
      "logps/chosen": -261.7895202636719,
      "logps/rejected": -256.654541015625,
      "loss": -0.6579,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -1.3003475666046143,
      "rewards/margins": 27.02567481994629,
      "rewards/rejected": -28.326025009155273,
      "step": 660
    },
    {
      "epoch": 0.09863094361843074,
      "grad_norm": 196.77843728363385,
      "learning_rate": 4.926470588235295e-07,
      "logits/chosen": -2.1178371906280518,
      "logits/rejected": -1.819494605064392,
      "logps/chosen": -300.79901123046875,
      "logps/rejected": -277.1000061035156,
      "loss": -0.6159,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -5.687954902648926,
      "rewards/margins": 32.39912033081055,
      "rewards/rejected": -38.08707809448242,
      "step": 670
    },
    {
      "epoch": 0.10010304725452672,
      "grad_norm": 83.89891420544326,
      "learning_rate": 5e-07,
      "logits/chosen": -2.54667329788208,
      "logits/rejected": -2.125702142715454,
      "logps/chosen": -245.4879913330078,
      "logps/rejected": -219.63021850585938,
      "loss": -0.6196,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 3.8056387901306152,
      "rewards/margins": 30.783977508544922,
      "rewards/rejected": -26.978342056274414,
      "step": 680
    },
    {
      "epoch": 0.1015751508906227,
      "grad_norm": 71.90177062734337,
      "learning_rate": 4.999966985858302e-07,
      "logits/chosen": -2.5251574516296387,
      "logits/rejected": -2.4980368614196777,
      "logps/chosen": -259.50518798828125,
      "logps/rejected": -278.435791015625,
      "loss": -0.665,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -4.512556552886963,
      "rewards/margins": 9.098235130310059,
      "rewards/rejected": -13.610791206359863,
      "step": 690
    },
    {
      "epoch": 0.10304725452671869,
      "grad_norm": 64.70363649640434,
      "learning_rate": 4.999867944305156e-07,
      "logits/chosen": -2.5377964973449707,
      "logits/rejected": -2.4816040992736816,
      "logps/chosen": -193.4921417236328,
      "logps/rejected": -221.6757049560547,
      "loss": -0.6129,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": 7.878302574157715,
      "rewards/margins": 22.794921875,
      "rewards/rejected": -14.916620254516602,
      "step": 700
    },
    {
      "epoch": 0.10451935816281466,
      "grad_norm": 90.38149312696764,
      "learning_rate": 4.99970287795638e-07,
      "logits/chosen": -2.5726866722106934,
      "logits/rejected": -2.4421629905700684,
      "logps/chosen": -268.4377746582031,
      "logps/rejected": -342.8684997558594,
      "loss": -0.5823,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -9.545019149780273,
      "rewards/margins": 23.75004768371582,
      "rewards/rejected": -33.295066833496094,
      "step": 710
    },
    {
      "epoch": 0.10599146179891064,
      "grad_norm": 88.50165297262586,
      "learning_rate": 4.999471791171592e-07,
      "logits/chosen": -2.560494899749756,
      "logits/rejected": -2.5193042755126953,
      "logps/chosen": -272.555908203125,
      "logps/rejected": -252.1201934814453,
      "loss": -0.666,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -9.861135482788086,
      "rewards/margins": 29.37734031677246,
      "rewards/rejected": -39.23847579956055,
      "step": 720
    },
    {
      "epoch": 0.10746356543500662,
      "grad_norm": 92.42398574130007,
      "learning_rate": 4.999174690054098e-07,
      "logits/chosen": -2.66166353225708,
      "logits/rejected": -2.5551979541778564,
      "logps/chosen": -248.6725311279297,
      "logps/rejected": -350.3984375,
      "loss": -0.6409,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -17.568798065185547,
      "rewards/margins": 24.579463958740234,
      "rewards/rejected": -42.14826202392578,
      "step": 730
    },
    {
      "epoch": 0.10893566907110261,
      "grad_norm": 67.89240115188201,
      "learning_rate": 4.998811582450728e-07,
      "logits/chosen": -2.659043788909912,
      "logits/rejected": -2.5731635093688965,
      "logps/chosen": -308.89520263671875,
      "logps/rejected": -342.09234619140625,
      "loss": -0.7058,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 2.6172072887420654,
      "rewards/margins": 45.0538444519043,
      "rewards/rejected": -42.43663787841797,
      "step": 740
    },
    {
      "epoch": 0.11040777270719859,
      "grad_norm": 80.93036585972376,
      "learning_rate": 4.998382477951632e-07,
      "logits/chosen": -2.6353344917297363,
      "logits/rejected": -2.5524983406066895,
      "logps/chosen": -227.56784057617188,
      "logps/rejected": -280.0120544433594,
      "loss": -0.5903,
      "rewards/accuracies": 0.6999999284744263,
      "rewards/chosen": -4.001662254333496,
      "rewards/margins": 25.076396942138672,
      "rewards/rejected": -29.078060150146484,
      "step": 750
    },
    {
      "epoch": 0.11187987634329456,
      "grad_norm": 68.21377964464313,
      "learning_rate": 4.997887387890022e-07,
      "logits/chosen": -2.6018803119659424,
      "logits/rejected": -2.506791353225708,
      "logps/chosen": -361.37908935546875,
      "logps/rejected": -372.95953369140625,
      "loss": -0.7663,
      "rewards/accuracies": 0.7999999523162842,
      "rewards/chosen": 9.055999755859375,
      "rewards/margins": 43.741981506347656,
      "rewards/rejected": -34.68598175048828,
      "step": 760
    },
    {
      "epoch": 0.11335197997939055,
      "grad_norm": 55.28666169889597,
      "learning_rate": 4.997326325341876e-07,
      "logits/chosen": -2.6614468097686768,
      "logits/rejected": -2.454007148742676,
      "logps/chosen": -261.86407470703125,
      "logps/rejected": -267.33990478515625,
      "loss": -0.494,
      "rewards/accuracies": 0.3999999761581421,
      "rewards/chosen": -20.325733184814453,
      "rewards/margins": -0.9755716323852539,
      "rewards/rejected": -19.35015869140625,
      "step": 770
    },
    {
      "epoch": 0.11482408361548653,
      "grad_norm": 75.41315146901344,
      "learning_rate": 4.996699305125597e-07,
      "logits/chosen": -2.575697898864746,
      "logits/rejected": -2.399491786956787,
      "logps/chosen": -231.89816284179688,
      "logps/rejected": -260.6578063964844,
      "loss": -0.6036,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -0.5631122589111328,
      "rewards/margins": 41.74716567993164,
      "rewards/rejected": -42.31027603149414,
      "step": 780
    },
    {
      "epoch": 0.11629618725158251,
      "grad_norm": 77.45809065517068,
      "learning_rate": 4.996006343801608e-07,
      "logits/chosen": -2.5522513389587402,
      "logits/rejected": -2.3192028999328613,
      "logps/chosen": -232.4510498046875,
      "logps/rejected": -280.1426696777344,
      "loss": -0.6614,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 5.7166748046875,
      "rewards/margins": 49.51537322998047,
      "rewards/rejected": -43.7987060546875,
      "step": 790
    },
    {
      "epoch": 0.1177682908876785,
      "grad_norm": 218.69995541405078,
      "learning_rate": 4.99524745967193e-07,
      "logits/chosen": -2.5149917602539062,
      "logits/rejected": -2.299288749694824,
      "logps/chosen": -368.71075439453125,
      "logps/rejected": -339.7911376953125,
      "loss": -0.6611,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -5.4729743003845215,
      "rewards/margins": 42.43523025512695,
      "rewards/rejected": -47.90821075439453,
      "step": 800
    },
    {
      "epoch": 0.11924039452377447,
      "grad_norm": 62.33239133785411,
      "learning_rate": 4.994422672779687e-07,
      "logits/chosen": -2.5763466358184814,
      "logits/rejected": -2.449383020401001,
      "logps/chosen": -237.8682098388672,
      "logps/rejected": -302.69293212890625,
      "loss": -0.7069,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -3.9075217247009277,
      "rewards/margins": 37.10382843017578,
      "rewards/rejected": -41.01134490966797,
      "step": 810
    },
    {
      "epoch": 0.12071249815987045,
      "grad_norm": 60.28832707219097,
      "learning_rate": 4.993532004908588e-07,
      "logits/chosen": -2.5143983364105225,
      "logits/rejected": -2.253477096557617,
      "logps/chosen": -249.19509887695312,
      "logps/rejected": -263.1053466796875,
      "loss": -0.6356,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -9.858936309814453,
      "rewards/margins": 15.042834281921387,
      "rewards/rejected": -24.90176773071289,
      "step": 820
    },
    {
      "epoch": 0.12218460179596644,
      "grad_norm": 104.62222762691741,
      "learning_rate": 4.992575479582337e-07,
      "logits/chosen": -2.2509100437164307,
      "logits/rejected": -1.5742653608322144,
      "logps/chosen": -390.2452697753906,
      "logps/rejected": -416.19940185546875,
      "loss": -0.6659,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -7.916274070739746,
      "rewards/margins": 49.6270866394043,
      "rewards/rejected": -57.543365478515625,
      "step": 830
    },
    {
      "epoch": 0.12365670543206242,
      "grad_norm": 187.41284073322984,
      "learning_rate": 4.991553122064028e-07,
      "logits/chosen": -1.8326995372772217,
      "logits/rejected": -1.4321322441101074,
      "logps/chosen": -331.9021301269531,
      "logps/rejected": -286.65570068359375,
      "loss": -0.5734,
      "rewards/accuracies": 0.5666666626930237,
      "rewards/chosen": -21.966266632080078,
      "rewards/margins": 21.23002052307129,
      "rewards/rejected": -43.1962890625,
      "step": 840
    },
    {
      "epoch": 0.1251288090681584,
      "grad_norm": 109.18014957301934,
      "learning_rate": 4.990464959355464e-07,
      "logits/chosen": -2.083986759185791,
      "logits/rejected": -1.9516900777816772,
      "logps/chosen": -244.8618927001953,
      "logps/rejected": -295.02593994140625,
      "loss": -0.6343,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -2.067255735397339,
      "rewards/margins": 32.335479736328125,
      "rewards/rejected": -34.40273666381836,
      "step": 850
    },
    {
      "epoch": 0.1266009127042544,
      "grad_norm": 78.48238686798216,
      "learning_rate": 4.98931102019645e-07,
      "logits/chosen": -2.2667150497436523,
      "logits/rejected": -1.9983021020889282,
      "logps/chosen": -224.1129913330078,
      "logps/rejected": -239.37979125976562,
      "loss": -0.6915,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": 8.183572769165039,
      "rewards/margins": 50.136722564697266,
      "rewards/rejected": -41.953147888183594,
      "step": 860
    },
    {
      "epoch": 0.12807301634035037,
      "grad_norm": 103.35329276521371,
      "learning_rate": 4.988091335064037e-07,
      "logits/chosen": -2.2633535861968994,
      "logits/rejected": -1.9170093536376953,
      "logps/chosen": -251.37692260742188,
      "logps/rejected": -282.84735107421875,
      "loss": -0.6157,
      "rewards/accuracies": 0.76666659116745,
      "rewards/chosen": -10.325926780700684,
      "rewards/margins": 44.50645065307617,
      "rewards/rejected": -54.832374572753906,
      "step": 870
    },
    {
      "epoch": 0.12954511997644635,
      "grad_norm": 190.77460704598968,
      "learning_rate": 4.98680593617171e-07,
      "logits/chosen": -2.3459203243255615,
      "logits/rejected": -2.083291530609131,
      "logps/chosen": -301.2386169433594,
      "logps/rejected": -232.2847137451172,
      "loss": -0.6659,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -2.4001781940460205,
      "rewards/margins": 19.18419647216797,
      "rewards/rejected": -21.584375381469727,
      "step": 880
    },
    {
      "epoch": 0.1310172236125423,
      "grad_norm": 74.27540605480694,
      "learning_rate": 4.985454857468542e-07,
      "logits/chosen": -2.302178144454956,
      "logits/rejected": -2.161571502685547,
      "logps/chosen": -333.8761901855469,
      "logps/rejected": -315.20037841796875,
      "loss": -0.6667,
      "rewards/accuracies": 0.7000000476837158,
      "rewards/chosen": -6.975069999694824,
      "rewards/margins": 29.80960464477539,
      "rewards/rejected": -36.78467559814453,
      "step": 890
    },
    {
      "epoch": 0.1324893272486383,
      "grad_norm": 85.82599534621886,
      "learning_rate": 4.984038134638297e-07,
      "logits/chosen": -2.3358492851257324,
      "logits/rejected": -2.3326470851898193,
      "logps/chosen": -271.2342834472656,
      "logps/rejected": -286.1571044921875,
      "loss": -0.6258,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -11.574197769165039,
      "rewards/margins": 21.882526397705078,
      "rewards/rejected": -33.45672607421875,
      "step": 900
    },
    {
      "epoch": 0.13396143088473428,
      "grad_norm": 100.0060272489992,
      "learning_rate": 4.982555805098483e-07,
      "logits/chosen": -2.413405179977417,
      "logits/rejected": -2.20133638381958,
      "logps/chosen": -248.87673950195312,
      "logps/rejected": -194.1851348876953,
      "loss": -0.6866,
      "rewards/accuracies": 0.6666667461395264,
      "rewards/chosen": 1.6779158115386963,
      "rewards/margins": 25.04676628112793,
      "rewards/rejected": -23.368854522705078,
      "step": 910
    },
    {
      "epoch": 0.13543353452083026,
      "grad_norm": 72.81899138008602,
      "learning_rate": 4.981007907999372e-07,
      "logits/chosen": -2.332641124725342,
      "logits/rejected": -2.317354917526245,
      "logps/chosen": -202.06475830078125,
      "logps/rejected": -299.42852783203125,
      "loss": -0.5872,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 1.1460152864456177,
      "rewards/margins": 14.52123737335205,
      "rewards/rejected": -13.375221252441406,
      "step": 920
    },
    {
      "epoch": 0.13690563815692625,
      "grad_norm": 54.61538283764078,
      "learning_rate": 4.979394484222961e-07,
      "logits/chosen": -2.487978458404541,
      "logits/rejected": -2.455573558807373,
      "logps/chosen": -185.4193572998047,
      "logps/rejected": -305.05804443359375,
      "loss": 3.7868,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": 4.297365665435791,
      "rewards/margins": 19.31717872619629,
      "rewards/rejected": -15.019813537597656,
      "step": 930
    },
    {
      "epoch": 0.13837774179302223,
      "grad_norm": 85.36943638649856,
      "learning_rate": 4.977715576381888e-07,
      "logits/chosen": -2.4613728523254395,
      "logits/rejected": -2.3615028858184814,
      "logps/chosen": -294.49884033203125,
      "logps/rejected": -308.27337646484375,
      "loss": -0.6184,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -6.58510684967041,
      "rewards/margins": 22.565067291259766,
      "rewards/rejected": -29.15017318725586,
      "step": 940
    },
    {
      "epoch": 0.1398498454291182,
      "grad_norm": 84.52399346940548,
      "learning_rate": 4.975971228818315e-07,
      "logits/chosen": -2.627000331878662,
      "logits/rejected": -2.4849612712860107,
      "logps/chosen": -223.9296875,
      "logps/rejected": -274.06787109375,
      "loss": -0.6704,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": 2.388002872467041,
      "rewards/margins": 46.65497589111328,
      "rewards/rejected": -44.266971588134766,
      "step": 950
    },
    {
      "epoch": 0.1413219490652142,
      "grad_norm": 69.99316684520782,
      "learning_rate": 4.974161487602753e-07,
      "logits/chosen": -2.5379221439361572,
      "logits/rejected": -2.383770227432251,
      "logps/chosen": -260.72381591796875,
      "logps/rejected": -300.0539245605469,
      "loss": -0.6601,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -11.593948364257812,
      "rewards/margins": 34.75862503051758,
      "rewards/rejected": -46.35257339477539,
      "step": 960
    },
    {
      "epoch": 0.14279405270131018,
      "grad_norm": 115.67018132017328,
      "learning_rate": 4.972286400532842e-07,
      "logits/chosen": -2.222830057144165,
      "logits/rejected": -1.8678233623504639,
      "logps/chosen": -256.8113098144531,
      "logps/rejected": -336.9583740234375,
      "loss": -0.6007,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": -24.745698928833008,
      "rewards/margins": 50.47902297973633,
      "rewards/rejected": -75.22472381591797,
      "step": 970
    },
    {
      "epoch": 0.14426615633740617,
      "grad_norm": 113.41236590197003,
      "learning_rate": 4.970346017132097e-07,
      "logits/chosen": -2.2508227825164795,
      "logits/rejected": -1.6305630207061768,
      "logps/chosen": -323.94085693359375,
      "logps/rejected": -321.0826416015625,
      "loss": -0.6698,
      "rewards/accuracies": 0.6999999284744263,
      "rewards/chosen": -34.0479621887207,
      "rewards/margins": 60.823402404785156,
      "rewards/rejected": -94.8713607788086,
      "step": 980
    },
    {
      "epoch": 0.14573825997350212,
      "grad_norm": 73.49295945649625,
      "learning_rate": 4.96834038864859e-07,
      "logits/chosen": -2.615482807159424,
      "logits/rejected": -2.461414337158203,
      "logps/chosen": -313.1335754394531,
      "logps/rejected": -279.95391845703125,
      "loss": -0.6637,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -1.2118756771087646,
      "rewards/margins": 28.42221450805664,
      "rewards/rejected": -29.63408851623535,
      "step": 990
    },
    {
      "epoch": 0.1472103636095981,
      "grad_norm": 82.4801378293873,
      "learning_rate": 4.966269568053605e-07,
      "logits/chosen": -2.723997116088867,
      "logits/rejected": -2.7201600074768066,
      "logps/chosen": -309.8893737792969,
      "logps/rejected": -326.711181640625,
      "loss": -0.6497,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": 4.347428321838379,
      "rewards/margins": 30.64121437072754,
      "rewards/rejected": -26.29378318786621,
      "step": 1000
    },
    {
      "epoch": 0.1486824672456941,
      "grad_norm": 92.72057407959254,
      "learning_rate": 4.964133610040232e-07,
      "logits/chosen": -2.678412437438965,
      "logits/rejected": -2.5421254634857178,
      "logps/chosen": -256.5310974121094,
      "logps/rejected": -288.23016357421875,
      "loss": -0.6668,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": 0.07843919098377228,
      "rewards/margins": 21.754364013671875,
      "rewards/rejected": -21.67592430114746,
      "step": 1010
    },
    {
      "epoch": 0.15015457088179007,
      "grad_norm": 97.93198196834305,
      "learning_rate": 4.961932571021928e-07,
      "logits/chosen": -2.782731056213379,
      "logits/rejected": -2.668245792388916,
      "logps/chosen": -273.4408874511719,
      "logps/rejected": -256.2829895019531,
      "loss": -0.668,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -3.170576572418213,
      "rewards/margins": 18.73792839050293,
      "rewards/rejected": -21.90850257873535,
      "step": 1020
    },
    {
      "epoch": 0.15162667451788606,
      "grad_norm": 65.3158648801075,
      "learning_rate": 4.959666509131025e-07,
      "logits/chosen": -2.743921995162964,
      "logits/rejected": -2.6128218173980713,
      "logps/chosen": -265.0013427734375,
      "logps/rejected": -254.9213104248047,
      "loss": -0.6523,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": 3.4670372009277344,
      "rewards/margins": 28.93754005432129,
      "rewards/rejected": -25.470500946044922,
      "step": 1030
    },
    {
      "epoch": 0.15309877815398204,
      "grad_norm": 83.93394253158273,
      "learning_rate": 4.957335484217193e-07,
      "logits/chosen": -2.670915365219116,
      "logits/rejected": -2.7056334018707275,
      "logps/chosen": -293.8043212890625,
      "logps/rejected": -315.1792907714844,
      "loss": -0.722,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": 0.12588591873645782,
      "rewards/margins": 17.933124542236328,
      "rewards/rejected": -17.807239532470703,
      "step": 1040
    },
    {
      "epoch": 0.15457088179007802,
      "grad_norm": 88.08321019188826,
      "learning_rate": 4.954939557845862e-07,
      "logits/chosen": -2.783658266067505,
      "logits/rejected": -2.5253853797912598,
      "logps/chosen": -325.5037536621094,
      "logps/rejected": -343.7779541015625,
      "loss": -0.6666,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": 7.419310569763184,
      "rewards/margins": 14.69555377960205,
      "rewards/rejected": -7.276242256164551,
      "step": 1050
    },
    {
      "epoch": 0.156042985426174,
      "grad_norm": 77.92907812295783,
      "learning_rate": 4.952478793296594e-07,
      "logits/chosen": -2.6564650535583496,
      "logits/rejected": -2.546450614929199,
      "logps/chosen": -180.88294982910156,
      "logps/rejected": -245.7649383544922,
      "loss": -0.6775,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": 3.953479766845703,
      "rewards/margins": 41.709068298339844,
      "rewards/rejected": -37.75558853149414,
      "step": 1060
    },
    {
      "epoch": 0.15751508906227,
      "grad_norm": 74.40440662452112,
      "learning_rate": 4.949953255561411e-07,
      "logits/chosen": -2.5832340717315674,
      "logits/rejected": -2.2338662147521973,
      "logps/chosen": -315.12738037109375,
      "logps/rejected": -338.770263671875,
      "loss": -0.7506,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.2327992022037506,
      "rewards/margins": 60.792213439941406,
      "rewards/rejected": -60.559417724609375,
      "step": 1070
    },
    {
      "epoch": 0.15898719269836598,
      "grad_norm": 88.63199379840925,
      "learning_rate": 4.947363011343083e-07,
      "logits/chosen": -2.5007026195526123,
      "logits/rejected": -2.2570762634277344,
      "logps/chosen": -293.9660949707031,
      "logps/rejected": -295.04266357421875,
      "loss": -0.7159,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -5.067392349243164,
      "rewards/margins": 45.86968994140625,
      "rewards/rejected": -50.937076568603516,
      "step": 1080
    },
    {
      "epoch": 0.16045929633446196,
      "grad_norm": 91.9219230938425,
      "learning_rate": 4.944708129053362e-07,
      "logits/chosen": -2.6720571517944336,
      "logits/rejected": -2.597010850906372,
      "logps/chosen": -302.1465759277344,
      "logps/rejected": -241.475341796875,
      "loss": -0.7105,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": 0.020271683111786842,
      "rewards/margins": 27.3707218170166,
      "rewards/rejected": -27.350452423095703,
      "step": 1090
    },
    {
      "epoch": 0.16193139997055792,
      "grad_norm": 73.33642929030076,
      "learning_rate": 4.941988678811176e-07,
      "logits/chosen": -2.684147357940674,
      "logits/rejected": -2.579228639602661,
      "logps/chosen": -277.89495849609375,
      "logps/rejected": -264.5125732421875,
      "loss": -0.6422,
      "rewards/accuracies": 0.6999999284744263,
      "rewards/chosen": -2.131862163543701,
      "rewards/margins": 28.240859985351562,
      "rewards/rejected": -30.372722625732422,
      "step": 1100
    },
    {
      "epoch": 0.1634035036066539,
      "grad_norm": 61.24739042861282,
      "learning_rate": 4.939204732440777e-07,
      "logits/chosen": -2.8110690116882324,
      "logits/rejected": -2.603346109390259,
      "logps/chosen": -240.46334838867188,
      "logps/rejected": -197.5297088623047,
      "loss": -2.4979,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -4.5412139892578125,
      "rewards/margins": 26.754268646240234,
      "rewards/rejected": -31.295482635498047,
      "step": 1110
    },
    {
      "epoch": 0.16487560724274988,
      "grad_norm": 76.60505616844326,
      "learning_rate": 4.936356363469845e-07,
      "logits/chosen": -2.6718361377716064,
      "logits/rejected": -2.6052029132843018,
      "logps/chosen": -274.7420349121094,
      "logps/rejected": -325.4130859375,
      "loss": -0.5987,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -2.0547122955322266,
      "rewards/margins": 29.85129737854004,
      "rewards/rejected": -31.906015396118164,
      "step": 1120
    },
    {
      "epoch": 0.16634771087884587,
      "grad_norm": 67.32878587721628,
      "learning_rate": 4.933443647127546e-07,
      "logits/chosen": -2.7724578380584717,
      "logits/rejected": -2.5966920852661133,
      "logps/chosen": -251.1339569091797,
      "logps/rejected": -242.5947265625,
      "loss": -0.6175,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": 0.3924156129360199,
      "rewards/margins": 24.09902572631836,
      "rewards/rejected": -23.706607818603516,
      "step": 1130
    },
    {
      "epoch": 0.16781981451494185,
      "grad_norm": 72.41998259331599,
      "learning_rate": 4.930466660342543e-07,
      "logits/chosen": -2.626244306564331,
      "logits/rejected": -2.3822736740112305,
      "logps/chosen": -223.68246459960938,
      "logps/rejected": -225.4077606201172,
      "loss": -0.705,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 2.835132122039795,
      "rewards/margins": 41.56477737426758,
      "rewards/rejected": -38.729644775390625,
      "step": 1140
    },
    {
      "epoch": 0.16929191815103783,
      "grad_norm": 61.55646592978056,
      "learning_rate": 4.927425481740968e-07,
      "logits/chosen": -2.626924991607666,
      "logits/rejected": -2.2874865531921387,
      "logps/chosen": -230.90133666992188,
      "logps/rejected": -254.4603729248047,
      "loss": -0.7343,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": 2.3516526222229004,
      "rewards/margins": 46.355491638183594,
      "rewards/rejected": -44.00383758544922,
      "step": 1150
    },
    {
      "epoch": 0.17076402178713382,
      "grad_norm": 90.3151959054484,
      "learning_rate": 4.924320191644341e-07,
      "logits/chosen": -2.623302936553955,
      "logits/rejected": -2.529200315475464,
      "logps/chosen": -317.5823059082031,
      "logps/rejected": -327.7528076171875,
      "loss": -0.6757,
      "rewards/accuracies": 0.7000000476837158,
      "rewards/chosen": -12.876846313476562,
      "rewards/margins": 33.17280578613281,
      "rewards/rejected": -46.049652099609375,
      "step": 1160
    },
    {
      "epoch": 0.1722361254232298,
      "grad_norm": 75.76071342272695,
      "learning_rate": 4.921150872067452e-07,
      "logits/chosen": -2.6242690086364746,
      "logits/rejected": -2.216398239135742,
      "logps/chosen": -303.5771789550781,
      "logps/rejected": -295.40185546875,
      "loss": -0.786,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 3.625026226043701,
      "rewards/margins": 52.29256057739258,
      "rewards/rejected": -48.667537689208984,
      "step": 1170
    },
    {
      "epoch": 0.17370822905932579,
      "grad_norm": 122.73664126324307,
      "learning_rate": 4.917917606716194e-07,
      "logits/chosen": -2.4054417610168457,
      "logits/rejected": -2.1809921264648438,
      "logps/chosen": -208.2989959716797,
      "logps/rejected": -294.93341064453125,
      "loss": -0.6959,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -23.096435546875,
      "rewards/margins": 35.52253723144531,
      "rewards/rejected": -58.61896896362305,
      "step": 1180
    },
    {
      "epoch": 0.17518033269542177,
      "grad_norm": 124.8790317539603,
      "learning_rate": 4.914620480985352e-07,
      "logits/chosen": -2.7098617553710938,
      "logits/rejected": -2.4683048725128174,
      "logps/chosen": -355.46270751953125,
      "logps/rejected": -360.7904052734375,
      "loss": -0.6356,
      "rewards/accuracies": 0.8333333730697632,
      "rewards/chosen": -4.357626438140869,
      "rewards/margins": 47.87146759033203,
      "rewards/rejected": -52.229087829589844,
      "step": 1190
    },
    {
      "epoch": 0.17665243633151773,
      "grad_norm": 107.6739528881974,
      "learning_rate": 4.911259581956345e-07,
      "logits/chosen": -2.6603550910949707,
      "logits/rejected": -2.732883930206299,
      "logps/chosen": -277.72747802734375,
      "logps/rejected": -400.99920654296875,
      "loss": 4.4113,
      "rewards/accuracies": 0.5999999642372131,
      "rewards/chosen": 12.696221351623535,
      "rewards/margins": 20.855327606201172,
      "rewards/rejected": -8.159107208251953,
      "step": 1200
    },
    {
      "epoch": 0.1781245399676137,
      "grad_norm": 97.24957901739211,
      "learning_rate": 4.907834998394932e-07,
      "logits/chosen": -2.797956943511963,
      "logits/rejected": -2.6978983879089355,
      "logps/chosen": -200.1792449951172,
      "logps/rejected": -268.91912841796875,
      "loss": -0.641,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 7.516181945800781,
      "rewards/margins": 35.79815673828125,
      "rewards/rejected": -28.281972885131836,
      "step": 1210
    },
    {
      "epoch": 0.1795966436037097,
      "grad_norm": 84.22028706378119,
      "learning_rate": 4.904346820748862e-07,
      "logits/chosen": -2.806896686553955,
      "logits/rejected": -2.503955841064453,
      "logps/chosen": -292.9959716796875,
      "logps/rejected": -248.148681640625,
      "loss": -0.6784,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": 2.5019969940185547,
      "rewards/margins": 32.79449462890625,
      "rewards/rejected": -30.292495727539062,
      "step": 1220
    },
    {
      "epoch": 0.18106874723980568,
      "grad_norm": 61.24529486917335,
      "learning_rate": 4.900795141145487e-07,
      "logits/chosen": -2.9347567558288574,
      "logits/rejected": -2.6512258052825928,
      "logps/chosen": -184.76492309570312,
      "logps/rejected": -270.0002746582031,
      "loss": -0.7133,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": 4.262557506561279,
      "rewards/margins": 46.8855094909668,
      "rewards/rejected": -42.62295150756836,
      "step": 1230
    },
    {
      "epoch": 0.18254085087590166,
      "grad_norm": 72.13685504185246,
      "learning_rate": 4.897180053389332e-07,
      "logits/chosen": -2.828439235687256,
      "logits/rejected": -2.746650218963623,
      "logps/chosen": -241.13815307617188,
      "logps/rejected": -239.35009765625,
      "loss": -0.7205,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": 6.7129669189453125,
      "rewards/margins": 24.787017822265625,
      "rewards/rejected": -18.074050903320312,
      "step": 1240
    },
    {
      "epoch": 0.18401295451199765,
      "grad_norm": 147.34091780964883,
      "learning_rate": 4.89350165295961e-07,
      "logits/chosen": -2.7413105964660645,
      "logits/rejected": -2.672975778579712,
      "logps/chosen": -255.37478637695312,
      "logps/rejected": -333.3829040527344,
      "loss": -0.6487,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -1.0049980878829956,
      "rewards/margins": 52.42389678955078,
      "rewards/rejected": -53.42888641357422,
      "step": 1250
    },
    {
      "epoch": 0.18548505814809363,
      "grad_norm": 72.7244130914646,
      "learning_rate": 4.88976003700771e-07,
      "logits/chosen": -2.7688562870025635,
      "logits/rejected": -2.540863275527954,
      "logps/chosen": -351.5015563964844,
      "logps/rejected": -324.80938720703125,
      "loss": -0.6864,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 10.832496643066406,
      "rewards/margins": 35.56503677368164,
      "rewards/rejected": -24.732540130615234,
      "step": 1260
    },
    {
      "epoch": 0.1869571617841896,
      "grad_norm": 76.36422130509607,
      "learning_rate": 4.885955304354622e-07,
      "logits/chosen": -2.600578546524048,
      "logits/rejected": -2.4345011711120605,
      "logps/chosen": -243.2291259765625,
      "logps/rejected": -277.20367431640625,
      "loss": -0.7256,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": 8.633105278015137,
      "rewards/margins": 36.317867279052734,
      "rewards/rejected": -27.68475914001465,
      "step": 1270
    },
    {
      "epoch": 0.1884292654202856,
      "grad_norm": 91.07503608926177,
      "learning_rate": 4.882087555488331e-07,
      "logits/chosen": -2.6976864337921143,
      "logits/rejected": -2.448218822479248,
      "logps/chosen": -318.2763671875,
      "logps/rejected": -271.00018310546875,
      "loss": -0.6697,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": 1.6309995651245117,
      "rewards/margins": 46.72700119018555,
      "rewards/rejected": -45.09599685668945,
      "step": 1280
    },
    {
      "epoch": 0.18990136905638158,
      "grad_norm": 80.37513599102812,
      "learning_rate": 4.878156892561167e-07,
      "logits/chosen": -2.6494293212890625,
      "logits/rejected": -2.490159511566162,
      "logps/chosen": -216.50497436523438,
      "logps/rejected": -279.2850646972656,
      "loss": -0.6363,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -12.130294799804688,
      "rewards/margins": 22.382308959960938,
      "rewards/rejected": -34.512603759765625,
      "step": 1290
    },
    {
      "epoch": 0.19137347269247756,
      "grad_norm": 359.415705892699,
      "learning_rate": 4.874163419387099e-07,
      "logits/chosen": -2.705488681793213,
      "logits/rejected": -2.5450594425201416,
      "logps/chosen": -193.6772918701172,
      "logps/rejected": -260.7572326660156,
      "loss": -0.7475,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -1.4891043901443481,
      "rewards/margins": 33.805023193359375,
      "rewards/rejected": -35.294124603271484,
      "step": 1300
    },
    {
      "epoch": 0.19284557632857352,
      "grad_norm": 60.44824246983375,
      "learning_rate": 4.870107241438999e-07,
      "logits/chosen": -2.629612922668457,
      "logits/rejected": -2.430769920349121,
      "logps/chosen": -255.6735076904297,
      "logps/rejected": -297.3590087890625,
      "loss": -0.6565,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 2.1552319526672363,
      "rewards/margins": 43.86231994628906,
      "rewards/rejected": -41.70709228515625,
      "step": 1310
    },
    {
      "epoch": 0.1943176799646695,
      "grad_norm": 83.70517149400183,
      "learning_rate": 4.865988465845852e-07,
      "logits/chosen": -2.6949524879455566,
      "logits/rejected": -2.412956953048706,
      "logps/chosen": -270.3111877441406,
      "logps/rejected": -281.77288818359375,
      "loss": -0.6131,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -6.196536540985107,
      "rewards/margins": 32.236244201660156,
      "rewards/rejected": -38.432777404785156,
      "step": 1320
    },
    {
      "epoch": 0.1957897836007655,
      "grad_norm": 78.46835847160601,
      "learning_rate": 4.861807201389933e-07,
      "logits/chosen": -2.719102382659912,
      "logits/rejected": -2.372081995010376,
      "logps/chosen": -270.2271728515625,
      "logps/rejected": -279.7436828613281,
      "loss": -0.7123,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": 0.9091800451278687,
      "rewards/margins": 65.81986999511719,
      "rewards/rejected": -64.91069030761719,
      "step": 1330
    },
    {
      "epoch": 0.19726188723686147,
      "grad_norm": 78.43578036342166,
      "learning_rate": 4.857563558503925e-07,
      "logits/chosen": -2.5899641513824463,
      "logits/rejected": -2.4410719871520996,
      "logps/chosen": -264.52130126953125,
      "logps/rejected": -316.6982116699219,
      "loss": -0.6532,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": 7.075021266937256,
      "rewards/margins": 29.954580307006836,
      "rewards/rejected": -22.87955665588379,
      "step": 1340
    },
    {
      "epoch": 0.19873399087295746,
      "grad_norm": 116.44178929294159,
      "learning_rate": 4.853257649268014e-07,
      "logits/chosen": -2.5544726848602295,
      "logits/rejected": -2.119459867477417,
      "logps/chosen": -231.9049530029297,
      "logps/rejected": -227.8389129638672,
      "loss": -0.6844,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 1.4845632314682007,
      "rewards/margins": 50.256248474121094,
      "rewards/rejected": -48.77168655395508,
      "step": 1350
    },
    {
      "epoch": 0.20020609450905344,
      "grad_norm": 62.00673329459614,
      "learning_rate": 4.848889587406915e-07,
      "logits/chosen": -2.4986958503723145,
      "logits/rejected": -2.5062966346740723,
      "logps/chosen": -270.48443603515625,
      "logps/rejected": -306.4799499511719,
      "loss": -0.7182,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -9.07248306274414,
      "rewards/margins": 32.678977966308594,
      "rewards/rejected": -41.75147247314453,
      "step": 1360
    },
    {
      "epoch": 0.20167819814514942,
      "grad_norm": 76.91663459171959,
      "learning_rate": 4.84445948828688e-07,
      "logits/chosen": -2.323050022125244,
      "logits/rejected": -2.059093952178955,
      "logps/chosen": -292.814208984375,
      "logps/rejected": -186.75045776367188,
      "loss": -0.6244,
      "rewards/accuracies": 0.5333333015441895,
      "rewards/chosen": -5.726569175720215,
      "rewards/margins": 23.842098236083984,
      "rewards/rejected": -29.56867027282715,
      "step": 1370
    },
    {
      "epoch": 0.2031503017812454,
      "grad_norm": 76.22646533549366,
      "learning_rate": 4.839967468912645e-07,
      "logits/chosen": -2.5608062744140625,
      "logits/rejected": -2.5182080268859863,
      "logps/chosen": -314.24774169921875,
      "logps/rejected": -312.85137939453125,
      "loss": -0.6615,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -5.522100448608398,
      "rewards/margins": 25.072267532348633,
      "rewards/rejected": -30.594369888305664,
      "step": 1380
    },
    {
      "epoch": 0.2046224054173414,
      "grad_norm": 227.87841162940236,
      "learning_rate": 4.83541364792434e-07,
      "logits/chosen": -2.492464542388916,
      "logits/rejected": -2.3928744792938232,
      "logps/chosen": -229.90286254882812,
      "logps/rejected": -277.5281066894531,
      "loss": -0.6883,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -4.213377952575684,
      "rewards/margins": 38.57929229736328,
      "rewards/rejected": -42.79267501831055,
      "step": 1390
    },
    {
      "epoch": 0.20609450905343737,
      "grad_norm": 120.2999861618777,
      "learning_rate": 4.83079814559436e-07,
      "logits/chosen": -2.5247902870178223,
      "logits/rejected": -2.3615386486053467,
      "logps/chosen": -304.0498962402344,
      "logps/rejected": -289.92559814453125,
      "loss": -0.8577,
      "rewards/accuracies": 0.6999999284744263,
      "rewards/chosen": -11.205354690551758,
      "rewards/margins": 31.1839599609375,
      "rewards/rejected": -42.389312744140625,
      "step": 1400
    },
    {
      "epoch": 0.20756661268953333,
      "grad_norm": 62.59763702746446,
      "learning_rate": 4.826121083824181e-07,
      "logits/chosen": -2.469452381134033,
      "logits/rejected": -2.2727584838867188,
      "logps/chosen": -302.0731506347656,
      "logps/rejected": -304.88421630859375,
      "loss": -0.701,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -10.510722160339355,
      "rewards/margins": 43.43576431274414,
      "rewards/rejected": -53.94648361206055,
      "step": 1410
    },
    {
      "epoch": 0.20903871632562931,
      "grad_norm": 87.76719883179676,
      "learning_rate": 4.82138258614115e-07,
      "logits/chosen": -2.561288356781006,
      "logits/rejected": -2.3799843788146973,
      "logps/chosen": -292.7859191894531,
      "logps/rejected": -313.80194091796875,
      "loss": -0.6458,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.9916560649871826,
      "rewards/margins": 28.520343780517578,
      "rewards/rejected": -30.51199722290039,
      "step": 1420
    },
    {
      "epoch": 0.2105108199617253,
      "grad_norm": 170.16018194423896,
      "learning_rate": 4.816582777695212e-07,
      "logits/chosen": -2.6641011238098145,
      "logits/rejected": -2.419689655303955,
      "logps/chosen": -221.0267333984375,
      "logps/rejected": -286.5566101074219,
      "loss": -0.7181,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 6.767381191253662,
      "rewards/margins": 46.53803253173828,
      "rewards/rejected": -39.770652770996094,
      "step": 1430
    },
    {
      "epoch": 0.21198292359782128,
      "grad_norm": 86.73234316292547,
      "learning_rate": 4.811721785255612e-07,
      "logits/chosen": -2.306082248687744,
      "logits/rejected": -2.2250263690948486,
      "logps/chosen": -205.35238647460938,
      "logps/rejected": -245.49716186523438,
      "loss": -0.7851,
      "rewards/accuracies": 0.6999999284744263,
      "rewards/chosen": -6.832254886627197,
      "rewards/margins": 42.48173904418945,
      "rewards/rejected": -49.313995361328125,
      "step": 1440
    },
    {
      "epoch": 0.21345502723391727,
      "grad_norm": 77.47203827520778,
      "learning_rate": 4.806799737207546e-07,
      "logits/chosen": -2.3669934272766113,
      "logits/rejected": -2.2257132530212402,
      "logps/chosen": -224.83871459960938,
      "logps/rejected": -277.8377380371094,
      "loss": -0.7235,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -7.207655429840088,
      "rewards/margins": 36.7170524597168,
      "rewards/rejected": -43.924705505371094,
      "step": 1450
    },
    {
      "epoch": 0.21492713087001325,
      "grad_norm": 101.3351955360075,
      "learning_rate": 4.801816763548766e-07,
      "logits/chosen": -2.5346155166625977,
      "logits/rejected": -2.1646103858947754,
      "logps/chosen": -287.0006103515625,
      "logps/rejected": -322.3687744140625,
      "loss": -0.6775,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 4.042638301849365,
      "rewards/margins": 45.53026580810547,
      "rewards/rejected": -41.48762512207031,
      "step": 1460
    },
    {
      "epoch": 0.21639923450610923,
      "grad_norm": 75.04249118191855,
      "learning_rate": 4.796772995886151e-07,
      "logits/chosen": -2.561363697052002,
      "logits/rejected": -2.5328352451324463,
      "logps/chosen": -263.40423583984375,
      "logps/rejected": -269.92730712890625,
      "loss": -0.643,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 6.277713775634766,
      "rewards/margins": 24.70981216430664,
      "rewards/rejected": -18.43209457397461,
      "step": 1470
    },
    {
      "epoch": 0.21787133814220522,
      "grad_norm": 76.25048499830696,
      "learning_rate": 4.791668567432229e-07,
      "logits/chosen": -2.5150701999664307,
      "logits/rejected": -2.252511978149414,
      "logps/chosen": -297.79290771484375,
      "logps/rejected": -286.78033447265625,
      "loss": -22.4192,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": 2.0505223274230957,
      "rewards/margins": 37.63640594482422,
      "rewards/rejected": -35.58588409423828,
      "step": 1480
    },
    {
      "epoch": 0.2193434417783012,
      "grad_norm": 92.22196553263032,
      "learning_rate": 4.78650361300166e-07,
      "logits/chosen": -2.4397177696228027,
      "logits/rejected": -2.274834156036377,
      "logps/chosen": -333.6665954589844,
      "logps/rejected": -364.16290283203125,
      "loss": -0.7585,
      "rewards/accuracies": 0.8333333730697632,
      "rewards/chosen": 4.076385498046875,
      "rewards/margins": 46.87431335449219,
      "rewards/rejected": -42.79792404174805,
      "step": 1490
    },
    {
      "epoch": 0.22081554541439719,
      "grad_norm": 87.14665256513209,
      "learning_rate": 4.781278269007675e-07,
      "logits/chosen": -2.362628221511841,
      "logits/rejected": -1.907284140586853,
      "logps/chosen": -356.5301513671875,
      "logps/rejected": -387.0435791015625,
      "loss": -0.7381,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -14.297991752624512,
      "rewards/margins": 73.7328872680664,
      "rewards/rejected": -88.0308837890625,
      "step": 1500
    },
    {
      "epoch": 0.22228764905049314,
      "grad_norm": 199.23618551596277,
      "learning_rate": 4.775992673458469e-07,
      "logits/chosen": -2.284015655517578,
      "logits/rejected": -2.0400185585021973,
      "logps/chosen": -292.8704833984375,
      "logps/rejected": -313.0919494628906,
      "loss": 12.8276,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": -5.792512893676758,
      "rewards/margins": 60.96533966064453,
      "rewards/rejected": -66.75785827636719,
      "step": 1510
    },
    {
      "epoch": 0.22375975268658913,
      "grad_norm": 104.57301056098619,
      "learning_rate": 4.770646965953564e-07,
      "logits/chosen": -2.358999013900757,
      "logits/rejected": -2.141723155975342,
      "logps/chosen": -296.458251953125,
      "logps/rejected": -326.75445556640625,
      "loss": -0.6826,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 11.847020149230957,
      "rewards/margins": 57.59717559814453,
      "rewards/rejected": -45.75015640258789,
      "step": 1520
    },
    {
      "epoch": 0.2252318563226851,
      "grad_norm": 91.4144190822726,
      "learning_rate": 4.765241287680116e-07,
      "logits/chosen": -2.603083610534668,
      "logits/rejected": -2.3442018032073975,
      "logps/chosen": -331.6548767089844,
      "logps/rejected": -246.1962432861328,
      "loss": -0.7178,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 12.898735046386719,
      "rewards/margins": 44.98307800292969,
      "rewards/rejected": -32.08434295654297,
      "step": 1530
    },
    {
      "epoch": 0.2267039599587811,
      "grad_norm": 150.81930534017388,
      "learning_rate": 4.759775781409187e-07,
      "logits/chosen": -2.4958221912384033,
      "logits/rejected": -2.26798939704895,
      "logps/chosen": -291.4118957519531,
      "logps/rejected": -270.84661865234375,
      "loss": -0.6444,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -8.145350456237793,
      "rewards/margins": 22.866413116455078,
      "rewards/rejected": -31.011768341064453,
      "step": 1540
    },
    {
      "epoch": 0.22817606359487708,
      "grad_norm": 91.5271846521091,
      "learning_rate": 4.7542505914919775e-07,
      "logits/chosen": -2.407763957977295,
      "logits/rejected": -2.3077902793884277,
      "logps/chosen": -291.1492614746094,
      "logps/rejected": -307.9754943847656,
      "loss": -0.7586,
      "rewards/accuracies": 0.8333333730697632,
      "rewards/chosen": 5.496190071105957,
      "rewards/margins": 54.667076110839844,
      "rewards/rejected": -49.17088317871094,
      "step": 1550
    },
    {
      "epoch": 0.22964816723097306,
      "grad_norm": 86.09523272728596,
      "learning_rate": 4.7486658638560076e-07,
      "logits/chosen": -2.390676259994507,
      "logits/rejected": -2.3858399391174316,
      "logps/chosen": -259.97784423828125,
      "logps/rejected": -279.2264709472656,
      "loss": -1.0874,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -4.5579071044921875,
      "rewards/margins": 17.47983741760254,
      "rewards/rejected": -22.03774070739746,
      "step": 1560
    },
    {
      "epoch": 0.23112027086706904,
      "grad_norm": 84.4002283065593,
      "learning_rate": 4.7430217460012703e-07,
      "logits/chosen": -2.2219419479370117,
      "logits/rejected": -2.042577028274536,
      "logps/chosen": -297.7833557128906,
      "logps/rejected": -282.4963684082031,
      "loss": -0.6553,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -13.171992301940918,
      "rewards/margins": 46.453125,
      "rewards/rejected": -59.6251220703125,
      "step": 1570
    },
    {
      "epoch": 0.23259237450316503,
      "grad_norm": 78.2578718175767,
      "learning_rate": 4.7373183869963295e-07,
      "logits/chosen": -2.568427562713623,
      "logits/rejected": -2.4184021949768066,
      "logps/chosen": -335.6342468261719,
      "logps/rejected": -330.76190185546875,
      "loss": -0.6905,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 10.552172660827637,
      "rewards/margins": 33.60314178466797,
      "rewards/rejected": -23.050973892211914,
      "step": 1580
    },
    {
      "epoch": 0.234064478139261,
      "grad_norm": 3784.9599496248275,
      "learning_rate": 4.7315559374743896e-07,
      "logits/chosen": -2.63429594039917,
      "logits/rejected": -2.552780866622925,
      "logps/chosen": -245.27963256835938,
      "logps/rejected": -309.32861328125,
      "loss": -0.2034,
      "rewards/accuracies": 0.7999999523162842,
      "rewards/chosen": 6.639822483062744,
      "rewards/margins": 30.23357582092285,
      "rewards/rejected": -23.593753814697266,
      "step": 1590
    },
    {
      "epoch": 0.235536581775357,
      "grad_norm": 57.567066323755334,
      "learning_rate": 4.725734549629308e-07,
      "logits/chosen": -2.3962364196777344,
      "logits/rejected": -2.202939748764038,
      "logps/chosen": -242.0668182373047,
      "logps/rejected": -298.4100646972656,
      "loss": -0.7175,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": 1.2525441646575928,
      "rewards/margins": 45.838340759277344,
      "rewards/rejected": -44.58579635620117,
      "step": 1600
    },
    {
      "epoch": 0.23700868541145298,
      "grad_norm": 60.53974703153272,
      "learning_rate": 4.719854377211585e-07,
      "logits/chosen": -2.577446699142456,
      "logits/rejected": -2.533982992172241,
      "logps/chosen": -253.6029052734375,
      "logps/rejected": -221.40695190429688,
      "loss": -0.6766,
      "rewards/accuracies": 0.5999999642372131,
      "rewards/chosen": 1.321341633796692,
      "rewards/margins": 25.407787322998047,
      "rewards/rejected": -24.086444854736328,
      "step": 1610
    },
    {
      "epoch": 0.23848078904754894,
      "grad_norm": 64.54581324681916,
      "learning_rate": 4.713915575524296e-07,
      "logits/chosen": -2.602355718612671,
      "logits/rejected": -2.600691795349121,
      "logps/chosen": -214.05557250976562,
      "logps/rejected": -228.7794647216797,
      "loss": -0.6169,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -11.509217262268066,
      "rewards/margins": 16.350555419921875,
      "rewards/rejected": -27.85977554321289,
      "step": 1620
    },
    {
      "epoch": 0.23995289268364492,
      "grad_norm": 106.36990338961671,
      "learning_rate": 4.7079183014189937e-07,
      "logits/chosen": -2.4466190338134766,
      "logits/rejected": -2.100706100463867,
      "logps/chosen": -233.1989288330078,
      "logps/rejected": -270.4334411621094,
      "loss": -0.66,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -3.4575438499450684,
      "rewards/margins": 55.49211883544922,
      "rewards/rejected": -58.94966506958008,
      "step": 1630
    },
    {
      "epoch": 0.2414249963197409,
      "grad_norm": 86.99159179285415,
      "learning_rate": 4.7018627132915634e-07,
      "logits/chosen": -2.3698010444641113,
      "logits/rejected": -2.1125988960266113,
      "logps/chosen": -282.465087890625,
      "logps/rejected": -265.3690490722656,
      "loss": -0.6409,
      "rewards/accuracies": 0.6999999284744263,
      "rewards/chosen": -9.7558012008667,
      "rewards/margins": 36.68387985229492,
      "rewards/rejected": -46.43967819213867,
      "step": 1640
    },
    {
      "epoch": 0.2428970999558369,
      "grad_norm": 108.89898069040578,
      "learning_rate": 4.695748971078042e-07,
      "logits/chosen": -2.579191207885742,
      "logits/rejected": -2.184810161590576,
      "logps/chosen": -287.9367370605469,
      "logps/rejected": -232.45101928710938,
      "loss": -0.6756,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 5.003921031951904,
      "rewards/margins": 32.68029022216797,
      "rewards/rejected": -27.67637062072754,
      "step": 1650
    },
    {
      "epoch": 0.24436920359193287,
      "grad_norm": 121.63741236457415,
      "learning_rate": 4.689577236250389e-07,
      "logits/chosen": -2.4219136238098145,
      "logits/rejected": -2.094757080078125,
      "logps/chosen": -282.84515380859375,
      "logps/rejected": -303.36181640625,
      "loss": -0.6741,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -13.486727714538574,
      "rewards/margins": 33.22724533081055,
      "rewards/rejected": -46.71397399902344,
      "step": 1660
    },
    {
      "epoch": 0.24584130722802885,
      "grad_norm": 75.20960833320248,
      "learning_rate": 4.683347671812228e-07,
      "logits/chosen": -2.604654550552368,
      "logits/rejected": -2.1868820190429688,
      "logps/chosen": -268.11676025390625,
      "logps/rejected": -227.364990234375,
      "loss": -0.7415,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -4.763229846954346,
      "rewards/margins": 51.2635498046875,
      "rewards/rejected": -56.02677536010742,
      "step": 1670
    },
    {
      "epoch": 0.24731341086412484,
      "grad_norm": 116.790653310887,
      "learning_rate": 4.677060442294537e-07,
      "logits/chosen": -2.609236717224121,
      "logits/rejected": -2.4223642349243164,
      "logps/chosen": -320.8695983886719,
      "logps/rejected": -322.0245361328125,
      "loss": -0.6794,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -0.7632617950439453,
      "rewards/margins": 28.655385971069336,
      "rewards/rejected": -29.41864585876465,
      "step": 1680
    },
    {
      "epoch": 0.24878551450022082,
      "grad_norm": 101.90936826704923,
      "learning_rate": 4.6707157137513056e-07,
      "logits/chosen": -2.649864673614502,
      "logits/rejected": -2.476078510284424,
      "logps/chosen": -295.62109375,
      "logps/rejected": -274.580078125,
      "loss": -0.6015,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -10.90587043762207,
      "rewards/margins": 35.41828536987305,
      "rewards/rejected": -46.32415771484375,
      "step": 1690
    },
    {
      "epoch": 0.2502576181363168,
      "grad_norm": 82.16661152006604,
      "learning_rate": 4.664313653755147e-07,
      "logits/chosen": -2.7364373207092285,
      "logits/rejected": -2.614427328109741,
      "logps/chosen": -399.4339904785156,
      "logps/rejected": -329.20587158203125,
      "loss": -0.7074,
      "rewards/accuracies": 0.6999999284744263,
      "rewards/chosen": 4.191736698150635,
      "rewards/margins": 28.914871215820312,
      "rewards/rejected": -24.723133087158203,
      "step": 1700
    },
    {
      "epoch": 0.25172972177241276,
      "grad_norm": 69.51376341037215,
      "learning_rate": 4.6578544313928735e-07,
      "logits/chosen": -2.481086015701294,
      "logits/rejected": -2.1925387382507324,
      "logps/chosen": -262.9289855957031,
      "logps/rejected": -320.6444396972656,
      "loss": -0.7074,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.3571496903896332,
      "rewards/margins": 57.485084533691406,
      "rewards/rejected": -57.12793731689453,
      "step": 1710
    },
    {
      "epoch": 0.2532018254085088,
      "grad_norm": 75.58025098290041,
      "learning_rate": 4.6513382172610324e-07,
      "logits/chosen": -2.4851760864257812,
      "logits/rejected": -2.204223394393921,
      "logps/chosen": -244.80996704101562,
      "logps/rejected": -286.06182861328125,
      "loss": -0.6537,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": 1.0297876596450806,
      "rewards/margins": 52.5792236328125,
      "rewards/rejected": -51.54943084716797,
      "step": 1720
    },
    {
      "epoch": 0.25467392904460473,
      "grad_norm": 74.39389726997848,
      "learning_rate": 4.6447651834613955e-07,
      "logits/chosen": -2.653064250946045,
      "logits/rejected": -2.5133509635925293,
      "logps/chosen": -265.446533203125,
      "logps/rejected": -300.1502685546875,
      "loss": -0.6481,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -7.019134521484375,
      "rewards/margins": 25.946155548095703,
      "rewards/rejected": -32.965293884277344,
      "step": 1730
    },
    {
      "epoch": 0.25614603268070074,
      "grad_norm": 72.42632708688211,
      "learning_rate": 4.638135503596419e-07,
      "logits/chosen": -2.5673577785491943,
      "logits/rejected": -2.6088123321533203,
      "logps/chosen": -265.07232666015625,
      "logps/rejected": -321.1754455566406,
      "loss": -0.5929,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": 3.85579252243042,
      "rewards/margins": 11.460622787475586,
      "rewards/rejected": -7.60482931137085,
      "step": 1740
    },
    {
      "epoch": 0.2576181363167967,
      "grad_norm": 158.32607505374898,
      "learning_rate": 4.6314493527646553e-07,
      "logits/chosen": -2.637463331222534,
      "logits/rejected": -2.3413801193237305,
      "logps/chosen": -328.34100341796875,
      "logps/rejected": -297.5247497558594,
      "loss": -0.6892,
      "rewards/accuracies": 0.5666667222976685,
      "rewards/chosen": 3.1566123962402344,
      "rewards/margins": 30.86942481994629,
      "rewards/rejected": -27.71280860900879,
      "step": 1750
    },
    {
      "epoch": 0.2590902399528927,
      "grad_norm": 42.22765829328495,
      "learning_rate": 4.624706907556129e-07,
      "logits/chosen": -2.593924045562744,
      "logits/rejected": -2.372157573699951,
      "logps/chosen": -244.11441040039062,
      "logps/rejected": -237.04953002929688,
      "loss": -0.7022,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -2.999920606613159,
      "rewards/margins": 39.863059997558594,
      "rewards/rejected": -42.86298370361328,
      "step": 1760
    },
    {
      "epoch": 0.26056234358898867,
      "grad_norm": 108.47204559841038,
      "learning_rate": 4.617908346047673e-07,
      "logits/chosen": -2.508075475692749,
      "logits/rejected": -2.3684747219085693,
      "logps/chosen": -218.1444091796875,
      "logps/rejected": -262.3788146972656,
      "loss": -0.6148,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 1.4345585107803345,
      "rewards/margins": 31.179866790771484,
      "rewards/rejected": -29.745309829711914,
      "step": 1770
    },
    {
      "epoch": 0.2620344472250846,
      "grad_norm": 84.14554111419795,
      "learning_rate": 4.6110538477982265e-07,
      "logits/chosen": -2.621854543685913,
      "logits/rejected": -2.463266134262085,
      "logps/chosen": -233.0678253173828,
      "logps/rejected": -296.0079650878906,
      "loss": -0.7425,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 2.8683202266693115,
      "rewards/margins": 51.43017578125,
      "rewards/rejected": -48.561866760253906,
      "step": 1780
    },
    {
      "epoch": 0.26350655086118063,
      "grad_norm": 96.70549260153847,
      "learning_rate": 4.6041435938440887e-07,
      "logits/chosen": -2.7175393104553223,
      "logits/rejected": -2.5751967430114746,
      "logps/chosen": -242.29568481445312,
      "logps/rejected": -269.5915222167969,
      "loss": -0.6331,
      "rewards/accuracies": 0.73333340883255,
      "rewards/chosen": 7.399426460266113,
      "rewards/margins": 22.541229248046875,
      "rewards/rejected": -15.141804695129395,
      "step": 1790
    },
    {
      "epoch": 0.2649786544972766,
      "grad_norm": 95.66531967238417,
      "learning_rate": 4.5971777666941445e-07,
      "logits/chosen": -2.737618923187256,
      "logits/rejected": -2.7057337760925293,
      "logps/chosen": -279.45611572265625,
      "logps/rejected": -294.10150146484375,
      "loss": -0.6231,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 6.000481605529785,
      "rewards/margins": 37.17029571533203,
      "rewards/rejected": -31.169815063476562,
      "step": 1800
    },
    {
      "epoch": 0.2664507581333726,
      "grad_norm": 85.49102034750727,
      "learning_rate": 4.5901565503250373e-07,
      "logits/chosen": -2.5968377590179443,
      "logits/rejected": -2.470646381378174,
      "logps/chosen": -299.7174987792969,
      "logps/rejected": -327.16082763671875,
      "loss": -1.4066,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": 21.12428855895996,
      "rewards/margins": 70.94327545166016,
      "rewards/rejected": -49.81898880004883,
      "step": 1810
    },
    {
      "epoch": 0.26792286176946856,
      "grad_norm": 184.17944946142285,
      "learning_rate": 4.583080130176312e-07,
      "logits/chosen": -2.557767868041992,
      "logits/rejected": -2.269334077835083,
      "logps/chosen": -276.9215087890625,
      "logps/rejected": -259.26190185546875,
      "loss": -0.7371,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -14.017718315124512,
      "rewards/margins": 34.24626159667969,
      "rewards/rejected": -48.26398468017578,
      "step": 1820
    },
    {
      "epoch": 0.26939496540556457,
      "grad_norm": 65.89018639452621,
      "learning_rate": 4.575948693145518e-07,
      "logits/chosen": -2.2914812564849854,
      "logits/rejected": -2.2028698921203613,
      "logps/chosen": -320.04681396484375,
      "logps/rejected": -335.24932861328125,
      "loss": -0.7236,
      "rewards/accuracies": 0.8333333730697632,
      "rewards/chosen": -9.95166301727295,
      "rewards/margins": 42.620948791503906,
      "rewards/rejected": -52.5726203918457,
      "step": 1830
    },
    {
      "epoch": 0.2708670690416605,
      "grad_norm": 233.4604160534317,
      "learning_rate": 4.568762427583275e-07,
      "logits/chosen": -2.4256174564361572,
      "logits/rejected": -2.444009304046631,
      "logps/chosen": -274.3991394042969,
      "logps/rejected": -334.6045837402344,
      "loss": -0.6918,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -10.113186836242676,
      "rewards/margins": 36.228492736816406,
      "rewards/rejected": -46.34168243408203,
      "step": 1840
    },
    {
      "epoch": 0.27233917267775654,
      "grad_norm": 90.98667697709801,
      "learning_rate": 4.561521523288293e-07,
      "logits/chosen": -2.3830056190490723,
      "logits/rejected": -2.1382339000701904,
      "logps/chosen": -317.91900634765625,
      "logps/rejected": -369.0251159667969,
      "loss": -0.2117,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -1.2548984289169312,
      "rewards/margins": 56.83732986450195,
      "rewards/rejected": -58.09223175048828,
      "step": 1850
    },
    {
      "epoch": 0.2738112763138525,
      "grad_norm": 136.663201068596,
      "learning_rate": 4.554226171502365e-07,
      "logits/chosen": -2.7044453620910645,
      "logits/rejected": -2.323603868484497,
      "logps/chosen": -315.4891357421875,
      "logps/rejected": -335.3113098144531,
      "loss": -0.6723,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -8.43090534210205,
      "rewards/margins": 37.2746696472168,
      "rewards/rejected": -45.7055778503418,
      "step": 1860
    },
    {
      "epoch": 0.2752833799499485,
      "grad_norm": 63.82844137196988,
      "learning_rate": 4.546876564905313e-07,
      "logits/chosen": -2.540470600128174,
      "logits/rejected": -2.6573739051818848,
      "logps/chosen": -230.170654296875,
      "logps/rejected": -384.15972900390625,
      "loss": -0.6208,
      "rewards/accuracies": 0.7333332300186157,
      "rewards/chosen": -5.673816680908203,
      "rewards/margins": 32.7302131652832,
      "rewards/rejected": -38.404029846191406,
      "step": 1870
    },
    {
      "epoch": 0.27675548358604446,
      "grad_norm": 80.6960115208755,
      "learning_rate": 4.5394728976099015e-07,
      "logits/chosen": -2.6163740158081055,
      "logits/rejected": -2.3720853328704834,
      "logps/chosen": -258.30877685546875,
      "logps/rejected": -297.8040466308594,
      "loss": 0.0644,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 4.141503810882568,
      "rewards/margins": 74.14186096191406,
      "rewards/rejected": -70.0003433227539,
      "step": 1880
    },
    {
      "epoch": 0.2782275872221404,
      "grad_norm": 240.56599870220595,
      "learning_rate": 4.532015365156705e-07,
      "logits/chosen": -2.6057381629943848,
      "logits/rejected": -2.4289793968200684,
      "logps/chosen": -309.7799987792969,
      "logps/rejected": -304.7105407714844,
      "loss": -0.6749,
      "rewards/accuracies": 0.7333332300186157,
      "rewards/chosen": -14.647315979003906,
      "rewards/margins": 42.67723846435547,
      "rewards/rejected": -57.324554443359375,
      "step": 1890
    },
    {
      "epoch": 0.2796996908582364,
      "grad_norm": 121.72154236884339,
      "learning_rate": 4.524504164508951e-07,
      "logits/chosen": -2.6673378944396973,
      "logits/rejected": -2.553657293319702,
      "logps/chosen": -307.00201416015625,
      "logps/rejected": -311.3681335449219,
      "loss": -0.6321,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -8.17476749420166,
      "rewards/margins": 24.92894172668457,
      "rewards/rejected": -33.10370635986328,
      "step": 1900
    },
    {
      "epoch": 0.2811717944943324,
      "grad_norm": 114.18484590378357,
      "learning_rate": 4.5169394940473137e-07,
      "logits/chosen": -2.7401843070983887,
      "logits/rejected": -2.4330644607543945,
      "logps/chosen": -296.49920654296875,
      "logps/rejected": -341.11297607421875,
      "loss": -0.6445,
      "rewards/accuracies": 0.7999999523162842,
      "rewards/chosen": -6.659232139587402,
      "rewards/margins": 45.62446594238281,
      "rewards/rejected": -52.2837028503418,
      "step": 1910
    },
    {
      "epoch": 0.2826438981304284,
      "grad_norm": 100.51392901111909,
      "learning_rate": 4.509321553564676e-07,
      "logits/chosen": -2.6525959968566895,
      "logits/rejected": -2.5785818099975586,
      "logps/chosen": -289.2851257324219,
      "logps/rejected": -306.8851318359375,
      "loss": -23.6467,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -6.633322715759277,
      "rewards/margins": 27.63372230529785,
      "rewards/rejected": -34.26704406738281,
      "step": 1920
    },
    {
      "epoch": 0.28411600176652435,
      "grad_norm": 77.9538580813883,
      "learning_rate": 4.501650544260848e-07,
      "logits/chosen": -2.4575130939483643,
      "logits/rejected": -2.5299606323242188,
      "logps/chosen": -228.7445068359375,
      "logps/rejected": -248.77969360351562,
      "loss": -3.12,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": 22.30492401123047,
      "rewards/margins": 31.33700942993164,
      "rewards/rejected": -9.032086372375488,
      "step": 1930
    },
    {
      "epoch": 0.28558810540262036,
      "grad_norm": 90.06491826614855,
      "learning_rate": 4.4939266687372636e-07,
      "logits/chosen": -2.5200092792510986,
      "logits/rejected": -2.4306371212005615,
      "logps/chosen": -348.6275329589844,
      "logps/rejected": -288.7826843261719,
      "loss": -0.6137,
      "rewards/accuracies": 0.76666659116745,
      "rewards/chosen": 3.8136889934539795,
      "rewards/margins": 30.207134246826172,
      "rewards/rejected": -26.393447875976562,
      "step": 1940
    },
    {
      "epoch": 0.2870602090387163,
      "grad_norm": 161.84898174167265,
      "learning_rate": 4.4861501309916185e-07,
      "logits/chosen": -2.7069907188415527,
      "logits/rejected": -2.464186191558838,
      "logps/chosen": -259.2839660644531,
      "logps/rejected": -222.9775390625,
      "loss": -0.748,
      "rewards/accuracies": 0.8333333730697632,
      "rewards/chosen": 5.976466178894043,
      "rewards/margins": 41.492401123046875,
      "rewards/rejected": -35.51593017578125,
      "step": 1950
    },
    {
      "epoch": 0.28853231267481233,
      "grad_norm": 206.7534815315281,
      "learning_rate": 4.478321136412487e-07,
      "logits/chosen": -2.3993654251098633,
      "logits/rejected": -2.371485471725464,
      "logps/chosen": -280.9446716308594,
      "logps/rejected": -336.62567138671875,
      "loss": -0.6202,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -6.090167999267578,
      "rewards/margins": 26.8797607421875,
      "rewards/rejected": -32.96992874145508,
      "step": 1960
    },
    {
      "epoch": 0.2900044163109083,
      "grad_norm": 90.4851211903041,
      "learning_rate": 4.4704398917738996e-07,
      "logits/chosen": -2.5603883266448975,
      "logits/rejected": -2.3935017585754395,
      "logps/chosen": -274.2432861328125,
      "logps/rejected": -338.32647705078125,
      "loss": -0.6889,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": 2.8059568405151367,
      "rewards/margins": 44.36222457885742,
      "rewards/rejected": -41.556270599365234,
      "step": 1970
    },
    {
      "epoch": 0.29147651994700424,
      "grad_norm": 116.90109711413876,
      "learning_rate": 4.4625066052298766e-07,
      "logits/chosen": -2.522430896759033,
      "logits/rejected": -2.4903817176818848,
      "logps/chosen": -244.3166961669922,
      "logps/rejected": -333.81878662109375,
      "loss": -0.5296,
      "rewards/accuracies": 0.5666667222976685,
      "rewards/chosen": -1.9700263738632202,
      "rewards/margins": 20.9844970703125,
      "rewards/rejected": -22.954524993896484,
      "step": 1980
    },
    {
      "epoch": 0.29294862358310025,
      "grad_norm": 61.793798517358454,
      "learning_rate": 4.454521486308936e-07,
      "logits/chosen": -2.6634294986724854,
      "logits/rejected": -2.5194132328033447,
      "logps/chosen": -330.0706787109375,
      "logps/rejected": -315.38134765625,
      "loss": -0.688,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -9.376073837280273,
      "rewards/margins": 30.618215560913086,
      "rewards/rejected": -39.994293212890625,
      "step": 1990
    },
    {
      "epoch": 0.2944207272191962,
      "grad_norm": 213.21256244619167,
      "learning_rate": 4.4464847459085553e-07,
      "logits/chosen": -2.561723470687866,
      "logits/rejected": -2.379157304763794,
      "logps/chosen": -281.9725646972656,
      "logps/rejected": -312.1358947753906,
      "loss": -12.9254,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": 2.5227885246276855,
      "rewards/margins": 50.054237365722656,
      "rewards/rejected": -47.53144073486328,
      "step": 2000
    },
    {
      "epoch": 0.2958928308552922,
      "grad_norm": 170.07177092531822,
      "learning_rate": 4.438396596289604e-07,
      "logits/chosen": -2.5364232063293457,
      "logits/rejected": -2.2993409633636475,
      "logps/chosen": -286.29608154296875,
      "logps/rejected": -312.2135009765625,
      "loss": -0.6867,
      "rewards/accuracies": 0.7333332300186157,
      "rewards/chosen": -11.821969985961914,
      "rewards/margins": 54.963584899902344,
      "rewards/rejected": -66.78556060791016,
      "step": 2010
    },
    {
      "epoch": 0.2973649344913882,
      "grad_norm": 95.11758562693238,
      "learning_rate": 4.430257251070736e-07,
      "logits/chosen": -2.464569568634033,
      "logits/rejected": -2.4491922855377197,
      "logps/chosen": -249.4795379638672,
      "logps/rejected": -245.88894653320312,
      "loss": -0.8713,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -8.901618003845215,
      "rewards/margins": 29.257038116455078,
      "rewards/rejected": -38.15865707397461,
      "step": 2020
    },
    {
      "epoch": 0.2988370381274842,
      "grad_norm": 97.2841083349037,
      "learning_rate": 4.422066925222748e-07,
      "logits/chosen": -2.4157471656799316,
      "logits/rejected": -2.1479756832122803,
      "logps/chosen": -243.72354125976562,
      "logps/rejected": -274.057373046875,
      "loss": -0.6869,
      "rewards/accuracies": 0.7999999523162842,
      "rewards/chosen": -1.2049791812896729,
      "rewards/margins": 51.067691802978516,
      "rewards/rejected": -52.272674560546875,
      "step": 2030
    },
    {
      "epoch": 0.30030914176358015,
      "grad_norm": 75.1140991127299,
      "learning_rate": 4.4138258350629033e-07,
      "logits/chosen": -2.743858814239502,
      "logits/rejected": -2.420362949371338,
      "logps/chosen": -324.70404052734375,
      "logps/rejected": -306.7781066894531,
      "loss": -0.678,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": 14.638076782226562,
      "rewards/margins": 39.89789962768555,
      "rewards/rejected": -25.259822845458984,
      "step": 2040
    },
    {
      "epoch": 0.30178124539967616,
      "grad_norm": 68.23833656180686,
      "learning_rate": 4.405534198249216e-07,
      "logits/chosen": -2.6563239097595215,
      "logits/rejected": -2.3331286907196045,
      "logps/chosen": -245.87265014648438,
      "logps/rejected": -252.4813232421875,
      "loss": -0.7291,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": 4.756333827972412,
      "rewards/margins": 48.290931701660156,
      "rewards/rejected": -43.53459930419922,
      "step": 2050
    },
    {
      "epoch": 0.3032533490357721,
      "grad_norm": 66.68219165908684,
      "learning_rate": 4.3971922337747045e-07,
      "logits/chosen": -2.5940747261047363,
      "logits/rejected": -2.4913887977600098,
      "logps/chosen": -270.41400146484375,
      "logps/rejected": -251.3420867919922,
      "loss": -0.7534,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 5.685196876525879,
      "rewards/margins": 25.664775848388672,
      "rewards/rejected": -19.97957992553711,
      "step": 2060
    },
    {
      "epoch": 0.3047254526718681,
      "grad_norm": 75.14976357893167,
      "learning_rate": 4.388800161961606e-07,
      "logits/chosen": -2.5797200202941895,
      "logits/rejected": -2.2242190837860107,
      "logps/chosen": -301.2087707519531,
      "logps/rejected": -296.1734619140625,
      "loss": -0.6601,
      "rewards/accuracies": 0.5666666030883789,
      "rewards/chosen": -6.068563938140869,
      "rewards/margins": 41.722511291503906,
      "rewards/rejected": -47.79106903076172,
      "step": 2070
    },
    {
      "epoch": 0.3061975563079641,
      "grad_norm": 112.40821227225159,
      "learning_rate": 4.380358204455559e-07,
      "logits/chosen": -2.373633623123169,
      "logits/rejected": -2.126237392425537,
      "logps/chosen": -223.82608032226562,
      "logps/rejected": -287.7281188964844,
      "loss": -0.6505,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -6.345137596130371,
      "rewards/margins": 36.56678009033203,
      "rewards/rejected": -42.91191482543945,
      "step": 2080
    },
    {
      "epoch": 0.30766965994406004,
      "grad_norm": 87.65984156249228,
      "learning_rate": 4.3718665842197494e-07,
      "logits/chosen": -2.2904887199401855,
      "logits/rejected": -1.926640272140503,
      "logps/chosen": -242.6881103515625,
      "logps/rejected": -294.6116638183594,
      "loss": -0.6563,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -14.420097351074219,
      "rewards/margins": 52.94926071166992,
      "rewards/rejected": -67.3693618774414,
      "step": 2090
    },
    {
      "epoch": 0.30914176358015605,
      "grad_norm": 82.29639395986348,
      "learning_rate": 4.363325525529019e-07,
      "logits/chosen": -2.248056411743164,
      "logits/rejected": -2.084132432937622,
      "logps/chosen": -297.8560485839844,
      "logps/rejected": -279.3362121582031,
      "loss": -0.608,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -16.757455825805664,
      "rewards/margins": 30.81332778930664,
      "rewards/rejected": -47.57078170776367,
      "step": 2100
    },
    {
      "epoch": 0.310613867216252,
      "grad_norm": 66.26622395570135,
      "learning_rate": 4.354735253963947e-07,
      "logits/chosen": -2.5714266300201416,
      "logits/rejected": -2.3725790977478027,
      "logps/chosen": -242.72830200195312,
      "logps/rejected": -276.6007995605469,
      "loss": -0.6718,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": 3.1119489669799805,
      "rewards/margins": 51.044677734375,
      "rewards/rejected": -47.93273162841797,
      "step": 2110
    },
    {
      "epoch": 0.312085970852348,
      "grad_norm": 60.94550807867775,
      "learning_rate": 4.3460959964048854e-07,
      "logits/chosen": -2.607111930847168,
      "logits/rejected": -2.4299206733703613,
      "logps/chosen": -310.88494873046875,
      "logps/rejected": -319.04180908203125,
      "loss": -0.6543,
      "rewards/accuracies": 0.6999999284744263,
      "rewards/chosen": -2.5861129760742188,
      "rewards/margins": 33.406883239746094,
      "rewards/rejected": -35.99300003051758,
      "step": 2120
    },
    {
      "epoch": 0.31355807448844397,
      "grad_norm": 73.64571080974636,
      "learning_rate": 4.337407981025974e-07,
      "logits/chosen": -2.632869005203247,
      "logits/rejected": -2.3964123725891113,
      "logps/chosen": -338.06573486328125,
      "logps/rejected": -386.76776123046875,
      "loss": -0.6349,
      "rewards/accuracies": 0.7333332300186157,
      "rewards/chosen": -17.05575180053711,
      "rewards/margins": 30.949771881103516,
      "rewards/rejected": -48.00552749633789,
      "step": 2130
    },
    {
      "epoch": 0.31503017812454,
      "grad_norm": 92.11652257675588,
      "learning_rate": 4.3286714372891086e-07,
      "logits/chosen": -2.6898276805877686,
      "logits/rejected": -2.5013351440429688,
      "logps/chosen": -236.6582794189453,
      "logps/rejected": -285.166015625,
      "loss": -0.7885,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": 10.560157775878906,
      "rewards/margins": 44.00127410888672,
      "rewards/rejected": -33.44110870361328,
      "step": 2140
    },
    {
      "epoch": 0.31650228176063594,
      "grad_norm": 102.17118049606704,
      "learning_rate": 4.319886595937885e-07,
      "logits/chosen": -2.6576383113861084,
      "logits/rejected": -2.4556071758270264,
      "logps/chosen": -321.7157287597656,
      "logps/rejected": -366.9571228027344,
      "loss": -0.7971,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": 13.748800277709961,
      "rewards/margins": 63.32960891723633,
      "rewards/rejected": -49.58080291748047,
      "step": 2150
    },
    {
      "epoch": 0.31797438539673195,
      "grad_norm": 73.28013266137452,
      "learning_rate": 4.3110536889914996e-07,
      "logits/chosen": -2.844325542449951,
      "logits/rejected": -2.607870578765869,
      "logps/chosen": -286.1172790527344,
      "logps/rejected": -292.04302978515625,
      "loss": -0.7009,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 10.867314338684082,
      "rewards/margins": 31.247425079345703,
      "rewards/rejected": -20.380109786987305,
      "step": 2160
    },
    {
      "epoch": 0.3194464890328279,
      "grad_norm": 105.49210301438332,
      "learning_rate": 4.302172949738626e-07,
      "logits/chosen": -2.5759360790252686,
      "logits/rejected": -2.318214178085327,
      "logps/chosen": -232.32937622070312,
      "logps/rejected": -321.3745422363281,
      "loss": -0.8111,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 5.288226127624512,
      "rewards/margins": 62.27608108520508,
      "rewards/rejected": -56.98784637451172,
      "step": 2170
    },
    {
      "epoch": 0.3209185926689239,
      "grad_norm": 104.88120015878164,
      "learning_rate": 4.2932446127312516e-07,
      "logits/chosen": -2.486461639404297,
      "logits/rejected": -2.3816890716552734,
      "logps/chosen": -242.31631469726562,
      "logps/rejected": -356.07415771484375,
      "loss": -0.6832,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 3.3620247840881348,
      "rewards/margins": 54.11565399169922,
      "rewards/rejected": -50.753631591796875,
      "step": 2180
    },
    {
      "epoch": 0.3223906963050199,
      "grad_norm": 104.42170582226233,
      "learning_rate": 4.2842689137784825e-07,
      "logits/chosen": -2.601975917816162,
      "logits/rejected": -2.2936434745788574,
      "logps/chosen": -320.7564697265625,
      "logps/rejected": -273.587646484375,
      "loss": -0.7268,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -1.2749035358428955,
      "rewards/margins": 42.10265350341797,
      "rewards/rejected": -43.37755584716797,
      "step": 2190
    },
    {
      "epoch": 0.32386279994111583,
      "grad_norm": 91.86484923117204,
      "learning_rate": 4.2752460899403175e-07,
      "logits/chosen": -2.5057363510131836,
      "logits/rejected": -2.1185107231140137,
      "logps/chosen": -226.27197265625,
      "logps/rejected": -303.4430236816406,
      "loss": -0.7317,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -9.893965721130371,
      "rewards/margins": 51.885459899902344,
      "rewards/rejected": -61.77943801879883,
      "step": 2200
    },
    {
      "epoch": 0.32533490357721184,
      "grad_norm": 100.25424936150958,
      "learning_rate": 4.2661763795213824e-07,
      "logits/chosen": -2.3606557846069336,
      "logits/rejected": -2.116250514984131,
      "logps/chosen": -233.66293334960938,
      "logps/rejected": -314.6795654296875,
      "loss": -0.7335,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 5.207980155944824,
      "rewards/margins": 65.5657730102539,
      "rewards/rejected": -60.3577880859375,
      "step": 2210
    },
    {
      "epoch": 0.3268070072133078,
      "grad_norm": 82.68437215609919,
      "learning_rate": 4.2570600220646425e-07,
      "logits/chosen": -2.6146700382232666,
      "logits/rejected": -2.3862128257751465,
      "logps/chosen": -278.12176513671875,
      "logps/rejected": -301.33502197265625,
      "loss": 3.9102,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -1.2459224462509155,
      "rewards/margins": 40.943668365478516,
      "rewards/rejected": -42.1895866394043,
      "step": 2220
    },
    {
      "epoch": 0.3282791108494038,
      "grad_norm": 2308.8890187851257,
      "learning_rate": 4.247897258345071e-07,
      "logits/chosen": -2.68940806388855,
      "logits/rejected": -2.3189053535461426,
      "logps/chosen": -331.98590087890625,
      "logps/rejected": -282.62457275390625,
      "loss": 3.2253,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -4.8536295890808105,
      "rewards/margins": 16.505260467529297,
      "rewards/rejected": -21.358890533447266,
      "step": 2230
    },
    {
      "epoch": 0.32975121448549977,
      "grad_norm": 60.3429335261467,
      "learning_rate": 4.238688330363292e-07,
      "logits/chosen": -2.567509174346924,
      "logits/rejected": -2.212477207183838,
      "logps/chosen": -213.06375122070312,
      "logps/rejected": -258.57965087890625,
      "loss": -0.6827,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": 2.9950978755950928,
      "rewards/margins": 59.56203079223633,
      "rewards/rejected": -56.566932678222656,
      "step": 2240
    },
    {
      "epoch": 0.3312233181215958,
      "grad_norm": 115.06514207741161,
      "learning_rate": 4.2294334813391874e-07,
      "logits/chosen": -2.614647150039673,
      "logits/rejected": -2.480909824371338,
      "logps/chosen": -286.1263427734375,
      "logps/rejected": -352.03900146484375,
      "loss": -0.7129,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": 18.064687728881836,
      "rewards/margins": 32.290504455566406,
      "rewards/rejected": -14.225814819335938,
      "step": 2250
    },
    {
      "epoch": 0.33269542175769173,
      "grad_norm": 85.0831225261269,
      "learning_rate": 4.220132955705476e-07,
      "logits/chosen": -2.6029670238494873,
      "logits/rejected": -2.3861308097839355,
      "logps/chosen": -254.7776336669922,
      "logps/rejected": -244.82711791992188,
      "loss": -0.6418,
      "rewards/accuracies": 0.6333332657814026,
      "rewards/chosen": 3.1470093727111816,
      "rewards/margins": 25.698070526123047,
      "rewards/rejected": -22.551063537597656,
      "step": 2260
    },
    {
      "epoch": 0.33416752539378775,
      "grad_norm": 79.2847933117751,
      "learning_rate": 4.2107869991012536e-07,
      "logits/chosen": -2.6607441902160645,
      "logits/rejected": -1.9417260885238647,
      "logps/chosen": -323.16033935546875,
      "logps/rejected": -344.9427795410156,
      "loss": -4.4084,
      "rewards/accuracies": 0.9333332777023315,
      "rewards/chosen": 6.050381660461426,
      "rewards/margins": 96.9163589477539,
      "rewards/rejected": -90.86597442626953,
      "step": 2270
    },
    {
      "epoch": 0.3356396290298837,
      "grad_norm": 124.69943322712251,
      "learning_rate": 4.201395858365509e-07,
      "logits/chosen": -2.480309009552002,
      "logits/rejected": -2.3694138526916504,
      "logps/chosen": -255.0525665283203,
      "logps/rejected": -328.5937194824219,
      "loss": -0.6876,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -21.11088752746582,
      "rewards/margins": 20.13485336303711,
      "rewards/rejected": -41.2457389831543,
      "step": 2280
    },
    {
      "epoch": 0.33711173266597966,
      "grad_norm": 129.0336811857592,
      "learning_rate": 4.191959781530603e-07,
      "logits/chosen": -2.8166344165802,
      "logits/rejected": -2.351555824279785,
      "logps/chosen": -264.4281311035156,
      "logps/rejected": -286.0342712402344,
      "loss": -0.7718,
      "rewards/accuracies": 0.8333333730697632,
      "rewards/chosen": -0.4663974642753601,
      "rewards/margins": 60.45280838012695,
      "rewards/rejected": -60.9192008972168,
      "step": 2290
    },
    {
      "epoch": 0.33858383630207567,
      "grad_norm": 70.55009313925287,
      "learning_rate": 4.1824790178157184e-07,
      "logits/chosen": -2.6250510215759277,
      "logits/rejected": -2.284123420715332,
      "logps/chosen": -306.263671875,
      "logps/rejected": -306.02691650390625,
      "loss": -0.7409,
      "rewards/accuracies": 0.76666659116745,
      "rewards/chosen": -9.272634506225586,
      "rewards/margins": 59.4998664855957,
      "rewards/rejected": -68.77249908447266,
      "step": 2300
    },
    {
      "epoch": 0.3400559399381716,
      "grad_norm": 64.54577947814656,
      "learning_rate": 4.172953817620275e-07,
      "logits/chosen": -2.5459072589874268,
      "logits/rejected": -2.2810449600219727,
      "logps/chosen": -242.42098999023438,
      "logps/rejected": -287.89886474609375,
      "loss": -0.6601,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -12.315505981445312,
      "rewards/margins": 54.3329963684082,
      "rewards/rejected": -66.64849853515625,
      "step": 2310
    },
    {
      "epoch": 0.34152804357426764,
      "grad_norm": 88.98938252792318,
      "learning_rate": 4.1633844325173215e-07,
      "logits/chosen": -2.763577699661255,
      "logits/rejected": -2.282698392868042,
      "logps/chosen": -305.0289001464844,
      "logps/rejected": -343.5362548828125,
      "loss": -0.7727,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -6.678229331970215,
      "rewards/margins": 45.769325256347656,
      "rewards/rejected": -52.44755935668945,
      "step": 2320
    },
    {
      "epoch": 0.3430001472103636,
      "grad_norm": 67.29191277025303,
      "learning_rate": 4.153771115246886e-07,
      "logits/chosen": -2.6084609031677246,
      "logits/rejected": -2.3912127017974854,
      "logps/chosen": -272.96563720703125,
      "logps/rejected": -272.52203369140625,
      "loss": -0.6664,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -9.72681713104248,
      "rewards/margins": 39.58460998535156,
      "rewards/rejected": -49.31142807006836,
      "step": 2330
    },
    {
      "epoch": 0.3444722508464596,
      "grad_norm": 76.78142039537157,
      "learning_rate": 4.144114119709303e-07,
      "logits/chosen": -2.5755510330200195,
      "logits/rejected": -2.425553798675537,
      "logps/chosen": -296.18096923828125,
      "logps/rejected": -310.77520751953125,
      "loss": -0.6659,
      "rewards/accuracies": 0.6999999284744263,
      "rewards/chosen": 8.817647933959961,
      "rewards/margins": 46.66798400878906,
      "rewards/rejected": -37.85033416748047,
      "step": 2340
    },
    {
      "epoch": 0.34594435448255556,
      "grad_norm": 114.87547435654835,
      "learning_rate": 4.134413700958509e-07,
      "logits/chosen": -2.469229221343994,
      "logits/rejected": -2.2215735912323,
      "logps/chosen": -289.657470703125,
      "logps/rejected": -321.3540954589844,
      "loss": -0.7095,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -6.12044620513916,
      "rewards/margins": 49.37657928466797,
      "rewards/rejected": -55.49702072143555,
      "step": 2350
    },
    {
      "epoch": 0.34741645811865157,
      "grad_norm": 63.8193152308524,
      "learning_rate": 4.1246701151953014e-07,
      "logits/chosen": -2.469897747039795,
      "logits/rejected": -2.215470314025879,
      "logps/chosen": -257.5547180175781,
      "logps/rejected": -245.970947265625,
      "loss": -0.6479,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -19.136707305908203,
      "rewards/margins": 19.173721313476562,
      "rewards/rejected": -38.31043243408203,
      "step": 2360
    },
    {
      "epoch": 0.34888856175474753,
      "grad_norm": 186.48831725717017,
      "learning_rate": 4.1148836197605774e-07,
      "logits/chosen": -2.5723843574523926,
      "logits/rejected": -2.2800140380859375,
      "logps/chosen": -275.5301818847656,
      "logps/rejected": -325.1964416503906,
      "loss": -0.6143,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -16.126646041870117,
      "rewards/margins": 65.03861236572266,
      "rewards/rejected": -81.16526794433594,
      "step": 2370
    },
    {
      "epoch": 0.35036066539084354,
      "grad_norm": 70.57259993651707,
      "learning_rate": 4.105054473128536e-07,
      "logits/chosen": -2.446237325668335,
      "logits/rejected": -2.357332229614258,
      "logps/chosen": -197.72628784179688,
      "logps/rejected": -233.57089233398438,
      "loss": -0.6279,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": 8.902010917663574,
      "rewards/margins": 38.1904411315918,
      "rewards/rejected": -29.288433074951172,
      "step": 2380
    },
    {
      "epoch": 0.3518327690269395,
      "grad_norm": 120.18973971657847,
      "learning_rate": 4.0951829348998477e-07,
      "logits/chosen": -2.6326401233673096,
      "logits/rejected": -2.4508490562438965,
      "logps/chosen": -299.4619445800781,
      "logps/rejected": -286.69146728515625,
      "loss": -0.9324,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": 4.497607707977295,
      "rewards/margins": 41.91070556640625,
      "rewards/rejected": -37.4130973815918,
      "step": 2390
    },
    {
      "epoch": 0.35330487266303545,
      "grad_norm": 76.06511246483704,
      "learning_rate": 4.0852692657948027e-07,
      "logits/chosen": -2.7557437419891357,
      "logits/rejected": -2.555698871612549,
      "logps/chosen": -236.10116577148438,
      "logps/rejected": -254.6735076904297,
      "loss": -0.7419,
      "rewards/accuracies": 0.6666667461395264,
      "rewards/chosen": -1.6319061517715454,
      "rewards/margins": 36.38689422607422,
      "rewards/rejected": -38.018798828125,
      "step": 2400
    },
    {
      "epoch": 0.35477697629913146,
      "grad_norm": 1113169.4959831291,
      "learning_rate": 4.075313727646422e-07,
      "logits/chosen": -2.7166664600372314,
      "logits/rejected": -2.432234287261963,
      "logps/chosen": -267.32672119140625,
      "logps/rejected": -364.2220764160156,
      "loss": 17.8787,
      "rewards/accuracies": 0.9000000953674316,
      "rewards/chosen": 12.352350234985352,
      "rewards/margins": 93.65645599365234,
      "rewards/rejected": -81.30411529541016,
      "step": 2410
    },
    {
      "epoch": 0.3562490799352274,
      "grad_norm": 71.3000880189185,
      "learning_rate": 4.0653165833935433e-07,
      "logits/chosen": -2.6669535636901855,
      "logits/rejected": -2.4376749992370605,
      "logps/chosen": -280.75177001953125,
      "logps/rejected": -239.6605682373047,
      "loss": -0.8877,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": -6.493041038513184,
      "rewards/margins": 33.5233039855957,
      "rewards/rejected": -40.0163459777832,
      "step": 2420
    },
    {
      "epoch": 0.35772118357132343,
      "grad_norm": 113.46729363614004,
      "learning_rate": 4.0552780970738755e-07,
      "logits/chosen": -2.708122730255127,
      "logits/rejected": -2.526247978210449,
      "logps/chosen": -240.5119171142578,
      "logps/rejected": -363.91644287109375,
      "loss": -0.7606,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": 1.116400122642517,
      "rewards/margins": 60.89400100708008,
      "rewards/rejected": -59.7775993347168,
      "step": 2430
    },
    {
      "epoch": 0.3591932872074194,
      "grad_norm": 97.16933848510878,
      "learning_rate": 4.045198533817028e-07,
      "logits/chosen": -2.723405599594116,
      "logits/rejected": -2.5663044452667236,
      "logps/chosen": -251.7305145263672,
      "logps/rejected": -248.3118133544922,
      "loss": -0.6902,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": 1.1811068058013916,
      "rewards/margins": 27.645965576171875,
      "rewards/rejected": -26.464855194091797,
      "step": 2440
    },
    {
      "epoch": 0.3606653908435154,
      "grad_norm": 226.36614100457035,
      "learning_rate": 4.0350781598375027e-07,
      "logits/chosen": -2.8735432624816895,
      "logits/rejected": -2.51082181930542,
      "logps/chosen": -376.7576904296875,
      "logps/rejected": -303.46575927734375,
      "loss": -0.6834,
      "rewards/accuracies": 0.7999999523162842,
      "rewards/chosen": 7.72605037689209,
      "rewards/margins": 49.9319953918457,
      "rewards/rejected": -42.20594787597656,
      "step": 2450
    },
    {
      "epoch": 0.36213749447961135,
      "grad_norm": 196.38218583036655,
      "learning_rate": 4.024917242427669e-07,
      "logits/chosen": -2.695909023284912,
      "logits/rejected": -2.436196804046631,
      "logps/chosen": -304.55999755859375,
      "logps/rejected": -335.4374084472656,
      "loss": -0.6876,
      "rewards/accuracies": 0.7999999523162842,
      "rewards/chosen": -12.202890396118164,
      "rewards/margins": 57.85411834716797,
      "rewards/rejected": -70.0570068359375,
      "step": 2460
    },
    {
      "epoch": 0.36360959811570737,
      "grad_norm": 84.31929671404994,
      "learning_rate": 4.0147160499507006e-07,
      "logits/chosen": -2.5878779888153076,
      "logits/rejected": -2.430908679962158,
      "logps/chosen": -356.7157287597656,
      "logps/rejected": -326.66400146484375,
      "loss": -0.6995,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -17.047378540039062,
      "rewards/margins": 40.2503776550293,
      "rewards/rejected": -57.297760009765625,
      "step": 2470
    },
    {
      "epoch": 0.3650817017518033,
      "grad_norm": 81.56745111563727,
      "learning_rate": 4.004474851833488e-07,
      "logits/chosen": -2.8771743774414062,
      "logits/rejected": -2.6473116874694824,
      "logps/chosen": -399.46954345703125,
      "logps/rejected": -331.3290710449219,
      "loss": -0.7048,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 13.612970352172852,
      "rewards/margins": 51.560325622558594,
      "rewards/rejected": -37.947364807128906,
      "step": 2480
    },
    {
      "epoch": 0.36655380538789933,
      "grad_norm": 95.73692802870434,
      "learning_rate": 3.9941939185595255e-07,
      "logits/chosen": -2.615170478820801,
      "logits/rejected": -2.334066390991211,
      "logps/chosen": -257.15740966796875,
      "logps/rejected": -322.5339050292969,
      "loss": 47.8659,
      "rewards/accuracies": 0.6666667461395264,
      "rewards/chosen": -8.909269332885742,
      "rewards/margins": 51.12077713012695,
      "rewards/rejected": -60.03004837036133,
      "step": 2490
    },
    {
      "epoch": 0.3680259090239953,
      "grad_norm": 85.51405327470115,
      "learning_rate": 3.9838735216617615e-07,
      "logits/chosen": -2.8057920932769775,
      "logits/rejected": -2.557541608810425,
      "logps/chosen": -257.7837829589844,
      "logps/rejected": -246.78439331054688,
      "loss": -0.5914,
      "rewards/accuracies": 0.6333333849906921,
      "rewards/chosen": -2.8867850303649902,
      "rewards/margins": 30.05050277709961,
      "rewards/rejected": -32.937294006347656,
      "step": 2500
    },
    {
      "epoch": 0.36949801266009125,
      "grad_norm": 86.14386550900443,
      "learning_rate": 3.9735139337154334e-07,
      "logits/chosen": -2.6930439472198486,
      "logits/rejected": -2.42246675491333,
      "logps/chosen": -212.5624542236328,
      "logps/rejected": -226.11599731445312,
      "loss": -0.5879,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -11.608402252197266,
      "rewards/margins": 32.09349060058594,
      "rewards/rejected": -43.70188522338867,
      "step": 2510
    },
    {
      "epoch": 0.37097011629618726,
      "grad_norm": 93.96830875534579,
      "learning_rate": 3.963115428330864e-07,
      "logits/chosen": -2.6680264472961426,
      "logits/rejected": -2.6381094455718994,
      "logps/chosen": -350.1546936035156,
      "logps/rejected": -365.5354309082031,
      "loss": -0.6984,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.5466006994247437,
      "rewards/margins": 24.105560302734375,
      "rewards/rejected": -25.65216064453125,
      "step": 2520
    },
    {
      "epoch": 0.3724422199322832,
      "grad_norm": 87.63902347155386,
      "learning_rate": 3.9526782801462384e-07,
      "logits/chosen": -2.7743194103240967,
      "logits/rejected": -2.4380762577056885,
      "logps/chosen": -306.7317810058594,
      "logps/rejected": -243.2174530029297,
      "loss": -0.7811,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 10.688423156738281,
      "rewards/margins": 61.9720344543457,
      "rewards/rejected": -51.283607482910156,
      "step": 2530
    },
    {
      "epoch": 0.3739143235683792,
      "grad_norm": 81.84572585247038,
      "learning_rate": 3.9422027648203474e-07,
      "logits/chosen": -2.4617257118225098,
      "logits/rejected": -2.357091188430786,
      "logps/chosen": -224.5398406982422,
      "logps/rejected": -269.66644287109375,
      "loss": -0.4013,
      "rewards/accuracies": 0.6999999284744263,
      "rewards/chosen": -11.5752534866333,
      "rewards/margins": 33.252830505371094,
      "rewards/rejected": -44.82808303833008,
      "step": 2540
    },
    {
      "epoch": 0.3753864272044752,
      "grad_norm": 94.69279208637856,
      "learning_rate": 3.9316891590253086e-07,
      "logits/chosen": -2.708138942718506,
      "logits/rejected": -2.371427536010742,
      "logps/chosen": -301.3210754394531,
      "logps/rejected": -292.19354248046875,
      "loss": -0.6727,
      "rewards/accuracies": 0.7000000476837158,
      "rewards/chosen": -12.867619514465332,
      "rewards/margins": 35.19382095336914,
      "rewards/rejected": -48.061439514160156,
      "step": 2550
    },
    {
      "epoch": 0.3768585308405712,
      "grad_norm": 59.802317622964786,
      "learning_rate": 3.921137740439261e-07,
      "logits/chosen": -2.667099714279175,
      "logits/rejected": -2.488518476486206,
      "logps/chosen": -259.38916015625,
      "logps/rejected": -254.8114471435547,
      "loss": -0.6953,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -5.237673759460449,
      "rewards/margins": 30.24190902709961,
      "rewards/rejected": -35.479583740234375,
      "step": 2560
    },
    {
      "epoch": 0.37833063447666715,
      "grad_norm": 88.02748933279608,
      "learning_rate": 3.910548787739024e-07,
      "logits/chosen": -2.7191176414489746,
      "logits/rejected": -2.570641279220581,
      "logps/chosen": -299.3561096191406,
      "logps/rejected": -342.9287414550781,
      "loss": -0.729,
      "rewards/accuracies": 0.7999999523162842,
      "rewards/chosen": -11.544007301330566,
      "rewards/margins": 47.69414138793945,
      "rewards/rejected": -59.2381477355957,
      "step": 2570
    },
    {
      "epoch": 0.37980273811276316,
      "grad_norm": 81.88728809555371,
      "learning_rate": 3.8999225805927483e-07,
      "logits/chosen": -2.7220566272735596,
      "logits/rejected": -2.285637378692627,
      "logps/chosen": -267.3989562988281,
      "logps/rejected": -288.13287353515625,
      "loss": -0.6425,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": -7.172521114349365,
      "rewards/margins": 39.57453155517578,
      "rewards/rejected": -46.74705123901367,
      "step": 2580
    },
    {
      "epoch": 0.3812748417488591,
      "grad_norm": 93.99405446836177,
      "learning_rate": 3.889259399652519e-07,
      "logits/chosen": -2.7984509468078613,
      "logits/rejected": -2.3128771781921387,
      "logps/chosen": -328.20355224609375,
      "logps/rejected": -333.05487060546875,
      "loss": -0.8111,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -6.571817874908447,
      "rewards/margins": 58.2531852722168,
      "rewards/rejected": -64.82500457763672,
      "step": 2590
    },
    {
      "epoch": 0.38274694538495513,
      "grad_norm": 72.84447057778507,
      "learning_rate": 3.8785595265469497e-07,
      "logits/chosen": -2.5414271354675293,
      "logits/rejected": -2.512065887451172,
      "logps/chosen": -275.2658996582031,
      "logps/rejected": -274.9867248535156,
      "loss": -0.7104,
      "rewards/accuracies": 0.5999999642372131,
      "rewards/chosen": -12.302694320678711,
      "rewards/margins": 32.15816116333008,
      "rewards/rejected": -44.460853576660156,
      "step": 2600
    },
    {
      "epoch": 0.3842190490210511,
      "grad_norm": 123.20815491664627,
      "learning_rate": 3.867823243873743e-07,
      "logits/chosen": -2.859454870223999,
      "logits/rejected": -2.436458110809326,
      "logps/chosen": -231.8162841796875,
      "logps/rejected": -262.0898132324219,
      "loss": -0.7986,
      "rewards/accuracies": 0.8333333730697632,
      "rewards/chosen": -10.004448890686035,
      "rewards/margins": 50.27470397949219,
      "rewards/rejected": -60.279151916503906,
      "step": 2610
    },
    {
      "epoch": 0.38569115265714704,
      "grad_norm": 81.78242830681914,
      "learning_rate": 3.8570508351922234e-07,
      "logits/chosen": -2.6597347259521484,
      "logits/rejected": -2.3492586612701416,
      "logps/chosen": -309.7577819824219,
      "logps/rejected": -325.4248962402344,
      "loss": -0.7212,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -18.79320526123047,
      "rewards/margins": 47.975318908691406,
      "rewards/rejected": -66.76850891113281,
      "step": 2620
    },
    {
      "epoch": 0.38716325629324305,
      "grad_norm": 94.31968594009591,
      "learning_rate": 3.8462425850158533e-07,
      "logits/chosen": -2.429105758666992,
      "logits/rejected": -2.1637160778045654,
      "logps/chosen": -262.8623962402344,
      "logps/rejected": -333.33868408203125,
      "loss": -0.6798,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -24.762741088867188,
      "rewards/margins": 47.04191970825195,
      "rewards/rejected": -71.8046646118164,
      "step": 2630
    },
    {
      "epoch": 0.388635359929339,
      "grad_norm": 113.89264368761899,
      "learning_rate": 3.8353987788047136e-07,
      "logits/chosen": -2.6344387531280518,
      "logits/rejected": -2.24543833732605,
      "logps/chosen": -307.3946228027344,
      "logps/rejected": -310.21820068359375,
      "loss": -0.7407,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": 0.1294742375612259,
      "rewards/margins": 45.322147369384766,
      "rewards/rejected": -45.19267272949219,
      "step": 2640
    },
    {
      "epoch": 0.390107463565435,
      "grad_norm": 85.63125615716802,
      "learning_rate": 3.824519702957969e-07,
      "logits/chosen": -2.6995201110839844,
      "logits/rejected": -2.342404365539551,
      "logps/chosen": -351.5836486816406,
      "logps/rejected": -289.8951110839844,
      "loss": -0.6325,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -0.7395766973495483,
      "rewards/margins": 33.60118865966797,
      "rewards/rejected": -34.34076690673828,
      "step": 2650
    },
    {
      "epoch": 0.391579567201531,
      "grad_norm": 69.70682576858229,
      "learning_rate": 3.8136056448063016e-07,
      "logits/chosen": -2.843043804168701,
      "logits/rejected": -2.5410046577453613,
      "logps/chosen": -273.81842041015625,
      "logps/rejected": -307.7102966308594,
      "loss": -0.7051,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": 1.7177295684814453,
      "rewards/margins": 50.21589660644531,
      "rewards/rejected": -48.4981689453125,
      "step": 2660
    },
    {
      "epoch": 0.393051670837627,
      "grad_norm": 212.8534244777239,
      "learning_rate": 3.802656892604319e-07,
      "logits/chosen": -2.634061813354492,
      "logits/rejected": -2.148177146911621,
      "logps/chosen": -228.0393524169922,
      "logps/rejected": -253.26736450195312,
      "loss": -0.7789,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": 1.9767882823944092,
      "rewards/margins": 56.949607849121094,
      "rewards/rejected": -54.972816467285156,
      "step": 2670
    },
    {
      "epoch": 0.39452377447372294,
      "grad_norm": 125.57637743892589,
      "learning_rate": 3.791673735522949e-07,
      "logits/chosen": -2.5828258991241455,
      "logits/rejected": -2.2705206871032715,
      "logps/chosen": -260.47393798828125,
      "logps/rejected": -360.3503112792969,
      "loss": -0.7296,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": 2.568230628967285,
      "rewards/margins": 61.191490173339844,
      "rewards/rejected": -58.623252868652344,
      "step": 2680
    },
    {
      "epoch": 0.39599587810981896,
      "grad_norm": 58.8226366238563,
      "learning_rate": 3.7806564636417936e-07,
      "logits/chosen": -2.427912712097168,
      "logits/rejected": -1.8779224157333374,
      "logps/chosen": -258.04058837890625,
      "logps/rejected": -303.82635498046875,
      "loss": -0.646,
      "rewards/accuracies": 0.6999999284744263,
      "rewards/chosen": 8.11634635925293,
      "rewards/margins": 63.62653732299805,
      "rewards/rejected": -55.51019287109375,
      "step": 2690
    },
    {
      "epoch": 0.3974679817459149,
      "grad_norm": 113.92087593449376,
      "learning_rate": 3.769605367941472e-07,
      "logits/chosen": -2.3446450233459473,
      "logits/rejected": -1.8881025314331055,
      "logps/chosen": -259.82867431640625,
      "logps/rejected": -296.55535888671875,
      "loss": -0.7582,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.9248573780059814,
      "rewards/margins": 52.08809280395508,
      "rewards/rejected": -54.01294708251953,
      "step": 2700
    },
    {
      "epoch": 0.39894008538201087,
      "grad_norm": 88.75301864020446,
      "learning_rate": 3.7585207402959377e-07,
      "logits/chosen": -2.5556421279907227,
      "logits/rejected": -2.180230140686035,
      "logps/chosen": -238.5950469970703,
      "logps/rejected": -287.8726501464844,
      "loss": -0.6986,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 2.949800968170166,
      "rewards/margins": 54.378997802734375,
      "rewards/rejected": -51.42919921875,
      "step": 2710
    },
    {
      "epoch": 0.4004121890181069,
      "grad_norm": 66.85108482002897,
      "learning_rate": 3.747402873464764e-07,
      "logits/chosen": -2.6250362396240234,
      "logits/rejected": -1.8463218212127686,
      "logps/chosen": -268.00701904296875,
      "logps/rejected": -291.17352294921875,
      "loss": -0.7609,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 8.148056030273438,
      "rewards/margins": 65.4305191040039,
      "rewards/rejected": -57.28246307373047,
      "step": 2720
    },
    {
      "epoch": 0.40188429265420283,
      "grad_norm": 216.84411140134145,
      "learning_rate": 3.7362520610854147e-07,
      "logits/chosen": -2.6109461784362793,
      "logits/rejected": -2.311800718307495,
      "logps/chosen": -196.44293212890625,
      "logps/rejected": -251.0883331298828,
      "loss": -0.6767,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 10.695627212524414,
      "rewards/margins": 45.51787567138672,
      "rewards/rejected": -34.82225036621094,
      "step": 2730
    },
    {
      "epoch": 0.40335639629029885,
      "grad_norm": 133.8498202108483,
      "learning_rate": 3.725068597665491e-07,
      "logits/chosen": -2.4188225269317627,
      "logits/rejected": -2.2328085899353027,
      "logps/chosen": -222.53756713867188,
      "logps/rejected": -338.2305603027344,
      "loss": -0.6856,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -23.1396541595459,
      "rewards/margins": 42.336639404296875,
      "rewards/rejected": -65.4762954711914,
      "step": 2740
    },
    {
      "epoch": 0.4048284999263948,
      "grad_norm": 104.92520865950972,
      "learning_rate": 3.71385277857495e-07,
      "logits/chosen": -2.3749561309814453,
      "logits/rejected": -2.1617014408111572,
      "logps/chosen": -279.67218017578125,
      "logps/rejected": -331.0541687011719,
      "loss": -4.2866,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -17.058372497558594,
      "rewards/margins": 30.91916275024414,
      "rewards/rejected": -47.97753143310547,
      "step": 2750
    },
    {
      "epoch": 0.4063006035624908,
      "grad_norm": 77.29097051991874,
      "learning_rate": 3.702604900038302e-07,
      "logits/chosen": -2.646713972091675,
      "logits/rejected": -2.2171335220336914,
      "logps/chosen": -248.16500854492188,
      "logps/rejected": -247.7259063720703,
      "loss": -0.6374,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -9.13017463684082,
      "rewards/margins": 36.27518844604492,
      "rewards/rejected": -45.405364990234375,
      "step": 2760
    },
    {
      "epoch": 0.40777270719858677,
      "grad_norm": 84.23075146329907,
      "learning_rate": 3.691325259126794e-07,
      "logits/chosen": -2.5689868927001953,
      "logits/rejected": -2.2306017875671387,
      "logps/chosen": -239.86978149414062,
      "logps/rejected": -323.3037414550781,
      "loss": -0.6569,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -6.626364707946777,
      "rewards/margins": 52.96110153198242,
      "rewards/rejected": -59.58746337890625,
      "step": 2770
    },
    {
      "epoch": 0.4092448108346828,
      "grad_norm": 85.55496376779018,
      "learning_rate": 3.6800141537505556e-07,
      "logits/chosen": -2.6908607482910156,
      "logits/rejected": -2.406304121017456,
      "logps/chosen": -239.12069702148438,
      "logps/rejected": -330.7021179199219,
      "loss": -0.7037,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 0.5555886030197144,
      "rewards/margins": 44.39179992675781,
      "rewards/rejected": -43.836212158203125,
      "step": 2780
    },
    {
      "epoch": 0.41071691447077874,
      "grad_norm": 97.41494020321413,
      "learning_rate": 3.6686718826507363e-07,
      "logits/chosen": -2.76045823097229,
      "logits/rejected": -2.5821468830108643,
      "logps/chosen": -302.7779846191406,
      "logps/rejected": -304.2327575683594,
      "loss": -0.6782,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": 4.135973930358887,
      "rewards/margins": 33.44560623168945,
      "rewards/rejected": -29.309635162353516,
      "step": 2790
    },
    {
      "epoch": 0.41218901810687475,
      "grad_norm": 75.18912366904597,
      "learning_rate": 3.6572987453916114e-07,
      "logits/chosen": -2.730023145675659,
      "logits/rejected": -2.4736034870147705,
      "logps/chosen": -231.3747100830078,
      "logps/rejected": -230.52316284179688,
      "loss": -0.7597,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 6.887289524078369,
      "rewards/margins": 38.67536163330078,
      "rewards/rejected": -31.788066864013672,
      "step": 2800
    },
    {
      "epoch": 0.4136611217429707,
      "grad_norm": 80.15087249685706,
      "learning_rate": 3.645895042352672e-07,
      "logits/chosen": -2.736933708190918,
      "logits/rejected": -2.1236047744750977,
      "logps/chosen": -270.4303894042969,
      "logps/rejected": -333.68096923828125,
      "loss": -0.7628,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 19.42034912109375,
      "rewards/margins": 77.29800415039062,
      "rewards/rejected": -57.877662658691406,
      "step": 2810
    },
    {
      "epoch": 0.41513322537906666,
      "grad_norm": 111.49174584979443,
      "learning_rate": 3.634461074720695e-07,
      "logits/chosen": -2.6423327922821045,
      "logits/rejected": -2.434622049331665,
      "logps/chosen": -249.50997924804688,
      "logps/rejected": -294.5934143066406,
      "loss": -0.7998,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": -5.743453025817871,
      "rewards/margins": 48.73863220214844,
      "rewards/rejected": -54.482078552246094,
      "step": 2820
    },
    {
      "epoch": 0.4166053290151627,
      "grad_norm": 127.60023817002428,
      "learning_rate": 3.622997144481781e-07,
      "logits/chosen": -2.644674062728882,
      "logits/rejected": -2.0337328910827637,
      "logps/chosen": -318.67083740234375,
      "logps/rejected": -275.48431396484375,
      "loss": -0.6781,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -25.577239990234375,
      "rewards/margins": 38.82292556762695,
      "rewards/rejected": -64.40016174316406,
      "step": 2830
    },
    {
      "epoch": 0.41807743265125863,
      "grad_norm": 132.59540419006566,
      "learning_rate": 3.611503554413383e-07,
      "logits/chosen": -2.879002332687378,
      "logits/rejected": -2.320209503173828,
      "logps/chosen": -297.0292663574219,
      "logps/rejected": -322.1534729003906,
      "loss": -0.6916,
      "rewards/accuracies": 0.5666667222976685,
      "rewards/chosen": -2.654829263687134,
      "rewards/margins": 45.274085998535156,
      "rewards/rejected": -47.928916931152344,
      "step": 2840
    },
    {
      "epoch": 0.41954953628735464,
      "grad_norm": 81.54386737108679,
      "learning_rate": 3.599980608076312e-07,
      "logits/chosen": -2.7010648250579834,
      "logits/rejected": -2.325174331665039,
      "logps/chosen": -294.64190673828125,
      "logps/rejected": -272.34686279296875,
      "loss": -0.6857,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -0.2103990614414215,
      "rewards/margins": 44.77183532714844,
      "rewards/rejected": -44.98223114013672,
      "step": 2850
    },
    {
      "epoch": 0.4210216399234506,
      "grad_norm": 94.14638878076416,
      "learning_rate": 3.5884286098067124e-07,
      "logits/chosen": -2.777280569076538,
      "logits/rejected": -2.35286545753479,
      "logps/chosen": -314.5992736816406,
      "logps/rejected": -320.1197814941406,
      "loss": -0.6476,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -4.099024295806885,
      "rewards/margins": 40.8216667175293,
      "rewards/rejected": -44.920692443847656,
      "step": 2860
    },
    {
      "epoch": 0.4224937435595466,
      "grad_norm": 71.67544624927321,
      "learning_rate": 3.5768478647080315e-07,
      "logits/chosen": -2.666503667831421,
      "logits/rejected": -2.5846028327941895,
      "logps/chosen": -301.4912109375,
      "logps/rejected": -313.23345947265625,
      "loss": -0.7032,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -5.412417411804199,
      "rewards/margins": 52.093589782714844,
      "rewards/rejected": -57.506011962890625,
      "step": 2870
    },
    {
      "epoch": 0.42396584719564256,
      "grad_norm": 101.02790514888798,
      "learning_rate": 3.565238678642957e-07,
      "logits/chosen": -2.423759698867798,
      "logits/rejected": -2.173614978790283,
      "logps/chosen": -333.2207946777344,
      "logps/rejected": -276.80010986328125,
      "loss": -0.6897,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -16.34337043762207,
      "rewards/margins": 45.225772857666016,
      "rewards/rejected": -61.56914138793945,
      "step": 2880
    },
    {
      "epoch": 0.4254379508317386,
      "grad_norm": 296.7032577790559,
      "learning_rate": 3.55360135822534e-07,
      "logits/chosen": -2.6985292434692383,
      "logits/rejected": -2.356548309326172,
      "logps/chosen": -290.6777648925781,
      "logps/rejected": -323.4270324707031,
      "loss": -0.7948,
      "rewards/accuracies": 0.76666659116745,
      "rewards/chosen": 0.23857459425926208,
      "rewards/margins": 53.69123458862305,
      "rewards/rejected": -53.45265579223633,
      "step": 2890
    },
    {
      "epoch": 0.42691005446783453,
      "grad_norm": 133.52776534786156,
      "learning_rate": 3.541936210812095e-07,
      "logits/chosen": -2.7000277042388916,
      "logits/rejected": -2.3520843982696533,
      "logps/chosen": -286.5193176269531,
      "logps/rejected": -288.99908447265625,
      "loss": -0.7726,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -13.305419921875,
      "rewards/margins": 53.116851806640625,
      "rewards/rejected": -66.42227172851562,
      "step": 2900
    },
    {
      "epoch": 0.42838215810393054,
      "grad_norm": 87.64243832827924,
      "learning_rate": 3.5302435444950894e-07,
      "logits/chosen": -2.8065645694732666,
      "logits/rejected": -2.517076015472412,
      "logps/chosen": -281.23828125,
      "logps/rejected": -273.15863037109375,
      "loss": -0.6591,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": 9.857844352722168,
      "rewards/margins": 51.78437423706055,
      "rewards/rejected": -41.92652893066406,
      "step": 2910
    },
    {
      "epoch": 0.4298542617400265,
      "grad_norm": 73.0056799720183,
      "learning_rate": 3.518523668092994e-07,
      "logits/chosen": -2.6742610931396484,
      "logits/rejected": -2.3831589221954346,
      "logps/chosen": -280.0317687988281,
      "logps/rejected": -285.3296813964844,
      "loss": -0.7166,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": 3.666325807571411,
      "rewards/margins": 52.02727508544922,
      "rewards/rejected": -48.36094665527344,
      "step": 2920
    },
    {
      "epoch": 0.43132636537612246,
      "grad_norm": 120.76520131622067,
      "learning_rate": 3.506776891143138e-07,
      "logits/chosen": -2.7074508666992188,
      "logits/rejected": -2.6369500160217285,
      "logps/chosen": -243.78457641601562,
      "logps/rejected": -341.9627685546875,
      "loss": -0.7256,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 13.19439697265625,
      "rewards/margins": 31.997966766357422,
      "rewards/rejected": -18.803569793701172,
      "step": 2930
    },
    {
      "epoch": 0.43279846901221847,
      "grad_norm": 68.71553436565667,
      "learning_rate": 3.495003523893329e-07,
      "logits/chosen": -2.5498576164245605,
      "logits/rejected": -2.540414333343506,
      "logps/chosen": -242.0194091796875,
      "logps/rejected": -260.10076904296875,
      "loss": -0.6655,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -0.2421816885471344,
      "rewards/margins": 26.442174911499023,
      "rewards/rejected": -26.684356689453125,
      "step": 2940
    },
    {
      "epoch": 0.4342705726483144,
      "grad_norm": 81.7819589998642,
      "learning_rate": 3.4832038772936574e-07,
      "logits/chosen": -2.7642226219177246,
      "logits/rejected": -2.5648632049560547,
      "logps/chosen": -250.544677734375,
      "logps/rejected": -270.638916015625,
      "loss": -0.7058,
      "rewards/accuracies": 0.533333420753479,
      "rewards/chosen": 7.783259391784668,
      "rewards/margins": 19.240196228027344,
      "rewards/rejected": -11.456938743591309,
      "step": 2950
    },
    {
      "epoch": 0.43574267628441044,
      "grad_norm": 147.9807467298251,
      "learning_rate": 3.471378262988288e-07,
      "logits/chosen": -2.686373472213745,
      "logits/rejected": -2.3986635208129883,
      "logps/chosen": -294.921630859375,
      "logps/rejected": -287.7221984863281,
      "loss": -0.7098,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -4.8899664878845215,
      "rewards/margins": 51.12627410888672,
      "rewards/rejected": -56.0162467956543,
      "step": 2960
    },
    {
      "epoch": 0.4372147799205064,
      "grad_norm": 96.44439315552242,
      "learning_rate": 3.459526993307225e-07,
      "logits/chosen": -2.4261579513549805,
      "logits/rejected": -2.134963274002075,
      "logps/chosen": -342.8690490722656,
      "logps/rejected": -308.73773193359375,
      "loss": -0.6019,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -9.165980339050293,
      "rewards/margins": 32.358070373535156,
      "rewards/rejected": -41.52405548095703,
      "step": 2970
    },
    {
      "epoch": 0.4386868835566024,
      "grad_norm": 74.96687868774019,
      "learning_rate": 3.4476503812580677e-07,
      "logits/chosen": -2.372666358947754,
      "logits/rejected": -1.9549150466918945,
      "logps/chosen": -320.19232177734375,
      "logps/rejected": -348.1940002441406,
      "loss": 74.6995,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -20.140460968017578,
      "rewards/margins": 26.598312377929688,
      "rewards/rejected": -46.73877716064453,
      "step": 2980
    },
    {
      "epoch": 0.44015898719269836,
      "grad_norm": 102.24776780211819,
      "learning_rate": 3.4357487405177367e-07,
      "logits/chosen": -2.569892406463623,
      "logits/rejected": -2.4708704948425293,
      "logps/chosen": -231.103759765625,
      "logps/rejected": -288.7547607421875,
      "loss": -0.6926,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": 1.0342962741851807,
      "rewards/margins": 42.33473587036133,
      "rewards/rejected": -41.300437927246094,
      "step": 2990
    },
    {
      "epoch": 0.44163109082879437,
      "grad_norm": 80.68217822133046,
      "learning_rate": 3.423822385424195e-07,
      "logits/chosen": -2.6748454570770264,
      "logits/rejected": -2.2513701915740967,
      "logps/chosen": -278.5334167480469,
      "logps/rejected": -332.339111328125,
      "loss": -0.8169,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": 2.4576449394226074,
      "rewards/margins": 64.17263793945312,
      "rewards/rejected": -61.714988708496094,
      "step": 3000
    },
    {
      "epoch": 0.4431031944648903,
      "grad_norm": 80.16737562304776,
      "learning_rate": 3.411871630968145e-07,
      "logits/chosen": -2.3873560428619385,
      "logits/rejected": -1.5910276174545288,
      "logps/chosen": -325.2021484375,
      "logps/rejected": -409.9442443847656,
      "loss": -0.6831,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": -9.126787185668945,
      "rewards/margins": 87.34750366210938,
      "rewards/rejected": -96.47429656982422,
      "step": 3010
    },
    {
      "epoch": 0.4445752981009863,
      "grad_norm": 413.8085614377934,
      "learning_rate": 3.3998967927847067e-07,
      "logits/chosen": -2.3671412467956543,
      "logits/rejected": -1.973791480064392,
      "logps/chosen": -365.7109680175781,
      "logps/rejected": -383.300537109375,
      "loss": -0.6637,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -17.73792266845703,
      "rewards/margins": 40.02997589111328,
      "rewards/rejected": -57.76789474487305,
      "step": 3020
    },
    {
      "epoch": 0.4460474017370823,
      "grad_norm": 115.67907275386725,
      "learning_rate": 3.3878981871450846e-07,
      "logits/chosen": -2.450517177581787,
      "logits/rejected": -2.182302236557007,
      "logps/chosen": -326.2355041503906,
      "logps/rejected": -377.9046936035156,
      "loss": -0.7071,
      "rewards/accuracies": 0.6999999284744263,
      "rewards/chosen": 2.779184341430664,
      "rewards/margins": 48.835044860839844,
      "rewards/rejected": -46.05585861206055,
      "step": 3030
    },
    {
      "epoch": 0.44751950537317825,
      "grad_norm": 142.9305280165796,
      "learning_rate": 3.375876130948211e-07,
      "logits/chosen": -2.300170421600342,
      "logits/rejected": -1.49214506149292,
      "logps/chosen": -284.2652587890625,
      "logps/rejected": -347.63323974609375,
      "loss": -0.7225,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": -11.260470390319824,
      "rewards/margins": 79.71625518798828,
      "rewards/rejected": -90.97673034667969,
      "step": 3040
    },
    {
      "epoch": 0.44899160900927426,
      "grad_norm": 96.04360776798958,
      "learning_rate": 3.3638309417123824e-07,
      "logits/chosen": -2.315545082092285,
      "logits/rejected": -1.7782764434814453,
      "logps/chosen": -314.16473388671875,
      "logps/rejected": -293.41534423828125,
      "loss": -0.6936,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -17.188386917114258,
      "rewards/margins": 43.2996711730957,
      "rewards/rejected": -60.48805618286133,
      "step": 3050
    },
    {
      "epoch": 0.4504637126453702,
      "grad_norm": 82.42504880915537,
      "learning_rate": 3.3517629375668615e-07,
      "logits/chosen": -2.322490692138672,
      "logits/rejected": -1.9889116287231445,
      "logps/chosen": -315.4691467285156,
      "logps/rejected": -344.6995849609375,
      "loss": -0.6679,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -10.202814102172852,
      "rewards/margins": 56.43828582763672,
      "rewards/rejected": -66.64109802246094,
      "step": 3060
    },
    {
      "epoch": 0.45193581628146623,
      "grad_norm": 101.68977365569663,
      "learning_rate": 3.3396724372434914e-07,
      "logits/chosen": -2.652897834777832,
      "logits/rejected": -1.9959911108016968,
      "logps/chosen": -355.628173828125,
      "logps/rejected": -277.19122314453125,
      "loss": -0.7121,
      "rewards/accuracies": 0.7999999523162842,
      "rewards/chosen": 1.5259668827056885,
      "rewards/margins": 51.608909606933594,
      "rewards/rejected": -50.08293914794922,
      "step": 3070
    },
    {
      "epoch": 0.4534079199175622,
      "grad_norm": 68.6876121706816,
      "learning_rate": 3.327559760068263e-07,
      "logits/chosen": -2.263030529022217,
      "logits/rejected": -2.005763292312622,
      "logps/chosen": -197.5602264404297,
      "logps/rejected": -314.7088928222656,
      "loss": -0.664,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -4.210844993591309,
      "rewards/margins": 59.7952880859375,
      "rewards/rejected": -64.0061264038086,
      "step": 3080
    },
    {
      "epoch": 0.4548800235536582,
      "grad_norm": 82.08690467121085,
      "learning_rate": 3.3154252259528883e-07,
      "logits/chosen": -2.6325697898864746,
      "logits/rejected": -1.578593373298645,
      "logps/chosen": -338.8136901855469,
      "logps/rejected": -340.54278564453125,
      "loss": -2.505,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": 4.713797569274902,
      "rewards/margins": 73.57369232177734,
      "rewards/rejected": -68.85990905761719,
      "step": 3090
    },
    {
      "epoch": 0.45635212718975415,
      "grad_norm": 102.59437784631194,
      "learning_rate": 3.303269155386351e-07,
      "logits/chosen": -2.15958833694458,
      "logits/rejected": -1.778428316116333,
      "logps/chosen": -291.5822448730469,
      "logps/rejected": -363.14898681640625,
      "loss": -0.6814,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -6.336532115936279,
      "rewards/margins": 69.19910430908203,
      "rewards/rejected": -75.53563690185547,
      "step": 3100
    },
    {
      "epoch": 0.45782423082585016,
      "grad_norm": 6441.712699498415,
      "learning_rate": 3.291091869426439e-07,
      "logits/chosen": -2.2635014057159424,
      "logits/rejected": -1.5306782722473145,
      "logps/chosen": -295.00408935546875,
      "logps/rejected": -323.4775390625,
      "loss": -0.1409,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": 22.24991226196289,
      "rewards/margins": 86.14424133300781,
      "rewards/rejected": -63.89434814453125,
      "step": 3110
    },
    {
      "epoch": 0.4592963344619461,
      "grad_norm": 83.45445940491435,
      "learning_rate": 3.27889368969127e-07,
      "logits/chosen": -2.3746657371520996,
      "logits/rejected": -1.9548835754394531,
      "logps/chosen": -283.87506103515625,
      "logps/rejected": -268.1792297363281,
      "loss": -0.7159,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 4.986113548278809,
      "rewards/margins": 63.373619079589844,
      "rewards/rejected": -58.38750457763672,
      "step": 3120
    },
    {
      "epoch": 0.4607684380980421,
      "grad_norm": 106.41484227196197,
      "learning_rate": 3.266674938350789e-07,
      "logits/chosen": -2.6442153453826904,
      "logits/rejected": -2.1574606895446777,
      "logps/chosen": -306.3396911621094,
      "logps/rejected": -295.5039978027344,
      "loss": -0.7672,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": 4.610134601593018,
      "rewards/margins": 47.56100845336914,
      "rewards/rejected": -42.95087432861328,
      "step": 3130
    },
    {
      "epoch": 0.4622405417341381,
      "grad_norm": 89.95701212088824,
      "learning_rate": 3.254435938118269e-07,
      "logits/chosen": -2.4572503566741943,
      "logits/rejected": -2.486487865447998,
      "logps/chosen": -304.4607849121094,
      "logps/rejected": -342.6962890625,
      "loss": -0.6369,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": 3.7462997436523438,
      "rewards/margins": 33.48993682861328,
      "rewards/rejected": -29.74363136291504,
      "step": 3140
    },
    {
      "epoch": 0.46371264537023404,
      "grad_norm": 77.43464890336979,
      "learning_rate": 3.242177012241778e-07,
      "logits/chosen": -2.400805950164795,
      "logits/rejected": -1.6492773294448853,
      "logps/chosen": -281.50140380859375,
      "logps/rejected": -263.4159851074219,
      "loss": -0.6812,
      "rewards/accuracies": 0.5666666626930237,
      "rewards/chosen": -10.936692237854004,
      "rewards/margins": 45.65433883666992,
      "rewards/rejected": -56.591033935546875,
      "step": 3150
    },
    {
      "epoch": 0.46518474900633006,
      "grad_norm": 88.33131143682643,
      "learning_rate": 3.229898484495649e-07,
      "logits/chosen": -2.511782169342041,
      "logits/rejected": -2.118090867996216,
      "logps/chosen": -188.48336791992188,
      "logps/rejected": -239.5282745361328,
      "loss": -0.7473,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": 4.1533589363098145,
      "rewards/margins": 40.969703674316406,
      "rewards/rejected": -36.81634521484375,
      "step": 3160
    },
    {
      "epoch": 0.466656852642426,
      "grad_norm": 149.8481796391133,
      "learning_rate": 3.2176006791719266e-07,
      "logits/chosen": -2.365854263305664,
      "logits/rejected": -1.8112577199935913,
      "logps/chosen": -235.55447387695312,
      "logps/rejected": -308.7928161621094,
      "loss": -0.7202,
      "rewards/accuracies": 0.7999999523162842,
      "rewards/chosen": -3.6007590293884277,
      "rewards/margins": 67.9854736328125,
      "rewards/rejected": -71.58623504638672,
      "step": 3170
    },
    {
      "epoch": 0.468128956278522,
      "grad_norm": 101.13602147129087,
      "learning_rate": 3.2052839210718007e-07,
      "logits/chosen": -2.4690487384796143,
      "logits/rejected": -2.4853668212890625,
      "logps/chosen": -235.9359588623047,
      "logps/rejected": -320.80450439453125,
      "loss": -0.7265,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": 0.44270095229148865,
      "rewards/margins": 39.651649475097656,
      "rewards/rejected": -39.208946228027344,
      "step": 3180
    },
    {
      "epoch": 0.469601059914618,
      "grad_norm": 106.54077888024827,
      "learning_rate": 3.1929485354970297e-07,
      "logits/chosen": -2.5685501098632812,
      "logits/rejected": -1.854379653930664,
      "logps/chosen": -287.9268493652344,
      "logps/rejected": -370.4284973144531,
      "loss": -0.8412,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": 4.224020957946777,
      "rewards/margins": 78.2432861328125,
      "rewards/rejected": -74.01924896240234,
      "step": 3190
    },
    {
      "epoch": 0.471073163550714,
      "grad_norm": 82.19495183773606,
      "learning_rate": 3.180594848241346e-07,
      "logits/chosen": -2.5115017890930176,
      "logits/rejected": -2.161999464035034,
      "logps/chosen": -238.5624542236328,
      "logps/rejected": -424.0438537597656,
      "loss": -0.6026,
      "rewards/accuracies": 0.7999999523162842,
      "rewards/chosen": -10.026851654052734,
      "rewards/margins": 47.29642105102539,
      "rewards/rejected": -57.323272705078125,
      "step": 3200
    },
    {
      "epoch": 0.47254526718680995,
      "grad_norm": 112.54013119236907,
      "learning_rate": 3.1682231855818565e-07,
      "logits/chosen": -2.521059513092041,
      "logits/rejected": -2.133840322494507,
      "logps/chosen": -203.39169311523438,
      "logps/rejected": -305.391845703125,
      "loss": -0.7034,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -1.9471960067749023,
      "rewards/margins": 64.62120056152344,
      "rewards/rejected": -66.56840515136719,
      "step": 3210
    },
    {
      "epoch": 0.47401737082290596,
      "grad_norm": 138.4822196795577,
      "learning_rate": 3.1558338742704175e-07,
      "logits/chosen": -2.489516258239746,
      "logits/rejected": -2.1804816722869873,
      "logps/chosen": -329.56817626953125,
      "logps/rejected": -294.7370910644531,
      "loss": -0.6785,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -12.298094749450684,
      "rewards/margins": 33.845130920410156,
      "rewards/rejected": -46.143226623535156,
      "step": 3220
    },
    {
      "epoch": 0.4754894744590019,
      "grad_norm": 114.58693847327399,
      "learning_rate": 3.1434272415250164e-07,
      "logits/chosen": -2.57947039604187,
      "logits/rejected": -2.2398529052734375,
      "logps/chosen": -303.3660888671875,
      "logps/rejected": -334.933349609375,
      "loss": -0.7445,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": 8.730567932128906,
      "rewards/margins": 60.29278564453125,
      "rewards/rejected": -51.56221389770508,
      "step": 3230
    },
    {
      "epoch": 0.47696157809509787,
      "grad_norm": 88.48816807517993,
      "learning_rate": 3.1310036150211155e-07,
      "logits/chosen": -2.4765632152557373,
      "logits/rejected": -2.236758232116699,
      "logps/chosen": -281.93133544921875,
      "logps/rejected": -324.3116455078125,
      "loss": -0.7836,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 9.110090255737305,
      "rewards/margins": 60.07036209106445,
      "rewards/rejected": -50.96027374267578,
      "step": 3240
    },
    {
      "epoch": 0.4784336817311939,
      "grad_norm": 69.33210847987922,
      "learning_rate": 3.1185633228830076e-07,
      "logits/chosen": -2.642122268676758,
      "logits/rejected": -1.9311221837997437,
      "logps/chosen": -305.46337890625,
      "logps/rejected": -332.5214538574219,
      "loss": -0.8088,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 4.2845354080200195,
      "rewards/margins": 79.90690612792969,
      "rewards/rejected": -75.62236022949219,
      "step": 3250
    },
    {
      "epoch": 0.47990578536728984,
      "grad_norm": 125.04237004175094,
      "learning_rate": 3.1061066936751483e-07,
      "logits/chosen": -2.5678629875183105,
      "logits/rejected": -2.171548843383789,
      "logps/chosen": -260.4076843261719,
      "logps/rejected": -341.443359375,
      "loss": -0.8068,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 8.692697525024414,
      "rewards/margins": 56.30988311767578,
      "rewards/rejected": -47.61717987060547,
      "step": 3260
    },
    {
      "epoch": 0.48137788900338585,
      "grad_norm": 84.27182219188026,
      "learning_rate": 3.0936340563934733e-07,
      "logits/chosen": -2.3942718505859375,
      "logits/rejected": -1.755292534828186,
      "logps/chosen": -262.45220947265625,
      "logps/rejected": -324.136962890625,
      "loss": -0.7488,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.3465851843357086,
      "rewards/margins": 72.87744140625,
      "rewards/rejected": -72.53084564208984,
      "step": 3270
    },
    {
      "epoch": 0.4828499926394818,
      "grad_norm": 61.286654187612385,
      "learning_rate": 3.0811457404567153e-07,
      "logits/chosen": -2.61860990524292,
      "logits/rejected": -2.054055690765381,
      "logps/chosen": -320.62164306640625,
      "logps/rejected": -289.1924743652344,
      "loss": -0.7532,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 4.254273414611816,
      "rewards/margins": 45.078834533691406,
      "rewards/rejected": -40.824562072753906,
      "step": 3280
    },
    {
      "epoch": 0.4843220962755778,
      "grad_norm": 72.42265674802357,
      "learning_rate": 3.068642075697699e-07,
      "logits/chosen": -2.441948413848877,
      "logits/rejected": -1.975001335144043,
      "logps/chosen": -268.97967529296875,
      "logps/rejected": -296.1715393066406,
      "loss": -0.6884,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -8.494482040405273,
      "rewards/margins": 44.50020980834961,
      "rewards/rejected": -52.99468231201172,
      "step": 3290
    },
    {
      "epoch": 0.4857941999116738,
      "grad_norm": 163.80950830499478,
      "learning_rate": 3.0561233923546336e-07,
      "logits/chosen": -2.351925849914551,
      "logits/rejected": -1.8425496816635132,
      "logps/chosen": -257.19732666015625,
      "logps/rejected": -303.4309387207031,
      "loss": -0.6688,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -14.490728378295898,
      "rewards/margins": 49.927574157714844,
      "rewards/rejected": -64.41830444335938,
      "step": 3300
    },
    {
      "epoch": 0.4872663035477698,
      "grad_norm": 83.7083842705087,
      "learning_rate": 3.0435900210623867e-07,
      "logits/chosen": -2.6125473976135254,
      "logits/rejected": -2.146824359893799,
      "logps/chosen": -250.01986694335938,
      "logps/rejected": -315.74871826171875,
      "loss": -0.7648,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 10.355220794677734,
      "rewards/margins": 55.91730499267578,
      "rewards/rejected": -45.56208038330078,
      "step": 3310
    },
    {
      "epoch": 0.48873840718386574,
      "grad_norm": 174.0862939302714,
      "learning_rate": 3.031042292843755e-07,
      "logits/chosen": -2.474438190460205,
      "logits/rejected": -1.7176011800765991,
      "logps/chosen": -288.0794677734375,
      "logps/rejected": -333.1150207519531,
      "loss": -0.7324,
      "rewards/accuracies": 0.8333333730697632,
      "rewards/chosen": -0.213603213429451,
      "rewards/margins": 56.461509704589844,
      "rewards/rejected": -56.67510986328125,
      "step": 3320
    },
    {
      "epoch": 0.4902105108199617,
      "grad_norm": 108.27531075307279,
      "learning_rate": 3.0184805391007205e-07,
      "logits/chosen": -2.1279587745666504,
      "logits/rejected": -2.1664645671844482,
      "logps/chosen": -301.41607666015625,
      "logps/rejected": -354.1761474609375,
      "loss": -0.7664,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -8.55003833770752,
      "rewards/margins": 27.62346839904785,
      "rewards/rejected": -36.17350769042969,
      "step": 3330
    },
    {
      "epoch": 0.4916826144560577,
      "grad_norm": 78.35016550260337,
      "learning_rate": 3.0059050916056977e-07,
      "logits/chosen": -2.6734557151794434,
      "logits/rejected": -1.9649978876113892,
      "logps/chosen": -277.696044921875,
      "logps/rejected": -248.39230346679688,
      "loss": -0.6769,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -13.985753059387207,
      "rewards/margins": 50.54861831665039,
      "rewards/rejected": -64.53437042236328,
      "step": 3340
    },
    {
      "epoch": 0.49315471809215367,
      "grad_norm": 68.2157325545065,
      "learning_rate": 2.9933162824927713e-07,
      "logits/chosen": -2.673866033554077,
      "logits/rejected": -2.192291021347046,
      "logps/chosen": -227.4352264404297,
      "logps/rejected": -273.68695068359375,
      "loss": -0.8079,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 3.2659103870391846,
      "rewards/margins": 53.953529357910156,
      "rewards/rejected": -50.68761444091797,
      "step": 3350
    },
    {
      "epoch": 0.4946268217282497,
      "grad_norm": 321.49773905035534,
      "learning_rate": 2.9807144442489234e-07,
      "logits/chosen": -2.601081609725952,
      "logits/rejected": -2.3438377380371094,
      "logps/chosen": -295.82879638671875,
      "logps/rejected": -322.96527099609375,
      "loss": -0.7492,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 12.285890579223633,
      "rewards/margins": 51.1326789855957,
      "rewards/rejected": -38.84678649902344,
      "step": 3360
    },
    {
      "epoch": 0.49609892536434563,
      "grad_norm": 113.35407949515881,
      "learning_rate": 2.9680999097052536e-07,
      "logits/chosen": -2.5352797508239746,
      "logits/rejected": -1.9774090051651,
      "logps/chosen": -200.98861694335938,
      "logps/rejected": -274.03717041015625,
      "loss": -0.7535,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 2.1646385192871094,
      "rewards/margins": 60.70086669921875,
      "rewards/rejected": -58.536224365234375,
      "step": 3370
    },
    {
      "epoch": 0.49757102900044164,
      "grad_norm": 69.10331820157218,
      "learning_rate": 2.9554730120281847e-07,
      "logits/chosen": -2.5201964378356934,
      "logits/rejected": -2.1685054302215576,
      "logps/chosen": -273.82843017578125,
      "logps/rejected": -275.40283203125,
      "loss": -0.9522,
      "rewards/accuracies": 0.73333340883255,
      "rewards/chosen": -9.506230354309082,
      "rewards/margins": 43.67967987060547,
      "rewards/rejected": -53.1859130859375,
      "step": 3380
    },
    {
      "epoch": 0.4990431326365376,
      "grad_norm": 166.67501328198483,
      "learning_rate": 2.942834084710668e-07,
      "logits/chosen": -2.611132860183716,
      "logits/rejected": -2.2054951190948486,
      "logps/chosen": -296.812744140625,
      "logps/rejected": -277.2122497558594,
      "loss": -0.6411,
      "rewards/accuracies": 0.5666666626930237,
      "rewards/chosen": -11.902681350708008,
      "rewards/margins": 41.9859733581543,
      "rewards/rejected": -53.88865280151367,
      "step": 3390
    },
    {
      "epoch": 0.5005152362726336,
      "grad_norm": 59.92975616996496,
      "learning_rate": 2.9301834615633766e-07,
      "logits/chosen": -2.7334187030792236,
      "logits/rejected": -2.633347988128662,
      "logps/chosen": -202.7835693359375,
      "logps/rejected": -241.5666961669922,
      "loss": -0.6967,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 12.660082817077637,
      "rewards/margins": 37.098323822021484,
      "rewards/rejected": -24.438241958618164,
      "step": 3400
    },
    {
      "epoch": 0.5019873399087296,
      "grad_norm": 108.73719890144606,
      "learning_rate": 2.917521476705879e-07,
      "logits/chosen": -2.6850502490997314,
      "logits/rejected": -2.2002174854278564,
      "logps/chosen": -315.414306640625,
      "logps/rejected": -344.98760986328125,
      "loss": -0.8777,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": 0.9118373990058899,
      "rewards/margins": 76.9712905883789,
      "rewards/rejected": -76.05946350097656,
      "step": 3410
    },
    {
      "epoch": 0.5034594435448255,
      "grad_norm": 75.74077971989041,
      "learning_rate": 2.904848464557827e-07,
      "logits/chosen": -2.4347777366638184,
      "logits/rejected": -2.2519500255584717,
      "logps/chosen": -307.3337097167969,
      "logps/rejected": -299.6181945800781,
      "loss": -0.7453,
      "rewards/accuracies": 0.46666669845581055,
      "rewards/chosen": -11.044910430908203,
      "rewards/margins": 33.85654830932617,
      "rewards/rejected": -44.901458740234375,
      "step": 3420
    },
    {
      "epoch": 0.5049315471809216,
      "grad_norm": 64.62455685178764,
      "learning_rate": 2.892164759830114e-07,
      "logits/chosen": -2.632577419281006,
      "logits/rejected": -2.1599032878875732,
      "logps/chosen": -266.697265625,
      "logps/rejected": -296.38482666015625,
      "loss": -0.7208,
      "rewards/accuracies": 0.76666659116745,
      "rewards/chosen": 6.910087585449219,
      "rewards/margins": 38.97590637207031,
      "rewards/rejected": -32.065818786621094,
      "step": 3430
    },
    {
      "epoch": 0.5064036508170175,
      "grad_norm": 89.23853168518566,
      "learning_rate": 2.8794706975160425e-07,
      "logits/chosen": -2.4853005409240723,
      "logits/rejected": -2.1473982334136963,
      "logps/chosen": -255.8648223876953,
      "logps/rejected": -280.06805419921875,
      "loss": -0.6329,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -4.648719310760498,
      "rewards/margins": 31.035213470458984,
      "rewards/rejected": -35.683937072753906,
      "step": 3440
    },
    {
      "epoch": 0.5078757544531135,
      "grad_norm": 101.487594132403,
      "learning_rate": 2.866766612882468e-07,
      "logits/chosen": -2.665196657180786,
      "logits/rejected": -2.261951446533203,
      "logps/chosen": -268.1519470214844,
      "logps/rejected": -329.4822998046875,
      "loss": -0.6982,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 5.988367557525635,
      "rewards/margins": 49.098426818847656,
      "rewards/rejected": -43.11005783081055,
      "step": 3450
    },
    {
      "epoch": 0.5093478580892095,
      "grad_norm": 76.80287797570007,
      "learning_rate": 2.8540528414609514e-07,
      "logits/chosen": -2.4245078563690186,
      "logits/rejected": -2.3439736366271973,
      "logps/chosen": -240.5614471435547,
      "logps/rejected": -342.6429138183594,
      "loss": -91.1691,
      "rewards/accuracies": 0.6999999284744263,
      "rewards/chosen": -9.160415649414062,
      "rewards/margins": 56.25688934326172,
      "rewards/rejected": -65.41731262207031,
      "step": 3460
    },
    {
      "epoch": 0.5108199617253054,
      "grad_norm": 159.97080327269342,
      "learning_rate": 2.8413297190388947e-07,
      "logits/chosen": -2.615940570831299,
      "logits/rejected": -2.3333375453948975,
      "logps/chosen": -260.0859375,
      "logps/rejected": -279.68218994140625,
      "loss": -0.7608,
      "rewards/accuracies": 0.73333340883255,
      "rewards/chosen": -1.307428002357483,
      "rewards/margins": 50.00926971435547,
      "rewards/rejected": -51.31669235229492,
      "step": 3470
    },
    {
      "epoch": 0.5122920653614015,
      "grad_norm": 91.71670703556696,
      "learning_rate": 2.8285975816506687e-07,
      "logits/chosen": -2.726797580718994,
      "logits/rejected": -2.676971912384033,
      "logps/chosen": -369.6214904785156,
      "logps/rejected": -322.4382019042969,
      "loss": -0.7922,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 13.88238525390625,
      "rewards/margins": 35.85356903076172,
      "rewards/rejected": -21.971187591552734,
      "step": 3480
    },
    {
      "epoch": 0.5137641689974974,
      "grad_norm": 68.05175944837372,
      "learning_rate": 2.815856765568746e-07,
      "logits/chosen": -2.7529959678649902,
      "logits/rejected": -2.520409345626831,
      "logps/chosen": -291.4873046875,
      "logps/rejected": -256.591552734375,
      "loss": -0.7796,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 8.161893844604492,
      "rewards/margins": 38.73536682128906,
      "rewards/rejected": -30.573471069335938,
      "step": 3490
    },
    {
      "epoch": 0.5152362726335934,
      "grad_norm": 177.10683898616344,
      "learning_rate": 2.803107607294811e-07,
      "logits/chosen": -2.7902185916900635,
      "logits/rejected": -2.3724935054779053,
      "logps/chosen": -255.7466583251953,
      "logps/rejected": -293.33868408203125,
      "loss": -0.6634,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": 11.051669120788574,
      "rewards/margins": 47.23113250732422,
      "rewards/rejected": -36.179466247558594,
      "step": 3500
    },
    {
      "epoch": 0.5167083762696894,
      "grad_norm": 80.00271891350668,
      "learning_rate": 2.7903504435508787e-07,
      "logits/chosen": -2.564509391784668,
      "logits/rejected": -2.3556227684020996,
      "logps/chosen": -317.4547424316406,
      "logps/rejected": -345.0162353515625,
      "loss": -0.678,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -0.48707637190818787,
      "rewards/margins": 46.31931686401367,
      "rewards/rejected": -46.806392669677734,
      "step": 3510
    },
    {
      "epoch": 0.5181804799057854,
      "grad_norm": 86.29170148280011,
      "learning_rate": 2.777585611270398e-07,
      "logits/chosen": -2.4675116539001465,
      "logits/rejected": -2.257948637008667,
      "logps/chosen": -271.17926025390625,
      "logps/rejected": -280.9930114746094,
      "loss": -1.0327,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": 14.995803833007812,
      "rewards/margins": 69.86456298828125,
      "rewards/rejected": -54.86875534057617,
      "step": 3520
    },
    {
      "epoch": 0.5196525835418814,
      "grad_norm": 81.1838739864705,
      "learning_rate": 2.7648134475893544e-07,
      "logits/chosen": -2.5696330070495605,
      "logits/rejected": -2.2927095890045166,
      "logps/chosen": -240.7815399169922,
      "logps/rejected": -258.56231689453125,
      "loss": 1.9045,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": 8.460527420043945,
      "rewards/margins": 22.227859497070312,
      "rewards/rejected": -13.76733112335205,
      "step": 3530
    },
    {
      "epoch": 0.5211246871779773,
      "grad_norm": 78.58938261847874,
      "learning_rate": 2.7520342898373657e-07,
      "logits/chosen": -2.134909152984619,
      "logits/rejected": -2.1686508655548096,
      "logps/chosen": -234.5335693359375,
      "logps/rejected": -261.7911071777344,
      "loss": -0.7267,
      "rewards/accuracies": 0.5666667222976685,
      "rewards/chosen": -21.94732666015625,
      "rewards/margins": 23.51199722290039,
      "rewards/rejected": -45.45931625366211,
      "step": 3540
    },
    {
      "epoch": 0.5225967908140733,
      "grad_norm": 88.45406981948815,
      "learning_rate": 2.73924847552877e-07,
      "logits/chosen": -2.4958319664001465,
      "logits/rejected": -2.0847251415252686,
      "logps/chosen": -261.79156494140625,
      "logps/rejected": -279.17694091796875,
      "loss": -0.7439,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": 1.5605026483535767,
      "rewards/margins": 58.964332580566406,
      "rewards/rejected": -57.40382766723633,
      "step": 3550
    },
    {
      "epoch": 0.5240688944501692,
      "grad_norm": 74.02072283780288,
      "learning_rate": 2.726456342353718e-07,
      "logits/chosen": -2.644763469696045,
      "logits/rejected": -2.4762775897979736,
      "logps/chosen": -316.66986083984375,
      "logps/rejected": -335.34967041015625,
      "loss": -0.7317,
      "rewards/accuracies": 0.6999999284744263,
      "rewards/chosen": 1.354274868965149,
      "rewards/margins": 37.888946533203125,
      "rewards/rejected": -36.534671783447266,
      "step": 3560
    },
    {
      "epoch": 0.5255409980862653,
      "grad_norm": 70.99601874478365,
      "learning_rate": 2.7136582281692443e-07,
      "logits/chosen": -2.5782177448272705,
      "logits/rejected": -2.2224764823913574,
      "logps/chosen": -268.8349609375,
      "logps/rejected": -248.9673309326172,
      "loss": -0.6301,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": 7.696728706359863,
      "rewards/margins": 52.05195236206055,
      "rewards/rejected": -44.355224609375,
      "step": 3570
    },
    {
      "epoch": 0.5270131017223613,
      "grad_norm": 95.60201899787185,
      "learning_rate": 2.700854470990357e-07,
      "logits/chosen": -2.5344653129577637,
      "logits/rejected": -2.2391438484191895,
      "logps/chosen": -267.438720703125,
      "logps/rejected": -323.77288818359375,
      "loss": -0.7415,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": 3.672468662261963,
      "rewards/margins": 65.34830474853516,
      "rewards/rejected": -61.67583084106445,
      "step": 3580
    },
    {
      "epoch": 0.5284852053584572,
      "grad_norm": 92.81208182719473,
      "learning_rate": 2.6880454089810954e-07,
      "logits/chosen": -2.8234734535217285,
      "logits/rejected": -2.3113009929656982,
      "logps/chosen": -289.82745361328125,
      "logps/rejected": -309.86669921875,
      "loss": -0.6583,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": 10.741256713867188,
      "rewards/margins": 40.7552375793457,
      "rewards/rejected": -30.013980865478516,
      "step": 3590
    },
    {
      "epoch": 0.5299573089945532,
      "grad_norm": 71.01275480021354,
      "learning_rate": 2.6752313804456124e-07,
      "logits/chosen": -2.7074646949768066,
      "logits/rejected": -2.279223918914795,
      "logps/chosen": -211.53359985351562,
      "logps/rejected": -205.438232421875,
      "loss": -0.713,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 13.569534301757812,
      "rewards/margins": 48.57689666748047,
      "rewards/rejected": -35.007362365722656,
      "step": 3600
    },
    {
      "epoch": 0.5314294126306492,
      "grad_norm": 77.31322268604994,
      "learning_rate": 2.66241272381923e-07,
      "logits/chosen": -2.6770730018615723,
      "logits/rejected": -2.49556303024292,
      "logps/chosen": -247.47720336914062,
      "logps/rejected": -224.8048858642578,
      "loss": -0.7164,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": 4.489247798919678,
      "rewards/margins": 32.67080307006836,
      "rewards/rejected": -28.181554794311523,
      "step": 3610
    },
    {
      "epoch": 0.5329015162667452,
      "grad_norm": 80.40974317214179,
      "learning_rate": 2.649589777659506e-07,
      "logits/chosen": -2.390718698501587,
      "logits/rejected": -2.1967577934265137,
      "logps/chosen": -201.43759155273438,
      "logps/rejected": -335.43048095703125,
      "loss": -0.7642,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 9.27869701385498,
      "rewards/margins": 50.43635940551758,
      "rewards/rejected": -41.15765380859375,
      "step": 3620
    },
    {
      "epoch": 0.5343736199028412,
      "grad_norm": 151.5254453409777,
      "learning_rate": 2.6367628806372893e-07,
      "logits/chosen": -2.6085829734802246,
      "logits/rejected": -1.949978232383728,
      "logps/chosen": -353.01385498046875,
      "logps/rejected": -303.0712890625,
      "loss": -0.7251,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": 1.272669792175293,
      "rewards/margins": 48.767364501953125,
      "rewards/rejected": -47.49469757080078,
      "step": 3630
    },
    {
      "epoch": 0.5358457235389371,
      "grad_norm": 293.32334278118316,
      "learning_rate": 2.623932371527776e-07,
      "logits/chosen": -2.6880764961242676,
      "logits/rejected": -1.9108976125717163,
      "logps/chosen": -311.4068298339844,
      "logps/rejected": -277.8924255371094,
      "loss": -0.7872,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": 9.134979248046875,
      "rewards/margins": 58.243324279785156,
      "rewards/rejected": -49.10834503173828,
      "step": 3640
    },
    {
      "epoch": 0.5373178271750331,
      "grad_norm": 156.10620293876423,
      "learning_rate": 2.611098589201563e-07,
      "logits/chosen": -2.4120378494262695,
      "logits/rejected": -1.9912668466567993,
      "logps/chosen": -223.871826171875,
      "logps/rejected": -278.7560729980469,
      "loss": -0.8252,
      "rewards/accuracies": 0.7999999523162842,
      "rewards/chosen": -5.805576801300049,
      "rewards/margins": 63.26725387573242,
      "rewards/rejected": -69.07283782958984,
      "step": 3650
    },
    {
      "epoch": 0.5387899308111291,
      "grad_norm": 597.747575021706,
      "learning_rate": 2.5982618726156965e-07,
      "logits/chosen": -2.355839252471924,
      "logits/rejected": -1.9262454509735107,
      "logps/chosen": -203.77993774414062,
      "logps/rejected": -281.4162902832031,
      "loss": -0.9396,
      "rewards/accuracies": 0.9333333969116211,
      "rewards/chosen": 12.699740409851074,
      "rewards/margins": 84.75579833984375,
      "rewards/rejected": -72.05606079101562,
      "step": 3660
    },
    {
      "epoch": 0.5402620344472251,
      "grad_norm": 104.51034302896224,
      "learning_rate": 2.5854225608047217e-07,
      "logits/chosen": -2.2367208003997803,
      "logits/rejected": -1.8190956115722656,
      "logps/chosen": -273.56890869140625,
      "logps/rejected": -311.8987121582031,
      "loss": -0.7876,
      "rewards/accuracies": 0.7999999523162842,
      "rewards/chosen": -7.094323635101318,
      "rewards/margins": 49.96450424194336,
      "rewards/rejected": -57.0588264465332,
      "step": 3670
    },
    {
      "epoch": 0.541734138083321,
      "grad_norm": 198.81752788734013,
      "learning_rate": 2.572580992871725e-07,
      "logits/chosen": -2.6190237998962402,
      "logits/rejected": -2.3143348693847656,
      "logps/chosen": -375.04876708984375,
      "logps/rejected": -358.738037109375,
      "loss": -0.743,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -5.9758734703063965,
      "rewards/margins": 39.722015380859375,
      "rewards/rejected": -45.6978874206543,
      "step": 3680
    },
    {
      "epoch": 0.543206241719417,
      "grad_norm": 61.23500502005178,
      "learning_rate": 2.5597375079793826e-07,
      "logits/chosen": -2.416537046432495,
      "logits/rejected": -2.0541045665740967,
      "logps/chosen": -244.35507202148438,
      "logps/rejected": -291.7746276855469,
      "loss": -0.6098,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.7105878591537476,
      "rewards/margins": 36.185935974121094,
      "rewards/rejected": -37.89652633666992,
      "step": 3690
    },
    {
      "epoch": 0.5446783453555131,
      "grad_norm": 85.02806345815891,
      "learning_rate": 2.5468924453409977e-07,
      "logits/chosen": -2.269230604171753,
      "logits/rejected": -1.8955150842666626,
      "logps/chosen": -253.9855499267578,
      "logps/rejected": -297.80780029296875,
      "loss": -0.5136,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -6.578619480133057,
      "rewards/margins": 42.02436828613281,
      "rewards/rejected": -48.602989196777344,
      "step": 3700
    },
    {
      "epoch": 0.546150448991609,
      "grad_norm": 72.11146905956008,
      "learning_rate": 2.534046144211544e-07,
      "logits/chosen": -2.4270167350769043,
      "logits/rejected": -2.2475571632385254,
      "logps/chosen": -306.38232421875,
      "logps/rejected": -322.11663818359375,
      "loss": -0.6656,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -15.332000732421875,
      "rewards/margins": 39.1548957824707,
      "rewards/rejected": -54.48688888549805,
      "step": 3710
    },
    {
      "epoch": 0.547622552627705,
      "grad_norm": 73.42624309616554,
      "learning_rate": 2.521198943878708e-07,
      "logits/chosen": -2.501641035079956,
      "logits/rejected": -2.596372127532959,
      "logps/chosen": -265.9951171875,
      "logps/rejected": -350.77655029296875,
      "loss": -0.6757,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -9.324424743652344,
      "rewards/margins": 14.207208633422852,
      "rewards/rejected": -23.531635284423828,
      "step": 3720
    },
    {
      "epoch": 0.5490946562638009,
      "grad_norm": 194.19174364792502,
      "learning_rate": 2.5083511836539213e-07,
      "logits/chosen": -2.629532814025879,
      "logits/rejected": -2.494886875152588,
      "logps/chosen": -281.5874938964844,
      "logps/rejected": -273.4043884277344,
      "loss": -0.6947,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.37146258354187,
      "rewards/margins": 17.307689666748047,
      "rewards/rejected": -19.679155349731445,
      "step": 3730
    },
    {
      "epoch": 0.550566759899897,
      "grad_norm": 82.60069687306489,
      "learning_rate": 2.495503202863407e-07,
      "logits/chosen": -2.521129608154297,
      "logits/rejected": -2.0944864749908447,
      "logps/chosen": -242.44833374023438,
      "logps/rejected": -252.85769653320312,
      "loss": -0.6772,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": -1.7784732580184937,
      "rewards/margins": 48.71028518676758,
      "rewards/rejected": -50.48875427246094,
      "step": 3740
    },
    {
      "epoch": 0.552038863535993,
      "grad_norm": 60.359132902476205,
      "learning_rate": 2.4826553408392104e-07,
      "logits/chosen": -2.5090136528015137,
      "logits/rejected": -2.185319423675537,
      "logps/chosen": -240.4409637451172,
      "logps/rejected": -276.98944091796875,
      "loss": -0.811,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 12.442364692687988,
      "rewards/margins": 59.82843780517578,
      "rewards/rejected": -47.38606643676758,
      "step": 3750
    },
    {
      "epoch": 0.5535109671720889,
      "grad_norm": 74.21040403061556,
      "learning_rate": 2.469807936910242e-07,
      "logits/chosen": -2.6539180278778076,
      "logits/rejected": -2.2991983890533447,
      "logps/chosen": -293.211181640625,
      "logps/rejected": -309.2485656738281,
      "loss": -0.7378,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": 10.049429893493652,
      "rewards/margins": 42.873878479003906,
      "rewards/rejected": -32.82445526123047,
      "step": 3760
    },
    {
      "epoch": 0.5549830708081849,
      "grad_norm": 57.9570803165221,
      "learning_rate": 2.456961330393313e-07,
      "logits/chosen": -2.492570161819458,
      "logits/rejected": -1.9844688177108765,
      "logps/chosen": -291.4251708984375,
      "logps/rejected": -264.16754150390625,
      "loss": -0.8123,
      "rewards/accuracies": 0.5666666626930237,
      "rewards/chosen": 5.353215217590332,
      "rewards/margins": 44.68406677246094,
      "rewards/rejected": -39.33086013793945,
      "step": 3770
    },
    {
      "epoch": 0.5564551744442808,
      "grad_norm": 105.18841887021016,
      "learning_rate": 2.444115860584174e-07,
      "logits/chosen": -2.4652833938598633,
      "logits/rejected": -2.0488338470458984,
      "logps/chosen": -315.0039978027344,
      "logps/rejected": -334.4344787597656,
      "loss": -0.7253,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -10.870089530944824,
      "rewards/margins": 46.954673767089844,
      "rewards/rejected": -57.82476806640625,
      "step": 3780
    },
    {
      "epoch": 0.5579272780803769,
      "grad_norm": 77.20195377569476,
      "learning_rate": 2.4312718667485523e-07,
      "logits/chosen": -2.643010139465332,
      "logits/rejected": -2.3389053344726562,
      "logps/chosen": -241.103271484375,
      "logps/rejected": -265.36865234375,
      "loss": -0.7257,
      "rewards/accuracies": 0.8333333730697632,
      "rewards/chosen": 16.45195770263672,
      "rewards/margins": 58.151702880859375,
      "rewards/rejected": -41.699745178222656,
      "step": 3790
    },
    {
      "epoch": 0.5593993817164729,
      "grad_norm": 72.43315849956089,
      "learning_rate": 2.418429688113194e-07,
      "logits/chosen": -2.591601848602295,
      "logits/rejected": -2.2089123725891113,
      "logps/chosen": -368.01708984375,
      "logps/rejected": -288.3812561035156,
      "loss": -0.6803,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -22.84748649597168,
      "rewards/margins": 42.637271881103516,
      "rewards/rejected": -65.48475646972656,
      "step": 3800
    },
    {
      "epoch": 0.5608714853525688,
      "grad_norm": 130.80333066054413,
      "learning_rate": 2.405589663856904e-07,
      "logits/chosen": -2.4428353309631348,
      "logits/rejected": -2.1469264030456543,
      "logps/chosen": -260.9988708496094,
      "logps/rejected": -287.84716796875,
      "loss": -0.7636,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 5.568016052246094,
      "rewards/margins": 46.314300537109375,
      "rewards/rejected": -40.74628448486328,
      "step": 3810
    },
    {
      "epoch": 0.5623435889886648,
      "grad_norm": 81.07562562993563,
      "learning_rate": 2.3927521331015865e-07,
      "logits/chosen": -2.500014543533325,
      "logits/rejected": -2.0926337242126465,
      "logps/chosen": -216.7418670654297,
      "logps/rejected": -286.13677978515625,
      "loss": -0.7788,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 1.9439260959625244,
      "rewards/margins": 70.53263854980469,
      "rewards/rejected": -68.58870697021484,
      "step": 3820
    },
    {
      "epoch": 0.5638156926247608,
      "grad_norm": 151.9106314780189,
      "learning_rate": 2.379917434903289e-07,
      "logits/chosen": -2.445439577102661,
      "logits/rejected": -2.261373996734619,
      "logps/chosen": -291.1381530761719,
      "logps/rejected": -343.917724609375,
      "loss": -0.7337,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -15.997273445129395,
      "rewards/margins": 42.571929931640625,
      "rewards/rejected": -58.5692024230957,
      "step": 3830
    },
    {
      "epoch": 0.5652877962608568,
      "grad_norm": 91.36665698044064,
      "learning_rate": 2.3670859082432458e-07,
      "logits/chosen": -2.66153621673584,
      "logits/rejected": -2.2737741470336914,
      "logps/chosen": -317.36444091796875,
      "logps/rejected": -322.4660339355469,
      "loss": -0.7001,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": 0.19059722125530243,
      "rewards/margins": 69.879150390625,
      "rewards/rejected": -69.68855285644531,
      "step": 3840
    },
    {
      "epoch": 0.5667598998969527,
      "grad_norm": 117.70727512204294,
      "learning_rate": 2.35425789201893e-07,
      "logits/chosen": -2.3893704414367676,
      "logits/rejected": -1.7847896814346313,
      "logps/chosen": -295.84027099609375,
      "logps/rejected": -351.02606201171875,
      "loss": -0.7875,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -14.942224502563477,
      "rewards/margins": 62.154014587402344,
      "rewards/rejected": -77.09623718261719,
      "step": 3850
    },
    {
      "epoch": 0.5682320035330487,
      "grad_norm": 108.51347736904056,
      "learning_rate": 2.3414337250350982e-07,
      "logits/chosen": -2.4072446823120117,
      "logits/rejected": -2.0115487575531006,
      "logps/chosen": -248.2409210205078,
      "logps/rejected": -294.29803466796875,
      "loss": 247.4278,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -11.621305465698242,
      "rewards/margins": 54.57768630981445,
      "rewards/rejected": -66.19898986816406,
      "step": 3860
    },
    {
      "epoch": 0.5697041071691447,
      "grad_norm": 129.74589514639132,
      "learning_rate": 2.3286137459948428e-07,
      "logits/chosen": -2.6357874870300293,
      "logits/rejected": -2.309662342071533,
      "logps/chosen": -202.0946044921875,
      "logps/rejected": -294.37982177734375,
      "loss": 2.9558,
      "rewards/accuracies": 0.7999999523162842,
      "rewards/chosen": -3.0756754875183105,
      "rewards/margins": 74.68962860107422,
      "rewards/rejected": -77.76530456542969,
      "step": 3870
    },
    {
      "epoch": 0.5711762108052407,
      "grad_norm": 399.0574936818435,
      "learning_rate": 2.3157982934906463e-07,
      "logits/chosen": -2.779195785522461,
      "logits/rejected": -2.5508666038513184,
      "logps/chosen": -313.29058837890625,
      "logps/rejected": -335.32562255859375,
      "loss": -0.8387,
      "rewards/accuracies": 0.76666659116745,
      "rewards/chosen": 7.008143424987793,
      "rewards/margins": 46.900978088378906,
      "rewards/rejected": -39.89284133911133,
      "step": 3880
    },
    {
      "epoch": 0.5726483144413367,
      "grad_norm": 140.99699005184812,
      "learning_rate": 2.3029877059954414e-07,
      "logits/chosen": -2.8618438243865967,
      "logits/rejected": -2.3823459148406982,
      "logps/chosen": -244.921630859375,
      "logps/rejected": -317.16619873046875,
      "loss": -0.8006,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": 12.424386978149414,
      "rewards/margins": 83.07490539550781,
      "rewards/rejected": -70.65052795410156,
      "step": 3890
    },
    {
      "epoch": 0.5741204180774326,
      "grad_norm": 119.175679854467,
      "learning_rate": 2.2901823218536693e-07,
      "logits/chosen": -2.8017196655273438,
      "logits/rejected": -2.3817191123962402,
      "logps/chosen": -254.01101684570312,
      "logps/rejected": -234.387939453125,
      "loss": -0.776,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -2.637476921081543,
      "rewards/margins": 34.891597747802734,
      "rewards/rejected": -37.529083251953125,
      "step": 3900
    },
    {
      "epoch": 0.5755925217135286,
      "grad_norm": 78.35448657878361,
      "learning_rate": 2.2773824792723428e-07,
      "logits/chosen": -2.6069254875183105,
      "logits/rejected": -2.0343565940856934,
      "logps/chosen": -229.3800811767578,
      "logps/rejected": -314.78741455078125,
      "loss": -1.0349,
      "rewards/accuracies": 0.9333333969116211,
      "rewards/chosen": 12.153387069702148,
      "rewards/margins": 95.13396453857422,
      "rewards/rejected": -82.98057556152344,
      "step": 3910
    },
    {
      "epoch": 0.5770646253496247,
      "grad_norm": 91.53323981221799,
      "learning_rate": 2.2645885163121156e-07,
      "logits/chosen": -2.684248685836792,
      "logits/rejected": -2.20408296585083,
      "logps/chosen": -273.9342041015625,
      "logps/rejected": -343.41748046875,
      "loss": -0.8172,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 9.871353149414062,
      "rewards/margins": 82.29325103759766,
      "rewards/rejected": -72.42190551757812,
      "step": 3920
    },
    {
      "epoch": 0.5785367289857206,
      "grad_norm": 249.85844253523535,
      "learning_rate": 2.2518007708783512e-07,
      "logits/chosen": -2.4635813236236572,
      "logits/rejected": -2.030609130859375,
      "logps/chosen": -278.921630859375,
      "logps/rejected": -404.1416015625,
      "loss": -0.636,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -12.669642448425293,
      "rewards/margins": 70.37755584716797,
      "rewards/rejected": -83.04719543457031,
      "step": 3930
    },
    {
      "epoch": 0.5800088326218166,
      "grad_norm": 99.67895744202157,
      "learning_rate": 2.2390195807122027e-07,
      "logits/chosen": -2.490687608718872,
      "logits/rejected": -2.4030768871307373,
      "logps/chosen": -257.94219970703125,
      "logps/rejected": -253.24856567382812,
      "loss": -0.6705,
      "rewards/accuracies": 0.6999999284744263,
      "rewards/chosen": 9.064730644226074,
      "rewards/margins": 31.893178939819336,
      "rewards/rejected": -22.828449249267578,
      "step": 3940
    },
    {
      "epoch": 0.5814809362579125,
      "grad_norm": 54.57094885257461,
      "learning_rate": 2.2262452833816886e-07,
      "logits/chosen": -2.528273105621338,
      "logits/rejected": -1.8314393758773804,
      "logps/chosen": -268.04034423828125,
      "logps/rejected": -325.70989990234375,
      "loss": -0.6906,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -10.292831420898438,
      "rewards/margins": 64.19205474853516,
      "rewards/rejected": -74.4848861694336,
      "step": 3950
    },
    {
      "epoch": 0.5829530398940085,
      "grad_norm": 81.62349856785619,
      "learning_rate": 2.2134782162727778e-07,
      "logits/chosen": -2.5449752807617188,
      "logits/rejected": -2.3111162185668945,
      "logps/chosen": -256.30908203125,
      "logps/rejected": -267.25115966796875,
      "loss": -0.7529,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -3.7794196605682373,
      "rewards/margins": 36.057342529296875,
      "rewards/rejected": -39.836761474609375,
      "step": 3960
    },
    {
      "epoch": 0.5844251435301046,
      "grad_norm": 72.48831093486496,
      "learning_rate": 2.2007187165804822e-07,
      "logits/chosen": -2.4071402549743652,
      "logits/rejected": -2.232578992843628,
      "logps/chosen": -290.33599853515625,
      "logps/rejected": -361.264404296875,
      "loss": 1234.0538,
      "rewards/accuracies": 0.76666659116745,
      "rewards/chosen": 6.289002895355225,
      "rewards/margins": 20.371612548828125,
      "rewards/rejected": -14.082612991333008,
      "step": 3970
    },
    {
      "epoch": 0.5858972471662005,
      "grad_norm": 94.71613577530866,
      "learning_rate": 2.1879671212999437e-07,
      "logits/chosen": -2.629178524017334,
      "logits/rejected": -2.310617685317993,
      "logps/chosen": -264.2568054199219,
      "logps/rejected": -260.29534912109375,
      "loss": -0.7993,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": 10.025006294250488,
      "rewards/margins": 46.22150802612305,
      "rewards/rejected": -36.196502685546875,
      "step": 3980
    },
    {
      "epoch": 0.5873693508022965,
      "grad_norm": 101.06833008294606,
      "learning_rate": 2.1752237672175433e-07,
      "logits/chosen": -2.7145843505859375,
      "logits/rejected": -2.444279193878174,
      "logps/chosen": -273.4049987792969,
      "logps/rejected": -291.19232177734375,
      "loss": -0.6743,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.6604251861572266,
      "rewards/margins": 18.980188369750977,
      "rewards/rejected": -19.640613555908203,
      "step": 3990
    },
    {
      "epoch": 0.5888414544383924,
      "grad_norm": 98.89319254722871,
      "learning_rate": 2.162488990901998e-07,
      "logits/chosen": -2.531237840652466,
      "logits/rejected": -2.344883680343628,
      "logps/chosen": -272.01336669921875,
      "logps/rejected": -294.5602111816406,
      "loss": -0.6839,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -6.978217124938965,
      "rewards/margins": 37.963809967041016,
      "rewards/rejected": -44.94202423095703,
      "step": 4000
    },
    {
      "epoch": 0.5903135580744885,
      "grad_norm": 115.82240046698574,
      "learning_rate": 2.1497631286954764e-07,
      "logits/chosen": -2.6702678203582764,
      "logits/rejected": -2.2400927543640137,
      "logps/chosen": -336.38922119140625,
      "logps/rejected": -312.2210693359375,
      "loss": -0.7123,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 8.4286470413208,
      "rewards/margins": 61.051063537597656,
      "rewards/rejected": -52.62241744995117,
      "step": 4010
    },
    {
      "epoch": 0.5917856617105844,
      "grad_norm": 97.74676480586396,
      "learning_rate": 2.1370465167047118e-07,
      "logits/chosen": -2.6766347885131836,
      "logits/rejected": -2.3707282543182373,
      "logps/chosen": -237.0137939453125,
      "logps/rejected": -235.32632446289062,
      "loss": -0.7048,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": 7.574594974517822,
      "rewards/margins": 50.63014221191406,
      "rewards/rejected": -43.05554962158203,
      "step": 4020
    },
    {
      "epoch": 0.5932577653466804,
      "grad_norm": 64.20316794345563,
      "learning_rate": 2.124339490792128e-07,
      "logits/chosen": -2.586503267288208,
      "logits/rejected": -2.2107067108154297,
      "logps/chosen": -283.1888732910156,
      "logps/rejected": -370.0911865234375,
      "loss": -0.7278,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -11.70268726348877,
      "rewards/margins": 51.036766052246094,
      "rewards/rejected": -62.73944854736328,
      "step": 4030
    },
    {
      "epoch": 0.5947298689827764,
      "grad_norm": 128.0908503525908,
      "learning_rate": 2.1116423865669703e-07,
      "logits/chosen": -2.7052388191223145,
      "logits/rejected": -2.108238935470581,
      "logps/chosen": -268.6873474121094,
      "logps/rejected": -309.52960205078125,
      "loss": -0.7546,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": 2.1367053985595703,
      "rewards/margins": 63.8824577331543,
      "rewards/rejected": -61.745750427246094,
      "step": 4040
    },
    {
      "epoch": 0.5962019726188724,
      "grad_norm": 146.93209198885847,
      "learning_rate": 2.0989555393764354e-07,
      "logits/chosen": -2.4266486167907715,
      "logits/rejected": -2.1463608741760254,
      "logps/chosen": -390.39581298828125,
      "logps/rejected": -354.31256103515625,
      "loss": -0.6742,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": 1.0637459754943848,
      "rewards/margins": 34.88615798950195,
      "rewards/rejected": -33.822418212890625,
      "step": 4050
    },
    {
      "epoch": 0.5976740762549684,
      "grad_norm": 558.9961030615849,
      "learning_rate": 2.0862792842968214e-07,
      "logits/chosen": -2.65617036819458,
      "logits/rejected": -2.284430980682373,
      "logps/chosen": -305.40447998046875,
      "logps/rejected": -250.5668487548828,
      "loss": -0.8593,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 12.305736541748047,
      "rewards/margins": 40.835079193115234,
      "rewards/rejected": -28.529342651367188,
      "step": 4060
    },
    {
      "epoch": 0.5991461798910643,
      "grad_norm": 93.23591952459391,
      "learning_rate": 2.0736139561246713e-07,
      "logits/chosen": -2.4903743267059326,
      "logits/rejected": -2.1680848598480225,
      "logps/chosen": -256.2744140625,
      "logps/rejected": -321.8411865234375,
      "loss": -0.6988,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5421493649482727,
      "rewards/margins": 48.9697380065918,
      "rewards/rejected": -49.51188659667969,
      "step": 4070
    },
    {
      "epoch": 0.6006182835271603,
      "grad_norm": 131.09182317271433,
      "learning_rate": 2.060959889367938e-07,
      "logits/chosen": -2.6299538612365723,
      "logits/rejected": -2.0922653675079346,
      "logps/chosen": -280.4296875,
      "logps/rejected": -352.35528564453125,
      "loss": -0.5669,
      "rewards/accuracies": 0.9333333969116211,
      "rewards/chosen": 15.282691955566406,
      "rewards/margins": 86.17539978027344,
      "rewards/rejected": -70.89271545410156,
      "step": 4080
    },
    {
      "epoch": 0.6020903871632562,
      "grad_norm": 94.72142720332153,
      "learning_rate": 2.0483174182371435e-07,
      "logits/chosen": -2.3635802268981934,
      "logits/rejected": -2.203770160675049,
      "logps/chosen": -281.474365234375,
      "logps/rejected": -302.9625549316406,
      "loss": -0.6876,
      "rewards/accuracies": 0.5333333015441895,
      "rewards/chosen": 5.9915995597839355,
      "rewards/margins": 22.965068817138672,
      "rewards/rejected": -16.97347068786621,
      "step": 4090
    },
    {
      "epoch": 0.6035624907993523,
      "grad_norm": 101.74813197509012,
      "learning_rate": 2.0356868766365536e-07,
      "logits/chosen": -2.706233501434326,
      "logits/rejected": -2.3220996856689453,
      "logps/chosen": -293.4935607910156,
      "logps/rejected": -371.00946044921875,
      "loss": 0.0159,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": 2.9281668663024902,
      "rewards/margins": 42.636146545410156,
      "rewards/rejected": -39.707984924316406,
      "step": 4100
    },
    {
      "epoch": 0.6050345944354483,
      "grad_norm": 114.40796060104553,
      "learning_rate": 2.023068598155363e-07,
      "logits/chosen": -2.4973232746124268,
      "logits/rejected": -2.2430973052978516,
      "logps/chosen": -323.6395263671875,
      "logps/rejected": -279.77032470703125,
      "loss": -0.6546,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -14.270988464355469,
      "rewards/margins": 32.70659637451172,
      "rewards/rejected": -46.97758483886719,
      "step": 4110
    },
    {
      "epoch": 0.6065066980715442,
      "grad_norm": 90.36350111349283,
      "learning_rate": 2.010462916058875e-07,
      "logits/chosen": -2.5094168186187744,
      "logits/rejected": -2.1453211307525635,
      "logps/chosen": -339.91802978515625,
      "logps/rejected": -381.2345275878906,
      "loss": -0.7202,
      "rewards/accuracies": 0.7000000476837158,
      "rewards/chosen": -2.581681966781616,
      "rewards/margins": 52.6976432800293,
      "rewards/rejected": -55.279327392578125,
      "step": 4120
    },
    {
      "epoch": 0.6079788017076402,
      "grad_norm": 74.5330333846038,
      "learning_rate": 1.9978701632797118e-07,
      "logits/chosen": -2.4298129081726074,
      "logits/rejected": -2.2981948852539062,
      "logps/chosen": -239.7661895751953,
      "logps/rejected": -283.0516357421875,
      "loss": -0.6824,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -6.571986198425293,
      "rewards/margins": 27.097787857055664,
      "rewards/rejected": -33.669776916503906,
      "step": 4130
    },
    {
      "epoch": 0.6094509053437362,
      "grad_norm": 48.58751342650849,
      "learning_rate": 1.9852906724090127e-07,
      "logits/chosen": -2.5505728721618652,
      "logits/rejected": -2.188162326812744,
      "logps/chosen": -221.0631561279297,
      "logps/rejected": -297.77740478515625,
      "loss": -0.7129,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": 1.4795809984207153,
      "rewards/margins": 56.841026306152344,
      "rewards/rejected": -55.3614387512207,
      "step": 4140
    },
    {
      "epoch": 0.6109230089798322,
      "grad_norm": 69.27079663299222,
      "learning_rate": 1.9727247756876534e-07,
      "logits/chosen": -2.717017650604248,
      "logits/rejected": -2.660250425338745,
      "logps/chosen": -229.2417755126953,
      "logps/rejected": -256.8539733886719,
      "loss": -0.638,
      "rewards/accuracies": 0.5333333015441895,
      "rewards/chosen": 7.670281410217285,
      "rewards/margins": 4.665963649749756,
      "rewards/rejected": 3.004316806793213,
      "step": 4150
    },
    {
      "epoch": 0.6123951126159282,
      "grad_norm": 58.045652535494014,
      "learning_rate": 1.9601728049974683e-07,
      "logits/chosen": -2.524672031402588,
      "logits/rejected": -2.259888172149658,
      "logps/chosen": -215.18930053710938,
      "logps/rejected": -388.11602783203125,
      "loss": -0.7192,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": 6.279829978942871,
      "rewards/margins": 82.31028747558594,
      "rewards/rejected": -76.03046417236328,
      "step": 4160
    },
    {
      "epoch": 0.6138672162520241,
      "grad_norm": 72.15670726618168,
      "learning_rate": 1.94763509185249e-07,
      "logits/chosen": -2.601207733154297,
      "logits/rejected": -2.3223538398742676,
      "logps/chosen": -261.0362854003906,
      "logps/rejected": -309.42242431640625,
      "loss": -0.7857,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -0.3807029724121094,
      "rewards/margins": 40.5506706237793,
      "rewards/rejected": -40.931373596191406,
      "step": 4170
    },
    {
      "epoch": 0.6153393198881201,
      "grad_norm": 166.7617704271573,
      "learning_rate": 1.935111967390189e-07,
      "logits/chosen": -2.7128820419311523,
      "logits/rejected": -2.223550796508789,
      "logps/chosen": -245.0447235107422,
      "logps/rejected": -246.5872344970703,
      "loss": -0.7389,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 8.429567337036133,
      "rewards/margins": 62.115028381347656,
      "rewards/rejected": -53.68546676635742,
      "step": 4180
    },
    {
      "epoch": 0.6168114235242161,
      "grad_norm": 84.52351367265412,
      "learning_rate": 1.922603762362729e-07,
      "logits/chosen": -2.497328758239746,
      "logits/rejected": -2.210002899169922,
      "logps/chosen": -232.84054565429688,
      "logps/rejected": -274.37200927734375,
      "loss": -0.6948,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -2.9002914428710938,
      "rewards/margins": 36.874114990234375,
      "rewards/rejected": -39.77440643310547,
      "step": 4190
    },
    {
      "epoch": 0.6182835271603121,
      "grad_norm": 345.944872246729,
      "learning_rate": 1.9101108071282342e-07,
      "logits/chosen": -2.5073533058166504,
      "logits/rejected": -2.17555832862854,
      "logps/chosen": -284.5340881347656,
      "logps/rejected": -296.3728942871094,
      "loss": -0.7263,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": -5.2348480224609375,
      "rewards/margins": 59.985191345214844,
      "rewards/rejected": -65.22003936767578,
      "step": 4200
    },
    {
      "epoch": 0.619755630796408,
      "grad_norm": 80.66338428129431,
      "learning_rate": 1.8976334316420576e-07,
      "logits/chosen": -2.6845405101776123,
      "logits/rejected": -2.2517361640930176,
      "logps/chosen": -345.73681640625,
      "logps/rejected": -340.09234619140625,
      "loss": -0.8281,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -1.4843990802764893,
      "rewards/margins": 57.43983840942383,
      "rewards/rejected": -58.924232482910156,
      "step": 4210
    },
    {
      "epoch": 0.621227734432504,
      "grad_norm": 284.673880494317,
      "learning_rate": 1.8851719654480748e-07,
      "logits/chosen": -2.6678075790405273,
      "logits/rejected": -2.258589744567871,
      "logps/chosen": -263.13531494140625,
      "logps/rejected": -300.5755920410156,
      "loss": -0.7134,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": 0.5573064684867859,
      "rewards/margins": 48.216400146484375,
      "rewards/rejected": -47.659095764160156,
      "step": 4220
    },
    {
      "epoch": 0.6226998380686001,
      "grad_norm": 69.05459317132183,
      "learning_rate": 1.8727267376699735e-07,
      "logits/chosen": -2.4860708713531494,
      "logits/rejected": -2.100111722946167,
      "logps/chosen": -327.36248779296875,
      "logps/rejected": -312.01458740234375,
      "loss": -0.7137,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 1.1522334814071655,
      "rewards/margins": 60.47835159301758,
      "rewards/rejected": -59.32611083984375,
      "step": 4230
    },
    {
      "epoch": 0.624171941704696,
      "grad_norm": 97.18543710059535,
      "learning_rate": 1.8602980770025645e-07,
      "logits/chosen": -2.477691411972046,
      "logits/rejected": -2.105161666870117,
      "logps/chosen": -287.7856750488281,
      "logps/rejected": -266.0497131347656,
      "loss": -0.8161,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": 20.386432647705078,
      "rewards/margins": 63.044647216796875,
      "rewards/rejected": -42.6582145690918,
      "step": 4240
    },
    {
      "epoch": 0.625644045340792,
      "grad_norm": 110.62460695534239,
      "learning_rate": 1.8478863117031007e-07,
      "logits/chosen": -2.246354579925537,
      "logits/rejected": -1.4650384187698364,
      "logps/chosen": -252.0066680908203,
      "logps/rejected": -329.66510009765625,
      "loss": -0.697,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -19.22934341430664,
      "rewards/margins": 60.640769958496094,
      "rewards/rejected": -79.8701171875,
      "step": 4250
    },
    {
      "epoch": 0.6271161489768879,
      "grad_norm": 118.78695056720997,
      "learning_rate": 1.8354917695826026e-07,
      "logits/chosen": -2.343855619430542,
      "logits/rejected": -2.029350996017456,
      "logps/chosen": -357.9104919433594,
      "logps/rejected": -378.22943115234375,
      "loss": -0.7369,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -13.506457328796387,
      "rewards/margins": 52.53751754760742,
      "rewards/rejected": -66.04398345947266,
      "step": 4260
    },
    {
      "epoch": 0.6285882526129839,
      "grad_norm": 243.68999411327633,
      "learning_rate": 1.8231147779972074e-07,
      "logits/chosen": -2.276017665863037,
      "logits/rejected": -1.8933336734771729,
      "logps/chosen": -225.41934204101562,
      "logps/rejected": -325.94097900390625,
      "loss": -0.6722,
      "rewards/accuracies": 0.6999999284744263,
      "rewards/chosen": -13.621973991394043,
      "rewards/margins": 72.26348876953125,
      "rewards/rejected": -85.88548278808594,
      "step": 4270
    },
    {
      "epoch": 0.63006035624908,
      "grad_norm": 80.02788631118838,
      "learning_rate": 1.8107556638395168e-07,
      "logits/chosen": -2.3434817790985107,
      "logits/rejected": -1.7130438089370728,
      "logps/chosen": -295.4878845214844,
      "logps/rejected": -356.1160888671875,
      "loss": -0.7701,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -4.1242995262146,
      "rewards/margins": 71.02518463134766,
      "rewards/rejected": -75.14948272705078,
      "step": 4280
    },
    {
      "epoch": 0.6315324598851759,
      "grad_norm": 142.9418826014825,
      "learning_rate": 1.798414753529971e-07,
      "logits/chosen": -2.3559134006500244,
      "logits/rejected": -2.12322735786438,
      "logps/chosen": -305.84527587890625,
      "logps/rejected": -320.21539306640625,
      "loss": -0.8053,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": 1.519500494003296,
      "rewards/margins": 55.85350799560547,
      "rewards/rejected": -54.33400344848633,
      "step": 4290
    },
    {
      "epoch": 0.6330045635212719,
      "grad_norm": 49.38883328369479,
      "learning_rate": 1.7860923730082152e-07,
      "logits/chosen": -2.5418763160705566,
      "logits/rejected": -2.307471752166748,
      "logps/chosen": -346.92352294921875,
      "logps/rejected": -291.11602783203125,
      "loss": -0.5812,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": 0.2757602632045746,
      "rewards/margins": 30.343013763427734,
      "rewards/rejected": -30.0672550201416,
      "step": 4300
    },
    {
      "epoch": 0.6344766671573678,
      "grad_norm": 129.40115787498328,
      "learning_rate": 1.7737888477245052e-07,
      "logits/chosen": -2.3783793449401855,
      "logits/rejected": -1.9540958404541016,
      "logps/chosen": -307.227294921875,
      "logps/rejected": -352.06024169921875,
      "loss": -1.1774,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -5.902768611907959,
      "rewards/margins": 52.674293518066406,
      "rewards/rejected": -58.577056884765625,
      "step": 4310
    },
    {
      "epoch": 0.6359487707934639,
      "grad_norm": 329.1970183835269,
      "learning_rate": 1.761504502631102e-07,
      "logits/chosen": -2.330719470977783,
      "logits/rejected": -1.840345025062561,
      "logps/chosen": -287.28057861328125,
      "logps/rejected": -371.9903869628906,
      "loss": -0.6882,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": 3.0561225414276123,
      "rewards/margins": 85.01544189453125,
      "rewards/rejected": -81.95930480957031,
      "step": 4320
    },
    {
      "epoch": 0.6374208744295599,
      "grad_norm": 128.6647363497441,
      "learning_rate": 1.749239662173693e-07,
      "logits/chosen": -2.161825180053711,
      "logits/rejected": -1.8902103900909424,
      "logps/chosen": -219.62149047851562,
      "logps/rejected": -273.25958251953125,
      "loss": -0.7742,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -18.40206527709961,
      "rewards/margins": 50.282493591308594,
      "rewards/rejected": -68.68455505371094,
      "step": 4330
    },
    {
      "epoch": 0.6388929780656558,
      "grad_norm": 186.69635807161538,
      "learning_rate": 1.7369946502828245e-07,
      "logits/chosen": -2.569502830505371,
      "logits/rejected": -1.9900968074798584,
      "logps/chosen": -262.0874938964844,
      "logps/rejected": -257.25201416015625,
      "loss": -0.8058,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": 16.41998863220215,
      "rewards/margins": 66.08404541015625,
      "rewards/rejected": -49.6640625,
      "step": 4340
    },
    {
      "epoch": 0.6403650817017518,
      "grad_norm": 81.32648233291513,
      "learning_rate": 1.7247697903653395e-07,
      "logits/chosen": -2.4426143169403076,
      "logits/rejected": -1.8840391635894775,
      "logps/chosen": -320.10516357421875,
      "logps/rejected": -369.4241027832031,
      "loss": -0.7568,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": 1.315777063369751,
      "rewards/margins": 81.34208679199219,
      "rewards/rejected": -80.02630615234375,
      "step": 4350
    },
    {
      "epoch": 0.6418371853378478,
      "grad_norm": 82.401423413955,
      "learning_rate": 1.7125654052958465e-07,
      "logits/chosen": -2.357712984085083,
      "logits/rejected": -1.6829559803009033,
      "logps/chosen": -257.79498291015625,
      "logps/rejected": -294.05792236328125,
      "loss": -0.7084,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -1.295220136642456,
      "rewards/margins": 60.586936950683594,
      "rewards/rejected": -61.88215255737305,
      "step": 4360
    },
    {
      "epoch": 0.6433092889739438,
      "grad_norm": 171.80977502152558,
      "learning_rate": 1.7003818174081832e-07,
      "logits/chosen": -2.5210585594177246,
      "logits/rejected": -1.595571756362915,
      "logps/chosen": -258.17864990234375,
      "logps/rejected": -292.80889892578125,
      "loss": -0.8297,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": -6.9971022605896,
      "rewards/margins": 85.52076721191406,
      "rewards/rejected": -92.51786804199219,
      "step": 4370
    },
    {
      "epoch": 0.6447813926100397,
      "grad_norm": 122.2164436669237,
      "learning_rate": 1.6882193484869067e-07,
      "logits/chosen": -2.419943332672119,
      "logits/rejected": -1.9102904796600342,
      "logps/chosen": -259.5379333496094,
      "logps/rejected": -282.40338134765625,
      "loss": -0.664,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -4.47910213470459,
      "rewards/margins": 36.37079620361328,
      "rewards/rejected": -40.84989547729492,
      "step": 4380
    },
    {
      "epoch": 0.6462534962461357,
      "grad_norm": 131.48845289402988,
      "learning_rate": 1.676078319758796e-07,
      "logits/chosen": -2.4326586723327637,
      "logits/rejected": -1.8618568181991577,
      "logps/chosen": -265.17388916015625,
      "logps/rejected": -261.5268859863281,
      "loss": -0.6909,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": 3.7738537788391113,
      "rewards/margins": 56.420867919921875,
      "rewards/rejected": -52.64701461791992,
      "step": 4390
    },
    {
      "epoch": 0.6477255998822317,
      "grad_norm": 101.44414372861019,
      "learning_rate": 1.6639590518843643e-07,
      "logits/chosen": -2.4426586627960205,
      "logits/rejected": -1.8241908550262451,
      "logps/chosen": -319.66778564453125,
      "logps/rejected": -428.209716796875,
      "loss": -0.7246,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": 10.618200302124023,
      "rewards/margins": 89.67259979248047,
      "rewards/rejected": -79.05439758300781,
      "step": 4400
    },
    {
      "epoch": 0.6491977035183277,
      "grad_norm": 282.1821945224619,
      "learning_rate": 1.6518618649493932e-07,
      "logits/chosen": -2.2738630771636963,
      "logits/rejected": -1.7976261377334595,
      "logps/chosen": -270.2701416015625,
      "logps/rejected": -342.27886962890625,
      "loss": -0.7264,
      "rewards/accuracies": 0.8333333730697632,
      "rewards/chosen": -5.16711950302124,
      "rewards/margins": 61.493003845214844,
      "rewards/rejected": -66.66012573242188,
      "step": 4410
    },
    {
      "epoch": 0.6506698071544237,
      "grad_norm": 51.00882530695201,
      "learning_rate": 1.6397870784564777e-07,
      "logits/chosen": -2.1950833797454834,
      "logits/rejected": -1.8651145696640015,
      "logps/chosen": -225.31808471679688,
      "logps/rejected": -369.024169921875,
      "loss": -0.7357,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -5.118387222290039,
      "rewards/margins": 81.08039855957031,
      "rewards/rejected": -86.19878387451172,
      "step": 4420
    },
    {
      "epoch": 0.6521419107905196,
      "grad_norm": 108.54267359717052,
      "learning_rate": 1.6277350113165887e-07,
      "logits/chosen": -2.3435375690460205,
      "logits/rejected": -1.6967179775238037,
      "logps/chosen": -285.5338439941406,
      "logps/rejected": -282.6825256347656,
      "loss": -0.6829,
      "rewards/accuracies": 0.6999999284744263,
      "rewards/chosen": -14.304408073425293,
      "rewards/margins": 52.61775588989258,
      "rewards/rejected": -66.92216491699219,
      "step": 4430
    },
    {
      "epoch": 0.6536140144266156,
      "grad_norm": 66.13628792933109,
      "learning_rate": 1.615705981840646e-07,
      "logits/chosen": -2.449232339859009,
      "logits/rejected": -1.7199681997299194,
      "logps/chosen": -330.1246643066406,
      "logps/rejected": -370.2337646484375,
      "loss": 0.0198,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.9943825006484985,
      "rewards/margins": 66.8294448852539,
      "rewards/rejected": -68.8238296508789,
      "step": 4440
    },
    {
      "epoch": 0.6550861180627117,
      "grad_norm": 77.51366723223568,
      "learning_rate": 1.6037003077311178e-07,
      "logits/chosen": -2.4296023845672607,
      "logits/rejected": -2.2446255683898926,
      "logps/chosen": -251.9778594970703,
      "logps/rejected": -336.67169189453125,
      "loss": -0.6271,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 0.6302841901779175,
      "rewards/margins": 61.348388671875,
      "rewards/rejected": -60.718101501464844,
      "step": 4450
    },
    {
      "epoch": 0.6565582216988076,
      "grad_norm": 116.76053064723517,
      "learning_rate": 1.591718306073625e-07,
      "logits/chosen": -2.4617578983306885,
      "logits/rejected": -1.8178918361663818,
      "logps/chosen": -456.7242126464844,
      "logps/rejected": -367.6231689453125,
      "loss": -0.716,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -11.52933406829834,
      "rewards/margins": 43.88861083984375,
      "rewards/rejected": -55.41794967651367,
      "step": 4460
    },
    {
      "epoch": 0.6580303253349036,
      "grad_norm": 217.46745990764964,
      "learning_rate": 1.5797602933285672e-07,
      "logits/chosen": -2.383301258087158,
      "logits/rejected": -1.5741980075836182,
      "logps/chosen": -286.13037109375,
      "logps/rejected": -322.68182373046875,
      "loss": -0.7603,
      "rewards/accuracies": 0.7333332300186157,
      "rewards/chosen": 8.460561752319336,
      "rewards/margins": 69.29878234863281,
      "rewards/rejected": -60.838218688964844,
      "step": 4470
    },
    {
      "epoch": 0.6595024289709995,
      "grad_norm": 90.12959327738713,
      "learning_rate": 1.5678265853227679e-07,
      "logits/chosen": -2.3807246685028076,
      "logits/rejected": -2.005905866622925,
      "logps/chosen": -296.81536865234375,
      "logps/rejected": -413.56298828125,
      "loss": -0.7826,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": 6.4788665771484375,
      "rewards/margins": 83.92039489746094,
      "rewards/rejected": -77.4415283203125,
      "step": 4480
    },
    {
      "epoch": 0.6609745326070955,
      "grad_norm": 157.26136394354373,
      "learning_rate": 1.5559174972411271e-07,
      "logits/chosen": -2.6120848655700684,
      "logits/rejected": -2.030663013458252,
      "logps/chosen": -339.408935546875,
      "logps/rejected": -379.10430908203125,
      "loss": -0.7665,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": 1.4827969074249268,
      "rewards/margins": 78.4524154663086,
      "rewards/rejected": -76.9696273803711,
      "step": 4490
    },
    {
      "epoch": 0.6624466362431916,
      "grad_norm": 101.39692012879294,
      "learning_rate": 1.5440333436183022e-07,
      "logits/chosen": -2.40238094329834,
      "logits/rejected": -1.7370355129241943,
      "logps/chosen": -224.8983154296875,
      "logps/rejected": -291.4688415527344,
      "loss": -0.746,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": 4.850283622741699,
      "rewards/margins": 88.3060531616211,
      "rewards/rejected": -83.45576477050781,
      "step": 4500
    },
    {
      "epoch": 0.6639187398792875,
      "grad_norm": 80.10983403976344,
      "learning_rate": 1.532174438330399e-07,
      "logits/chosen": -2.455435276031494,
      "logits/rejected": -2.0948588848114014,
      "logps/chosen": -326.8215637207031,
      "logps/rejected": -283.53509521484375,
      "loss": -0.7567,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": 10.640438079833984,
      "rewards/margins": 48.885581970214844,
      "rewards/rejected": -38.245140075683594,
      "step": 4510
    },
    {
      "epoch": 0.6653908435153835,
      "grad_norm": 74.2540926648995,
      "learning_rate": 1.5203410945866807e-07,
      "logits/chosen": -2.535553216934204,
      "logits/rejected": -1.9870595932006836,
      "logps/chosen": -248.607666015625,
      "logps/rejected": -329.76922607421875,
      "loss": -0.8091,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 9.019762992858887,
      "rewards/margins": 62.166831970214844,
      "rewards/rejected": -53.147071838378906,
      "step": 4520
    },
    {
      "epoch": 0.6668629471514794,
      "grad_norm": 77.41817168755189,
      "learning_rate": 1.5085336249212982e-07,
      "logits/chosen": -2.1303772926330566,
      "logits/rejected": -1.5159367322921753,
      "logps/chosen": -271.12225341796875,
      "logps/rejected": -309.1958312988281,
      "loss": -0.7818,
      "rewards/accuracies": 0.8333333730697632,
      "rewards/chosen": 5.459566116333008,
      "rewards/margins": 76.95426177978516,
      "rewards/rejected": -71.49469757080078,
      "step": 4530
    },
    {
      "epoch": 0.6683350507875755,
      "grad_norm": 169.15523507680783,
      "learning_rate": 1.4967523411850314e-07,
      "logits/chosen": -2.354499101638794,
      "logits/rejected": -1.983231782913208,
      "logps/chosen": -240.00454711914062,
      "logps/rejected": -315.48101806640625,
      "loss": -0.7714,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -6.63753604888916,
      "rewards/margins": 47.39200973510742,
      "rewards/rejected": -54.02954864501953,
      "step": 4540
    },
    {
      "epoch": 0.6698071544236714,
      "grad_norm": 98.22185093832319,
      "learning_rate": 1.484997554537057e-07,
      "logits/chosen": -2.5225775241851807,
      "logits/rejected": -2.1235642433166504,
      "logps/chosen": -283.7577209472656,
      "logps/rejected": -365.1656799316406,
      "loss": -0.7282,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": 5.006906986236572,
      "rewards/margins": 51.4794807434082,
      "rewards/rejected": -46.472572326660156,
      "step": 4550
    },
    {
      "epoch": 0.6712792580597674,
      "grad_norm": 111.18041559527387,
      "learning_rate": 1.4732695754367287e-07,
      "logits/chosen": -2.3815202713012695,
      "logits/rejected": -1.834180474281311,
      "logps/chosen": -293.9458923339844,
      "logps/rejected": -351.29901123046875,
      "loss": -0.723,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 5.619406223297119,
      "rewards/margins": 66.82183074951172,
      "rewards/rejected": -61.202430725097656,
      "step": 4560
    },
    {
      "epoch": 0.6727513616958634,
      "grad_norm": 119.16954884116619,
      "learning_rate": 1.4615687136353787e-07,
      "logits/chosen": -2.400749683380127,
      "logits/rejected": -1.7755588293075562,
      "logps/chosen": -294.52099609375,
      "logps/rejected": -291.78619384765625,
      "loss": -0.7127,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 7.406964302062988,
      "rewards/margins": 66.5722885131836,
      "rewards/rejected": -59.16532516479492,
      "step": 4570
    },
    {
      "epoch": 0.6742234653319593,
      "grad_norm": 159.3606803806806,
      "learning_rate": 1.4498952781681328e-07,
      "logits/chosen": -2.4394922256469727,
      "logits/rejected": -1.992449402809143,
      "logps/chosen": -229.252197265625,
      "logps/rejected": -259.01324462890625,
      "loss": -0.7708,
      "rewards/accuracies": 0.76666659116745,
      "rewards/chosen": 9.654232025146484,
      "rewards/margins": 53.11510467529297,
      "rewards/rejected": -43.46087646484375,
      "step": 4580
    },
    {
      "epoch": 0.6756955689680554,
      "grad_norm": 89.37685763873026,
      "learning_rate": 1.4382495773457544e-07,
      "logits/chosen": -2.4052324295043945,
      "logits/rejected": -2.135270833969116,
      "logps/chosen": -269.48699951171875,
      "logps/rejected": -258.02032470703125,
      "loss": -0.6409,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 2.905945301055908,
      "rewards/margins": 40.62530517578125,
      "rewards/rejected": -37.719364166259766,
      "step": 4590
    },
    {
      "epoch": 0.6771676726041513,
      "grad_norm": 66.36617447040922,
      "learning_rate": 1.4266319187464965e-07,
      "logits/chosen": -2.373192071914673,
      "logits/rejected": -2.0762810707092285,
      "logps/chosen": -206.6453094482422,
      "logps/rejected": -251.49130249023438,
      "loss": -0.8121,
      "rewards/accuracies": 0.7000000476837158,
      "rewards/chosen": 10.212796211242676,
      "rewards/margins": 48.98048782348633,
      "rewards/rejected": -38.76769256591797,
      "step": 4600
    },
    {
      "epoch": 0.6786397762402473,
      "grad_norm": 83.58948615857852,
      "learning_rate": 1.415042609207981e-07,
      "logits/chosen": -2.3542544841766357,
      "logits/rejected": -2.025891065597534,
      "logps/chosen": -224.2681884765625,
      "logps/rejected": -233.4315185546875,
      "loss": -0.774,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": 4.350973606109619,
      "rewards/margins": 44.082298278808594,
      "rewards/rejected": -39.7313232421875,
      "step": 4610
    },
    {
      "epoch": 0.6801118798763433,
      "grad_norm": 86.64879232851234,
      "learning_rate": 1.4034819548190936e-07,
      "logits/chosen": -2.338951826095581,
      "logits/rejected": -1.6386487483978271,
      "logps/chosen": -253.71475219726562,
      "logps/rejected": -292.8167724609375,
      "loss": -0.7735,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -3.4050240516662598,
      "rewards/margins": 72.18621826171875,
      "rewards/rejected": -75.59124755859375,
      "step": 4620
    },
    {
      "epoch": 0.6815839835124393,
      "grad_norm": 116.0593636926135,
      "learning_rate": 1.3919502609119004e-07,
      "logits/chosen": -2.2734997272491455,
      "logits/rejected": -1.980929970741272,
      "logps/chosen": -308.57513427734375,
      "logps/rejected": -331.3267822265625,
      "loss": -0.7286,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -5.829428672790527,
      "rewards/margins": 32.661903381347656,
      "rewards/rejected": -38.491329193115234,
      "step": 4630
    },
    {
      "epoch": 0.6830560871485353,
      "grad_norm": 137.45222670570644,
      "learning_rate": 1.380447832053583e-07,
      "logits/chosen": -2.452688694000244,
      "logits/rejected": -1.7120354175567627,
      "logps/chosen": -239.8981475830078,
      "logps/rejected": -314.863525390625,
      "loss": -0.7367,
      "rewards/accuracies": 0.7999999523162842,
      "rewards/chosen": 6.026345729827881,
      "rewards/margins": 75.89201354980469,
      "rewards/rejected": -69.86567687988281,
      "step": 4640
    },
    {
      "epoch": 0.6845281907846312,
      "grad_norm": 104.6462316819005,
      "learning_rate": 1.3689749720383934e-07,
      "logits/chosen": -2.3534820079803467,
      "logits/rejected": -2.056891918182373,
      "logps/chosen": -263.1820983886719,
      "logps/rejected": -265.2998352050781,
      "loss": -0.7658,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": 10.4490385055542,
      "rewards/margins": 30.453105926513672,
      "rewards/rejected": -20.004070281982422,
      "step": 4650
    },
    {
      "epoch": 0.6860002944207272,
      "grad_norm": 94.51173371989093,
      "learning_rate": 1.357531983879633e-07,
      "logits/chosen": -2.21866512298584,
      "logits/rejected": -1.84636652469635,
      "logps/chosen": -224.2871551513672,
      "logps/rejected": -253.46240234375,
      "loss": -0.7236,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -12.539409637451172,
      "rewards/margins": 48.87172317504883,
      "rewards/rejected": -61.4111328125,
      "step": 4660
    },
    {
      "epoch": 0.6874723980568233,
      "grad_norm": 188.62605774777796,
      "learning_rate": 1.3461191698016482e-07,
      "logits/chosen": -2.0118026733398438,
      "logits/rejected": -1.7495771646499634,
      "logps/chosen": -284.718505859375,
      "logps/rejected": -307.4160461425781,
      "loss": -0.7811,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -7.216207027435303,
      "rewards/margins": 61.4105224609375,
      "rewards/rejected": -68.62672424316406,
      "step": 4670
    },
    {
      "epoch": 0.6889445016929192,
      "grad_norm": 127.43085800242524,
      "learning_rate": 1.3347368312318475e-07,
      "logits/chosen": -2.3878819942474365,
      "logits/rejected": -2.1704912185668945,
      "logps/chosen": -327.8782043457031,
      "logps/rejected": -338.9222106933594,
      "loss": 1.6309,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": 6.213817596435547,
      "rewards/margins": 32.28771209716797,
      "rewards/rejected": -26.073898315429688,
      "step": 4680
    },
    {
      "epoch": 0.6904166053290152,
      "grad_norm": 92.0241582063313,
      "learning_rate": 1.3233852687927415e-07,
      "logits/chosen": -2.427248477935791,
      "logits/rejected": -1.9343907833099365,
      "logps/chosen": -293.6287841796875,
      "logps/rejected": -364.5401306152344,
      "loss": -0.6842,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -2.395965576171875,
      "rewards/margins": 55.271697998046875,
      "rewards/rejected": -57.66766357421875,
      "step": 4690
    },
    {
      "epoch": 0.6918887089651111,
      "grad_norm": 135.66734577512358,
      "learning_rate": 1.3120647822940035e-07,
      "logits/chosen": -2.4400973320007324,
      "logits/rejected": -1.8270124197006226,
      "logps/chosen": -257.6181335449219,
      "logps/rejected": -291.64154052734375,
      "loss": -0.7964,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": 11.05014419555664,
      "rewards/margins": 68.42149353027344,
      "rewards/rejected": -57.3713493347168,
      "step": 4700
    },
    {
      "epoch": 0.6933608126012071,
      "grad_norm": 71.05341386856844,
      "learning_rate": 1.3007756707245488e-07,
      "logits/chosen": -2.313220977783203,
      "logits/rejected": -2.0815072059631348,
      "logps/chosen": -240.5254669189453,
      "logps/rejected": -266.96710205078125,
      "loss": -0.7057,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -0.41453665494918823,
      "rewards/margins": 33.547515869140625,
      "rewards/rejected": -33.96205139160156,
      "step": 4710
    },
    {
      "epoch": 0.6948329162373031,
      "grad_norm": 104.70853122567155,
      "learning_rate": 1.2895182322446415e-07,
      "logits/chosen": -2.2702622413635254,
      "logits/rejected": -1.6405136585235596,
      "logps/chosen": -288.81585693359375,
      "logps/rejected": -286.9456481933594,
      "loss": -0.7753,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.9577770233154297,
      "rewards/margins": 63.376251220703125,
      "rewards/rejected": -62.41847610473633,
      "step": 4720
    },
    {
      "epoch": 0.6963050198733991,
      "grad_norm": 137.24994793173164,
      "learning_rate": 1.2782927641780167e-07,
      "logits/chosen": -2.4866690635681152,
      "logits/rejected": -2.1798348426818848,
      "logps/chosen": -314.9397277832031,
      "logps/rejected": -356.5355529785156,
      "loss": -0.7023,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": 5.7956013679504395,
      "rewards/margins": 41.23109436035156,
      "rewards/rejected": -35.43549346923828,
      "step": 4730
    },
    {
      "epoch": 0.6977771235094951,
      "grad_norm": 104.80965911017248,
      "learning_rate": 1.2670995630040288e-07,
      "logits/chosen": -2.265240430831909,
      "logits/rejected": -1.8815486431121826,
      "logps/chosen": -247.31826782226562,
      "logps/rejected": -275.379638671875,
      "loss": -0.6295,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -8.050859451293945,
      "rewards/margins": 49.565269470214844,
      "rewards/rejected": -57.61613082885742,
      "step": 4740
    },
    {
      "epoch": 0.699249227145591,
      "grad_norm": 146.8999281047024,
      "learning_rate": 1.2559389243498213e-07,
      "logits/chosen": -2.1898202896118164,
      "logits/rejected": -1.8124983310699463,
      "logps/chosen": -308.8436584472656,
      "logps/rejected": -358.0823059082031,
      "loss": -0.5866,
      "rewards/accuracies": 0.9000000953674316,
      "rewards/chosen": 23.008464813232422,
      "rewards/margins": 84.87181854248047,
      "rewards/rejected": -61.86335372924805,
      "step": 4750
    },
    {
      "epoch": 0.7007213307816871,
      "grad_norm": 83.85629996316227,
      "learning_rate": 1.2448111429825198e-07,
      "logits/chosen": -2.1404201984405518,
      "logits/rejected": -1.7062619924545288,
      "logps/chosen": -313.7795104980469,
      "logps/rejected": -378.5100402832031,
      "loss": -0.6691,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.8579015731811523,
      "rewards/margins": 91.99330139160156,
      "rewards/rejected": -94.85121154785156,
      "step": 4760
    },
    {
      "epoch": 0.702193434417783,
      "grad_norm": 232.65016976080182,
      "learning_rate": 1.2337165128014443e-07,
      "logits/chosen": -2.4453041553497314,
      "logits/rejected": -1.9615719318389893,
      "logps/chosen": -284.23297119140625,
      "logps/rejected": -310.29608154296875,
      "loss": -0.6929,
      "rewards/accuracies": 0.5666666030883789,
      "rewards/chosen": -5.019478797912598,
      "rewards/margins": 35.153282165527344,
      "rewards/rejected": -40.172760009765625,
      "step": 4770
    },
    {
      "epoch": 0.703665538053879,
      "grad_norm": 161.51020502755307,
      "learning_rate": 1.2226553268303494e-07,
      "logits/chosen": -2.1652488708496094,
      "logits/rejected": -1.9029653072357178,
      "logps/chosen": -221.1796112060547,
      "logps/rejected": -270.84918212890625,
      "loss": 106.1658,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -4.396166801452637,
      "rewards/margins": 22.249530792236328,
      "rewards/rejected": -26.645700454711914,
      "step": 4780
    },
    {
      "epoch": 0.705137641689975,
      "grad_norm": 88.00385629707807,
      "learning_rate": 1.2116278772096835e-07,
      "logits/chosen": -2.26037335395813,
      "logits/rejected": -1.6925718784332275,
      "logps/chosen": -324.34246826171875,
      "logps/rejected": -324.9882507324219,
      "loss": -0.7247,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": 5.619498252868652,
      "rewards/margins": 68.7273941040039,
      "rewards/rejected": -63.1078987121582,
      "step": 4790
    },
    {
      "epoch": 0.7066097453260709,
      "grad_norm": 76.22820327983533,
      "learning_rate": 1.2006344551888736e-07,
      "logits/chosen": -2.3812601566314697,
      "logits/rejected": -2.0506374835968018,
      "logps/chosen": -185.99099731445312,
      "logps/rejected": -241.4409637451172,
      "loss": -0.7857,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 2.454683542251587,
      "rewards/margins": 45.96783447265625,
      "rewards/rejected": -43.51314926147461,
      "step": 4800
    },
    {
      "epoch": 0.708081848962167,
      "grad_norm": 101.34008477722098,
      "learning_rate": 1.1896753511186364e-07,
      "logits/chosen": -2.407715320587158,
      "logits/rejected": -1.9829556941986084,
      "logps/chosen": -292.2974853515625,
      "logps/rejected": -221.4503631591797,
      "loss": -0.7388,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -0.8350914120674133,
      "rewards/margins": 34.97935104370117,
      "rewards/rejected": -35.8144416809082,
      "step": 4810
    },
    {
      "epoch": 0.7095539525982629,
      "grad_norm": 176.71255597471117,
      "learning_rate": 1.1787508544433002e-07,
      "logits/chosen": -2.26106595993042,
      "logits/rejected": -1.951155424118042,
      "logps/chosen": -255.847412109375,
      "logps/rejected": -339.22344970703125,
      "loss": -0.7734,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": -5.114264488220215,
      "rewards/margins": 71.21052551269531,
      "rewards/rejected": -76.32478332519531,
      "step": 4820
    },
    {
      "epoch": 0.7110260562343589,
      "grad_norm": 69.22681560570555,
      "learning_rate": 1.1678612536931718e-07,
      "logits/chosen": -2.350593090057373,
      "logits/rejected": -1.7576704025268555,
      "logps/chosen": -304.4311828613281,
      "logps/rejected": -344.8041076660156,
      "loss": -0.7055,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 7.712233066558838,
      "rewards/margins": 79.54559326171875,
      "rewards/rejected": -71.8333511352539,
      "step": 4830
    },
    {
      "epoch": 0.7124981598704548,
      "grad_norm": 96.07707631958094,
      "learning_rate": 1.1570068364769081e-07,
      "logits/chosen": -2.155362367630005,
      "logits/rejected": -1.8739850521087646,
      "logps/chosen": -280.16033935546875,
      "logps/rejected": -345.90142822265625,
      "loss": -0.6387,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -17.639495849609375,
      "rewards/margins": 33.631134033203125,
      "rewards/rejected": -51.2706298828125,
      "step": 4840
    },
    {
      "epoch": 0.7139702635065509,
      "grad_norm": 51.66750100515644,
      "learning_rate": 1.146187889473924e-07,
      "logits/chosen": -2.2019548416137695,
      "logits/rejected": -1.832624077796936,
      "logps/chosen": -299.6923828125,
      "logps/rejected": -327.6891784667969,
      "loss": -0.6932,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -16.468894958496094,
      "rewards/margins": 51.188323974609375,
      "rewards/rejected": -67.65721130371094,
      "step": 4850
    },
    {
      "epoch": 0.7154423671426469,
      "grad_norm": 141.4135387095894,
      "learning_rate": 1.135404698426819e-07,
      "logits/chosen": -2.2950921058654785,
      "logits/rejected": -1.8973544836044312,
      "logps/chosen": -250.41006469726562,
      "logps/rejected": -307.833251953125,
      "loss": -0.7051,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 3.4658310413360596,
      "rewards/margins": 56.4177131652832,
      "rewards/rejected": -52.951881408691406,
      "step": 4860
    },
    {
      "epoch": 0.7169144707787428,
      "grad_norm": 97.80495323080882,
      "learning_rate": 1.1246575481338305e-07,
      "logits/chosen": -2.485769271850586,
      "logits/rejected": -1.844927430152893,
      "logps/chosen": -318.64166259765625,
      "logps/rejected": -326.5401916503906,
      "loss": -0.918,
      "rewards/accuracies": 0.76666659116745,
      "rewards/chosen": 0.34953194856643677,
      "rewards/margins": 61.27960968017578,
      "rewards/rejected": -60.930076599121094,
      "step": 4870
    },
    {
      "epoch": 0.7183865744148388,
      "grad_norm": 72.99398258653757,
      "learning_rate": 1.1139467224413132e-07,
      "logits/chosen": -2.2852017879486084,
      "logits/rejected": -1.8058357238769531,
      "logps/chosen": -228.010009765625,
      "logps/rejected": -258.35882568359375,
      "loss": -0.5192,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": 11.944993019104004,
      "rewards/margins": 54.289276123046875,
      "rewards/rejected": -42.344276428222656,
      "step": 4880
    },
    {
      "epoch": 0.7198586780509347,
      "grad_norm": 71.80623903537845,
      "learning_rate": 1.1032725042362393e-07,
      "logits/chosen": -2.294329881668091,
      "logits/rejected": -1.718645691871643,
      "logps/chosen": -300.4781188964844,
      "logps/rejected": -272.0849914550781,
      "loss": -0.7645,
      "rewards/accuracies": 0.5333333611488342,
      "rewards/chosen": -20.55608558654785,
      "rewards/margins": 13.591865539550781,
      "rewards/rejected": -34.147945404052734,
      "step": 4890
    },
    {
      "epoch": 0.7213307816870308,
      "grad_norm": 132.27775178954104,
      "learning_rate": 1.0926351754387336e-07,
      "logits/chosen": -2.1916277408599854,
      "logits/rejected": -2.0671372413635254,
      "logps/chosen": -316.6535949707031,
      "logps/rejected": -305.7307434082031,
      "loss": -0.7279,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -4.036540985107422,
      "rewards/margins": 29.590993881225586,
      "rewards/rejected": -33.627540588378906,
      "step": 4900
    },
    {
      "epoch": 0.7228028853231268,
      "grad_norm": 64.10961446412661,
      "learning_rate": 1.0820350169946174e-07,
      "logits/chosen": -2.3097481727600098,
      "logits/rejected": -2.1056294441223145,
      "logps/chosen": -234.2753448486328,
      "logps/rejected": -295.0716857910156,
      "loss": -0.7879,
      "rewards/accuracies": 0.7999999523162842,
      "rewards/chosen": 3.082174777984619,
      "rewards/margins": 47.49871063232422,
      "rewards/rejected": -44.416534423828125,
      "step": 4910
    },
    {
      "epoch": 0.7242749889592227,
      "grad_norm": 85.59219946966638,
      "learning_rate": 1.0714723088679983e-07,
      "logits/chosen": -2.3036623001098633,
      "logits/rejected": -1.582182765007019,
      "logps/chosen": -322.2913818359375,
      "logps/rejected": -293.90838623046875,
      "loss": -0.7682,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -9.327914237976074,
      "rewards/margins": 67.60395050048828,
      "rewards/rejected": -76.93186950683594,
      "step": 4920
    },
    {
      "epoch": 0.7257470925953187,
      "grad_norm": 108.4856137374045,
      "learning_rate": 1.06094733003387e-07,
      "logits/chosen": -2.0047049522399902,
      "logits/rejected": -1.9288015365600586,
      "logps/chosen": -316.85797119140625,
      "logps/rejected": -323.1899108886719,
      "loss": -0.7108,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.2846875190734863,
      "rewards/margins": 38.18626022338867,
      "rewards/rejected": -39.470947265625,
      "step": 4930
    },
    {
      "epoch": 0.7272191962314147,
      "grad_norm": 2255.851248199864,
      "learning_rate": 1.0504603584707463e-07,
      "logits/chosen": -2.2087793350219727,
      "logits/rejected": -1.6204297542572021,
      "logps/chosen": -230.6611785888672,
      "logps/rejected": -291.7385559082031,
      "loss": -0.4782,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -8.937911987304688,
      "rewards/margins": 66.35737609863281,
      "rewards/rejected": -75.29528045654297,
      "step": 4940
    },
    {
      "epoch": 0.7286912998675107,
      "grad_norm": 278.2495480677591,
      "learning_rate": 1.0400116711533217e-07,
      "logits/chosen": -2.1279499530792236,
      "logits/rejected": -1.5661303997039795,
      "logps/chosen": -362.98565673828125,
      "logps/rejected": -324.5312805175781,
      "loss": -0.67,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -9.964164733886719,
      "rewards/margins": 27.314987182617188,
      "rewards/rejected": -37.27915573120117,
      "step": 4950
    },
    {
      "epoch": 0.7301634035036066,
      "grad_norm": 100.85608402318358,
      "learning_rate": 1.029601544045148e-07,
      "logits/chosen": -1.90776789188385,
      "logits/rejected": -1.574951410293579,
      "logps/chosen": -302.9644775390625,
      "logps/rejected": -289.6867370605469,
      "loss": -0.7605,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -14.523101806640625,
      "rewards/margins": 35.036067962646484,
      "rewards/rejected": -49.55916976928711,
      "step": 4960
    },
    {
      "epoch": 0.7316355071397026,
      "grad_norm": 76.77845451196963,
      "learning_rate": 1.0192302520913563e-07,
      "logits/chosen": -2.0869555473327637,
      "logits/rejected": -1.8605190515518188,
      "logps/chosen": -262.4488830566406,
      "logps/rejected": -301.84808349609375,
      "loss": -0.6892,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -3.8128371238708496,
      "rewards/margins": 36.08661651611328,
      "rewards/rejected": -39.899452209472656,
      "step": 4970
    },
    {
      "epoch": 0.7331076107757987,
      "grad_norm": 84.8151720917729,
      "learning_rate": 1.0088980692113872e-07,
      "logits/chosen": -2.093554973602295,
      "logits/rejected": -1.4057438373565674,
      "logps/chosen": -364.63531494140625,
      "logps/rejected": -355.93499755859375,
      "loss": -0.8649,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -15.828730583190918,
      "rewards/margins": 58.52103805541992,
      "rewards/rejected": -74.34976959228516,
      "step": 4980
    },
    {
      "epoch": 0.7345797144118946,
      "grad_norm": 137.85581702390556,
      "learning_rate": 9.986052682917611e-08,
      "logits/chosen": -2.4457671642303467,
      "logits/rejected": -2.0700790882110596,
      "logps/chosen": -314.89288330078125,
      "logps/rejected": -334.78179931640625,
      "loss": -0.5655,
      "rewards/accuracies": 0.5666666626930237,
      "rewards/chosen": -0.3338131010532379,
      "rewards/margins": 20.481714248657227,
      "rewards/rejected": -20.815526962280273,
      "step": 4990
    },
    {
      "epoch": 0.7360518180479906,
      "grad_norm": 102.55766247720688,
      "learning_rate": 9.883521211788682e-08,
      "logits/chosen": -2.1220436096191406,
      "logits/rejected": -1.5362423658370972,
      "logps/chosen": -312.0055847167969,
      "logps/rejected": -304.5025634765625,
      "loss": -0.7711,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -1.3952182531356812,
      "rewards/margins": 46.38567352294922,
      "rewards/rejected": -47.78089141845703,
      "step": 5000
    },
    {
      "epoch": 0.7375239216840865,
      "grad_norm": 87.33940774422727,
      "learning_rate": 9.781388986717898e-08,
      "logits/chosen": -2.1359963417053223,
      "logits/rejected": -1.5125343799591064,
      "logps/chosen": -214.4330596923828,
      "logps/rejected": -250.96792602539062,
      "loss": -0.7223,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": 0.6099538803100586,
      "rewards/margins": 70.59184265136719,
      "rewards/rejected": -69.98190307617188,
      "step": 5010
    },
    {
      "epoch": 0.7389960253201825,
      "grad_norm": 128.48546906498368,
      "learning_rate": 9.679658705151461e-08,
      "logits/chosen": -2.194247007369995,
      "logits/rejected": -1.4523541927337646,
      "logps/chosen": -242.8587188720703,
      "logps/rejected": -298.85565185546875,
      "loss": -0.7011,
      "rewards/accuracies": 0.6999999284744263,
      "rewards/chosen": 2.108475923538208,
      "rewards/margins": 75.58739471435547,
      "rewards/rejected": -73.47892761230469,
      "step": 5020
    },
    {
      "epoch": 0.7404681289562786,
      "grad_norm": 97.4937020194971,
      "learning_rate": 9.578333053919704e-08,
      "logits/chosen": -2.3135523796081543,
      "logits/rejected": -2.157606840133667,
      "logps/chosen": -232.5798797607422,
      "logps/rejected": -324.1941223144531,
      "loss": -0.6602,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.04815273359417915,
      "rewards/margins": 36.49648666381836,
      "rewards/rejected": -36.544639587402344,
      "step": 5030
    },
    {
      "epoch": 0.7419402325923745,
      "grad_norm": 69.44048465949824,
      "learning_rate": 9.477414709166182e-08,
      "logits/chosen": -2.4149398803710938,
      "logits/rejected": -1.614513635635376,
      "logps/chosen": -305.0122985839844,
      "logps/rejected": -346.1600036621094,
      "loss": -0.7398,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 8.206254959106445,
      "rewards/margins": 63.90106964111328,
      "rewards/rejected": -55.6948127746582,
      "step": 5040
    },
    {
      "epoch": 0.7434123362284705,
      "grad_norm": 212.88879697001346,
      "learning_rate": 9.376906336276894e-08,
      "logits/chosen": -2.3200109004974365,
      "logits/rejected": -1.836568832397461,
      "logps/chosen": -259.5028381347656,
      "logps/rejected": -326.04180908203125,
      "loss": -0.7453,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": 6.223690509796143,
      "rewards/margins": 44.73774719238281,
      "rewards/rejected": -38.514060974121094,
      "step": 5050
    },
    {
      "epoch": 0.7448844398645664,
      "grad_norm": 137.96435701669063,
      "learning_rate": 9.276810589809978e-08,
      "logits/chosen": -2.00022029876709,
      "logits/rejected": -1.8082927465438843,
      "logps/chosen": -197.70228576660156,
      "logps/rejected": -253.357177734375,
      "loss": -0.7163,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -2.434911012649536,
      "rewards/margins": 40.26336669921875,
      "rewards/rejected": -42.69827651977539,
      "step": 5060
    },
    {
      "epoch": 0.7463565435006625,
      "grad_norm": 114.65888368195407,
      "learning_rate": 9.177130113425562e-08,
      "logits/chosen": -2.1429388523101807,
      "logits/rejected": -1.570826768875122,
      "logps/chosen": -204.94900512695312,
      "logps/rejected": -247.0119171142578,
      "loss": -0.6471,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -13.629190444946289,
      "rewards/margins": 52.4788932800293,
      "rewards/rejected": -66.10807800292969,
      "step": 5070
    },
    {
      "epoch": 0.7478286471367585,
      "grad_norm": 139.6570252590884,
      "learning_rate": 9.077867539815948e-08,
      "logits/chosen": -2.5762887001037598,
      "logits/rejected": -2.043825626373291,
      "logps/chosen": -264.924072265625,
      "logps/rejected": -243.60122680664062,
      "loss": -0.7539,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 8.390881538391113,
      "rewards/margins": 39.90256118774414,
      "rewards/rejected": -31.511682510375977,
      "step": 5080
    },
    {
      "epoch": 0.7493007507728544,
      "grad_norm": 78.51069249716885,
      "learning_rate": 8.979025490636064e-08,
      "logits/chosen": -2.1928024291992188,
      "logits/rejected": -1.9431911706924438,
      "logps/chosen": -277.1522521972656,
      "logps/rejected": -302.0496520996094,
      "loss": -0.8196,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -9.275369644165039,
      "rewards/margins": 41.5316047668457,
      "rewards/rejected": -50.806976318359375,
      "step": 5090
    },
    {
      "epoch": 0.7507728544089504,
      "grad_norm": 87.80333167246222,
      "learning_rate": 8.880606576434249e-08,
      "logits/chosen": -2.451469898223877,
      "logits/rejected": -1.9593521356582642,
      "logps/chosen": -247.2191619873047,
      "logps/rejected": -279.9991760253906,
      "loss": -0.7478,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 9.918397903442383,
      "rewards/margins": 47.10988235473633,
      "rewards/rejected": -37.19148254394531,
      "step": 5100
    },
    {
      "epoch": 0.7522449580450463,
      "grad_norm": 66.27243647689522,
      "learning_rate": 8.782613396583285e-08,
      "logits/chosen": -2.416496992111206,
      "logits/rejected": -2.0649380683898926,
      "logps/chosen": -283.9575500488281,
      "logps/rejected": -262.5621643066406,
      "loss": -0.7335,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": 6.561954498291016,
      "rewards/margins": 29.752544403076172,
      "rewards/rejected": -23.190589904785156,
      "step": 5110
    },
    {
      "epoch": 0.7537170616811424,
      "grad_norm": 158.52927595368666,
      "learning_rate": 8.685048539211745e-08,
      "logits/chosen": -2.1655640602111816,
      "logits/rejected": -1.5940873622894287,
      "logps/chosen": -311.0968933105469,
      "logps/rejected": -332.5016784667969,
      "loss": -0.7567,
      "rewards/accuracies": 0.8333333730697632,
      "rewards/chosen": -12.993843078613281,
      "rewards/margins": 54.101348876953125,
      "rewards/rejected": -67.09519958496094,
      "step": 5120
    },
    {
      "epoch": 0.7551891653172383,
      "grad_norm": 118.74351765427632,
      "learning_rate": 8.587914581135672e-08,
      "logits/chosen": -2.147181272506714,
      "logits/rejected": -1.6785316467285156,
      "logps/chosen": -252.3111572265625,
      "logps/rejected": -310.37091064453125,
      "loss": -0.7598,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": 5.518768310546875,
      "rewards/margins": 57.34308624267578,
      "rewards/rejected": -51.824317932128906,
      "step": 5130
    },
    {
      "epoch": 0.7566612689533343,
      "grad_norm": 88.96269822648745,
      "learning_rate": 8.491214087790447e-08,
      "logits/chosen": -2.4968574047088623,
      "logits/rejected": -1.9349473714828491,
      "logps/chosen": -327.1272888183594,
      "logps/rejected": -399.276611328125,
      "loss": -0.7728,
      "rewards/accuracies": 0.7999999523162842,
      "rewards/chosen": 8.254644393920898,
      "rewards/margins": 68.18389129638672,
      "rewards/rejected": -59.92924880981445,
      "step": 5140
    },
    {
      "epoch": 0.7581333725894303,
      "grad_norm": 111.52842096989498,
      "learning_rate": 8.394949613163111e-08,
      "logits/chosen": -2.3991293907165527,
      "logits/rejected": -1.9896471500396729,
      "logps/chosen": -222.0389862060547,
      "logps/rejected": -244.4817657470703,
      "loss": -0.6929,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": 7.451291084289551,
      "rewards/margins": 45.87103271484375,
      "rewards/rejected": -38.41974639892578,
      "step": 5150
    },
    {
      "epoch": 0.7596054762255263,
      "grad_norm": 98.40160792506595,
      "learning_rate": 8.299123699724864e-08,
      "logits/chosen": -2.2390551567077637,
      "logits/rejected": -1.624415397644043,
      "logps/chosen": -339.3470153808594,
      "logps/rejected": -313.86480712890625,
      "loss": -0.7985,
      "rewards/accuracies": 0.8333333730697632,
      "rewards/chosen": 12.72245979309082,
      "rewards/margins": 60.08031463623047,
      "rewards/rejected": -47.35786056518555,
      "step": 5160
    },
    {
      "epoch": 0.7610775798616223,
      "grad_norm": 117.48284492858672,
      "learning_rate": 8.203738878363933e-08,
      "logits/chosen": -2.126404285430908,
      "logits/rejected": -1.389928936958313,
      "logps/chosen": -239.77670288085938,
      "logps/rejected": -259.6825256347656,
      "loss": -0.7368,
      "rewards/accuracies": 0.7999999523162842,
      "rewards/chosen": -2.296095609664917,
      "rewards/margins": 58.85325241088867,
      "rewards/rejected": -61.14935302734375,
      "step": 5170
    },
    {
      "epoch": 0.7625496834977182,
      "grad_norm": 106.40981699902026,
      "learning_rate": 8.108797668318743e-08,
      "logits/chosen": -2.0280838012695312,
      "logits/rejected": -1.4500586986541748,
      "logps/chosen": -236.4788360595703,
      "logps/rejected": -300.0771789550781,
      "loss": -0.6442,
      "rewards/accuracies": 0.6999999284744263,
      "rewards/chosen": -13.801615715026855,
      "rewards/margins": 52.92882537841797,
      "rewards/rejected": -66.73045349121094,
      "step": 5180
    },
    {
      "epoch": 0.7640217871338142,
      "grad_norm": 100.35835593981501,
      "learning_rate": 8.01430257711132e-08,
      "logits/chosen": -2.300967216491699,
      "logits/rejected": -1.4804694652557373,
      "logps/chosen": -252.414794921875,
      "logps/rejected": -302.7409362792969,
      "loss": -0.7434,
      "rewards/accuracies": 0.76666659116745,
      "rewards/chosen": 0.09500005096197128,
      "rewards/margins": 86.36992645263672,
      "rewards/rejected": -86.27493286132812,
      "step": 5190
    },
    {
      "epoch": 0.7654938907699103,
      "grad_norm": 110.47732688516378,
      "learning_rate": 7.92025610048114e-08,
      "logits/chosen": -2.296189069747925,
      "logits/rejected": -1.5087240934371948,
      "logps/chosen": -235.5931396484375,
      "logps/rejected": -263.14471435546875,
      "loss": -0.7788,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 3.1874423027038574,
      "rewards/margins": 65.69438934326172,
      "rewards/rejected": -62.5069465637207,
      "step": 5200
    },
    {
      "epoch": 0.7669659944060062,
      "grad_norm": 65.71931931606474,
      "learning_rate": 7.826660722319165e-08,
      "logits/chosen": -2.2692267894744873,
      "logits/rejected": -1.4216653108596802,
      "logps/chosen": -362.70111083984375,
      "logps/rejected": -342.90966796875,
      "loss": -0.7656,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 4.520699977874756,
      "rewards/margins": 58.619964599609375,
      "rewards/rejected": -54.0992546081543,
      "step": 5210
    },
    {
      "epoch": 0.7684380980421022,
      "grad_norm": 120.63910215693608,
      "learning_rate": 7.733518914602252e-08,
      "logits/chosen": -2.308825969696045,
      "logits/rejected": -1.4703242778778076,
      "logps/chosen": -339.43023681640625,
      "logps/rejected": -332.4958190917969,
      "loss": -0.7851,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 1.6489776372909546,
      "rewards/margins": 54.10001754760742,
      "rewards/rejected": -52.45104217529297,
      "step": 5220
    },
    {
      "epoch": 0.7699102016781981,
      "grad_norm": 68.91866991034165,
      "learning_rate": 7.640833137327882e-08,
      "logits/chosen": -2.1958792209625244,
      "logits/rejected": -1.3889541625976562,
      "logps/chosen": -271.56732177734375,
      "logps/rejected": -300.2256774902344,
      "loss": -0.7921,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 1.2850185632705688,
      "rewards/margins": 64.97767639160156,
      "rewards/rejected": -63.692657470703125,
      "step": 5230
    },
    {
      "epoch": 0.7713823053142941,
      "grad_norm": 74.41630286681959,
      "learning_rate": 7.548605838449151e-08,
      "logits/chosen": -1.9696900844573975,
      "logits/rejected": -1.7550268173217773,
      "logps/chosen": -249.27035522460938,
      "logps/rejected": -318.96856689453125,
      "loss": -0.8993,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -6.10817813873291,
      "rewards/margins": 35.36677932739258,
      "rewards/rejected": -41.474952697753906,
      "step": 5240
    },
    {
      "epoch": 0.7728544089503901,
      "grad_norm": 98.53794140564857,
      "learning_rate": 7.456839453810157e-08,
      "logits/chosen": -2.2343204021453857,
      "logits/rejected": -1.8729960918426514,
      "logps/chosen": -295.0408935546875,
      "logps/rejected": -345.82122802734375,
      "loss": -0.6996,
      "rewards/accuracies": 0.5999999642372131,
      "rewards/chosen": -5.5329179763793945,
      "rewards/margins": 38.89052963256836,
      "rewards/rejected": -44.42345428466797,
      "step": 5250
    },
    {
      "epoch": 0.7743265125864861,
      "grad_norm": 214.24280687610366,
      "learning_rate": 7.365536407081633e-08,
      "logits/chosen": -2.3593716621398926,
      "logits/rejected": -2.0165884494781494,
      "logps/chosen": -257.39288330078125,
      "logps/rejected": -346.29058837890625,
      "loss": -0.8668,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": 6.921254634857178,
      "rewards/margins": 71.91499328613281,
      "rewards/rejected": -64.99372863769531,
      "step": 5260
    },
    {
      "epoch": 0.7757986162225821,
      "grad_norm": 73.92009677775248,
      "learning_rate": 7.274699109696975e-08,
      "logits/chosen": -2.089122772216797,
      "logits/rejected": -1.7889842987060547,
      "logps/chosen": -244.0390625,
      "logps/rejected": -242.1707305908203,
      "loss": -0.6359,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -2.7404990196228027,
      "rewards/margins": 36.292396545410156,
      "rewards/rejected": -39.03289794921875,
      "step": 5270
    },
    {
      "epoch": 0.777270719858678,
      "grad_norm": 121.60440513271102,
      "learning_rate": 7.184329960788491e-08,
      "logits/chosen": -2.1056950092315674,
      "logits/rejected": -1.659100890159607,
      "logps/chosen": -213.1819305419922,
      "logps/rejected": -295.44610595703125,
      "loss": -0.7782,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -2.2414584159851074,
      "rewards/margins": 56.39984130859375,
      "rewards/rejected": -58.64130783081055,
      "step": 5280
    },
    {
      "epoch": 0.7787428234947741,
      "grad_norm": 81.93478383206556,
      "learning_rate": 7.094431347124092e-08,
      "logits/chosen": -2.0099313259124756,
      "logits/rejected": -1.5348535776138306,
      "logps/chosen": -248.3863983154297,
      "logps/rejected": -292.2684631347656,
      "loss": -0.8007,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 10.063932418823242,
      "rewards/margins": 76.46867370605469,
      "rewards/rejected": -66.40474700927734,
      "step": 5290
    },
    {
      "epoch": 0.78021492713087,
      "grad_norm": 103.63656529738545,
      "learning_rate": 7.00500564304424e-08,
      "logits/chosen": -2.266536235809326,
      "logits/rejected": -1.6821407079696655,
      "logps/chosen": -358.4216003417969,
      "logps/rejected": -360.35089111328125,
      "loss": -0.8053,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": 17.065505981445312,
      "rewards/margins": 58.616607666015625,
      "rewards/rejected": -41.55109786987305,
      "step": 5300
    },
    {
      "epoch": 0.781687030766966,
      "grad_norm": 100.71966952597194,
      "learning_rate": 6.916055210399219e-08,
      "logits/chosen": -2.5128676891326904,
      "logits/rejected": -2.232588291168213,
      "logps/chosen": -295.70794677734375,
      "logps/rejected": -390.49224853515625,
      "loss": -0.7167,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": 14.753250122070312,
      "rewards/margins": 53.91606903076172,
      "rewards/rejected": -39.16282653808594,
      "step": 5310
    },
    {
      "epoch": 0.783159134403062,
      "grad_norm": 150.2814505974642,
      "learning_rate": 6.827582398486797e-08,
      "logits/chosen": -2.016505718231201,
      "logits/rejected": -1.5840662717819214,
      "logps/chosen": -405.02001953125,
      "logps/rejected": -358.10308837890625,
      "loss": -0.7033,
      "rewards/accuracies": 0.6999999284744263,
      "rewards/chosen": -9.399382591247559,
      "rewards/margins": 56.071990966796875,
      "rewards/rejected": -65.47136688232422,
      "step": 5320
    },
    {
      "epoch": 0.7846312380391579,
      "grad_norm": 85.12813284690415,
      "learning_rate": 6.739589543990118e-08,
      "logits/chosen": -1.9922984838485718,
      "logits/rejected": -1.458070993423462,
      "logps/chosen": -298.188720703125,
      "logps/rejected": -277.27197265625,
      "loss": -0.7133,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -5.566129684448242,
      "rewards/margins": 48.83222198486328,
      "rewards/rejected": -54.398353576660156,
      "step": 5330
    },
    {
      "epoch": 0.786103341675254,
      "grad_norm": 117.97057340266417,
      "learning_rate": 6.652078970916037e-08,
      "logits/chosen": -2.412026882171631,
      "logits/rejected": -1.9143133163452148,
      "logps/chosen": -274.83392333984375,
      "logps/rejected": -343.00152587890625,
      "loss": -0.7806,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -1.4147021770477295,
      "rewards/margins": 64.3420639038086,
      "rewards/rejected": -65.75675964355469,
      "step": 5340
    },
    {
      "epoch": 0.7875754453113499,
      "grad_norm": 108.39333138282278,
      "learning_rate": 6.565052990533715e-08,
      "logits/chosen": -2.266096830368042,
      "logits/rejected": -1.9494587182998657,
      "logps/chosen": -267.88031005859375,
      "logps/rejected": -302.0382385253906,
      "loss": -0.7112,
      "rewards/accuracies": 0.7999999523162842,
      "rewards/chosen": -7.90502405166626,
      "rewards/margins": 46.792991638183594,
      "rewards/rejected": -54.6980094909668,
      "step": 5350
    },
    {
      "epoch": 0.7890475489474459,
      "grad_norm": 149.938069756433,
      "learning_rate": 6.478513901313615e-08,
      "logits/chosen": -1.8119449615478516,
      "logits/rejected": -1.5586328506469727,
      "logps/chosen": -247.27395629882812,
      "logps/rejected": -343.09503173828125,
      "loss": -0.627,
      "rewards/accuracies": 0.7000000476837158,
      "rewards/chosen": 0.5859317779541016,
      "rewards/margins": 48.279762268066406,
      "rewards/rejected": -47.69383239746094,
      "step": 5360
    },
    {
      "epoch": 0.7905196525835418,
      "grad_norm": 107.79503259774549,
      "learning_rate": 6.392463988866714e-08,
      "logits/chosen": -2.0127832889556885,
      "logits/rejected": -1.676841378211975,
      "logps/chosen": -279.77874755859375,
      "logps/rejected": -361.9905090332031,
      "loss": -0.6043,
      "rewards/accuracies": 0.7000000476837158,
      "rewards/chosen": 0.9934545755386353,
      "rewards/margins": 55.605491638183594,
      "rewards/rejected": -54.612037658691406,
      "step": 5370
    },
    {
      "epoch": 0.7919917562196379,
      "grad_norm": 189.27144712923064,
      "learning_rate": 6.306905525884227e-08,
      "logits/chosen": -2.4005022048950195,
      "logits/rejected": -2.0721888542175293,
      "logps/chosen": -279.23675537109375,
      "logps/rejected": -324.535400390625,
      "loss": -0.735,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -1.1525943279266357,
      "rewards/margins": 42.17296600341797,
      "rewards/rejected": -43.3255615234375,
      "step": 5380
    },
    {
      "epoch": 0.7934638598557339,
      "grad_norm": 81.52720595280475,
      "learning_rate": 6.221840772077525e-08,
      "logits/chosen": -2.0224361419677734,
      "logits/rejected": -1.556707739830017,
      "logps/chosen": -259.517578125,
      "logps/rejected": -283.36981201171875,
      "loss": -0.6867,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -15.958511352539062,
      "rewards/margins": 53.55073165893555,
      "rewards/rejected": -69.50924682617188,
      "step": 5390
    },
    {
      "epoch": 0.7949359634918298,
      "grad_norm": 117.37591868330784,
      "learning_rate": 6.137271974118468e-08,
      "logits/chosen": -2.1610922813415527,
      "logits/rejected": -1.7317039966583252,
      "logps/chosen": -267.01666259765625,
      "logps/rejected": -291.14910888671875,
      "loss": -0.7046,
      "rewards/accuracies": 0.7000000476837158,
      "rewards/chosen": 2.2026212215423584,
      "rewards/margins": 46.74576950073242,
      "rewards/rejected": -44.54315948486328,
      "step": 5400
    },
    {
      "epoch": 0.7964080671279258,
      "grad_norm": 114.85268576796491,
      "learning_rate": 6.05320136558011e-08,
      "logits/chosen": -2.2298219203948975,
      "logits/rejected": -1.8267402648925781,
      "logps/chosen": -254.42630004882812,
      "logps/rejected": -328.8345031738281,
      "loss": -0.6917,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.4060148298740387,
      "rewards/margins": 45.05670166015625,
      "rewards/rejected": -45.462711334228516,
      "step": 5410
    },
    {
      "epoch": 0.7978801707640217,
      "grad_norm": 96.98406410429226,
      "learning_rate": 5.969631166877607e-08,
      "logits/chosen": -2.1112864017486572,
      "logits/rejected": -1.492666244506836,
      "logps/chosen": -321.58905029296875,
      "logps/rejected": -312.4671630859375,
      "loss": -0.6924,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 14.221929550170898,
      "rewards/margins": 55.43073654174805,
      "rewards/rejected": -41.20880889892578,
      "step": 5420
    },
    {
      "epoch": 0.7993522744001178,
      "grad_norm": 113.23160525855415,
      "learning_rate": 5.8865635852096754e-08,
      "logits/chosen": -2.224186658859253,
      "logits/rejected": -1.786064863204956,
      "logps/chosen": -264.71051025390625,
      "logps/rejected": -270.12030029296875,
      "loss": -0.7605,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -2.7375988960266113,
      "rewards/margins": 44.7318115234375,
      "rewards/rejected": -47.46940612792969,
      "step": 5430
    },
    {
      "epoch": 0.8008243780362138,
      "grad_norm": 328.98073499155896,
      "learning_rate": 5.8040008145002344e-08,
      "logits/chosen": -2.4944465160369873,
      "logits/rejected": -1.3368237018585205,
      "logps/chosen": -315.5783386230469,
      "logps/rejected": -341.8042297363281,
      "loss": -0.8432,
      "rewards/accuracies": 0.966666579246521,
      "rewards/chosen": 17.478527069091797,
      "rewards/margins": 88.3083267211914,
      "rewards/rejected": -70.82980346679688,
      "step": 5440
    },
    {
      "epoch": 0.8022964816723097,
      "grad_norm": 101.8408827267506,
      "learning_rate": 5.721945035340511e-08,
      "logits/chosen": -2.2972052097320557,
      "logits/rejected": -1.6335198879241943,
      "logps/chosen": -251.82119750976562,
      "logps/rejected": -286.37554931640625,
      "loss": -0.7632,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": 4.074501991271973,
      "rewards/margins": 72.82389068603516,
      "rewards/rejected": -68.7493896484375,
      "step": 5450
    },
    {
      "epoch": 0.8037685853084057,
      "grad_norm": 96.54990465161197,
      "learning_rate": 5.640398414931399e-08,
      "logits/chosen": -1.7668726444244385,
      "logits/rejected": -1.72323739528656,
      "logps/chosen": -227.35305786132812,
      "logps/rejected": -314.2005310058594,
      "loss": -0.6519,
      "rewards/accuracies": 0.5666666626930237,
      "rewards/chosen": -11.062286376953125,
      "rewards/margins": 42.82783889770508,
      "rewards/rejected": -53.8901252746582,
      "step": 5460
    },
    {
      "epoch": 0.8052406889445017,
      "grad_norm": 78.76027695377145,
      "learning_rate": 5.5593631070262296e-08,
      "logits/chosen": -2.1194252967834473,
      "logits/rejected": -1.5858840942382812,
      "logps/chosen": -301.8524475097656,
      "logps/rejected": -359.39410400390625,
      "loss": -0.7688,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": 0.642661452293396,
      "rewards/margins": 74.3678207397461,
      "rewards/rejected": -73.72515869140625,
      "step": 5470
    },
    {
      "epoch": 0.8067127925805977,
      "grad_norm": 90.3427757919698,
      "learning_rate": 5.478841251873922e-08,
      "logits/chosen": -2.3569531440734863,
      "logits/rejected": -1.7924286127090454,
      "logps/chosen": -323.0899353027344,
      "logps/rejected": -311.77288818359375,
      "loss": -0.78,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 19.703365325927734,
      "rewards/margins": 66.11837768554688,
      "rewards/rejected": -46.415016174316406,
      "step": 5480
    },
    {
      "epoch": 0.8081848962166936,
      "grad_norm": 64.0118565190515,
      "learning_rate": 5.398834976162414e-08,
      "logits/chosen": -2.151620626449585,
      "logits/rejected": -1.727710485458374,
      "logps/chosen": -296.69189453125,
      "logps/rejected": -348.12237548828125,
      "loss": 56.7635,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 34.578163146972656,
      "rewards/margins": 84.95592498779297,
      "rewards/rejected": -50.377769470214844,
      "step": 5490
    },
    {
      "epoch": 0.8096569998527896,
      "grad_norm": 153.62187152044947,
      "learning_rate": 5.319346392962545e-08,
      "logits/chosen": -2.136197090148926,
      "logits/rejected": -2.172917127609253,
      "logps/chosen": -274.52813720703125,
      "logps/rejected": -310.9470520019531,
      "loss": -0.6909,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -1.539521336555481,
      "rewards/margins": 28.571975708007812,
      "rewards/rejected": -30.111495971679688,
      "step": 5500
    },
    {
      "epoch": 0.8111291034888857,
      "grad_norm": 95.18585918200773,
      "learning_rate": 5.24037760167218e-08,
      "logits/chosen": -2.3080227375030518,
      "logits/rejected": -1.7586380243301392,
      "logps/chosen": -318.279052734375,
      "logps/rejected": -337.3676452636719,
      "loss": -0.7977,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 13.823498725891113,
      "rewards/margins": 70.82807159423828,
      "rewards/rejected": -57.00457000732422,
      "step": 5510
    },
    {
      "epoch": 0.8126012071249816,
      "grad_norm": 89.13479623695014,
      "learning_rate": 5.161930687960808e-08,
      "logits/chosen": -2.1941018104553223,
      "logits/rejected": -1.4440172910690308,
      "logps/chosen": -317.6274108886719,
      "logps/rejected": -267.21588134765625,
      "loss": -0.6625,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -3.8953404426574707,
      "rewards/margins": 54.333106994628906,
      "rewards/rejected": -58.22844314575195,
      "step": 5520
    },
    {
      "epoch": 0.8140733107610776,
      "grad_norm": 159.8703918208317,
      "learning_rate": 5.0840077237144594e-08,
      "logits/chosen": -2.3228049278259277,
      "logits/rejected": -1.340940237045288,
      "logps/chosen": -283.53900146484375,
      "logps/rejected": -303.1983642578125,
      "loss": -0.9206,
      "rewards/accuracies": 0.7999999523162842,
      "rewards/chosen": 25.251033782958984,
      "rewards/margins": 90.15493774414062,
      "rewards/rejected": -64.90390014648438,
      "step": 5530
    },
    {
      "epoch": 0.8155454143971735,
      "grad_norm": 151.6481362108295,
      "learning_rate": 5.006610766980945e-08,
      "logits/chosen": -2.3938629627227783,
      "logits/rejected": -1.9302971363067627,
      "logps/chosen": -296.81817626953125,
      "logps/rejected": -314.52923583984375,
      "loss": -0.702,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 1.7738258838653564,
      "rewards/margins": 51.2370719909668,
      "rewards/rejected": -49.46324920654297,
      "step": 5540
    },
    {
      "epoch": 0.8170175180332695,
      "grad_norm": 94.7508294839783,
      "learning_rate": 4.929741861915571e-08,
      "logits/chosen": -2.482947826385498,
      "logits/rejected": -1.8282470703125,
      "logps/chosen": -262.51531982421875,
      "logps/rejected": -281.73577880859375,
      "loss": -0.7268,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": -1.6487325429916382,
      "rewards/margins": 54.388206481933594,
      "rewards/rejected": -56.03693771362305,
      "step": 5550
    },
    {
      "epoch": 0.8184896216693656,
      "grad_norm": 158.27935909127362,
      "learning_rate": 4.853403038727047e-08,
      "logits/chosen": -1.9222780466079712,
      "logits/rejected": -1.8222230672836304,
      "logps/chosen": -238.17031860351562,
      "logps/rejected": -371.2836608886719,
      "loss": -0.7234,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -9.094657897949219,
      "rewards/margins": 43.69096755981445,
      "rewards/rejected": -52.78562545776367,
      "step": 5560
    },
    {
      "epoch": 0.8199617253054615,
      "grad_norm": 136.49977997963381,
      "learning_rate": 4.777596313623966e-08,
      "logits/chosen": -2.175825834274292,
      "logits/rejected": -1.6960262060165405,
      "logps/chosen": -342.3695068359375,
      "logps/rejected": -356.53179931640625,
      "loss": -0.8652,
      "rewards/accuracies": 0.8333333730697632,
      "rewards/chosen": 15.054143905639648,
      "rewards/margins": 79.98979187011719,
      "rewards/rejected": -64.93563842773438,
      "step": 5570
    },
    {
      "epoch": 0.8214338289415575,
      "grad_norm": 99.51051955475644,
      "learning_rate": 4.702323688761492e-08,
      "logits/chosen": -2.0917694568634033,
      "logits/rejected": -1.4477622509002686,
      "logps/chosen": -276.0234680175781,
      "logps/rejected": -349.0508728027344,
      "loss": -0.7568,
      "rewards/accuracies": 0.8333333730697632,
      "rewards/chosen": -18.63760757446289,
      "rewards/margins": 81.61722564697266,
      "rewards/rejected": -100.25481414794922,
      "step": 5580
    },
    {
      "epoch": 0.8229059325776534,
      "grad_norm": 84.54614065816254,
      "learning_rate": 4.627587152188522e-08,
      "logits/chosen": -2.4279849529266357,
      "logits/rejected": -1.8469690084457397,
      "logps/chosen": -290.27447509765625,
      "logps/rejected": -274.65594482421875,
      "loss": -0.7586,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": -1.832317590713501,
      "rewards/margins": 53.417869567871094,
      "rewards/rejected": -55.25019073486328,
      "step": 5590
    },
    {
      "epoch": 0.8243780362137495,
      "grad_norm": 95.68786822278138,
      "learning_rate": 4.5533886777951395e-08,
      "logits/chosen": -2.3871984481811523,
      "logits/rejected": -1.9108858108520508,
      "logps/chosen": -320.6990661621094,
      "logps/rejected": -452.50897216796875,
      "loss": -0.7906,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 22.535449981689453,
      "rewards/margins": 67.96965789794922,
      "rewards/rejected": -45.434207916259766,
      "step": 5600
    },
    {
      "epoch": 0.8258501398498455,
      "grad_norm": 75.23502730580988,
      "learning_rate": 4.4797302252604916e-08,
      "logits/chosen": -2.280862331390381,
      "logits/rejected": -1.8155925273895264,
      "logps/chosen": -252.8149871826172,
      "logps/rejected": -219.4938201904297,
      "loss": -0.6312,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -5.28755521774292,
      "rewards/margins": 29.736156463623047,
      "rewards/rejected": -35.023712158203125,
      "step": 5610
    },
    {
      "epoch": 0.8273222434859414,
      "grad_norm": 286.0137515053568,
      "learning_rate": 4.4066137400010555e-08,
      "logits/chosen": -2.257211446762085,
      "logits/rejected": -1.39231276512146,
      "logps/chosen": -271.11187744140625,
      "logps/rejected": -349.4048767089844,
      "loss": -0.7851,
      "rewards/accuracies": 0.8333333730697632,
      "rewards/chosen": 3.881307601928711,
      "rewards/margins": 101.87802124023438,
      "rewards/rejected": -97.99671173095703,
      "step": 5620
    },
    {
      "epoch": 0.8287943471220374,
      "grad_norm": 88.67681065462575,
      "learning_rate": 4.3340411531192306e-08,
      "logits/chosen": -2.406048059463501,
      "logits/rejected": -2.0602266788482666,
      "logps/chosen": -293.98638916015625,
      "logps/rejected": -272.84356689453125,
      "loss": -0.7614,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": 22.151912689208984,
      "rewards/margins": 56.7090950012207,
      "rewards/rejected": -34.55717849731445,
      "step": 5630
    },
    {
      "epoch": 0.8302664507581333,
      "grad_norm": 122.52009381177714,
      "learning_rate": 4.262014381352363e-08,
      "logits/chosen": -2.1729819774627686,
      "logits/rejected": -1.5455762147903442,
      "logps/chosen": -285.13153076171875,
      "logps/rejected": -320.47100830078125,
      "loss": -0.7157,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": -2.1818225383758545,
      "rewards/margins": 61.5328369140625,
      "rewards/rejected": -63.71466064453125,
      "step": 5640
    },
    {
      "epoch": 0.8317385543942294,
      "grad_norm": 149.40543159059496,
      "learning_rate": 4.1905353270220795e-08,
      "logits/chosen": -2.10978364944458,
      "logits/rejected": -1.4540704488754272,
      "logps/chosen": -262.0694885253906,
      "logps/rejected": -346.611572265625,
      "loss": -0.6565,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 1.0179131031036377,
      "rewards/margins": 49.19779968261719,
      "rewards/rejected": -48.179893493652344,
      "step": 5650
    },
    {
      "epoch": 0.8332106580303253,
      "grad_norm": 150.5971638203285,
      "learning_rate": 4.119605877984089e-08,
      "logits/chosen": -2.314483404159546,
      "logits/rejected": -2.0508553981781006,
      "logps/chosen": -230.4044952392578,
      "logps/rejected": -284.71441650390625,
      "loss": -0.6287,
      "rewards/accuracies": 0.5333333015441895,
      "rewards/chosen": -11.441461563110352,
      "rewards/margins": 32.903724670410156,
      "rewards/rejected": -44.345184326171875,
      "step": 5660
    },
    {
      "epoch": 0.8346827616664213,
      "grad_norm": 109.03512005183273,
      "learning_rate": 4.049227907578284e-08,
      "logits/chosen": -2.118040084838867,
      "logits/rejected": -1.8914930820465088,
      "logps/chosen": -258.9620361328125,
      "logps/rejected": -387.27593994140625,
      "loss": -0.7536,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 4.346409797668457,
      "rewards/margins": 82.36874389648438,
      "rewards/rejected": -78.02233123779297,
      "step": 5670
    },
    {
      "epoch": 0.8361548653025173,
      "grad_norm": 163.27043713519825,
      "learning_rate": 3.979403274579313e-08,
      "logits/chosen": -2.269315242767334,
      "logits/rejected": -1.8081134557724,
      "logps/chosen": -303.896484375,
      "logps/rejected": -357.58172607421875,
      "loss": -0.7695,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": 13.891095161437988,
      "rewards/margins": 102.65971374511719,
      "rewards/rejected": -88.76862335205078,
      "step": 5680
    },
    {
      "epoch": 0.8376269689386133,
      "grad_norm": 78.43453849760272,
      "learning_rate": 3.9101338231474354e-08,
      "logits/chosen": -2.4337055683135986,
      "logits/rejected": -1.995911955833435,
      "logps/chosen": -312.57470703125,
      "logps/rejected": -344.0626220703125,
      "loss": -0.7323,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -17.19305992126465,
      "rewards/margins": 47.5669059753418,
      "rewards/rejected": -64.75996398925781,
      "step": 5690
    },
    {
      "epoch": 0.8390990725747093,
      "grad_norm": 84.80125267718056,
      "learning_rate": 3.841421382779827e-08,
      "logits/chosen": -2.2423367500305176,
      "logits/rejected": -1.8190889358520508,
      "logps/chosen": -251.7604522705078,
      "logps/rejected": -370.1092529296875,
      "loss": -0.9031,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": 5.381461143493652,
      "rewards/margins": 84.04764556884766,
      "rewards/rejected": -78.66618347167969,
      "step": 5700
    },
    {
      "epoch": 0.8405711762108052,
      "grad_norm": 90.15956577282084,
      "learning_rate": 3.773267768262289e-08,
      "logits/chosen": -2.177293300628662,
      "logits/rejected": -1.8908439874649048,
      "logps/chosen": -275.24835205078125,
      "logps/rejected": -318.5881042480469,
      "loss": -0.6751,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.099002480506897,
      "rewards/margins": 43.286155700683594,
      "rewards/rejected": -44.385154724121094,
      "step": 5710
    },
    {
      "epoch": 0.8420432798469012,
      "grad_norm": 83.35741408524098,
      "learning_rate": 3.7056747796212844e-08,
      "logits/chosen": -2.18552303314209,
      "logits/rejected": -1.497544288635254,
      "logps/chosen": -246.2370147705078,
      "logps/rejected": -290.31219482421875,
      "loss": -0.7706,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 5.612303733825684,
      "rewards/margins": 79.48788452148438,
      "rewards/rejected": -73.87559509277344,
      "step": 5720
    },
    {
      "epoch": 0.8435153834829971,
      "grad_norm": 148.07950986657085,
      "learning_rate": 3.63864420207643e-08,
      "logits/chosen": -2.187957763671875,
      "logits/rejected": -1.5948190689086914,
      "logps/chosen": -310.15618896484375,
      "logps/rejected": -358.4129333496094,
      "loss": -0.7588,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": -8.987142562866211,
      "rewards/margins": 71.47904968261719,
      "rewards/rejected": -80.46617126464844,
      "step": 5730
    },
    {
      "epoch": 0.8449874871190932,
      "grad_norm": 75.85979964644646,
      "learning_rate": 3.5721778059933096e-08,
      "logits/chosen": -2.0094504356384277,
      "logits/rejected": -1.8357727527618408,
      "logps/chosen": -291.5694274902344,
      "logps/rejected": -261.3731689453125,
      "loss": -0.6836,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -20.451644897460938,
      "rewards/margins": 27.806310653686523,
      "rewards/rejected": -48.25795364379883,
      "step": 5740
    },
    {
      "epoch": 0.8464595907551892,
      "grad_norm": 70.24729050726498,
      "learning_rate": 3.506277346836731e-08,
      "logits/chosen": -1.9825160503387451,
      "logits/rejected": -2.0510661602020264,
      "logps/chosen": -247.6156005859375,
      "logps/rejected": -399.41033935546875,
      "loss": 30.2132,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -15.885641098022461,
      "rewards/margins": 55.98344039916992,
      "rewards/rejected": -71.86908721923828,
      "step": 5750
    },
    {
      "epoch": 0.8479316943912851,
      "grad_norm": 204.53071800248296,
      "learning_rate": 3.440944565124371e-08,
      "logits/chosen": -2.150926113128662,
      "logits/rejected": -1.5553237199783325,
      "logps/chosen": -216.8142852783203,
      "logps/rejected": -273.75762939453125,
      "loss": -0.9064,
      "rewards/accuracies": 0.73333340883255,
      "rewards/chosen": 10.67918872833252,
      "rewards/margins": 49.98529052734375,
      "rewards/rejected": -39.30610275268555,
      "step": 5760
    },
    {
      "epoch": 0.8494037980273811,
      "grad_norm": 94.12740736539499,
      "learning_rate": 3.376181186380808e-08,
      "logits/chosen": -2.255796432495117,
      "logits/rejected": -1.9802026748657227,
      "logps/chosen": -281.370361328125,
      "logps/rejected": -264.36968994140625,
      "loss": -0.6737,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -14.425827026367188,
      "rewards/margins": 25.500131607055664,
      "rewards/rejected": -39.92595672607422,
      "step": 5770
    },
    {
      "epoch": 0.8508759016634772,
      "grad_norm": 132.72489561338728,
      "learning_rate": 3.311988921091935e-08,
      "logits/chosen": -2.418732166290283,
      "logits/rejected": -1.7738285064697266,
      "logps/chosen": -300.326904296875,
      "logps/rejected": -262.7984619140625,
      "loss": -0.6995,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 2.963822841644287,
      "rewards/margins": 43.79780960083008,
      "rewards/rejected": -40.833984375,
      "step": 5780
    },
    {
      "epoch": 0.8523480052995731,
      "grad_norm": 151.45909026901043,
      "learning_rate": 3.248369464659775e-08,
      "logits/chosen": -2.1550755500793457,
      "logits/rejected": -1.620525598526001,
      "logps/chosen": -221.2736358642578,
      "logps/rejected": -273.8992919921875,
      "loss": -0.6852,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.6718955039978027,
      "rewards/margins": 68.7145767211914,
      "rewards/rejected": -72.386474609375,
      "step": 5790
    },
    {
      "epoch": 0.8538201089356691,
      "grad_norm": 18882.304608683728,
      "learning_rate": 3.1853244973577306e-08,
      "logits/chosen": -2.342681646347046,
      "logits/rejected": -1.5942251682281494,
      "logps/chosen": -286.8219299316406,
      "logps/rejected": -450.5138244628906,
      "loss": -0.3812,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": 5.387963771820068,
      "rewards/margins": 50.358184814453125,
      "rewards/rejected": -44.97022247314453,
      "step": 5800
    },
    {
      "epoch": 0.855292212571765,
      "grad_norm": 133.9716550251656,
      "learning_rate": 3.122855684286185e-08,
      "logits/chosen": -2.152519941329956,
      "logits/rejected": -1.6545722484588623,
      "logps/chosen": -310.1380310058594,
      "logps/rejected": -382.76129150390625,
      "loss": -0.7648,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -2.41501784324646,
      "rewards/margins": 70.07633209228516,
      "rewards/rejected": -72.49134826660156,
      "step": 5810
    },
    {
      "epoch": 0.8567643162078611,
      "grad_norm": 153.72962054601234,
      "learning_rate": 3.060964675328545e-08,
      "logits/chosen": -2.1113152503967285,
      "logits/rejected": -1.7487062215805054,
      "logps/chosen": -271.75347900390625,
      "logps/rejected": -332.22857666015625,
      "loss": -0.779,
      "rewards/accuracies": 0.7999999523162842,
      "rewards/chosen": -12.753252029418945,
      "rewards/margins": 53.50651931762695,
      "rewards/rejected": -66.259765625,
      "step": 5820
    },
    {
      "epoch": 0.858236419843957,
      "grad_norm": 96.0430424653025,
      "learning_rate": 2.999653105107644e-08,
      "logits/chosen": -2.3742880821228027,
      "logits/rejected": -1.9559133052825928,
      "logps/chosen": -237.2894744873047,
      "logps/rejected": -307.9116516113281,
      "loss": -0.556,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 10.86506462097168,
      "rewards/margins": 77.41947937011719,
      "rewards/rejected": -66.55441284179688,
      "step": 5830
    },
    {
      "epoch": 0.859708523480053,
      "grad_norm": 122.07173354328424,
      "learning_rate": 2.9389225929425637e-08,
      "logits/chosen": -2.404451370239258,
      "logits/rejected": -2.016819477081299,
      "logps/chosen": -248.9982452392578,
      "logps/rejected": -261.70721435546875,
      "loss": -0.7831,
      "rewards/accuracies": 0.76666659116745,
      "rewards/chosen": 1.4284194707870483,
      "rewards/margins": 42.598228454589844,
      "rewards/rejected": -41.16980743408203,
      "step": 5840
    },
    {
      "epoch": 0.861180627116149,
      "grad_norm": 111.37975041917925,
      "learning_rate": 2.8787747428058977e-08,
      "logits/chosen": -2.129019021987915,
      "logits/rejected": -2.1207222938537598,
      "logps/chosen": -322.9551696777344,
      "logps/rejected": -365.7874450683594,
      "loss": -0.9223,
      "rewards/accuracies": 0.6999999284744263,
      "rewards/chosen": 13.7469482421875,
      "rewards/margins": 62.592124938964844,
      "rewards/rejected": -48.845176696777344,
      "step": 5850
    },
    {
      "epoch": 0.8626527307522449,
      "grad_norm": 80.72146722374441,
      "learning_rate": 2.8192111432813554e-08,
      "logits/chosen": -2.6486871242523193,
      "logits/rejected": -1.9362847805023193,
      "logps/chosen": -268.9266052246094,
      "logps/rejected": -337.00537109375,
      "loss": -0.4591,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": 6.510004997253418,
      "rewards/margins": 66.61495208740234,
      "rewards/rejected": -60.104942321777344,
      "step": 5860
    },
    {
      "epoch": 0.864124834388341,
      "grad_norm": 127.01859130978377,
      "learning_rate": 2.7602333675218443e-08,
      "logits/chosen": -2.5263547897338867,
      "logits/rejected": -2.1044723987579346,
      "logps/chosen": -254.7208251953125,
      "logps/rejected": -228.03244018554688,
      "loss": -0.7739,
      "rewards/accuracies": 0.6999999284744263,
      "rewards/chosen": 4.498696804046631,
      "rewards/margins": 38.01673126220703,
      "rewards/rejected": -33.518035888671875,
      "step": 5870
    },
    {
      "epoch": 0.8655969380244369,
      "grad_norm": 154.63044955295888,
      "learning_rate": 2.701842973207874e-08,
      "logits/chosen": -2.147864818572998,
      "logits/rejected": -1.7933530807495117,
      "logps/chosen": -274.31561279296875,
      "logps/rejected": -341.22479248046875,
      "loss": -0.7747,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 4.401491641998291,
      "rewards/margins": 68.10581970214844,
      "rewards/rejected": -63.7043342590332,
      "step": 5880
    },
    {
      "epoch": 0.8670690416605329,
      "grad_norm": 126.13266342507005,
      "learning_rate": 2.6440415025064417e-08,
      "logits/chosen": -2.343883752822876,
      "logits/rejected": -1.856532335281372,
      "logps/chosen": -324.1205749511719,
      "logps/rejected": -342.4364318847656,
      "loss": -0.6754,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": 0.42764702439308167,
      "rewards/margins": 62.497154235839844,
      "rewards/rejected": -62.06949996948242,
      "step": 5890
    },
    {
      "epoch": 0.8685411452966288,
      "grad_norm": 102.68099055453723,
      "learning_rate": 2.5868304820303027e-08,
      "logits/chosen": -2.233236789703369,
      "logits/rejected": -1.6277291774749756,
      "logps/chosen": -304.1766662597656,
      "logps/rejected": -320.6139831542969,
      "loss": -0.7957,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -3.0690019130706787,
      "rewards/margins": 63.12407302856445,
      "rewards/rejected": -66.19307708740234,
      "step": 5900
    },
    {
      "epoch": 0.8700132489327249,
      "grad_norm": 84.19959073046503,
      "learning_rate": 2.5302114227976513e-08,
      "logits/chosen": -2.4458529949188232,
      "logits/rejected": -2.0958681106567383,
      "logps/chosen": -287.1915588378906,
      "logps/rejected": -315.306396484375,
      "loss": -0.8641,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": 11.262953758239746,
      "rewards/margins": 57.09888458251953,
      "rewards/rejected": -45.83592987060547,
      "step": 5910
    },
    {
      "epoch": 0.8714853525688209,
      "grad_norm": 116.99614715412366,
      "learning_rate": 2.4741858201922065e-08,
      "logits/chosen": -2.3081841468811035,
      "logits/rejected": -1.8074767589569092,
      "logps/chosen": -277.1915283203125,
      "logps/rejected": -351.69195556640625,
      "loss": -0.7255,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": 3.0922937393188477,
      "rewards/margins": 67.7026596069336,
      "rewards/rejected": -64.6103744506836,
      "step": 5920
    },
    {
      "epoch": 0.8729574562049168,
      "grad_norm": 101.81154018650251,
      "learning_rate": 2.4187551539237073e-08,
      "logits/chosen": -2.2673916816711426,
      "logits/rejected": -1.6705923080444336,
      "logps/chosen": -285.9935302734375,
      "logps/rejected": -279.42901611328125,
      "loss": -0.7625,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": 3.109992265701294,
      "rewards/margins": 49.351173400878906,
      "rewards/rejected": -46.241172790527344,
      "step": 5930
    },
    {
      "epoch": 0.8744295598410128,
      "grad_norm": 135.1689408289447,
      "learning_rate": 2.363920887988849e-08,
      "logits/chosen": -2.2640440464019775,
      "logits/rejected": -1.697877287864685,
      "logps/chosen": -248.3556671142578,
      "logps/rejected": -342.02667236328125,
      "loss": -0.7176,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -18.522497177124023,
      "rewards/margins": 70.66897583007812,
      "rewards/rejected": -89.19148254394531,
      "step": 5940
    },
    {
      "epoch": 0.8759016634771087,
      "grad_norm": 77.57492755260326,
      "learning_rate": 2.3096844706326112e-08,
      "logits/chosen": -2.2563424110412598,
      "logits/rejected": -1.5565226078033447,
      "logps/chosen": -296.9709777832031,
      "logps/rejected": -317.47735595703125,
      "loss": -0.7492,
      "rewards/accuracies": 0.9333332777023315,
      "rewards/chosen": 4.003455638885498,
      "rewards/margins": 87.72456359863281,
      "rewards/rejected": -83.72111511230469,
      "step": 5950
    },
    {
      "epoch": 0.8773737671132048,
      "grad_norm": 115.1298056027288,
      "learning_rate": 2.2560473343100155e-08,
      "logits/chosen": -2.202230453491211,
      "logits/rejected": -1.8289365768432617,
      "logps/chosen": -255.3357391357422,
      "logps/rejected": -269.7892150878906,
      "loss": -0.7081,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -17.284791946411133,
      "rewards/margins": 51.39235305786133,
      "rewards/rejected": -68.67713928222656,
      "step": 5960
    },
    {
      "epoch": 0.8788458707493008,
      "grad_norm": 132.1426840794124,
      "learning_rate": 2.20301089564828e-08,
      "logits/chosen": -2.341587543487549,
      "logits/rejected": -1.5666840076446533,
      "logps/chosen": -275.98248291015625,
      "logps/rejected": -305.9757385253906,
      "loss": -0.7681,
      "rewards/accuracies": 0.8333333730697632,
      "rewards/chosen": 12.419610977172852,
      "rewards/margins": 73.03529357910156,
      "rewards/rejected": -60.615684509277344,
      "step": 5970
    },
    {
      "epoch": 0.8803179743853967,
      "grad_norm": 86.12342559883824,
      "learning_rate": 2.150576555409392e-08,
      "logits/chosen": -2.324397325515747,
      "logits/rejected": -1.6751896142959595,
      "logps/chosen": -190.828369140625,
      "logps/rejected": -222.99948120117188,
      "loss": -0.8146,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 9.166664123535156,
      "rewards/margins": 55.8420295715332,
      "rewards/rejected": -46.67536163330078,
      "step": 5980
    },
    {
      "epoch": 0.8817900780214927,
      "grad_norm": 104.04830164341841,
      "learning_rate": 2.0987456984531472e-08,
      "logits/chosen": -2.628462314605713,
      "logits/rejected": -2.1351020336151123,
      "logps/chosen": -254.72024536132812,
      "logps/rejected": -292.05322265625,
      "loss": -0.7494,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": 4.499943256378174,
      "rewards/margins": 58.52509689331055,
      "rewards/rejected": -54.025146484375,
      "step": 5990
    },
    {
      "epoch": 0.8832621816575887,
      "grad_norm": 138.14543713851077,
      "learning_rate": 2.0475196937005557e-08,
      "logits/chosen": -2.3378241062164307,
      "logits/rejected": -1.8990907669067383,
      "logps/chosen": -200.73715209960938,
      "logps/rejected": -334.19769287109375,
      "loss": -0.6271,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": 17.625381469726562,
      "rewards/margins": 72.57964324951172,
      "rewards/rejected": -54.95426559448242,
      "step": 6000
    },
    {
      "epoch": 0.8847342852936847,
      "grad_norm": 133.59524969657164,
      "learning_rate": 1.9968998940976784e-08,
      "logits/chosen": -2.340803623199463,
      "logits/rejected": -1.64697265625,
      "logps/chosen": -269.7743835449219,
      "logps/rejected": -312.74481201171875,
      "loss": -0.7011,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.20305243134498596,
      "rewards/margins": 72.11607360839844,
      "rewards/rejected": -72.31912994384766,
      "step": 6010
    },
    {
      "epoch": 0.8862063889297807,
      "grad_norm": 113.50878380800796,
      "learning_rate": 1.9468876365799054e-08,
      "logits/chosen": -2.252559185028076,
      "logits/rejected": -1.4916131496429443,
      "logps/chosen": -279.6479797363281,
      "logps/rejected": -279.4778137207031,
      "loss": -0.7795,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": 2.732151508331299,
      "rewards/margins": 67.8152847290039,
      "rewards/rejected": -65.08312225341797,
      "step": 6020
    },
    {
      "epoch": 0.8876784925658766,
      "grad_norm": 86.35223713673304,
      "learning_rate": 1.89748424203664e-08,
      "logits/chosen": -2.315373420715332,
      "logits/rejected": -1.8608535528182983,
      "logps/chosen": -300.5802307128906,
      "logps/rejected": -287.92938232421875,
      "loss": -0.8391,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 17.357128143310547,
      "rewards/margins": 52.151031494140625,
      "rewards/rejected": -34.79390335083008,
      "step": 6030
    },
    {
      "epoch": 0.8891505962019726,
      "grad_norm": 84.68742780563127,
      "learning_rate": 1.8486910152764102e-08,
      "logits/chosen": -2.283740520477295,
      "logits/rejected": -1.7487766742706299,
      "logps/chosen": -259.4682922363281,
      "logps/rejected": -315.3001403808594,
      "loss": -0.9698,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": 3.7430412769317627,
      "rewards/margins": 60.86800003051758,
      "rewards/rejected": -57.124961853027344,
      "step": 6040
    },
    {
      "epoch": 0.8906226998380686,
      "grad_norm": 273.04406164421266,
      "learning_rate": 1.8005092449924292e-08,
      "logits/chosen": -2.0152385234832764,
      "logits/rejected": -1.917507529258728,
      "logps/chosen": -238.93344116210938,
      "logps/rejected": -264.11334228515625,
      "loss": -0.6436,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -23.163488388061523,
      "rewards/margins": 23.024063110351562,
      "rewards/rejected": -46.18755340576172,
      "step": 6050
    },
    {
      "epoch": 0.8920948034741646,
      "grad_norm": 368.35897672283676,
      "learning_rate": 1.7529402037285312e-08,
      "logits/chosen": -2.2831244468688965,
      "logits/rejected": -1.9306867122650146,
      "logps/chosen": -265.66400146484375,
      "logps/rejected": -321.8489074707031,
      "loss": -0.754,
      "rewards/accuracies": 0.8333333730697632,
      "rewards/chosen": 15.182594299316406,
      "rewards/margins": 69.27645111083984,
      "rewards/rejected": -54.0938606262207,
      "step": 6060
    },
    {
      "epoch": 0.8935669071102605,
      "grad_norm": 164.61688697848226,
      "learning_rate": 1.7059851478455594e-08,
      "logits/chosen": -2.22601056098938,
      "logits/rejected": -1.7097816467285156,
      "logps/chosen": -266.8016052246094,
      "logps/rejected": -298.68438720703125,
      "loss": -0.8135,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 5.728878974914551,
      "rewards/margins": 62.31638717651367,
      "rewards/rejected": -56.58750534057617,
      "step": 6070
    },
    {
      "epoch": 0.8950390107463565,
      "grad_norm": 66.65248365280085,
      "learning_rate": 1.6596453174882145e-08,
      "logits/chosen": -1.9896303415298462,
      "logits/rejected": -1.8624740839004517,
      "logps/chosen": -207.6454620361328,
      "logps/rejected": -324.1514587402344,
      "loss": -0.8584,
      "rewards/accuracies": 0.5999999642372131,
      "rewards/chosen": -9.610658645629883,
      "rewards/margins": 48.109893798828125,
      "rewards/rejected": -57.720558166503906,
      "step": 6080
    },
    {
      "epoch": 0.8965111143824526,
      "grad_norm": 74.11984908375977,
      "learning_rate": 1.6139219365522732e-08,
      "logits/chosen": -2.320930004119873,
      "logits/rejected": -1.8024120330810547,
      "logps/chosen": -354.9090576171875,
      "logps/rejected": -382.1637878417969,
      "loss": -0.7877,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -3.388049364089966,
      "rewards/margins": 52.475852966308594,
      "rewards/rejected": -55.8639030456543,
      "step": 6090
    },
    {
      "epoch": 0.8979832180185485,
      "grad_norm": 66.62240163280188,
      "learning_rate": 1.5688162126522853e-08,
      "logits/chosen": -2.2722666263580322,
      "logits/rejected": -1.6595401763916016,
      "logps/chosen": -289.54583740234375,
      "logps/rejected": -269.2230224609375,
      "loss": -0.7767,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.3582178354263306,
      "rewards/margins": 61.02288055419922,
      "rewards/rejected": -62.38109588623047,
      "step": 6100
    },
    {
      "epoch": 0.8994553216546445,
      "grad_norm": 89.86765264837324,
      "learning_rate": 1.5243293370896554e-08,
      "logits/chosen": -2.3652002811431885,
      "logits/rejected": -1.8143867254257202,
      "logps/chosen": -271.22418212890625,
      "logps/rejected": -292.3133239746094,
      "loss": -0.6971,
      "rewards/accuracies": 0.5333333611488342,
      "rewards/chosen": 10.225584983825684,
      "rewards/margins": 49.28586959838867,
      "rewards/rejected": -39.0602912902832,
      "step": 6110
    },
    {
      "epoch": 0.9009274252907404,
      "grad_norm": 144.8508151352382,
      "learning_rate": 1.4804624848211933e-08,
      "logits/chosen": -2.2305145263671875,
      "logits/rejected": -1.8076874017715454,
      "logps/chosen": -270.6466979980469,
      "logps/rejected": -332.2124938964844,
      "loss": -0.6747,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -5.345417022705078,
      "rewards/margins": 32.88909149169922,
      "rewards/rejected": -38.23450469970703,
      "step": 6120
    },
    {
      "epoch": 0.9023995289268365,
      "grad_norm": 87.6873458626567,
      "learning_rate": 1.4372168144280744e-08,
      "logits/chosen": -2.291274309158325,
      "logits/rejected": -1.597228765487671,
      "logps/chosen": -265.6590576171875,
      "logps/rejected": -274.9820861816406,
      "loss": -0.7267,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -2.1281094551086426,
      "rewards/margins": 43.186256408691406,
      "rewards/rejected": -45.31435775756836,
      "step": 6130
    },
    {
      "epoch": 0.9038716325629325,
      "grad_norm": 92.32064084100472,
      "learning_rate": 1.3945934680852617e-08,
      "logits/chosen": -2.281552791595459,
      "logits/rejected": -1.388491153717041,
      "logps/chosen": -293.021728515625,
      "logps/rejected": -323.14752197265625,
      "loss": -0.793,
      "rewards/accuracies": 0.8666666746139526,
      "rewards/chosen": 11.804425239562988,
      "rewards/margins": 92.34114074707031,
      "rewards/rejected": -80.53669738769531,
      "step": 6140
    },
    {
      "epoch": 0.9053437361990284,
      "grad_norm": 136.3656962623486,
      "learning_rate": 1.3525935715313074e-08,
      "logits/chosen": -2.095292806625366,
      "logits/rejected": -1.702466368675232,
      "logps/chosen": -215.5361328125,
      "logps/rejected": -290.7070007324219,
      "loss": -0.6823,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -24.022085189819336,
      "rewards/margins": 41.512184143066406,
      "rewards/rejected": -65.53425598144531,
      "step": 6150
    },
    {
      "epoch": 0.9068158398351244,
      "grad_norm": 232.32682887598205,
      "learning_rate": 1.3112182340386412e-08,
      "logits/chosen": -2.093787908554077,
      "logits/rejected": -1.7462289333343506,
      "logps/chosen": -229.1507568359375,
      "logps/rejected": -257.6136474609375,
      "loss": -0.7,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -10.055268287658691,
      "rewards/margins": 57.6724739074707,
      "rewards/rejected": -67.72774505615234,
      "step": 6160
    },
    {
      "epoch": 0.9082879434712203,
      "grad_norm": 61.85619166184606,
      "learning_rate": 1.2704685483842626e-08,
      "logits/chosen": -2.360452175140381,
      "logits/rejected": -1.708276391029358,
      "logps/chosen": -192.33749389648438,
      "logps/rejected": -209.0330352783203,
      "loss": -0.7106,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 3.8913753032684326,
      "rewards/margins": 57.424400329589844,
      "rewards/rejected": -53.53303146362305,
      "step": 6170
    },
    {
      "epoch": 0.9097600471073164,
      "grad_norm": 117.01458340815451,
      "learning_rate": 1.2303455908208915e-08,
      "logits/chosen": -2.3984482288360596,
      "logits/rejected": -2.091489791870117,
      "logps/chosen": -228.8653106689453,
      "logps/rejected": -219.59201049804688,
      "loss": -0.7167,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": 2.795315980911255,
      "rewards/margins": 39.80274200439453,
      "rewards/rejected": -37.00742721557617,
      "step": 6180
    },
    {
      "epoch": 0.9112321507434124,
      "grad_norm": 144.44309638532752,
      "learning_rate": 1.1908504210485337e-08,
      "logits/chosen": -2.4019038677215576,
      "logits/rejected": -1.8438972234725952,
      "logps/chosen": -307.22967529296875,
      "logps/rejected": -281.276611328125,
      "loss": -0.7403,
      "rewards/accuracies": 0.7999999523162842,
      "rewards/chosen": 9.279439926147461,
      "rewards/margins": 43.76252746582031,
      "rewards/rejected": -34.48309326171875,
      "step": 6190
    },
    {
      "epoch": 0.9127042543795083,
      "grad_norm": 71.38658883208366,
      "learning_rate": 1.1519840821864968e-08,
      "logits/chosen": -2.2655680179595947,
      "logits/rejected": -1.768019437789917,
      "logps/chosen": -313.5616760253906,
      "logps/rejected": -356.56829833984375,
      "loss": -0.6985,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -1.076363205909729,
      "rewards/margins": 39.326141357421875,
      "rewards/rejected": -40.402503967285156,
      "step": 6200
    },
    {
      "epoch": 0.9141763580156043,
      "grad_norm": 236.77666032606575,
      "learning_rate": 1.113747600745829e-08,
      "logits/chosen": -2.371504545211792,
      "logits/rejected": -1.9415676593780518,
      "logps/chosen": -272.3446960449219,
      "logps/rejected": -390.1558837890625,
      "loss": -0.723,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": 4.292144775390625,
      "rewards/margins": 71.34015655517578,
      "rewards/rejected": -67.04800415039062,
      "step": 6210
    },
    {
      "epoch": 0.9156484616517003,
      "grad_norm": 119.2806146585152,
      "learning_rate": 1.0761419866022247e-08,
      "logits/chosen": -2.382429599761963,
      "logits/rejected": -2.1006393432617188,
      "logps/chosen": -242.2930145263672,
      "logps/rejected": -261.974853515625,
      "loss": -0.6876,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": 13.187830924987793,
      "rewards/margins": 49.577537536621094,
      "rewards/rejected": -36.38970947265625,
      "step": 6220
    },
    {
      "epoch": 0.9171205652877963,
      "grad_norm": 134.1642177289151,
      "learning_rate": 1.039168232969348e-08,
      "logits/chosen": -2.3489444255828857,
      "logits/rejected": -1.9187822341918945,
      "logps/chosen": -261.72894287109375,
      "logps/rejected": -287.34625244140625,
      "loss": -0.7073,
      "rewards/accuracies": 0.76666659116745,
      "rewards/chosen": 4.5326056480407715,
      "rewards/margins": 46.498939514160156,
      "rewards/rejected": -41.96633529663086,
      "step": 6230
    },
    {
      "epoch": 0.9185926689238922,
      "grad_norm": 116.47080949726845,
      "learning_rate": 1.0028273163725898e-08,
      "logits/chosen": -2.1793150901794434,
      "logits/rejected": -1.784123420715332,
      "logps/chosen": -340.27154541015625,
      "logps/rejected": -334.50628662109375,
      "loss": -0.7671,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -3.149559736251831,
      "rewards/margins": 56.6826171875,
      "rewards/rejected": -59.832176208496094,
      "step": 6240
    },
    {
      "epoch": 0.9200647725599882,
      "grad_norm": 283.8118753872778,
      "learning_rate": 9.671201966232889e-09,
      "logits/chosen": -2.1876354217529297,
      "logits/rejected": -2.0576915740966797,
      "logps/chosen": -237.8395538330078,
      "logps/rejected": -273.7745666503906,
      "loss": -0.6591,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -9.500242233276367,
      "rewards/margins": 29.381973266601562,
      "rewards/rejected": -38.88221740722656,
      "step": 6250
    },
    {
      "epoch": 0.9215368761960842,
      "grad_norm": 100.69388126854896,
      "learning_rate": 9.320478167933659e-09,
      "logits/chosen": -2.2262861728668213,
      "logits/rejected": -1.9294683933258057,
      "logps/chosen": -277.97076416015625,
      "logps/rejected": -330.37371826171875,
      "loss": -0.7203,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 4.644320487976074,
      "rewards/margins": 52.455055236816406,
      "rewards/rejected": -47.81072998046875,
      "step": 6260
    },
    {
      "epoch": 0.9230089798321802,
      "grad_norm": 65.76323278821008,
      "learning_rate": 8.976111031904377e-09,
      "logits/chosen": -2.6477818489074707,
      "logits/rejected": -2.066955804824829,
      "logps/chosen": -306.2950744628906,
      "logps/rejected": -274.1404113769531,
      "loss": -0.7521,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": 1.0136619806289673,
      "rewards/margins": 44.092838287353516,
      "rewards/rejected": -43.07917785644531,
      "step": 6270
    },
    {
      "epoch": 0.9244810834682762,
      "grad_norm": 93.53353151000151,
      "learning_rate": 8.638109653333419e-09,
      "logits/chosen": -2.548518657684326,
      "logits/rejected": -1.7308332920074463,
      "logps/chosen": -350.6437072753906,
      "logps/rejected": -340.27105712890625,
      "loss": -0.7776,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 7.257396697998047,
      "rewards/margins": 83.01739501953125,
      "rewards/rejected": -75.760009765625,
      "step": 6280
    },
    {
      "epoch": 0.9259531871043721,
      "grad_norm": 236.813210388252,
      "learning_rate": 8.306482959281103e-09,
      "logits/chosen": -2.06644606590271,
      "logits/rejected": -1.7106187343597412,
      "logps/chosen": -214.3295135498047,
      "logps/rejected": -237.26025390625,
      "loss": -0.7307,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": -5.25048828125,
      "rewards/margins": 56.21826934814453,
      "rewards/rejected": -61.4687614440918,
      "step": 6290
    },
    {
      "epoch": 0.9274252907404681,
      "grad_norm": 126.2118708377055,
      "learning_rate": 7.981239708443971e-09,
      "logits/chosen": -2.188920021057129,
      "logits/rejected": -1.5776751041412354,
      "logps/chosen": -295.75140380859375,
      "logps/rejected": -329.2916259765625,
      "loss": -0.76,
      "rewards/accuracies": 0.8333333730697632,
      "rewards/chosen": 4.978885173797607,
      "rewards/margins": 80.37007141113281,
      "rewards/rejected": -75.39118957519531,
      "step": 6300
    },
    {
      "epoch": 0.9288973943765642,
      "grad_norm": 110.128344243677,
      "learning_rate": 7.662388490923378e-09,
      "logits/chosen": -2.186687707901001,
      "logits/rejected": -1.5321046113967896,
      "logps/chosen": -297.8403015136719,
      "logps/rejected": -374.5328369140625,
      "loss": -0.87,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -13.389477729797363,
      "rewards/margins": 63.2852783203125,
      "rewards/rejected": -76.67475128173828,
      "step": 6310
    },
    {
      "epoch": 0.9303694980126601,
      "grad_norm": 62.10788698118774,
      "learning_rate": 7.3499377279988e-09,
      "logits/chosen": -2.3185267448425293,
      "logits/rejected": -1.9248993396759033,
      "logps/chosen": -270.59161376953125,
      "logps/rejected": -256.2392272949219,
      "loss": -0.6317,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -13.846537590026855,
      "rewards/margins": 34.31627655029297,
      "rewards/rejected": -48.162818908691406,
      "step": 6320
    },
    {
      "epoch": 0.9318416016487561,
      "grad_norm": 84.10334309889404,
      "learning_rate": 7.043895671905248e-09,
      "logits/chosen": -2.30346417427063,
      "logits/rejected": -2.0712122917175293,
      "logps/chosen": -243.4429168701172,
      "logps/rejected": -306.1946105957031,
      "loss": -0.713,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.2218431532382965,
      "rewards/margins": 37.28938293457031,
      "rewards/rejected": -37.067543029785156,
      "step": 6330
    },
    {
      "epoch": 0.933313705284852,
      "grad_norm": 108.37529986228502,
      "learning_rate": 6.744270405615371e-09,
      "logits/chosen": -2.24432635307312,
      "logits/rejected": -1.8119218349456787,
      "logps/chosen": -274.7903747558594,
      "logps/rejected": -315.25146484375,
      "loss": -0.9095,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": 23.60199546813965,
      "rewards/margins": 94.67326354980469,
      "rewards/rejected": -71.0712661743164,
      "step": 6340
    },
    {
      "epoch": 0.934785808920948,
      "grad_norm": 73.49520922151967,
      "learning_rate": 6.4510698426259165e-09,
      "logits/chosen": -2.4096789360046387,
      "logits/rejected": -1.880297303199768,
      "logps/chosen": -258.64208984375,
      "logps/rejected": -281.1539611816406,
      "loss": -0.8252,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": 8.757104873657227,
      "rewards/margins": 56.78657150268555,
      "rewards/rejected": -48.02946472167969,
      "step": 6350
    },
    {
      "epoch": 0.936257912557044,
      "grad_norm": 117.26522915636508,
      "learning_rate": 6.164301726748838e-09,
      "logits/chosen": -2.3384957313537598,
      "logits/rejected": -1.7874279022216797,
      "logps/chosen": -269.7473449707031,
      "logps/rejected": -274.350341796875,
      "loss": -0.7193,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 7.612980842590332,
      "rewards/margins": 68.10344696044922,
      "rewards/rejected": -60.4904670715332,
      "step": 6360
    },
    {
      "epoch": 0.93773001619314,
      "grad_norm": 74.35244889621339,
      "learning_rate": 5.883973631906736e-09,
      "logits/chosen": -2.2826719284057617,
      "logits/rejected": -1.7989610433578491,
      "logps/chosen": -267.5291748046875,
      "logps/rejected": -310.46966552734375,
      "loss": -0.7036,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -17.17521858215332,
      "rewards/margins": 41.40718078613281,
      "rewards/rejected": -58.58240509033203,
      "step": 6370
    },
    {
      "epoch": 0.939202119829236,
      "grad_norm": 111.20245502455114,
      "learning_rate": 5.610092961932766e-09,
      "logits/chosen": -2.1844449043273926,
      "logits/rejected": -2.008172035217285,
      "logps/chosen": -279.8955383300781,
      "logps/rejected": -315.6399841308594,
      "loss": -0.7819,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -4.45657205581665,
      "rewards/margins": 57.62977981567383,
      "rewards/rejected": -62.08635330200195,
      "step": 6380
    },
    {
      "epoch": 0.9406742234653319,
      "grad_norm": 91.79628106069006,
      "learning_rate": 5.3426669503751355e-09,
      "logits/chosen": -2.3842360973358154,
      "logits/rejected": -1.7594025135040283,
      "logps/chosen": -326.11895751953125,
      "logps/rejected": -302.1062927246094,
      "loss": -0.7729,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": 0.217681884765625,
      "rewards/margins": 61.229957580566406,
      "rewards/rejected": -61.01227569580078,
      "step": 6390
    },
    {
      "epoch": 0.942146327101428,
      "grad_norm": 145.5751603681799,
      "learning_rate": 5.081702660306025e-09,
      "logits/chosen": -2.43515682220459,
      "logits/rejected": -1.6579389572143555,
      "logps/chosen": -278.21478271484375,
      "logps/rejected": -351.71832275390625,
      "loss": -1.8922,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -10.224485397338867,
      "rewards/margins": 71.3990478515625,
      "rewards/rejected": -81.62353515625,
      "step": 6400
    },
    {
      "epoch": 0.9436184307375239,
      "grad_norm": 78.05438574690228,
      "learning_rate": 4.827206984135024e-09,
      "logits/chosen": -2.3385891914367676,
      "logits/rejected": -2.0188231468200684,
      "logps/chosen": -211.88858032226562,
      "logps/rejected": -259.54864501953125,
      "loss": -0.734,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 2.5906856060028076,
      "rewards/margins": 42.05583953857422,
      "rewards/rejected": -39.46515655517578,
      "step": 6410
    },
    {
      "epoch": 0.9450905343736199,
      "grad_norm": 91.41898507525372,
      "learning_rate": 4.579186643427241e-09,
      "logits/chosen": -2.2000410556793213,
      "logits/rejected": -1.710097312927246,
      "logps/chosen": -394.6190490722656,
      "logps/rejected": -334.26190185546875,
      "loss": -1.4909,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 5.625645637512207,
      "rewards/margins": 51.729156494140625,
      "rewards/rejected": -46.103515625,
      "step": 6420
    },
    {
      "epoch": 0.9465626380097159,
      "grad_norm": 95.67708959337492,
      "learning_rate": 4.337648188725618e-09,
      "logits/chosen": -2.246196746826172,
      "logits/rejected": -1.6747539043426514,
      "logps/chosen": -282.8218688964844,
      "logps/rejected": -295.25555419921875,
      "loss": -1.0542,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 9.983587265014648,
      "rewards/margins": 62.557594299316406,
      "rewards/rejected": -52.573997497558594,
      "step": 6430
    },
    {
      "epoch": 0.9480347416458119,
      "grad_norm": 88.60924397840357,
      "learning_rate": 4.102597999377927e-09,
      "logits/chosen": -2.2350571155548096,
      "logits/rejected": -1.816972017288208,
      "logps/chosen": -334.84283447265625,
      "logps/rejected": -399.1749572753906,
      "loss": -0.7463,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -4.4422197341918945,
      "rewards/margins": 53.0917854309082,
      "rewards/rejected": -57.53400802612305,
      "step": 6440
    },
    {
      "epoch": 0.9495068452819079,
      "grad_norm": 115.00895459305956,
      "learning_rate": 3.874042283368406e-09,
      "logits/chosen": -2.346817970275879,
      "logits/rejected": -1.9208383560180664,
      "logps/chosen": -247.54800415039062,
      "logps/rejected": -311.3568420410156,
      "loss": -0.7274,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -10.356578826904297,
      "rewards/margins": 42.065269470214844,
      "rewards/rejected": -52.421852111816406,
      "step": 6450
    },
    {
      "epoch": 0.9509789489180038,
      "grad_norm": 169.42060901943742,
      "learning_rate": 3.6519870771536952e-09,
      "logits/chosen": -2.1893887519836426,
      "logits/rejected": -2.153595209121704,
      "logps/chosen": -259.8551940917969,
      "logps/rejected": -362.0740966796875,
      "loss": -0.6936,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -5.575837135314941,
      "rewards/margins": 32.25075912475586,
      "rewards/rejected": -37.82659912109375,
      "step": 6460
    },
    {
      "epoch": 0.9524510525540998,
      "grad_norm": 99.95262621246927,
      "learning_rate": 3.436438245503409e-09,
      "logits/chosen": -2.1909940242767334,
      "logits/rejected": -1.7248694896697998,
      "logps/chosen": -298.1640930175781,
      "logps/rejected": -330.6931457519531,
      "loss": -0.7063,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": 2.519141435623169,
      "rewards/margins": 52.98332977294922,
      "rewards/rejected": -50.464195251464844,
      "step": 6470
    },
    {
      "epoch": 0.9539231561901957,
      "grad_norm": 88.24484087311127,
      "learning_rate": 3.2274014813452875e-09,
      "logits/chosen": -2.141745090484619,
      "logits/rejected": -1.6937854290008545,
      "logps/chosen": -238.09213256835938,
      "logps/rejected": -365.1256408691406,
      "loss": -0.7357,
      "rewards/accuracies": 0.76666659116745,
      "rewards/chosen": -12.985589981079102,
      "rewards/margins": 55.926963806152344,
      "rewards/rejected": -68.91255187988281,
      "step": 6480
    },
    {
      "epoch": 0.9553952598262918,
      "grad_norm": 122.26140153790637,
      "learning_rate": 3.0248823056148175e-09,
      "logits/chosen": -2.102341651916504,
      "logits/rejected": -2.1420738697052,
      "logps/chosen": -265.5021057128906,
      "logps/rejected": -320.6543273925781,
      "loss": 0.0443,
      "rewards/accuracies": 0.7000000476837158,
      "rewards/chosen": -7.671290397644043,
      "rewards/margins": 50.940696716308594,
      "rewards/rejected": -58.61198806762695,
      "step": 6490
    },
    {
      "epoch": 0.9568673634623878,
      "grad_norm": 1353.853713837786,
      "learning_rate": 2.828886067109404e-09,
      "logits/chosen": -2.2932066917419434,
      "logits/rejected": -1.804426908493042,
      "logps/chosen": -308.35125732421875,
      "logps/rejected": -335.9036560058594,
      "loss": -0.6367,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": -4.164955139160156,
      "rewards/margins": 58.80066680908203,
      "rewards/rejected": -62.96561813354492,
      "step": 6500
    },
    {
      "epoch": 0.9583394670984837,
      "grad_norm": 604.0594242454009,
      "learning_rate": 2.639417942347122e-09,
      "logits/chosen": -2.352555751800537,
      "logits/rejected": -1.7515252828598022,
      "logps/chosen": -222.1473388671875,
      "logps/rejected": -318.51422119140625,
      "loss": -0.7781,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 2.8718228340148926,
      "rewards/margins": 74.02092742919922,
      "rewards/rejected": -71.14910125732422,
      "step": 6510
    },
    {
      "epoch": 0.9598115707345797,
      "grad_norm": 113.298768417235,
      "learning_rate": 2.456482935429993e-09,
      "logits/chosen": -2.4401726722717285,
      "logits/rejected": -1.857866644859314,
      "logps/chosen": -276.2210388183594,
      "logps/rejected": -304.19696044921875,
      "loss": -0.7259,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 0.6905351877212524,
      "rewards/margins": 50.18748092651367,
      "rewards/rejected": -49.4969482421875,
      "step": 6520
    },
    {
      "epoch": 0.9612836743706757,
      "grad_norm": 73.21123679554442,
      "learning_rate": 2.2800858779118417e-09,
      "logits/chosen": -2.1220879554748535,
      "logits/rejected": -1.6453192234039307,
      "logps/chosen": -262.3907470703125,
      "logps/rejected": -304.8507995605469,
      "loss": -0.457,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -1.7461726665496826,
      "rewards/margins": 60.20293426513672,
      "rewards/rejected": -61.94911575317383,
      "step": 6530
    },
    {
      "epoch": 0.9627557780067717,
      "grad_norm": 389.81916311570745,
      "learning_rate": 2.1102314286705334e-09,
      "logits/chosen": -2.486177444458008,
      "logits/rejected": -2.2106292247772217,
      "logps/chosen": -326.3155517578125,
      "logps/rejected": -305.4042663574219,
      "loss": -0.7587,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": 10.102300643920898,
      "rewards/margins": 29.281429290771484,
      "rewards/rejected": -19.17913246154785,
      "step": 6540
    },
    {
      "epoch": 0.9642278816428677,
      "grad_norm": 536.2561935172404,
      "learning_rate": 1.9469240737852445e-09,
      "logits/chosen": -2.214005947113037,
      "logits/rejected": -2.0208189487457275,
      "logps/chosen": -186.35369873046875,
      "logps/rejected": -260.21160888671875,
      "loss": -0.7556,
      "rewards/accuracies": 0.7999999523162842,
      "rewards/chosen": 8.95404052734375,
      "rewards/margins": 62.0992546081543,
      "rewards/rejected": -53.14521408081055,
      "step": 6550
    },
    {
      "epoch": 0.9656999852789636,
      "grad_norm": 94.81692613790108,
      "learning_rate": 1.790168126417635e-09,
      "logits/chosen": -2.4521591663360596,
      "logits/rejected": -1.6262309551239014,
      "logps/chosen": -211.48880004882812,
      "logps/rejected": -259.4698791503906,
      "loss": -0.8543,
      "rewards/accuracies": 0.7999999523162842,
      "rewards/chosen": 11.022046089172363,
      "rewards/margins": 71.4749984741211,
      "rewards/rejected": -60.45295333862305,
      "step": 6560
    },
    {
      "epoch": 0.9671720889150596,
      "grad_norm": 112.31630028844862,
      "learning_rate": 1.6399677266982214e-09,
      "logits/chosen": -2.1532835960388184,
      "logits/rejected": -1.7811733484268188,
      "logps/chosen": -311.2800598144531,
      "logps/rejected": -325.8093566894531,
      "loss": -0.8,
      "rewards/accuracies": 0.7000000476837158,
      "rewards/chosen": -7.245842933654785,
      "rewards/margins": 51.746856689453125,
      "rewards/rejected": -58.992706298828125,
      "step": 6570
    },
    {
      "epoch": 0.9686441925511556,
      "grad_norm": 297.71034082918493,
      "learning_rate": 1.4963268416167685e-09,
      "logits/chosen": -2.202979803085327,
      "logits/rejected": -1.5451149940490723,
      "logps/chosen": -269.4142761230469,
      "logps/rejected": -304.4900207519531,
      "loss": -0.729,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 5.907937049865723,
      "rewards/margins": 73.07244873046875,
      "rewards/rejected": -67.16450500488281,
      "step": 6580
    },
    {
      "epoch": 0.9701162961872516,
      "grad_norm": 75.71362202089666,
      "learning_rate": 1.3592492649177056e-09,
      "logits/chosen": -2.304546594619751,
      "logits/rejected": -1.8261032104492188,
      "logps/chosen": -254.3771209716797,
      "logps/rejected": -284.8136901855469,
      "loss": -0.7152,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 7.427770137786865,
      "rewards/margins": 80.71339416503906,
      "rewards/rejected": -73.2856216430664,
      "step": 6590
    },
    {
      "epoch": 0.9715883998233475,
      "grad_norm": 69.98510253002826,
      "learning_rate": 1.2287386169998747e-09,
      "logits/chosen": -2.475227117538452,
      "logits/rejected": -1.7404963970184326,
      "logps/chosen": -258.05548095703125,
      "logps/rejected": -293.58660888671875,
      "loss": -0.688,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": 0.7432466745376587,
      "rewards/margins": 72.51031494140625,
      "rewards/rejected": -71.7670669555664,
      "step": 6600
    },
    {
      "epoch": 0.9730605034594435,
      "grad_norm": 101.38583923745865,
      "learning_rate": 1.1047983448209397e-09,
      "logits/chosen": -2.3193576335906982,
      "logits/rejected": -1.7693437337875366,
      "logps/chosen": -272.04833984375,
      "logps/rejected": -308.0151062011719,
      "loss": -0.8416,
      "rewards/accuracies": 0.8333333134651184,
      "rewards/chosen": 25.532018661499023,
      "rewards/margins": 65.42366027832031,
      "rewards/rejected": -39.891639709472656,
      "step": 6610
    },
    {
      "epoch": 0.9745326070955396,
      "grad_norm": 101.23909464168035,
      "learning_rate": 9.87431721806209e-10,
      "logits/chosen": -2.5541205406188965,
      "logits/rejected": -2.034274101257324,
      "logps/chosen": -258.8510437011719,
      "logps/rejected": -267.6814270019531,
      "loss": -0.8206,
      "rewards/accuracies": 0.7666667103767395,
      "rewards/chosen": 13.036738395690918,
      "rewards/margins": 56.21497344970703,
      "rewards/rejected": -43.17824172973633,
      "step": 6620
    },
    {
      "epoch": 0.9760047107316355,
      "grad_norm": 109.74631541860946,
      "learning_rate": 8.766418477623716e-10,
      "logits/chosen": -2.251767635345459,
      "logits/rejected": -1.6601579189300537,
      "logps/chosen": -258.80462646484375,
      "logps/rejected": -274.0628967285156,
      "loss": -0.7914,
      "rewards/accuracies": 0.7999999523162842,
      "rewards/chosen": 16.269901275634766,
      "rewards/margins": 65.83000183105469,
      "rewards/rejected": -49.56010818481445,
      "step": 6630
    },
    {
      "epoch": 0.9774768143677315,
      "grad_norm": 98.76345511831263,
      "learning_rate": 7.724316487954796e-10,
      "logits/chosen": -2.3322033882141113,
      "logits/rejected": -1.885703682899475,
      "logps/chosen": -239.4267120361328,
      "logps/rejected": -300.558349609375,
      "loss": -0.6713,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 7.8031744956970215,
      "rewards/margins": 56.65901565551758,
      "rewards/rejected": -48.855838775634766,
      "step": 6640
    },
    {
      "epoch": 0.9789489180038274,
      "grad_norm": 165.96098313100424,
      "learning_rate": 6.748038772337595e-10,
      "logits/chosen": -2.241995334625244,
      "logits/rejected": -1.653377890586853,
      "logps/chosen": -266.0572204589844,
      "logps/rejected": -298.50823974609375,
      "loss": -0.6345,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -0.5360725522041321,
      "rewards/margins": 48.91666793823242,
      "rewards/rejected": -49.4527473449707,
      "step": 6650
    },
    {
      "epoch": 0.9804210216399234,
      "grad_norm": 90.92425872773774,
      "learning_rate": 5.837611115549201e-10,
      "logits/chosen": -2.543891429901123,
      "logits/rejected": -1.6706173419952393,
      "logps/chosen": -309.26531982421875,
      "logps/rejected": -292.6114196777344,
      "loss": -0.7665,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": 13.822061538696289,
      "rewards/margins": 59.3404655456543,
      "rewards/rejected": -45.51840591430664,
      "step": 6660
    },
    {
      "epoch": 0.9818931252760195,
      "grad_norm": 143.76183688952105,
      "learning_rate": 4.993057563179303e-10,
      "logits/chosen": -2.4913487434387207,
      "logits/rejected": -1.7680962085723877,
      "logps/chosen": -334.53094482421875,
      "logps/rejected": -301.77032470703125,
      "loss": -0.8193,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": 1.2772624492645264,
      "rewards/margins": 63.0628547668457,
      "rewards/rejected": -61.78559112548828,
      "step": 6670
    },
    {
      "epoch": 0.9833652289121154,
      "grad_norm": 198.83076472726057,
      "learning_rate": 4.2144004209962445e-10,
      "logits/chosen": -2.237600803375244,
      "logits/rejected": -2.164032459259033,
      "logps/chosen": -310.2505798339844,
      "logps/rejected": -308.8753967285156,
      "loss": -0.8693,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": 16.49312973022461,
      "rewards/margins": 38.65729904174805,
      "rewards/rejected": -22.164165496826172,
      "step": 6680
    },
    {
      "epoch": 0.9848373325482114,
      "grad_norm": 192.48406649935333,
      "learning_rate": 3.501660254358607e-10,
      "logits/chosen": -2.153223752975464,
      "logits/rejected": -1.510780692100525,
      "logps/chosen": -240.48727416992188,
      "logps/rejected": -282.06719970703125,
      "loss": -0.7876,
      "rewards/accuracies": 0.6666666269302368,
      "rewards/chosen": -15.80981159210205,
      "rewards/margins": 68.04016876220703,
      "rewards/rejected": -83.84999084472656,
      "step": 6690
    },
    {
      "epoch": 0.9863094361843073,
      "grad_norm": 231.19812999078704,
      "learning_rate": 2.854855887669538e-10,
      "logits/chosen": -2.1755497455596924,
      "logits/rejected": -2.205728054046631,
      "logps/chosen": -320.7024841308594,
      "logps/rejected": -374.41302490234375,
      "loss": -0.726,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -7.06673526763916,
      "rewards/margins": 30.04939842224121,
      "rewards/rejected": -37.11613082885742,
      "step": 6700
    },
    {
      "epoch": 0.9877815398204034,
      "grad_norm": 79.5052567895229,
      "learning_rate": 2.274004403882146e-10,
      "logits/chosen": -2.4645824432373047,
      "logits/rejected": -1.7773373126983643,
      "logps/chosen": -267.2254638671875,
      "logps/rejected": -350.01971435546875,
      "loss": -0.7617,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": 3.878751039505005,
      "rewards/margins": 75.93704986572266,
      "rewards/rejected": -72.05828857421875,
      "step": 6710
    },
    {
      "epoch": 0.9892536434564994,
      "grad_norm": 93.9387154652332,
      "learning_rate": 1.7591211440468047e-10,
      "logits/chosen": -2.223320245742798,
      "logits/rejected": -1.6575326919555664,
      "logps/chosen": -253.0481414794922,
      "logps/rejected": -307.81103515625,
      "loss": -0.7253,
      "rewards/accuracies": 0.8666666150093079,
      "rewards/chosen": 14.505145072937012,
      "rewards/margins": 87.2362060546875,
      "rewards/rejected": -72.73106384277344,
      "step": 6720
    },
    {
      "epoch": 0.9907257470925953,
      "grad_norm": 81.75763586790833,
      "learning_rate": 1.3102197069067567e-10,
      "logits/chosen": -2.3184447288513184,
      "logits/rejected": -1.6614739894866943,
      "logps/chosen": -270.355224609375,
      "logps/rejected": -318.6822204589844,
      "loss": -0.8169,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.4444900751113892,
      "rewards/margins": 69.64049530029297,
      "rewards/rejected": -71.08499145507812,
      "step": 6730
    },
    {
      "epoch": 0.9921978507286913,
      "grad_norm": 89.94572601661541,
      "learning_rate": 9.273119485386783e-11,
      "logits/chosen": -2.346011161804199,
      "logits/rejected": -1.7475192546844482,
      "logps/chosen": -298.18267822265625,
      "logps/rejected": -371.02374267578125,
      "loss": -0.7239,
      "rewards/accuracies": 0.6666666865348816,
      "rewards/chosen": -13.411401748657227,
      "rewards/margins": 55.30180740356445,
      "rewards/rejected": -68.71321105957031,
      "step": 6740
    },
    {
      "epoch": 0.9936699543647873,
      "grad_norm": 102.64586022268126,
      "learning_rate": 6.104079820390407e-11,
      "logits/chosen": -2.350371837615967,
      "logits/rejected": -1.9945220947265625,
      "logps/chosen": -307.3117980957031,
      "logps/rejected": -338.32757568359375,
      "loss": -0.665,
      "rewards/accuracies": 0.7333333492279053,
      "rewards/chosen": 1.8768154382705688,
      "rewards/margins": 43.738624572753906,
      "rewards/rejected": -41.86180877685547,
      "step": 6750
    },
    {
      "epoch": 0.9951420580008833,
      "grad_norm": 108.75368330460739,
      "learning_rate": 3.595161772582123e-11,
      "logits/chosen": -2.4512977600097656,
      "logits/rejected": -1.8409268856048584,
      "logps/chosen": -217.0909423828125,
      "logps/rejected": -225.4525604248047,
      "loss": -0.7403,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 10.948321342468262,
      "rewards/margins": 47.81084442138672,
      "rewards/rejected": -36.862518310546875,
      "step": 6760
    },
    {
      "epoch": 0.9966141616369792,
      "grad_norm": 90.21975143691185,
      "learning_rate": 1.7464316057785866e-11,
      "logits/chosen": -2.2038464546203613,
      "logits/rejected": -1.7020190954208374,
      "logps/chosen": -296.98895263671875,
      "logps/rejected": -318.6799621582031,
      "loss": -0.7325,
      "rewards/accuracies": 0.6333333253860474,
      "rewards/chosen": -1.0464004278182983,
      "rewards/margins": 36.1985969543457,
      "rewards/rejected": -37.244998931884766,
      "step": 6770
    },
    {
      "epoch": 0.9980862652730752,
      "grad_norm": 148.60155994093634,
      "learning_rate": 5.5793814737192805e-12,
      "logits/chosen": -2.1566977500915527,
      "logits/rejected": -2.054730176925659,
      "logps/chosen": -298.15924072265625,
      "logps/rejected": -287.9697570800781,
      "loss": -0.6917,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -14.498701095581055,
      "rewards/margins": 30.409961700439453,
      "rewards/rejected": -44.90866470336914,
      "step": 6780
    },
    {
      "epoch": 0.9995583689091712,
      "grad_norm": 203.8571098276532,
      "learning_rate": 2.9712787039115617e-13,
      "logits/chosen": -2.2216525077819824,
      "logits/rejected": -1.7771928310394287,
      "logps/chosen": -230.14013671875,
      "logps/rejected": -265.4197082519531,
      "loss": -0.7882,
      "rewards/accuracies": 0.7333332896232605,
      "rewards/chosen": -4.7097344398498535,
      "rewards/margins": 52.28461837768555,
      "rewards/rejected": -56.994346618652344,
      "step": 6790
    },
    {
      "epoch": 1.0,
      "step": 6793,
      "total_flos": 0.0,
      "train_loss": 1.8018821151218305,
      "train_runtime": 26417.7774,
      "train_samples_per_second": 2.314,
      "train_steps_per_second": 0.257
    }
  ],
  "logging_steps": 10,
  "max_steps": 6793,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 2000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 3,
  "trial_name": null,
  "trial_params": null
}