{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.05592059275828324,
  "eval_steps": 50,
  "global_step": 200,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0002796029637914162,
      "grad_norm": 25.76739404251824,
      "learning_rate": 5e-08,
      "logits/chosen": -0.9193593263626099,
      "logits/rejected": -1.1508712768554688,
      "logps/chosen": -0.9903213977813721,
      "logps/rejected": -0.9371722936630249,
      "loss": 1.683,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.4758033752441406,
      "rewards/margins": -0.13287273049354553,
      "rewards/rejected": -2.342930793762207,
      "step": 1
    },
    {
      "epoch": 0.0005592059275828324,
      "grad_norm": 45.297284297927725,
      "learning_rate": 1e-07,
      "logits/chosen": -0.8994183540344238,
      "logits/rejected": -1.2252119779586792,
      "logps/chosen": -1.4980652332305908,
      "logps/rejected": -1.216281533241272,
      "loss": 1.7319,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -3.7451629638671875,
      "rewards/margins": -0.7044591903686523,
      "rewards/rejected": -3.040703773498535,
      "step": 2
    },
    {
      "epoch": 0.0008388088913742485,
      "grad_norm": 31.873718310326044,
      "learning_rate": 1.5e-07,
      "logits/chosen": -0.5185738205909729,
      "logits/rejected": -1.3382601737976074,
      "logps/chosen": -1.3677482604980469,
      "logps/rejected": -1.4751423597335815,
      "loss": 1.4124,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -3.419370412826538,
      "rewards/margins": 0.26848527789115906,
      "rewards/rejected": -3.6878557205200195,
      "step": 3
    },
    {
      "epoch": 0.0011184118551656648,
      "grad_norm": 48.966957322673935,
      "learning_rate": 2e-07,
      "logits/chosen": -0.6579636931419373,
      "logits/rejected": -1.3838244676589966,
      "logps/chosen": -0.9612038731575012,
      "logps/rejected": -1.2441552877426147,
      "loss": 1.4082,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.4030096530914307,
      "rewards/margins": 0.7073784470558167,
      "rewards/rejected": -3.1103880405426025,
      "step": 4
    },
    {
      "epoch": 0.001398014818957081,
      "grad_norm": 46.91034256260659,
      "learning_rate": 2.5e-07,
      "logits/chosen": -1.2698190212249756,
      "logits/rejected": -1.5386033058166504,
      "logps/chosen": -1.6302634477615356,
      "logps/rejected": -1.7429945468902588,
      "loss": 1.6189,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -4.075658798217773,
      "rewards/margins": 0.28182780742645264,
      "rewards/rejected": -4.357486248016357,
      "step": 5
    },
    {
      "epoch": 0.001677617782748497,
      "grad_norm": 73.65526070267136,
      "learning_rate": 3e-07,
      "logits/chosen": -0.8474517464637756,
      "logits/rejected": -0.7566354274749756,
      "logps/chosen": -1.1040312051773071,
      "logps/rejected": -1.1213470697402954,
      "loss": 1.3294,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.760077953338623,
      "rewards/margins": 0.04328957200050354,
      "rewards/rejected": -2.8033676147460938,
      "step": 6
    },
    {
      "epoch": 0.001957220746539913,
      "grad_norm": 24.84640983932336,
      "learning_rate": 3.5e-07,
      "logits/chosen": -0.8542510271072388,
      "logits/rejected": -0.34997448325157166,
      "logps/chosen": -0.7964720726013184,
      "logps/rejected": -0.8464012145996094,
      "loss": 1.583,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -1.991180181503296,
      "rewards/margins": 0.12482279539108276,
      "rewards/rejected": -2.1160030364990234,
      "step": 7
    },
    {
      "epoch": 0.0022368237103313295,
      "grad_norm": 22.841316781980318,
      "learning_rate": 4e-07,
      "logits/chosen": -0.7528856992721558,
      "logits/rejected": -1.2337926626205444,
      "logps/chosen": -1.1317272186279297,
      "logps/rejected": -1.4922257661819458,
      "loss": 1.3675,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.829317808151245,
      "rewards/margins": 0.9012465476989746,
      "rewards/rejected": -3.7305643558502197,
      "step": 8
    },
    {
      "epoch": 0.002516426674122746,
      "grad_norm": 87.07632980072657,
      "learning_rate": 4.5e-07,
      "logits/chosen": -0.9475423693656921,
      "logits/rejected": -1.4221036434173584,
      "logps/chosen": -1.2074404954910278,
      "logps/rejected": -0.9064420461654663,
      "loss": 1.7839,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -3.018601417541504,
      "rewards/margins": -0.7524962425231934,
      "rewards/rejected": -2.2661051750183105,
      "step": 9
    },
    {
      "epoch": 0.002796029637914162,
      "grad_norm": 27.009928075261808,
      "learning_rate": 5e-07,
      "logits/chosen": -0.6325011253356934,
      "logits/rejected": -1.1582834720611572,
      "logps/chosen": -0.7608655691146851,
      "logps/rejected": -0.8033685088157654,
      "loss": 1.3733,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.9021639823913574,
      "rewards/margins": 0.10625733435153961,
      "rewards/rejected": -2.0084214210510254,
      "step": 10
    },
    {
      "epoch": 0.003075632601705578,
      "grad_norm": 30.088075877499065,
      "learning_rate": 5.5e-07,
      "logits/chosen": -1.018723964691162,
      "logits/rejected": -1.0942765474319458,
      "logps/chosen": -0.9146077632904053,
      "logps/rejected": -0.8910419344902039,
      "loss": 1.2264,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.2865195274353027,
      "rewards/margins": -0.058914512395858765,
      "rewards/rejected": -2.227604866027832,
      "step": 11
    },
    {
      "epoch": 0.003355235565496994,
      "grad_norm": 69.74430608906756,
      "learning_rate": 6e-07,
      "logits/chosen": -1.0309629440307617,
      "logits/rejected": -1.2155206203460693,
      "logps/chosen": -1.868607521057129,
      "logps/rejected": -2.074054002761841,
      "loss": 1.3981,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -4.6715192794799805,
      "rewards/margins": 0.5136159062385559,
      "rewards/rejected": -5.1851348876953125,
      "step": 12
    },
    {
      "epoch": 0.0036348385292884105,
      "grad_norm": 33.254043165620104,
      "learning_rate": 6.5e-07,
      "logits/chosen": -1.0877211093902588,
      "logits/rejected": -1.1691397428512573,
      "logps/chosen": -1.4225409030914307,
      "logps/rejected": -1.5891215801239014,
      "loss": 1.9712,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.556352138519287,
      "rewards/margins": 0.4164515733718872,
      "rewards/rejected": -3.9728035926818848,
      "step": 13
    },
    {
      "epoch": 0.003914441493079826,
      "grad_norm": 21.755018885778018,
      "learning_rate": 7e-07,
      "logits/chosen": -0.5653956532478333,
      "logits/rejected": -1.170688271522522,
      "logps/chosen": -1.041928768157959,
      "logps/rejected": -1.546263337135315,
      "loss": 1.4989,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.6048219203948975,
      "rewards/margins": 1.2608366012573242,
      "rewards/rejected": -3.8656585216522217,
      "step": 14
    },
    {
      "epoch": 0.004194044456871243,
      "grad_norm": 69.81299826798852,
      "learning_rate": 7.5e-07,
      "logits/chosen": -0.4804845452308655,
      "logits/rejected": -0.8382622003555298,
      "logps/chosen": -1.0265494585037231,
      "logps/rejected": -1.0567755699157715,
      "loss": 1.4168,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.566373586654663,
      "rewards/margins": 0.07556554675102234,
      "rewards/rejected": -2.641939163208008,
      "step": 15
    },
    {
      "epoch": 0.004473647420662659,
      "grad_norm": 24.193090245288765,
      "learning_rate": 8e-07,
      "logits/chosen": -0.6370764970779419,
      "logits/rejected": -1.0934374332427979,
      "logps/chosen": -0.71354079246521,
      "logps/rejected": -1.11177396774292,
      "loss": 0.9962,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.783851981163025,
      "rewards/margins": 0.9955829977989197,
      "rewards/rejected": -2.7794349193573,
      "step": 16
    },
    {
      "epoch": 0.004753250384454075,
      "grad_norm": 25.26345718560276,
      "learning_rate": 8.499999999999999e-07,
      "logits/chosen": -0.9751855134963989,
      "logits/rejected": -1.1418442726135254,
      "logps/chosen": -1.2313238382339478,
      "logps/rejected": -1.6264705657958984,
      "loss": 1.4442,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.0783097743988037,
      "rewards/margins": 0.9878668189048767,
      "rewards/rejected": -4.066176414489746,
      "step": 17
    },
    {
      "epoch": 0.005032853348245492,
      "grad_norm": 15.702824197515223,
      "learning_rate": 9e-07,
      "logits/chosen": -0.9074665904045105,
      "logits/rejected": -1.053283929824829,
      "logps/chosen": -0.8471686244010925,
      "logps/rejected": -1.086425542831421,
      "loss": 1.4019,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.1179215908050537,
      "rewards/margins": 0.5981425046920776,
      "rewards/rejected": -2.716063976287842,
      "step": 18
    },
    {
      "epoch": 0.005312456312036908,
      "grad_norm": 26.78497329037122,
      "learning_rate": 9.499999999999999e-07,
      "logits/chosen": -0.8664759993553162,
      "logits/rejected": -0.9702961444854736,
      "logps/chosen": -0.9274067878723145,
      "logps/rejected": -1.054181694984436,
      "loss": 1.1298,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.318516969680786,
      "rewards/margins": 0.31693723797798157,
      "rewards/rejected": -2.6354541778564453,
      "step": 19
    },
    {
      "epoch": 0.005592059275828324,
      "grad_norm": 43.484383450898136,
      "learning_rate": 1e-06,
      "logits/chosen": -0.9217432141304016,
      "logits/rejected": -1.0792617797851562,
      "logps/chosen": -1.1316174268722534,
      "logps/rejected": -1.3112761974334717,
      "loss": 1.6817,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.8290436267852783,
      "rewards/margins": 0.44914710521698,
      "rewards/rejected": -3.2781906127929688,
      "step": 20
    },
    {
      "epoch": 0.0058716622396197396,
      "grad_norm": 23.72254452641318,
      "learning_rate": 9.999238475781957e-07,
      "logits/chosen": -0.892957866191864,
      "logits/rejected": -0.896960973739624,
      "logps/chosen": -1.0195448398590088,
      "logps/rejected": -1.172441005706787,
      "loss": 1.4088,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.5488619804382324,
      "rewards/margins": 0.3822404146194458,
      "rewards/rejected": -2.9311022758483887,
      "step": 21
    },
    {
      "epoch": 0.006151265203411156,
      "grad_norm": 19.983043621451948,
      "learning_rate": 9.996954135095478e-07,
      "logits/chosen": -0.815680742263794,
      "logits/rejected": -1.0532113313674927,
      "logps/chosen": -1.1461201906204224,
      "logps/rejected": -1.190143346786499,
      "loss": 1.4757,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.865300416946411,
      "rewards/margins": 0.11005795001983643,
      "rewards/rejected": -2.975358486175537,
      "step": 22
    },
    {
      "epoch": 0.006430868167202572,
      "grad_norm": 130.05670927021893,
      "learning_rate": 9.99314767377287e-07,
      "logits/chosen": -1.0124224424362183,
      "logits/rejected": -1.3622022867202759,
      "logps/chosen": -1.240678310394287,
      "logps/rejected": -1.411242961883545,
      "loss": 1.4839,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.1016955375671387,
      "rewards/margins": 0.42641204595565796,
      "rewards/rejected": -3.5281076431274414,
      "step": 23
    },
    {
      "epoch": 0.006710471130993988,
      "grad_norm": 22.122176134369795,
      "learning_rate": 9.98782025129912e-07,
      "logits/chosen": -1.276932716369629,
      "logits/rejected": -1.717280387878418,
      "logps/chosen": -1.536954402923584,
      "logps/rejected": -2.9943461418151855,
      "loss": 0.6631,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.84238600730896,
      "rewards/margins": 3.643479108810425,
      "rewards/rejected": -7.485865592956543,
      "step": 24
    },
    {
      "epoch": 0.006990074094785405,
      "grad_norm": 25.203974129539585,
      "learning_rate": 9.980973490458728e-07,
      "logits/chosen": -1.0255053043365479,
      "logits/rejected": -1.122296690940857,
      "logps/chosen": -1.0807580947875977,
      "logps/rejected": -1.0660605430603027,
      "loss": 1.6138,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -2.701895236968994,
      "rewards/margins": -0.036744117736816406,
      "rewards/rejected": -2.6651511192321777,
      "step": 25
    },
    {
      "epoch": 0.007269677058576821,
      "grad_norm": 27.512055310670853,
      "learning_rate": 9.972609476841365e-07,
      "logits/chosen": -1.2338676452636719,
      "logits/rejected": -1.3265719413757324,
      "logps/chosen": -1.2775177955627441,
      "logps/rejected": -1.486391544342041,
      "loss": 1.4048,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.1937947273254395,
      "rewards/margins": 0.5221841335296631,
      "rewards/rejected": -3.7159788608551025,
      "step": 26
    },
    {
      "epoch": 0.007549280022368237,
      "grad_norm": 137.0687602353488,
      "learning_rate": 9.96273075820661e-07,
      "logits/chosen": -0.8894615173339844,
      "logits/rejected": -1.346874713897705,
      "logps/chosen": -0.8310048580169678,
      "logps/rejected": -1.700638771057129,
      "loss": 1.2364,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.077512264251709,
      "rewards/margins": 2.1740849018096924,
      "rewards/rejected": -4.2515974044799805,
      "step": 27
    },
    {
      "epoch": 0.007828882986159653,
      "grad_norm": 17.619065074739723,
      "learning_rate": 9.95134034370785e-07,
      "logits/chosen": -1.0894310474395752,
      "logits/rejected": -1.202174425125122,
      "logps/chosen": -1.1609501838684082,
      "logps/rejected": -1.407175064086914,
      "loss": 1.4807,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.9023752212524414,
      "rewards/margins": 0.6155624389648438,
      "rewards/rejected": -3.517937660217285,
      "step": 28
    },
    {
      "epoch": 0.00810848594995107,
      "grad_norm": 110.88293466564413,
      "learning_rate": 9.938441702975689e-07,
      "logits/chosen": -0.601264476776123,
      "logits/rejected": -0.8674606084823608,
      "logps/chosen": -1.1357152462005615,
      "logps/rejected": -1.3777132034301758,
      "loss": 1.6135,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.839287757873535,
      "rewards/margins": 0.6049953699111938,
      "rewards/rejected": -3.4442832469940186,
      "step": 29
    },
    {
      "epoch": 0.008388088913742486,
      "grad_norm": 28.486621774152123,
      "learning_rate": 9.92403876506104e-07,
      "logits/chosen": -0.8283142447471619,
      "logits/rejected": -1.3725578784942627,
      "logps/chosen": -0.8853375911712646,
      "logps/rejected": -1.7784080505371094,
      "loss": 1.4251,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.213344097137451,
      "rewards/margins": 2.2326760292053223,
      "rewards/rejected": -4.446020126342773,
      "step": 30
    },
    {
      "epoch": 0.008667691877533901,
      "grad_norm": 40.53817922951689,
      "learning_rate": 9.90813591723832e-07,
      "logits/chosen": -1.3251934051513672,
      "logits/rejected": -1.2978826761245728,
      "logps/chosen": -1.1201034784317017,
      "logps/rejected": -1.6758404970169067,
      "loss": 1.1433,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.8002586364746094,
      "rewards/margins": 1.3893425464630127,
      "rewards/rejected": -4.189601421356201,
      "step": 31
    },
    {
      "epoch": 0.008947294841325318,
      "grad_norm": 30.2037356357338,
      "learning_rate": 9.890738003669027e-07,
      "logits/chosen": -0.8872109055519104,
      "logits/rejected": -1.0567409992218018,
      "logps/chosen": -1.3809734582901,
      "logps/rejected": -1.5892744064331055,
      "loss": 2.2798,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -3.4524335861206055,
      "rewards/margins": 0.5207523703575134,
      "rewards/rejected": -3.9731857776641846,
      "step": 32
    },
    {
      "epoch": 0.009226897805116735,
      "grad_norm": 22.08377585136328,
      "learning_rate": 9.871850323926177e-07,
      "logits/chosen": -0.6034784913063049,
      "logits/rejected": -1.3737246990203857,
      "logps/chosen": -0.7753689885139465,
      "logps/rejected": -1.4922962188720703,
      "loss": 1.186,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.938422441482544,
      "rewards/margins": 1.7923178672790527,
      "rewards/rejected": -3.7307403087615967,
      "step": 33
    },
    {
      "epoch": 0.00950650076890815,
      "grad_norm": 44.83487292747772,
      "learning_rate": 9.851478631379982e-07,
      "logits/chosen": -1.5287456512451172,
      "logits/rejected": -1.604817509651184,
      "logps/chosen": -1.2651934623718262,
      "logps/rejected": -1.606818675994873,
      "loss": 1.2006,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.1629838943481445,
      "rewards/margins": 0.8540627360343933,
      "rewards/rejected": -4.0170464515686035,
      "step": 34
    },
    {
      "epoch": 0.009786103732699567,
      "grad_norm": 33.18197115037115,
      "learning_rate": 9.82962913144534e-07,
      "logits/chosen": -0.9402756094932556,
      "logits/rejected": -1.3055014610290527,
      "logps/chosen": -1.1110124588012695,
      "logps/rejected": -1.350355625152588,
      "loss": 1.2388,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.777531147003174,
      "rewards/margins": 0.5983579158782959,
      "rewards/rejected": -3.3758890628814697,
      "step": 35
    },
    {
      "epoch": 0.010065706696490984,
      "grad_norm": 28.70546678811581,
      "learning_rate": 9.806308479691594e-07,
      "logits/chosen": -1.2600600719451904,
      "logits/rejected": -1.17176353931427,
      "logps/chosen": -0.9965181350708008,
      "logps/rejected": -1.3181788921356201,
      "loss": 1.4099,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.491295337677002,
      "rewards/margins": 0.8041520118713379,
      "rewards/rejected": -3.29544734954834,
      "step": 36
    },
    {
      "epoch": 0.010345309660282399,
      "grad_norm": 21.048051889450377,
      "learning_rate": 9.781523779815178e-07,
      "logits/chosen": -0.8446118831634521,
      "logits/rejected": -0.9734996557235718,
      "logps/chosen": -1.013080358505249,
      "logps/rejected": -1.1583538055419922,
      "loss": 1.5673,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.532700777053833,
      "rewards/margins": 0.3631836771965027,
      "rewards/rejected": -2.8958845138549805,
      "step": 37
    },
    {
      "epoch": 0.010624912624073815,
      "grad_norm": 23.549736337048298,
      "learning_rate": 9.755282581475767e-07,
      "logits/chosen": -0.772028386592865,
      "logits/rejected": -0.821832001209259,
      "logps/chosen": -1.1050798892974854,
      "logps/rejected": -1.126805305480957,
      "loss": 1.6001,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.762699842453003,
      "rewards/margins": 0.0543133020401001,
      "rewards/rejected": -2.8170132637023926,
      "step": 38
    },
    {
      "epoch": 0.010904515587865232,
      "grad_norm": 31.678751895679902,
      "learning_rate": 9.727592877996584e-07,
      "logits/chosen": -1.2923513650894165,
      "logits/rejected": -0.9500499963760376,
      "logps/chosen": -1.3736703395843506,
      "logps/rejected": -1.8005601167678833,
      "loss": 1.3401,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.434175968170166,
      "rewards/margins": 1.0672242641448975,
      "rewards/rejected": -4.501400470733643,
      "step": 39
    },
    {
      "epoch": 0.011184118551656647,
      "grad_norm": 42.14350999647913,
      "learning_rate": 9.698463103929541e-07,
      "logits/chosen": -0.9315403699874878,
      "logits/rejected": -0.9749379754066467,
      "logps/chosen": -1.1820948123931885,
      "logps/rejected": -1.1733828783035278,
      "loss": 1.1077,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.9552369117736816,
      "rewards/margins": -0.021779626607894897,
      "rewards/rejected": -2.933457374572754,
      "step": 40
    },
    {
      "epoch": 0.011463721515448064,
      "grad_norm": 28.8262776267842,
      "learning_rate": 9.667902132486008e-07,
      "logits/chosen": -1.2946910858154297,
      "logits/rejected": -1.3090943098068237,
      "logps/chosen": -1.1675629615783691,
      "logps/rejected": -1.324582576751709,
      "loss": 1.4377,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.918907403945923,
      "rewards/margins": 0.39254891872406006,
      "rewards/rejected": -3.3114564418792725,
      "step": 41
    },
    {
      "epoch": 0.011743324479239479,
      "grad_norm": 106.3933529697075,
      "learning_rate": 9.635919272833937e-07,
      "logits/chosen": -0.8468477725982666,
      "logits/rejected": -0.87651127576828,
      "logps/chosen": -1.5985316038131714,
      "logps/rejected": -1.475963830947876,
      "loss": 1.5327,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -3.9963290691375732,
      "rewards/margins": -0.3064194321632385,
      "rewards/rejected": -3.6899096965789795,
      "step": 42
    },
    {
      "epoch": 0.012022927443030896,
      "grad_norm": 16.873173603232406,
      "learning_rate": 9.602524267262202e-07,
      "logits/chosen": -0.6168065667152405,
      "logits/rejected": -0.7960647940635681,
      "logps/chosen": -1.0298948287963867,
      "logps/rejected": -1.0422132015228271,
      "loss": 1.0135,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -2.574737310409546,
      "rewards/margins": 0.030795753002166748,
      "rewards/rejected": -2.6055331230163574,
      "step": 43
    },
    {
      "epoch": 0.012302530406822313,
      "grad_norm": 22.346008825908783,
      "learning_rate": 9.567727288213004e-07,
      "logits/chosen": -0.8090029954910278,
      "logits/rejected": -0.7225205898284912,
      "logps/chosen": -1.0612010955810547,
      "logps/rejected": -1.0335237979888916,
      "loss": 1.4244,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -2.6530027389526367,
      "rewards/margins": -0.06919330358505249,
      "rewards/rejected": -2.5838096141815186,
      "step": 44
    },
    {
      "epoch": 0.012582133370613728,
      "grad_norm": 23.528648829320613,
      "learning_rate": 9.531538935183249e-07,
      "logits/chosen": -1.1082680225372314,
      "logits/rejected": -1.311267614364624,
      "logps/chosen": -1.0698087215423584,
      "logps/rejected": -1.3603800535202026,
      "loss": 1.1636,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.6745219230651855,
      "rewards/margins": 0.7264281511306763,
      "rewards/rejected": -3.4009499549865723,
      "step": 45
    },
    {
      "epoch": 0.012861736334405145,
      "grad_norm": 36.88944148482263,
      "learning_rate": 9.493970231495834e-07,
      "logits/chosen": -0.9574987292289734,
      "logits/rejected": -1.4005671739578247,
      "logps/chosen": -1.1250247955322266,
      "logps/rejected": -1.6039255857467651,
      "loss": 1.3524,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.8125619888305664,
      "rewards/margins": 1.1972520351409912,
      "rewards/rejected": -4.009814262390137,
      "step": 46
    },
    {
      "epoch": 0.013141339298196561,
      "grad_norm": 34.07967217969598,
      "learning_rate": 9.455032620941839e-07,
      "logits/chosen": -0.575081467628479,
      "logits/rejected": -1.0301098823547363,
      "logps/chosen": -0.6966181993484497,
      "logps/rejected": -0.7595060467720032,
      "loss": 1.1816,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.7415456771850586,
      "rewards/margins": 0.15721949934959412,
      "rewards/rejected": -1.8987650871276855,
      "step": 47
    },
    {
      "epoch": 0.013420942261987976,
      "grad_norm": 37.33338028904019,
      "learning_rate": 9.414737964294634e-07,
      "logits/chosen": -0.48685717582702637,
      "logits/rejected": -0.9498811960220337,
      "logps/chosen": -0.8811389207839966,
      "logps/rejected": -1.4033122062683105,
      "loss": 0.9144,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.202847480773926,
      "rewards/margins": 1.3054332733154297,
      "rewards/rejected": -3.5082805156707764,
      "step": 48
    },
    {
      "epoch": 0.013700545225779393,
      "grad_norm": 20.112352494523353,
      "learning_rate": 9.373098535696978e-07,
      "logits/chosen": -0.9682779908180237,
      "logits/rejected": -1.1488547325134277,
      "logps/chosen": -1.1651062965393066,
      "logps/rejected": -1.3421710729599,
      "loss": 1.5188,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.9127655029296875,
      "rewards/margins": 0.4426620602607727,
      "rewards/rejected": -3.3554277420043945,
      "step": 49
    },
    {
      "epoch": 0.01398014818957081,
      "grad_norm": 33.55406806691841,
      "learning_rate": 9.330127018922193e-07,
      "logits/chosen": -1.2889659404754639,
      "logits/rejected": -0.9354056119918823,
      "logps/chosen": -1.3529675006866455,
      "logps/rejected": -1.1981086730957031,
      "loss": 1.645,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -3.3824188709259033,
      "rewards/margins": -0.38714712858200073,
      "rewards/rejected": -2.995271682739258,
      "step": 50
    },
    {
      "epoch": 0.01398014818957081,
      "eval_logits/chosen": -1.184069275856018,
      "eval_logits/rejected": -1.346756100654602,
      "eval_logps/chosen": -1.1177828311920166,
      "eval_logps/rejected": -1.4929856061935425,
      "eval_loss": 1.2563124895095825,
      "eval_rewards/accuracies": 0.7027027010917664,
      "eval_rewards/chosen": -2.794457197189331,
      "eval_rewards/margins": 0.9380068182945251,
      "eval_rewards/rejected": -3.73246431350708,
      "eval_runtime": 345.1606,
      "eval_samples_per_second": 3.847,
      "eval_steps_per_second": 0.322,
      "step": 50
    },
    {
      "epoch": 0.014259751153362225,
      "grad_norm": 22.5880454834462,
      "learning_rate": 9.285836503510562e-07,
      "logits/chosen": -0.8200809955596924,
      "logits/rejected": -0.8582978248596191,
      "logps/chosen": -1.1703073978424072,
      "logps/rejected": -1.458601951599121,
      "loss": 0.9907,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.9257683753967285,
      "rewards/margins": 0.7207364439964294,
      "rewards/rejected": -3.6465048789978027,
      "step": 51
    },
    {
      "epoch": 0.014539354117153642,
      "grad_norm": 45.098078544435936,
      "learning_rate": 9.240240480782129e-07,
      "logits/chosen": -0.7370530962944031,
      "logits/rejected": -1.368110179901123,
      "logps/chosen": -0.9416132569313049,
      "logps/rejected": -1.0832104682922363,
      "loss": 1.697,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.3540329933166504,
      "rewards/margins": 0.35399293899536133,
      "rewards/rejected": -2.708026170730591,
      "step": 52
    },
    {
      "epoch": 0.014818957080945059,
      "grad_norm": 101.10764848570363,
      "learning_rate": 9.19335283972712e-07,
      "logits/chosen": -0.5560005903244019,
      "logits/rejected": -0.7855780124664307,
      "logps/chosen": -1.0373799800872803,
      "logps/rejected": -1.0468213558197021,
      "loss": 1.3264,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.5934500694274902,
      "rewards/margins": 0.023603498935699463,
      "rewards/rejected": -2.617053508758545,
      "step": 53
    },
    {
      "epoch": 0.015098560044736474,
      "grad_norm": 64.45755933217201,
      "learning_rate": 9.145187862775208e-07,
      "logits/chosen": -1.421772837638855,
      "logits/rejected": -1.4179284572601318,
      "logps/chosen": -1.214949607849121,
      "logps/rejected": -2.0096287727355957,
      "loss": 1.2303,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.0373740196228027,
      "rewards/margins": 1.9866979122161865,
      "rewards/rejected": -5.02407169342041,
      "step": 54
    },
    {
      "epoch": 0.01537816300852789,
      "grad_norm": 24.722247357252545,
      "learning_rate": 9.095760221444959e-07,
      "logits/chosen": -1.1340514421463013,
      "logits/rejected": -1.4643547534942627,
      "logps/chosen": -1.0777888298034668,
      "logps/rejected": -1.2248834371566772,
      "loss": 1.163,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.694471836090088,
      "rewards/margins": 0.3677366375923157,
      "rewards/rejected": -3.062208652496338,
      "step": 55
    },
    {
      "epoch": 0.015657765972319306,
      "grad_norm": 88.82834717216237,
      "learning_rate": 9.045084971874737e-07,
      "logits/chosen": -1.0740933418273926,
      "logits/rejected": -0.9167161583900452,
      "logps/chosen": -1.2093803882598877,
      "logps/rejected": -1.6125764846801758,
      "loss": 1.2562,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -3.023451089859009,
      "rewards/margins": 1.0079903602600098,
      "rewards/rejected": -4.031441688537598,
      "step": 56
    },
    {
      "epoch": 0.015937368936110724,
      "grad_norm": 18.31655486222315,
      "learning_rate": 8.993177550236463e-07,
      "logits/chosen": -0.6805390119552612,
      "logits/rejected": -0.7955325841903687,
      "logps/chosen": -1.2940752506256104,
      "logps/rejected": -2.059095859527588,
      "loss": 1.2213,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.2351880073547363,
      "rewards/margins": 1.912551760673523,
      "rewards/rejected": -5.147739410400391,
      "step": 57
    },
    {
      "epoch": 0.01621697189990214,
      "grad_norm": 22.784684254467784,
      "learning_rate": 8.940053768033608e-07,
      "logits/chosen": -0.5054349303245544,
      "logits/rejected": -1.086111068725586,
      "logps/chosen": -0.8294824957847595,
      "logps/rejected": -1.2600980997085571,
      "loss": 1.1366,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.0737061500549316,
      "rewards/margins": 1.0765390396118164,
      "rewards/rejected": -3.150245189666748,
      "step": 58
    },
    {
      "epoch": 0.016496574863693554,
      "grad_norm": 24.23188699617836,
      "learning_rate": 8.885729807284854e-07,
      "logits/chosen": -0.9133630990982056,
      "logits/rejected": -1.0163180828094482,
      "logps/chosen": -1.049289345741272,
      "logps/rejected": -1.134240984916687,
      "loss": 1.0682,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.623223304748535,
      "rewards/margins": 0.21237897872924805,
      "rewards/rejected": -2.8356025218963623,
      "step": 59
    },
    {
      "epoch": 0.016776177827484973,
      "grad_norm": 28.93680920583823,
      "learning_rate": 8.83022221559489e-07,
      "logits/chosen": -1.0600225925445557,
      "logits/rejected": -1.2378225326538086,
      "logps/chosen": -1.2599135637283325,
      "logps/rejected": -1.3674981594085693,
      "loss": 0.9747,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.1497838497161865,
      "rewards/margins": 0.2689616084098816,
      "rewards/rejected": -3.418745517730713,
      "step": 60
    },
    {
      "epoch": 0.017055780791276388,
      "grad_norm": 29.885312194374713,
      "learning_rate": 8.773547901113861e-07,
      "logits/chosen": -0.7806551456451416,
      "logits/rejected": -1.628957748413086,
      "logps/chosen": -0.7070074677467346,
      "logps/rejected": -1.1015799045562744,
      "loss": 0.7853,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7675187587738037,
      "rewards/margins": 0.9864310026168823,
      "rewards/rejected": -2.7539496421813965,
      "step": 61
    },
    {
      "epoch": 0.017335383755067803,
      "grad_norm": 24.972205075082606,
      "learning_rate": 8.71572412738697e-07,
      "logits/chosen": -1.0657659769058228,
      "logits/rejected": -1.4624931812286377,
      "logps/chosen": -1.1949583292007446,
      "logps/rejected": -1.8615357875823975,
      "loss": 0.631,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.987395763397217,
      "rewards/margins": 1.6664433479309082,
      "rewards/rejected": -4.653839111328125,
      "step": 62
    },
    {
      "epoch": 0.01761498671885922,
      "grad_norm": 21.933003035104587,
      "learning_rate": 8.656768508095852e-07,
      "logits/chosen": -1.1191598176956177,
      "logits/rejected": -1.2448678016662598,
      "logps/chosen": -1.0887205600738525,
      "logps/rejected": -1.6507292985916138,
      "loss": 0.887,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.7218017578125,
      "rewards/margins": 1.4050216674804688,
      "rewards/rejected": -4.126823425292969,
      "step": 63
    },
    {
      "epoch": 0.017894589682650636,
      "grad_norm": 24.060712897137755,
      "learning_rate": 8.596699001693255e-07,
      "logits/chosen": -0.798321008682251,
      "logits/rejected": -1.0583261251449585,
      "logps/chosen": -0.7978274822235107,
      "logps/rejected": -1.1801316738128662,
      "loss": 1.3542,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9945688247680664,
      "rewards/margins": 0.9557601809501648,
      "rewards/rejected": -2.950329065322876,
      "step": 64
    },
    {
      "epoch": 0.01817419264644205,
      "grad_norm": 35.64070939669877,
      "learning_rate": 8.535533905932737e-07,
      "logits/chosen": -0.9407416582107544,
      "logits/rejected": -1.2474677562713623,
      "logps/chosen": -1.1173073053359985,
      "logps/rejected": -1.4939210414886475,
      "loss": 1.1944,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.7932682037353516,
      "rewards/margins": 0.9415344595909119,
      "rewards/rejected": -3.734802722930908,
      "step": 65
    },
    {
      "epoch": 0.01845379561023347,
      "grad_norm": 23.204374003534983,
      "learning_rate": 8.473291852294986e-07,
      "logits/chosen": -0.7950379252433777,
      "logits/rejected": -1.1784076690673828,
      "logps/chosen": -0.8873813152313232,
      "logps/rejected": -1.0771222114562988,
      "loss": 1.291,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.2184531688690186,
      "rewards/margins": 0.4743524193763733,
      "rewards/rejected": -2.692805767059326,
      "step": 66
    },
    {
      "epoch": 0.018733398574024885,
      "grad_norm": 22.342402065645,
      "learning_rate": 8.409991800312492e-07,
      "logits/chosen": -0.6992224454879761,
      "logits/rejected": -1.0948431491851807,
      "logps/chosen": -1.1709400415420532,
      "logps/rejected": -1.746230959892273,
      "loss": 1.0885,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.9273500442504883,
      "rewards/margins": 1.4382274150848389,
      "rewards/rejected": -4.365577220916748,
      "step": 67
    },
    {
      "epoch": 0.0190130015378163,
      "grad_norm": 26.635960985863385,
      "learning_rate": 8.34565303179429e-07,
      "logits/chosen": -0.9120014905929565,
      "logits/rejected": -1.2207849025726318,
      "logps/chosen": -1.022168755531311,
      "logps/rejected": -1.3500474691390991,
      "loss": 1.0317,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.555421829223633,
      "rewards/margins": 0.819696843624115,
      "rewards/rejected": -3.3751187324523926,
      "step": 68
    },
    {
      "epoch": 0.01929260450160772,
      "grad_norm": 216.32442664714605,
      "learning_rate": 8.280295144952536e-07,
      "logits/chosen": -0.9009850025177002,
      "logits/rejected": -0.9338705539703369,
      "logps/chosen": -1.7400050163269043,
      "logps/rejected": -3.0889453887939453,
      "loss": 1.6279,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.350012302398682,
      "rewards/margins": 3.3723511695861816,
      "rewards/rejected": -7.722363471984863,
      "step": 69
    },
    {
      "epoch": 0.019572207465399134,
      "grad_norm": 33.24100380368566,
      "learning_rate": 8.213938048432696e-07,
      "logits/chosen": -1.1465697288513184,
      "logits/rejected": -1.5220608711242676,
      "logps/chosen": -1.5942764282226562,
      "logps/rejected": -2.3567376136779785,
      "loss": 0.9686,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.9856910705566406,
      "rewards/margins": 1.9061533212661743,
      "rewards/rejected": -5.891844272613525,
      "step": 70
    },
    {
      "epoch": 0.01985181042919055,
      "grad_norm": 17.831061474851452,
      "learning_rate": 8.146601955249187e-07,
      "logits/chosen": -0.7144116163253784,
      "logits/rejected": -1.260697603225708,
      "logps/chosen": -0.7787595987319946,
      "logps/rejected": -1.2630029916763306,
      "loss": 1.2457,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9468989372253418,
      "rewards/margins": 1.2106084823608398,
      "rewards/rejected": -3.1575074195861816,
      "step": 71
    },
    {
      "epoch": 0.020131413392981967,
      "grad_norm": 32.219733178240375,
      "learning_rate": 8.07830737662829e-07,
      "logits/chosen": -1.14467191696167,
      "logits/rejected": -0.7406169176101685,
      "logps/chosen": -1.5345901250839233,
      "logps/rejected": -1.2437353134155273,
      "loss": 1.2542,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -3.836475372314453,
      "rewards/margins": -0.7271367907524109,
      "rewards/rejected": -3.1093382835388184,
      "step": 72
    },
    {
      "epoch": 0.020411016356773382,
      "grad_norm": 28.263777851446193,
      "learning_rate": 8.009075115760242e-07,
      "logits/chosen": -0.8964699506759644,
      "logits/rejected": -1.3684449195861816,
      "logps/chosen": -0.913366436958313,
      "logps/rejected": -1.5431159734725952,
      "loss": 1.0506,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.2834160327911377,
      "rewards/margins": 1.5743739604949951,
      "rewards/rejected": -3.857790231704712,
      "step": 73
    },
    {
      "epoch": 0.020690619320564797,
      "grad_norm": 33.62623441801084,
      "learning_rate": 7.938926261462365e-07,
      "logits/chosen": -1.5208332538604736,
      "logits/rejected": -1.4947750568389893,
      "logps/chosen": -1.6027369499206543,
      "logps/rejected": -2.2491843700408936,
      "loss": 1.1593,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.006842136383057,
      "rewards/margins": 1.6161189079284668,
      "rewards/rejected": -5.622961044311523,
      "step": 74
    },
    {
      "epoch": 0.020970222284356216,
      "grad_norm": 19.73073540290438,
      "learning_rate": 7.86788218175523e-07,
      "logits/chosen": -0.8135213851928711,
      "logits/rejected": -1.1303890943527222,
      "logps/chosen": -1.1947009563446045,
      "logps/rejected": -1.178377628326416,
      "loss": 1.0676,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.986752510070801,
      "rewards/margins": -0.04080846905708313,
      "rewards/rejected": -2.945943832397461,
      "step": 75
    },
    {
      "epoch": 0.02124982524814763,
      "grad_norm": 30.725692444381682,
      "learning_rate": 7.795964517353733e-07,
      "logits/chosen": -0.5928746461868286,
      "logits/rejected": -1.3386216163635254,
      "logps/chosen": -1.1336582899093628,
      "logps/rejected": -2.486417770385742,
      "loss": 1.2958,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.8341457843780518,
      "rewards/margins": 3.3818984031677246,
      "rewards/rejected": -6.2160444259643555,
      "step": 76
    },
    {
      "epoch": 0.021529428211939046,
      "grad_norm": 19.993312804225585,
      "learning_rate": 7.723195175075135e-07,
      "logits/chosen": -1.0590918064117432,
      "logits/rejected": -1.4984095096588135,
      "logps/chosen": -1.341080904006958,
      "logps/rejected": -2.226268768310547,
      "loss": 0.7399,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.3527023792266846,
      "rewards/margins": 2.212970018386841,
      "rewards/rejected": -5.565672397613525,
      "step": 77
    },
    {
      "epoch": 0.021809031175730464,
      "grad_norm": 27.58218498128165,
      "learning_rate": 7.649596321166024e-07,
      "logits/chosen": -0.903189480304718,
      "logits/rejected": -1.4045133590698242,
      "logps/chosen": -1.2869590520858765,
      "logps/rejected": -2.1190972328186035,
      "loss": 1.1142,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.217397689819336,
      "rewards/margins": 2.080345392227173,
      "rewards/rejected": -5.29774284362793,
      "step": 78
    },
    {
      "epoch": 0.02208863413952188,
      "grad_norm": 33.76140161445492,
      "learning_rate": 7.575190374550271e-07,
      "logits/chosen": -1.1929422616958618,
      "logits/rejected": -1.402430534362793,
      "logps/chosen": -1.4159026145935059,
      "logps/rejected": -1.8726705312728882,
      "loss": 1.0192,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.5397567749023438,
      "rewards/margins": 1.1419193744659424,
      "rewards/rejected": -4.681675910949707,
      "step": 79
    },
    {
      "epoch": 0.022368237103313295,
      "grad_norm": 23.027800645867977,
      "learning_rate": 7.5e-07,
      "logits/chosen": -0.6945698857307434,
      "logits/rejected": -1.40185546875,
      "logps/chosen": -1.4425044059753418,
      "logps/rejected": -2.2865071296691895,
      "loss": 0.9812,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.6062612533569336,
      "rewards/margins": 2.110006809234619,
      "rewards/rejected": -5.7162675857543945,
      "step": 80
    },
    {
      "epoch": 0.02264784006710471,
      "grad_norm": 38.27741144565889,
      "learning_rate": 7.424048101231686e-07,
      "logits/chosen": -1.2587817907333374,
      "logits/rejected": -1.0885329246520996,
      "logps/chosen": -1.2346124649047852,
      "logps/rejected": -1.6860368251800537,
      "loss": 0.7012,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.086531162261963,
      "rewards/margins": 1.1285609006881714,
      "rewards/rejected": -4.215092182159424,
      "step": 81
    },
    {
      "epoch": 0.022927443030896128,
      "grad_norm": 90.96248649883246,
      "learning_rate": 7.347357813929454e-07,
      "logits/chosen": -1.2520544528961182,
      "logits/rejected": -1.0464751720428467,
      "logps/chosen": -2.1459150314331055,
      "logps/rejected": -2.046426296234131,
      "loss": 1.5015,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -5.364788055419922,
      "rewards/margins": -0.24872243404388428,
      "rewards/rejected": -5.116065502166748,
      "step": 82
    },
    {
      "epoch": 0.023207045994687543,
      "grad_norm": 100.9433648678187,
      "learning_rate": 7.269952498697734e-07,
      "logits/chosen": -1.059312105178833,
      "logits/rejected": -0.9108483791351318,
      "logps/chosen": -1.4121086597442627,
      "logps/rejected": -1.6452100276947021,
      "loss": 1.3536,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -3.530271530151367,
      "rewards/margins": 0.5827533602714539,
      "rewards/rejected": -4.113024711608887,
      "step": 83
    },
    {
      "epoch": 0.023486648958478958,
      "grad_norm": 173.9823816167412,
      "learning_rate": 7.191855733945386e-07,
      "logits/chosen": -0.9303680658340454,
      "logits/rejected": -1.1252155303955078,
      "logps/chosen": -1.284766674041748,
      "logps/rejected": -1.3653523921966553,
      "loss": 2.4014,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -3.211916923522949,
      "rewards/margins": 0.2014639973640442,
      "rewards/rejected": -3.4133810997009277,
      "step": 84
    },
    {
      "epoch": 0.023766251922270377,
      "grad_norm": 28.477804453675674,
      "learning_rate": 7.113091308703497e-07,
      "logits/chosen": -0.7400285601615906,
      "logits/rejected": -0.847143292427063,
      "logps/chosen": -1.8564958572387695,
      "logps/rejected": -2.197305917739868,
      "loss": 1.3282,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -4.641240119934082,
      "rewards/margins": 0.8520248532295227,
      "rewards/rejected": -5.493264675140381,
      "step": 85
    },
    {
      "epoch": 0.024045854886061792,
      "grad_norm": 158.57710825565218,
      "learning_rate": 7.033683215379002e-07,
      "logits/chosen": -1.1289122104644775,
      "logits/rejected": -1.7158691883087158,
      "logps/chosen": -1.2116670608520508,
      "logps/rejected": -2.2744312286376953,
      "loss": 1.1088,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.029167652130127,
      "rewards/margins": 2.6569106578826904,
      "rewards/rejected": -5.686078071594238,
      "step": 86
    },
    {
      "epoch": 0.024325457849853207,
      "grad_norm": 14.96513597899858,
      "learning_rate": 6.953655642446367e-07,
      "logits/chosen": -0.6412773728370667,
      "logits/rejected": -1.4269566535949707,
      "logps/chosen": -0.985595703125,
      "logps/rejected": -2.8099608421325684,
      "loss": 0.6137,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.4639892578125,
      "rewards/margins": 4.5609130859375,
      "rewards/rejected": -7.02490234375,
      "step": 87
    },
    {
      "epoch": 0.024605060813644625,
      "grad_norm": 69.31853470031655,
      "learning_rate": 6.87303296707956e-07,
      "logits/chosen": -1.252925157546997,
      "logits/rejected": -1.3233586549758911,
      "logps/chosen": -1.3767462968826294,
      "logps/rejected": -1.7665659189224243,
      "loss": 1.2815,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.441865921020508,
      "rewards/margins": 0.9745489358901978,
      "rewards/rejected": -4.416414737701416,
      "step": 88
    },
    {
      "epoch": 0.02488466377743604,
      "grad_norm": 27.097255688460947,
      "learning_rate": 6.7918397477265e-07,
      "logits/chosen": -0.8277978301048279,
      "logits/rejected": -1.0574264526367188,
      "logps/chosen": -1.1671627759933472,
      "logps/rejected": -2.2953076362609863,
      "loss": 1.0056,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.9179069995880127,
      "rewards/margins": 2.820362091064453,
      "rewards/rejected": -5.738268852233887,
      "step": 89
    },
    {
      "epoch": 0.025164266741227456,
      "grad_norm": 52.839387510999444,
      "learning_rate": 6.710100716628344e-07,
      "logits/chosen": -1.40108323097229,
      "logits/rejected": -1.4667222499847412,
      "logps/chosen": -1.4003353118896484,
      "logps/rejected": -1.6744577884674072,
      "loss": 1.3688,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -3.500838279724121,
      "rewards/margins": 0.6853060126304626,
      "rewards/rejected": -4.1861443519592285,
      "step": 90
    },
    {
      "epoch": 0.025443869705018874,
      "grad_norm": 21.925148847234777,
      "learning_rate": 6.627840772285783e-07,
      "logits/chosen": -1.0636601448059082,
      "logits/rejected": -1.0789152383804321,
      "logps/chosen": -1.1292977333068848,
      "logps/rejected": -1.3410239219665527,
      "loss": 1.4889,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.823244094848633,
      "rewards/margins": 0.5293155908584595,
      "rewards/rejected": -3.3525595664978027,
      "step": 91
    },
    {
      "epoch": 0.02572347266881029,
      "grad_norm": 26.167972640340828,
      "learning_rate": 6.545084971874736e-07,
      "logits/chosen": -0.6311432719230652,
      "logits/rejected": -1.1260812282562256,
      "logps/chosen": -1.1044063568115234,
      "logps/rejected": -1.6531506776809692,
      "loss": 1.2341,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.7610158920288086,
      "rewards/margins": 1.3718609809875488,
      "rewards/rejected": -4.132876873016357,
      "step": 92
    },
    {
      "epoch": 0.026003075632601704,
      "grad_norm": 19.955245570389728,
      "learning_rate": 6.461858523613684e-07,
      "logits/chosen": -0.9683464765548706,
      "logits/rejected": -0.9185085296630859,
      "logps/chosen": -0.9664384722709656,
      "logps/rejected": -1.6319332122802734,
      "loss": 0.8303,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.4160962104797363,
      "rewards/margins": 1.6637368202209473,
      "rewards/rejected": -4.079833030700684,
      "step": 93
    },
    {
      "epoch": 0.026282678596393123,
      "grad_norm": 38.679628570882045,
      "learning_rate": 6.378186779084995e-07,
      "logits/chosen": -0.8759285807609558,
      "logits/rejected": -0.6558728218078613,
      "logps/chosen": -1.1541013717651367,
      "logps/rejected": -1.992755651473999,
      "loss": 0.598,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.885253429412842,
      "rewards/margins": 2.0966360569000244,
      "rewards/rejected": -4.981889247894287,
      "step": 94
    },
    {
      "epoch": 0.026562281560184538,
      "grad_norm": 43.75173307796015,
      "learning_rate": 6.294095225512604e-07,
      "logits/chosen": -0.9964724779129028,
      "logits/rejected": -0.7991459369659424,
      "logps/chosen": -1.0949715375900269,
      "logps/rejected": -1.5087236166000366,
      "loss": 1.0437,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.737428665161133,
      "rewards/margins": 1.034380316734314,
      "rewards/rejected": -3.7718091011047363,
      "step": 95
    },
    {
      "epoch": 0.026841884523975953,
      "grad_norm": 22.420799786964153,
      "learning_rate": 6.209609477998338e-07,
      "logits/chosen": -0.4085179269313812,
      "logits/rejected": -0.7962716221809387,
      "logps/chosen": -1.0953500270843506,
      "logps/rejected": -1.046759843826294,
      "loss": 1.5788,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.738374710083008,
      "rewards/margins": -0.12147501111030579,
      "rewards/rejected": -2.6168999671936035,
      "step": 96
    },
    {
      "epoch": 0.02712148748776737,
      "grad_norm": 28.1032386388583,
      "learning_rate": 6.124755271719326e-07,
      "logits/chosen": -0.96661376953125,
      "logits/rejected": -1.131087064743042,
      "logps/chosen": -1.2935943603515625,
      "logps/rejected": -1.4725990295410156,
      "loss": 1.1007,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -3.233985662460327,
      "rewards/margins": 0.4475119113922119,
      "rewards/rejected": -3.681497573852539,
      "step": 97
    },
    {
      "epoch": 0.027401090451558786,
      "grad_norm": 51.25959864452938,
      "learning_rate": 6.039558454088795e-07,
      "logits/chosen": -1.0819038152694702,
      "logits/rejected": -1.3961520195007324,
      "logps/chosen": -1.1702494621276855,
      "logps/rejected": -2.0412793159484863,
      "loss": 0.8748,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.925623655319214,
      "rewards/margins": 2.1775741577148438,
      "rewards/rejected": -5.103198051452637,
      "step": 98
    },
    {
      "epoch": 0.0276806934153502,
      "grad_norm": 44.86957712355451,
      "learning_rate": 5.954044976882723e-07,
      "logits/chosen": -0.4054487347602844,
      "logits/rejected": -0.9896742105484009,
      "logps/chosen": -0.8635058999061584,
      "logps/rejected": -1.2418280839920044,
      "loss": 1.0457,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.1587648391723633,
      "rewards/margins": 0.9458054900169373,
      "rewards/rejected": -3.1045703887939453,
      "step": 99
    },
    {
      "epoch": 0.02796029637914162,
      "grad_norm": 28.90572894211588,
      "learning_rate": 5.868240888334652e-07,
      "logits/chosen": -0.7194627523422241,
      "logits/rejected": -1.274187445640564,
      "logps/chosen": -0.822409987449646,
      "logps/rejected": -1.5885546207427979,
      "loss": 0.8722,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.0560250282287598,
      "rewards/margins": 1.9153616428375244,
      "rewards/rejected": -3.971386432647705,
      "step": 100
    },
    {
      "epoch": 0.02796029637914162,
      "eval_logits/chosen": -1.2195580005645752,
      "eval_logits/rejected": -1.3816862106323242,
      "eval_logps/chosen": -1.2307511568069458,
      "eval_logps/rejected": -1.8937082290649414,
      "eval_loss": 1.0619382858276367,
      "eval_rewards/accuracies": 0.7319819927215576,
      "eval_rewards/chosen": -3.0768775939941406,
      "eval_rewards/margins": 1.6573928594589233,
      "eval_rewards/rejected": -4.7342705726623535,
      "eval_runtime": 343.8898,
      "eval_samples_per_second": 3.862,
      "eval_steps_per_second": 0.323,
      "step": 100
    },
    {
      "epoch": 0.028239899342933035,
      "grad_norm": 23.014771898473285,
      "learning_rate": 5.782172325201155e-07,
      "logits/chosen": -1.019005298614502,
      "logits/rejected": -1.587357759475708,
      "logps/chosen": -0.8991877436637878,
      "logps/rejected": -2.31307053565979,
      "loss": 1.1455,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.247969388961792,
      "rewards/margins": 3.5347068309783936,
      "rewards/rejected": -5.7826762199401855,
      "step": 101
    },
    {
      "epoch": 0.02851950230672445,
      "grad_norm": 129.77518240929777,
      "learning_rate": 5.695865504800327e-07,
      "logits/chosen": -0.8380779027938843,
      "logits/rejected": -1.340008020401001,
      "logps/chosen": -1.1220433712005615,
      "logps/rejected": -1.7773716449737549,
      "loss": 0.8694,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.8051083087921143,
      "rewards/margins": 1.6383205652236938,
      "rewards/rejected": -4.443428993225098,
      "step": 102
    },
    {
      "epoch": 0.02879910527051587,
      "grad_norm": 172.3569574432423,
      "learning_rate": 5.609346717025737e-07,
      "logits/chosen": -1.052437424659729,
      "logits/rejected": -1.4004924297332764,
      "logps/chosen": -0.9889116287231445,
      "logps/rejected": -1.5602869987487793,
      "loss": 1.7629,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.4722790718078613,
      "rewards/margins": 1.428438425064087,
      "rewards/rejected": -3.9007177352905273,
      "step": 103
    },
    {
      "epoch": 0.029078708234307284,
      "grad_norm": 36.623523960138435,
      "learning_rate": 5.522642316338268e-07,
      "logits/chosen": -1.1740097999572754,
      "logits/rejected": -0.980044424533844,
      "logps/chosen": -1.2009364366531372,
      "logps/rejected": -1.3514597415924072,
      "loss": 1.2361,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -3.0023412704467773,
      "rewards/margins": 0.3763081431388855,
      "rewards/rejected": -3.3786492347717285,
      "step": 104
    },
    {
      "epoch": 0.0293583111980987,
      "grad_norm": 77.37683529524608,
      "learning_rate": 5.435778713738292e-07,
      "logits/chosen": -1.2499001026153564,
      "logits/rejected": -1.2681267261505127,
      "logps/chosen": -1.7606796026229858,
      "logps/rejected": -2.930025577545166,
      "loss": 1.1121,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -4.401699066162109,
      "rewards/margins": 2.9233648777008057,
      "rewards/rejected": -7.325064182281494,
      "step": 105
    },
    {
      "epoch": 0.029637914161890117,
      "grad_norm": 147.76082371794118,
      "learning_rate": 5.348782368720625e-07,
      "logits/chosen": -0.7781787514686584,
      "logits/rejected": -1.1500039100646973,
      "logps/chosen": -1.7407994270324707,
      "logps/rejected": -5.4726667404174805,
      "loss": 1.2898,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.351998805999756,
      "rewards/margins": 9.329668045043945,
      "rewards/rejected": -13.68166732788086,
      "step": 106
    },
    {
      "epoch": 0.029917517125681532,
      "grad_norm": 23.97972069856743,
      "learning_rate": 5.26167978121472e-07,
      "logits/chosen": -0.7871793508529663,
      "logits/rejected": -1.1882908344268799,
      "logps/chosen": -0.5037890672683716,
      "logps/rejected": -1.5715763568878174,
      "loss": 0.6191,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2594726085662842,
      "rewards/margins": 2.669468402862549,
      "rewards/rejected": -3.928941249847412,
      "step": 107
    },
    {
      "epoch": 0.030197120089472947,
      "grad_norm": 26.15247415705612,
      "learning_rate": 5.174497483512505e-07,
      "logits/chosen": -1.1062333583831787,
      "logits/rejected": -1.6174620389938354,
      "logps/chosen": -1.3177549839019775,
      "logps/rejected": -1.8563097715377808,
      "loss": 0.941,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.2943873405456543,
      "rewards/margins": 1.3463870286941528,
      "rewards/rejected": -4.640774726867676,
      "step": 108
    },
    {
      "epoch": 0.030476723053264366,
      "grad_norm": 36.12906136492366,
      "learning_rate": 5.087262032186418e-07,
      "logits/chosen": -1.320471167564392,
      "logits/rejected": -1.2250688076019287,
      "logps/chosen": -1.1929512023925781,
      "logps/rejected": -1.7405405044555664,
      "loss": 0.6462,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.9823780059814453,
      "rewards/margins": 1.3689732551574707,
      "rewards/rejected": -4.351351261138916,
      "step": 109
    },
    {
      "epoch": 0.03075632601705578,
      "grad_norm": 21.26364702784651,
      "learning_rate": 5e-07,
      "logits/chosen": -0.6797312498092651,
      "logits/rejected": -1.517352819442749,
      "logps/chosen": -1.253048062324524,
      "logps/rejected": -2.614882707595825,
      "loss": 0.7045,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.132620334625244,
      "rewards/margins": 3.4045865535736084,
      "rewards/rejected": -6.537206649780273,
      "step": 110
    },
    {
      "epoch": 0.031035928980847196,
      "grad_norm": 44.22722460589391,
      "learning_rate": 4.912737967813582e-07,
      "logits/chosen": -0.8700337409973145,
      "logits/rejected": -1.606900930404663,
      "logps/chosen": -1.6089668273925781,
      "logps/rejected": -2.9295287132263184,
      "loss": 0.8187,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.022417068481445,
      "rewards/margins": 3.3014047145843506,
      "rewards/rejected": -7.323822021484375,
      "step": 111
    },
    {
      "epoch": 0.03131553194463861,
      "grad_norm": 76.80253216440869,
      "learning_rate": 4.825502516487496e-07,
      "logits/chosen": -0.8108619451522827,
      "logits/rejected": -1.4683247804641724,
      "logps/chosen": -1.2750149965286255,
      "logps/rejected": -1.875692367553711,
      "loss": 1.1174,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.187537670135498,
      "rewards/margins": 1.501693606376648,
      "rewards/rejected": -4.689230918884277,
      "step": 112
    },
    {
      "epoch": 0.03159513490843003,
      "grad_norm": 39.5980607890587,
      "learning_rate": 4.7383202187852804e-07,
      "logits/chosen": -1.134104609489441,
      "logits/rejected": -1.3199536800384521,
      "logps/chosen": -1.1517322063446045,
      "logps/rejected": -2.180983543395996,
      "loss": 1.4293,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.879330635070801,
      "rewards/margins": 2.5731282234191895,
      "rewards/rejected": -5.45245885848999,
      "step": 113
    },
    {
      "epoch": 0.03187473787222145,
      "grad_norm": 159.282123008416,
      "learning_rate": 4.6512176312793735e-07,
      "logits/chosen": -1.2752604484558105,
      "logits/rejected": -1.6413378715515137,
      "logps/chosen": -0.9552395343780518,
      "logps/rejected": -2.1065025329589844,
      "loss": 1.8414,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.38809871673584,
      "rewards/margins": 2.878157377243042,
      "rewards/rejected": -5.266256332397461,
      "step": 114
    },
    {
      "epoch": 0.03215434083601286,
      "grad_norm": 23.36900105938972,
      "learning_rate": 4.5642212862617085e-07,
      "logits/chosen": -1.099194049835205,
      "logits/rejected": -1.1428616046905518,
      "logps/chosen": -1.1194630861282349,
      "logps/rejected": -1.9647703170776367,
      "loss": 1.211,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.7986578941345215,
      "rewards/margins": 2.1132683753967285,
      "rewards/rejected": -4.911925792694092,
      "step": 115
    },
    {
      "epoch": 0.03243394379980428,
      "grad_norm": 22.810174582850898,
      "learning_rate": 4.477357683661733e-07,
      "logits/chosen": -1.2737977504730225,
      "logits/rejected": -1.3265190124511719,
      "logps/chosen": -1.2955751419067383,
      "logps/rejected": -2.3141541481018066,
      "loss": 1.1194,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.2389378547668457,
      "rewards/margins": 2.546447515487671,
      "rewards/rejected": -5.785386085510254,
      "step": 116
    },
    {
      "epoch": 0.0327135467635957,
      "grad_norm": 155.39469755085068,
      "learning_rate": 4.390653282974263e-07,
      "logits/chosen": -1.007920503616333,
      "logits/rejected": -1.2567858695983887,
      "logps/chosen": -1.3624416589736938,
      "logps/rejected": -1.7886241674423218,
      "loss": 1.4514,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.406104326248169,
      "rewards/margins": 1.0654561519622803,
      "rewards/rejected": -4.471560478210449,
      "step": 117
    },
    {
      "epoch": 0.03299314972738711,
      "grad_norm": 28.639395965705603,
      "learning_rate": 4.304134495199674e-07,
      "logits/chosen": -0.8960734605789185,
      "logits/rejected": -0.5021066069602966,
      "logps/chosen": -0.928629994392395,
      "logps/rejected": -1.1891580820083618,
      "loss": 0.8406,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.3215749263763428,
      "rewards/margins": 0.651320219039917,
      "rewards/rejected": -2.9728951454162598,
      "step": 118
    },
    {
      "epoch": 0.03327275269117853,
      "grad_norm": 43.81099025759765,
      "learning_rate": 4.2178276747988444e-07,
      "logits/chosen": -1.010365605354309,
      "logits/rejected": -1.6597810983657837,
      "logps/chosen": -1.1012680530548096,
      "logps/rejected": -3.7689075469970703,
      "loss": 1.0053,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.7531704902648926,
      "rewards/margins": 6.669097900390625,
      "rewards/rejected": -9.422268867492676,
      "step": 119
    },
    {
      "epoch": 0.033552355654969945,
      "grad_norm": 50.65816817248901,
      "learning_rate": 4.131759111665348e-07,
      "logits/chosen": -0.8199977874755859,
      "logits/rejected": -1.105729341506958,
      "logps/chosen": -1.1784420013427734,
      "logps/rejected": -1.7748521566390991,
      "loss": 1.0382,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.9461052417755127,
      "rewards/margins": 1.4910252094268799,
      "rewards/rejected": -4.437130928039551,
      "step": 120
    },
    {
      "epoch": 0.03383195861876136,
      "grad_norm": 135.269469018394,
      "learning_rate": 4.0459550231172755e-07,
      "logits/chosen": -1.0090067386627197,
      "logits/rejected": -1.2103219032287598,
      "logps/chosen": -1.3218321800231934,
      "logps/rejected": -1.8139877319335938,
      "loss": 1.3587,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.3045804500579834,
      "rewards/margins": 1.23038911819458,
      "rewards/rejected": -4.534969806671143,
      "step": 121
    },
    {
      "epoch": 0.034111561582552775,
      "grad_norm": 39.54642498391402,
      "learning_rate": 3.960441545911204e-07,
      "logits/chosen": -0.7739511132240295,
      "logits/rejected": -1.1406965255737305,
      "logps/chosen": -1.6368045806884766,
      "logps/rejected": -1.7438288927078247,
      "loss": 1.1795,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -4.092011451721191,
      "rewards/margins": 0.2675609588623047,
      "rewards/rejected": -4.359572410583496,
      "step": 122
    },
    {
      "epoch": 0.034391164546344194,
      "grad_norm": 100.27265763472896,
      "learning_rate": 3.8752447282806755e-07,
      "logits/chosen": -1.0782852172851562,
      "logits/rejected": -1.3873653411865234,
      "logps/chosen": -1.1215283870697021,
      "logps/rejected": -2.216892719268799,
      "loss": 1.3417,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.803821086883545,
      "rewards/margins": 2.738410472869873,
      "rewards/rejected": -5.542231559753418,
      "step": 123
    },
    {
      "epoch": 0.034670767510135606,
      "grad_norm": 58.20614675830375,
      "learning_rate": 3.790390522001662e-07,
      "logits/chosen": -0.778974711894989,
      "logits/rejected": -1.2259283065795898,
      "logps/chosen": -1.4575412273406982,
      "logps/rejected": -2.0206165313720703,
      "loss": 0.7091,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.643852949142456,
      "rewards/margins": 1.4076881408691406,
      "rewards/rejected": -5.051541328430176,
      "step": 124
    },
    {
      "epoch": 0.034950370473927024,
      "grad_norm": 14.601199122985598,
      "learning_rate": 3.7059047744873955e-07,
      "logits/chosen": -0.8316655158996582,
      "logits/rejected": -1.5047085285186768,
      "logps/chosen": -1.2063120603561401,
      "logps/rejected": -2.6588189601898193,
      "loss": 0.4785,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.015780448913574,
      "rewards/margins": 3.6312668323516846,
      "rewards/rejected": -6.64704704284668,
      "step": 125
    },
    {
      "epoch": 0.03522997343771844,
      "grad_norm": 27.93055728093027,
      "learning_rate": 3.621813220915004e-07,
      "logits/chosen": -1.099701166152954,
      "logits/rejected": -1.5104920864105225,
      "logps/chosen": -1.2890055179595947,
      "logps/rejected": -2.336859703063965,
      "loss": 0.6663,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.2225139141082764,
      "rewards/margins": 2.6196351051330566,
      "rewards/rejected": -5.842149257659912,
      "step": 126
    },
    {
      "epoch": 0.035509576401509854,
      "grad_norm": 38.5610788021146,
      "learning_rate": 3.5381414763863163e-07,
      "logits/chosen": -1.0062170028686523,
      "logits/rejected": -1.459620714187622,
      "logps/chosen": -1.9963696002960205,
      "logps/rejected": -2.577173948287964,
      "loss": 1.0607,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.990923881530762,
      "rewards/margins": 1.4520111083984375,
      "rewards/rejected": -6.442934989929199,
      "step": 127
    },
    {
      "epoch": 0.03578917936530127,
      "grad_norm": 64.49312626407205,
      "learning_rate": 3.454915028125263e-07,
      "logits/chosen": -0.9179176092147827,
      "logits/rejected": -1.1796026229858398,
      "logps/chosen": -1.013648271560669,
      "logps/rejected": -1.823269248008728,
      "loss": 1.0723,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.534120559692383,
      "rewards/margins": 2.024052143096924,
      "rewards/rejected": -4.558172702789307,
      "step": 128
    },
    {
      "epoch": 0.03606878232909269,
      "grad_norm": 36.00805300468051,
      "learning_rate": 3.3721592277142173e-07,
      "logits/chosen": -1.0871798992156982,
      "logits/rejected": -1.1635856628417969,
      "logps/chosen": -1.6440746784210205,
      "logps/rejected": -2.153071641921997,
      "loss": 1.0645,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -4.110186576843262,
      "rewards/margins": 1.2724924087524414,
      "rewards/rejected": -5.382678985595703,
      "step": 129
    },
    {
      "epoch": 0.0363483852928841,
      "grad_norm": 18.299398093476555,
      "learning_rate": 3.2898992833716563e-07,
      "logits/chosen": -0.852850079536438,
      "logits/rejected": -1.2836261987686157,
      "logps/chosen": -1.0674067735671997,
      "logps/rejected": -2.2412667274475098,
      "loss": 0.8613,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.6685171127319336,
      "rewards/margins": 2.934649705886841,
      "rewards/rejected": -5.603166580200195,
      "step": 130
    },
    {
      "epoch": 0.03662798825667552,
      "grad_norm": 52.07726812332011,
      "learning_rate": 3.2081602522734985e-07,
      "logits/chosen": -1.0853538513183594,
      "logits/rejected": -1.274430274963379,
      "logps/chosen": -1.4096601009368896,
      "logps/rejected": -1.5677156448364258,
      "loss": 1.65,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -3.5241503715515137,
      "rewards/margins": 0.3951385021209717,
      "rewards/rejected": -3.9192888736724854,
      "step": 131
    },
    {
      "epoch": 0.03690759122046694,
      "grad_norm": 17.512442562016194,
      "learning_rate": 3.1269670329204393e-07,
      "logits/chosen": -1.1719293594360352,
      "logits/rejected": -0.9959172606468201,
      "logps/chosen": -1.1080865859985352,
      "logps/rejected": -2.0133416652679443,
      "loss": 0.7374,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.770216464996338,
      "rewards/margins": 2.2631378173828125,
      "rewards/rejected": -5.033354759216309,
      "step": 132
    },
    {
      "epoch": 0.03718719418425835,
      "grad_norm": 684.8675181225219,
      "learning_rate": 3.0463443575536317e-07,
      "logits/chosen": -1.3861017227172852,
      "logits/rejected": -1.1517585515975952,
      "logps/chosen": -2.434314250946045,
      "logps/rejected": -1.6802737712860107,
      "loss": 2.4108,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -6.085785388946533,
      "rewards/margins": -1.8851011991500854,
      "rewards/rejected": -4.200684070587158,
      "step": 133
    },
    {
      "epoch": 0.03746679714804977,
      "grad_norm": 203.35746911786714,
      "learning_rate": 2.9663167846209996e-07,
      "logits/chosen": -0.6332231163978577,
      "logits/rejected": -1.2013721466064453,
      "logps/chosen": -0.8640561699867249,
      "logps/rejected": -1.9115948677062988,
      "loss": 1.0114,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.1601402759552,
      "rewards/margins": 2.618846893310547,
      "rewards/rejected": -4.778987407684326,
      "step": 134
    },
    {
      "epoch": 0.03774640011184119,
      "grad_norm": 48.99403271441584,
      "learning_rate": 2.8869086912965036e-07,
      "logits/chosen": -1.1275267601013184,
      "logits/rejected": -1.376509189605713,
      "logps/chosen": -1.562412977218628,
      "logps/rejected": -3.6972084045410156,
      "loss": 0.8296,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.9060323238372803,
      "rewards/margins": 5.3369879722595215,
      "rewards/rejected": -9.243021011352539,
      "step": 135
    },
    {
      "epoch": 0.0380260030756326,
      "grad_norm": 27.827124034061338,
      "learning_rate": 2.808144266054612e-07,
      "logits/chosen": -0.7335802912712097,
      "logits/rejected": -1.3153456449508667,
      "logps/chosen": -1.1802624464035034,
      "logps/rejected": -2.093515157699585,
      "loss": 1.0364,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.9506561756134033,
      "rewards/margins": 2.2831313610076904,
      "rewards/rejected": -5.233787536621094,
      "step": 136
    },
    {
      "epoch": 0.03830560603942402,
      "grad_norm": 41.51238681345992,
      "learning_rate": 2.730047501302266e-07,
      "logits/chosen": -0.9400544166564941,
      "logits/rejected": -1.42076575756073,
      "logps/chosen": -1.4838162660598755,
      "logps/rejected": -2.123250722885132,
      "loss": 1.4503,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.709540843963623,
      "rewards/margins": 1.598586082458496,
      "rewards/rejected": -5.308126926422119,
      "step": 137
    },
    {
      "epoch": 0.03858520900321544,
      "grad_norm": 22.114098288460713,
      "learning_rate": 2.6526421860705473e-07,
      "logits/chosen": -0.8485064506530762,
      "logits/rejected": -1.127300500869751,
      "logps/chosen": -1.4366434812545776,
      "logps/rejected": -1.9873528480529785,
      "loss": 0.6841,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.591608762741089,
      "rewards/margins": 1.3767732381820679,
      "rewards/rejected": -4.968381881713867,
      "step": 138
    },
    {
      "epoch": 0.03886481196700685,
      "grad_norm": 20.628330217139027,
      "learning_rate": 2.575951898768315e-07,
      "logits/chosen": -1.1259043216705322,
      "logits/rejected": -1.3020617961883545,
      "logps/chosen": -1.4316058158874512,
      "logps/rejected": -2.6916065216064453,
      "loss": 0.845,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.579014301300049,
      "rewards/margins": 3.1500015258789062,
      "rewards/rejected": -6.729016304016113,
      "step": 139
    },
    {
      "epoch": 0.03914441493079827,
      "grad_norm": 21.796146073482507,
      "learning_rate": 2.500000000000001e-07,
      "logits/chosen": -1.3260260820388794,
      "logits/rejected": -1.5544166564941406,
      "logps/chosen": -2.0802931785583496,
      "logps/rejected": -2.217221260070801,
      "loss": 1.0337,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -5.200733661651611,
      "rewards/margins": 0.3423195481300354,
      "rewards/rejected": -5.543053150177002,
      "step": 140
    },
    {
      "epoch": 0.039424017894589686,
      "grad_norm": 18.878252064796683,
      "learning_rate": 2.424809625449729e-07,
      "logits/chosen": -0.8346128463745117,
      "logits/rejected": -1.0135633945465088,
      "logps/chosen": -1.6469535827636719,
      "logps/rejected": -2.0207669734954834,
      "loss": 0.379,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.11738395690918,
      "rewards/margins": 0.9345333576202393,
      "rewards/rejected": -5.05191707611084,
      "step": 141
    },
    {
      "epoch": 0.0397036208583811,
      "grad_norm": 15.618943765371364,
      "learning_rate": 2.350403678833976e-07,
      "logits/chosen": -1.1391526460647583,
      "logits/rejected": -1.3611162900924683,
      "logps/chosen": -1.1667499542236328,
      "logps/rejected": -2.9371497631073,
      "loss": 0.363,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.916875123977661,
      "rewards/margins": 4.425999164581299,
      "rewards/rejected": -7.342874526977539,
      "step": 142
    },
    {
      "epoch": 0.039983223822172516,
      "grad_norm": 67.95024500967284,
      "learning_rate": 2.2768048249248644e-07,
      "logits/chosen": -0.9159025549888611,
      "logits/rejected": -1.5903668403625488,
      "logps/chosen": -1.0491520166397095,
      "logps/rejected": -3.3000054359436035,
      "loss": 0.7062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.622879981994629,
      "rewards/margins": 5.627133846282959,
      "rewards/rejected": -8.250014305114746,
      "step": 143
    },
    {
      "epoch": 0.040262826785963934,
      "grad_norm": 24.48650844009346,
      "learning_rate": 2.2040354826462664e-07,
      "logits/chosen": -0.948284387588501,
      "logits/rejected": -1.2763409614562988,
      "logps/chosen": -1.060044527053833,
      "logps/rejected": -2.7557332515716553,
      "loss": 0.4762,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.650111436843872,
      "rewards/margins": 4.239221572875977,
      "rewards/rejected": -6.8893327713012695,
      "step": 144
    },
    {
      "epoch": 0.040542429749755346,
      "grad_norm": 22.244412776379363,
      "learning_rate": 2.1321178182447709e-07,
      "logits/chosen": -0.8840318322181702,
      "logits/rejected": -0.11764158308506012,
      "logps/chosen": -1.242844581604004,
      "logps/rejected": -1.281248927116394,
      "loss": 0.9514,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -3.1071114540100098,
      "rewards/margins": 0.09601074457168579,
      "rewards/rejected": -3.20312237739563,
      "step": 145
    },
    {
      "epoch": 0.040822032713546764,
      "grad_norm": 33.28893137790503,
      "learning_rate": 2.0610737385376348e-07,
      "logits/chosen": -1.2970675230026245,
      "logits/rejected": -1.1824913024902344,
      "logps/chosen": -1.404200553894043,
      "logps/rejected": -2.006523609161377,
      "loss": 0.9091,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.5105013847351074,
      "rewards/margins": 1.505807638168335,
      "rewards/rejected": -5.016308784484863,
      "step": 146
    },
    {
      "epoch": 0.04110163567733818,
      "grad_norm": 62.45940480786672,
      "learning_rate": 1.990924884239758e-07,
      "logits/chosen": -0.7262787222862244,
      "logits/rejected": -0.9536744952201843,
      "logps/chosen": -1.3462529182434082,
      "logps/rejected": -2.189612865447998,
      "loss": 1.1719,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.3656325340270996,
      "rewards/margins": 2.1083993911743164,
      "rewards/rejected": -5.474031448364258,
      "step": 147
    },
    {
      "epoch": 0.041381238641129595,
      "grad_norm": 14.195827731761723,
      "learning_rate": 1.9216926233717084e-07,
      "logits/chosen": -0.7084841132164001,
      "logits/rejected": -1.271304965019226,
      "logps/chosen": -1.093338131904602,
      "logps/rejected": -2.5351109504699707,
      "loss": 0.4317,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.7333450317382812,
      "rewards/margins": 3.6044318675994873,
      "rewards/rejected": -6.337777137756348,
      "step": 148
    },
    {
      "epoch": 0.04166084160492101,
      "grad_norm": 30.52003756938978,
      "learning_rate": 1.8533980447508135e-07,
      "logits/chosen": -0.8182432055473328,
      "logits/rejected": -1.2552411556243896,
      "logps/chosen": -1.5775116682052612,
      "logps/rejected": -1.9700381755828857,
      "loss": 0.9821,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.9437789916992188,
      "rewards/margins": 0.9813161492347717,
      "rewards/rejected": -4.925095558166504,
      "step": 149
    },
    {
      "epoch": 0.04194044456871243,
      "grad_norm": 53.98320994568875,
      "learning_rate": 1.7860619515673032e-07,
      "logits/chosen": -0.9220970869064331,
      "logits/rejected": -0.8754558563232422,
      "logps/chosen": -1.3505525588989258,
      "logps/rejected": -1.8634326457977295,
      "loss": 1.0404,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.3763811588287354,
      "rewards/margins": 1.282200574874878,
      "rewards/rejected": -4.658581733703613,
      "step": 150
    },
    {
      "epoch": 0.04194044456871243,
      "eval_logits/chosen": -1.208183765411377,
      "eval_logits/rejected": -1.363861083984375,
      "eval_logps/chosen": -1.3818128108978271,
      "eval_logps/rejected": -2.246399164199829,
      "eval_loss": 0.9883071780204773,
      "eval_rewards/accuracies": 0.7545045018196106,
      "eval_rewards/chosen": -3.4545319080352783,
      "eval_rewards/margins": 2.161465644836426,
      "eval_rewards/rejected": -5.615997791290283,
      "eval_runtime": 343.7097,
      "eval_samples_per_second": 3.864,
      "eval_steps_per_second": 0.323,
      "step": 150
    },
    {
      "epoch": 0.04222004753250384,
      "grad_norm": 220.36984630306912,
      "learning_rate": 1.7197048550474641e-07,
      "logits/chosen": -0.6710684299468994,
      "logits/rejected": -0.9249241948127747,
      "logps/chosen": -1.366889476776123,
      "logps/rejected": -3.0947508811950684,
      "loss": 1.8539,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.4172236919403076,
      "rewards/margins": 4.3196539878845215,
      "rewards/rejected": -7.73687744140625,
      "step": 151
    },
    {
      "epoch": 0.04249965049629526,
      "grad_norm": 70.6167555600515,
      "learning_rate": 1.6543469682057104e-07,
      "logits/chosen": -1.052169919013977,
      "logits/rejected": -1.4687204360961914,
      "logps/chosen": -1.1982924938201904,
      "logps/rejected": -1.9964299201965332,
      "loss": 0.7174,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.9957308769226074,
      "rewards/margins": 1.9953436851501465,
      "rewards/rejected": -4.991075038909912,
      "step": 152
    },
    {
      "epoch": 0.04277925346008668,
      "grad_norm": 14.934556421376993,
      "learning_rate": 1.590008199687508e-07,
      "logits/chosen": -1.2788257598876953,
      "logits/rejected": -1.3499538898468018,
      "logps/chosen": -1.4407329559326172,
      "logps/rejected": -2.7239699363708496,
      "loss": 0.964,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.601832389831543,
      "rewards/margins": 3.208092212677002,
      "rewards/rejected": -6.809924602508545,
      "step": 153
    },
    {
      "epoch": 0.04305885642387809,
      "grad_norm": 34.17784688286548,
      "learning_rate": 1.5267081477050131e-07,
      "logits/chosen": -0.9582281112670898,
      "logits/rejected": -1.459122896194458,
      "logps/chosen": -1.7836196422576904,
      "logps/rejected": -1.8749334812164307,
      "loss": 0.774,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -4.459049224853516,
      "rewards/margins": 0.22828447818756104,
      "rewards/rejected": -4.687334060668945,
      "step": 154
    },
    {
      "epoch": 0.04333845938766951,
      "grad_norm": 16.121695263258527,
      "learning_rate": 1.4644660940672627e-07,
      "logits/chosen": -0.6466392874717712,
      "logits/rejected": -1.4516868591308594,
      "logps/chosen": -1.236275315284729,
      "logps/rejected": -3.074626922607422,
      "loss": 0.8325,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.0906882286071777,
      "rewards/margins": 4.595878601074219,
      "rewards/rejected": -7.686567306518555,
      "step": 155
    },
    {
      "epoch": 0.04361806235146093,
      "grad_norm": 59.092861255129954,
      "learning_rate": 1.4033009983067452e-07,
      "logits/chosen": -1.0997824668884277,
      "logits/rejected": -0.8644530773162842,
      "logps/chosen": -1.5466465950012207,
      "logps/rejected": -1.9648511409759521,
      "loss": 1.2282,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.8666162490844727,
      "rewards/margins": 1.0455113649368286,
      "rewards/rejected": -4.91212797164917,
      "step": 156
    },
    {
      "epoch": 0.04389766531525234,
      "grad_norm": 124.7156154616911,
      "learning_rate": 1.3432314919041477e-07,
      "logits/chosen": -1.2434520721435547,
      "logits/rejected": -0.9779855608940125,
      "logps/chosen": -2.0951521396636963,
      "logps/rejected": -1.2733700275421143,
      "loss": 2.0045,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -5.237880706787109,
      "rewards/margins": -2.054455280303955,
      "rewards/rejected": -3.183424949645996,
      "step": 157
    },
    {
      "epoch": 0.04417726827904376,
      "grad_norm": 17.359377980307887,
      "learning_rate": 1.284275872613028e-07,
      "logits/chosen": -0.8045494556427002,
      "logits/rejected": -1.271446943283081,
      "logps/chosen": -1.310563087463379,
      "logps/rejected": -2.273303508758545,
      "loss": 0.6038,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.2764077186584473,
      "rewards/margins": 2.406850814819336,
      "rewards/rejected": -5.683258533477783,
      "step": 158
    },
    {
      "epoch": 0.04445687124283517,
      "grad_norm": 41.376354047075345,
      "learning_rate": 1.22645209888614e-07,
      "logits/chosen": -0.7052321434020996,
      "logits/rejected": -1.4795576333999634,
      "logps/chosen": -1.3022897243499756,
      "logps/rejected": -2.3771719932556152,
      "loss": 0.9528,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.2557244300842285,
      "rewards/margins": 2.6872057914733887,
      "rewards/rejected": -5.942930221557617,
      "step": 159
    },
    {
      "epoch": 0.04473647420662659,
      "grad_norm": 45.99787159929172,
      "learning_rate": 1.1697777844051104e-07,
      "logits/chosen": -0.9138689041137695,
      "logits/rejected": -1.5214874744415283,
      "logps/chosen": -1.44037663936615,
      "logps/rejected": -2.119154214859009,
      "loss": 1.4147,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -3.6009416580200195,
      "rewards/margins": 1.696943759918213,
      "rewards/rejected": -5.297885417938232,
      "step": 160
    },
    {
      "epoch": 0.04501607717041801,
      "grad_norm": 39.8925791182464,
      "learning_rate": 1.1142701927151454e-07,
      "logits/chosen": -1.4353740215301514,
      "logits/rejected": -1.4286375045776367,
      "logps/chosen": -1.4594804048538208,
      "logps/rejected": -2.6095705032348633,
      "loss": 0.7317,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.6487011909484863,
      "rewards/margins": 2.87522554397583,
      "rewards/rejected": -6.523926734924316,
      "step": 161
    },
    {
      "epoch": 0.04529568013420942,
      "grad_norm": 31.35281628244166,
      "learning_rate": 1.0599462319663904e-07,
      "logits/chosen": -1.1067039966583252,
      "logits/rejected": -1.1960399150848389,
      "logps/chosen": -1.6614606380462646,
      "logps/rejected": -2.211236000061035,
      "loss": 0.9855,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.153651714324951,
      "rewards/margins": 1.374437928199768,
      "rewards/rejected": -5.52808952331543,
      "step": 162
    },
    {
      "epoch": 0.04557528309800084,
      "grad_norm": 75.21576214554203,
      "learning_rate": 1.0068224497635369e-07,
      "logits/chosen": -1.2265667915344238,
      "logits/rejected": -1.0434250831604004,
      "logps/chosen": -1.1752564907073975,
      "logps/rejected": -2.1756644248962402,
      "loss": 0.7941,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.9381415843963623,
      "rewards/margins": 2.501019239425659,
      "rewards/rejected": -5.4391608238220215,
      "step": 163
    },
    {
      "epoch": 0.045854886061792256,
      "grad_norm": 93.40285204475232,
      "learning_rate": 9.549150281252632e-08,
      "logits/chosen": -0.831551194190979,
      "logits/rejected": -0.7609004974365234,
      "logps/chosen": -1.2593779563903809,
      "logps/rejected": -1.790838599205017,
      "loss": 1.5802,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -3.1484451293945312,
      "rewards/margins": 1.3286514282226562,
      "rewards/rejected": -4.4770965576171875,
      "step": 164
    },
    {
      "epoch": 0.04613448902558367,
      "grad_norm": 41.48157239220995,
      "learning_rate": 9.042397785550404e-08,
      "logits/chosen": -0.6658945679664612,
      "logits/rejected": -0.9514182806015015,
      "logps/chosen": -1.3134242296218872,
      "logps/rejected": -1.9456892013549805,
      "loss": 0.5563,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.2835607528686523,
      "rewards/margins": 1.580662488937378,
      "rewards/rejected": -4.864223003387451,
      "step": 165
    },
    {
      "epoch": 0.046414091989375086,
      "grad_norm": 49.840157280127464,
      "learning_rate": 8.548121372247919e-08,
      "logits/chosen": -0.9251920580863953,
      "logits/rejected": -1.4082067012786865,
      "logps/chosen": -1.3225380182266235,
      "logps/rejected": -3.200735092163086,
      "loss": 0.7022,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.306344985961914,
      "rewards/margins": 4.695493221282959,
      "rewards/rejected": -8.001838684082031,
      "step": 166
    },
    {
      "epoch": 0.046693694953166505,
      "grad_norm": 118.05732935497699,
      "learning_rate": 8.066471602728803e-08,
      "logits/chosen": -1.2742832899093628,
      "logits/rejected": -1.172432541847229,
      "logps/chosen": -1.6827492713928223,
      "logps/rejected": -1.412048578262329,
      "loss": 1.6709,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -4.206873416900635,
      "rewards/margins": -0.6767516732215881,
      "rewards/rejected": -3.5301215648651123,
      "step": 167
    },
    {
      "epoch": 0.046973297916957917,
      "grad_norm": 16.707525761245098,
      "learning_rate": 7.597595192178702e-08,
      "logits/chosen": -1.2708326578140259,
      "logits/rejected": -1.250439167022705,
      "logps/chosen": -0.9043286442756653,
      "logps/rejected": -2.5250792503356934,
      "loss": 0.4602,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.2608213424682617,
      "rewards/margins": 4.051877021789551,
      "rewards/rejected": -6.3126983642578125,
      "step": 168
    },
    {
      "epoch": 0.047252900880749335,
      "grad_norm": 43.907524013668834,
      "learning_rate": 7.141634964894388e-08,
      "logits/chosen": -0.9978333711624146,
      "logits/rejected": -1.234339714050293,
      "logps/chosen": -1.7665379047393799,
      "logps/rejected": -2.1084702014923096,
      "loss": 0.9423,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.41634464263916,
      "rewards/margins": 0.8548306226730347,
      "rewards/rejected": -5.271175384521484,
      "step": 169
    },
    {
      "epoch": 0.047532503844540754,
      "grad_norm": 88.65537871518796,
      "learning_rate": 6.698729810778064e-08,
      "logits/chosen": -0.8296370506286621,
      "logits/rejected": -0.7557154297828674,
      "logps/chosen": -1.1506996154785156,
      "logps/rejected": -1.4387410879135132,
      "loss": 1.2953,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.876749038696289,
      "rewards/margins": 0.7201035022735596,
      "rewards/rejected": -3.5968525409698486,
      "step": 170
    },
    {
      "epoch": 0.047812106808332165,
      "grad_norm": 35.86474595345899,
      "learning_rate": 6.269014643030212e-08,
      "logits/chosen": -1.3039546012878418,
      "logits/rejected": -1.6143985986709595,
      "logps/chosen": -2.294726610183716,
      "logps/rejected": -2.714628219604492,
      "loss": 0.951,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -5.73681640625,
      "rewards/margins": 1.0497541427612305,
      "rewards/rejected": -6.7865705490112305,
      "step": 171
    },
    {
      "epoch": 0.048091709772123584,
      "grad_norm": 47.72518452969935,
      "learning_rate": 5.8526203570536504e-08,
      "logits/chosen": -1.2507563829421997,
      "logits/rejected": -1.449178695678711,
      "logps/chosen": -1.8437649011611938,
      "logps/rejected": -3.4956912994384766,
      "loss": 1.271,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.60941219329834,
      "rewards/margins": 4.129816055297852,
      "rewards/rejected": -8.739228248596191,
      "step": 172
    },
    {
      "epoch": 0.048371312735915,
      "grad_norm": 45.10015403750292,
      "learning_rate": 5.44967379058161e-08,
      "logits/chosen": -1.5573780536651611,
      "logits/rejected": -1.247377634048462,
      "logps/chosen": -0.8558065891265869,
      "logps/rejected": -2.1532318592071533,
      "loss": 0.7389,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.1395163536071777,
      "rewards/margins": 3.243563175201416,
      "rewards/rejected": -5.383079528808594,
      "step": 173
    },
    {
      "epoch": 0.048650915699706414,
      "grad_norm": 126.90491606998833,
      "learning_rate": 5.060297685041659e-08,
      "logits/chosen": -1.0825918912887573,
      "logits/rejected": -1.1961251497268677,
      "logps/chosen": -1.5061373710632324,
      "logps/rejected": -2.8128504753112793,
      "loss": 1.415,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.76534366607666,
      "rewards/margins": 3.266782283782959,
      "rewards/rejected": -7.032125949859619,
      "step": 174
    },
    {
      "epoch": 0.04893051866349783,
      "grad_norm": 104.83325539903379,
      "learning_rate": 4.684610648167503e-08,
      "logits/chosen": -0.2594190239906311,
      "logits/rejected": -1.2538113594055176,
      "logps/chosen": -0.8970815539360046,
      "logps/rejected": -3.357142925262451,
      "loss": 0.7837,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.242703914642334,
      "rewards/margins": 6.150154113769531,
      "rewards/rejected": -8.392858505249023,
      "step": 175
    },
    {
      "epoch": 0.04921012162728925,
      "grad_norm": 25.391300351126056,
      "learning_rate": 4.322727117869951e-08,
      "logits/chosen": -1.0320615768432617,
      "logits/rejected": -1.3764903545379639,
      "logps/chosen": -1.2408421039581299,
      "logps/rejected": -3.5107479095458984,
      "loss": 0.701,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.102104902267456,
      "rewards/margins": 5.674765110015869,
      "rewards/rejected": -8.776870727539062,
      "step": 176
    },
    {
      "epoch": 0.04948972459108066,
      "grad_norm": 100.44588632718167,
      "learning_rate": 3.974757327377981e-08,
      "logits/chosen": -1.1055123805999756,
      "logits/rejected": -0.956331729888916,
      "logps/chosen": -1.6190693378448486,
      "logps/rejected": -2.0674889087677,
      "loss": 0.8604,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -4.047673225402832,
      "rewards/margins": 1.1210490465164185,
      "rewards/rejected": -5.168722629547119,
      "step": 177
    },
    {
      "epoch": 0.04976932755487208,
      "grad_norm": 23.973360234647668,
      "learning_rate": 3.6408072716606345e-08,
      "logits/chosen": -0.6234188079833984,
      "logits/rejected": -0.7233335375785828,
      "logps/chosen": -1.2935428619384766,
      "logps/rejected": -1.6678102016448975,
      "loss": 1.0504,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.2338571548461914,
      "rewards/margins": 0.9356681704521179,
      "rewards/rejected": -4.169525146484375,
      "step": 178
    },
    {
      "epoch": 0.0500489305186635,
      "grad_norm": 128.60647901400966,
      "learning_rate": 3.3209786751399184e-08,
      "logits/chosen": -0.6642372608184814,
      "logits/rejected": -1.3921313285827637,
      "logps/chosen": -1.1783480644226074,
      "logps/rejected": -4.460213661193848,
      "loss": 1.2305,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.9458699226379395,
      "rewards/margins": 8.20466423034668,
      "rewards/rejected": -11.150533676147461,
      "step": 179
    },
    {
      "epoch": 0.05032853348245491,
      "grad_norm": 45.48438050770713,
      "learning_rate": 3.015368960704584e-08,
      "logits/chosen": -0.753233790397644,
      "logits/rejected": -0.880287766456604,
      "logps/chosen": -1.1847925186157227,
      "logps/rejected": -2.643932342529297,
      "loss": 1.6424,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.9619812965393066,
      "rewards/margins": 3.6478493213653564,
      "rewards/rejected": -6.609830856323242,
      "step": 180
    },
    {
      "epoch": 0.05060813644624633,
      "grad_norm": 43.3062398465464,
      "learning_rate": 2.7240712200341576e-08,
      "logits/chosen": -1.2592175006866455,
      "logits/rejected": -1.2679963111877441,
      "logps/chosen": -1.8530656099319458,
      "logps/rejected": -1.859635353088379,
      "loss": 1.8695,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -4.632663726806641,
      "rewards/margins": 0.01642441749572754,
      "rewards/rejected": -4.649088382720947,
      "step": 181
    },
    {
      "epoch": 0.05088773941003775,
      "grad_norm": 34.019147990522235,
      "learning_rate": 2.4471741852423233e-08,
      "logits/chosen": -1.0714620351791382,
      "logits/rejected": -1.2085020542144775,
      "logps/chosen": -1.3924332857131958,
      "logps/rejected": -1.9933589696884155,
      "loss": 1.1236,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.481083393096924,
      "rewards/margins": 1.5023138523101807,
      "rewards/rejected": -4.983397483825684,
      "step": 182
    },
    {
      "epoch": 0.05116734237382916,
      "grad_norm": 38.25765530969567,
      "learning_rate": 2.1847622018482282e-08,
      "logits/chosen": -1.323840856552124,
      "logits/rejected": -1.2305158376693726,
      "logps/chosen": -1.737386703491211,
      "logps/rejected": -2.449878692626953,
      "loss": 1.3231,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -4.343466758728027,
      "rewards/margins": 1.7812304496765137,
      "rewards/rejected": -6.124697208404541,
      "step": 183
    },
    {
      "epoch": 0.05144694533762058,
      "grad_norm": 21.832900395742424,
      "learning_rate": 1.936915203084055e-08,
      "logits/chosen": -1.102403163909912,
      "logits/rejected": -0.8794164061546326,
      "logps/chosen": -1.1961630582809448,
      "logps/rejected": -2.0587728023529053,
      "loss": 0.5583,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.990407705307007,
      "rewards/margins": 2.156524181365967,
      "rewards/rejected": -5.146932125091553,
      "step": 184
    },
    {
      "epoch": 0.051726548301412,
      "grad_norm": 97.17645669564288,
      "learning_rate": 1.7037086855465898e-08,
      "logits/chosen": -0.8995164036750793,
      "logits/rejected": -1.224024772644043,
      "logps/chosen": -1.5391126871109009,
      "logps/rejected": -3.2247936725616455,
      "loss": 1.0714,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.8477818965911865,
      "rewards/margins": 4.214201927185059,
      "rewards/rejected": -8.061984062194824,
      "step": 185
    },
    {
      "epoch": 0.05200615126520341,
      "grad_norm": 46.501514059890596,
      "learning_rate": 1.4852136862001763e-08,
      "logits/chosen": -1.3521168231964111,
      "logits/rejected": -1.7020211219787598,
      "logps/chosen": -1.7328448295593262,
      "logps/rejected": -2.3388752937316895,
      "loss": 1.4423,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -4.332111835479736,
      "rewards/margins": 1.5150765180587769,
      "rewards/rejected": -5.847188472747803,
      "step": 186
    },
    {
      "epoch": 0.05228575422899483,
      "grad_norm": 43.90284836540054,
      "learning_rate": 1.2814967607382432e-08,
      "logits/chosen": -0.7001566886901855,
      "logits/rejected": -0.8944656848907471,
      "logps/chosen": -1.2563323974609375,
      "logps/rejected": -1.8060386180877686,
      "loss": 0.4237,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.1408309936523438,
      "rewards/margins": 1.374265432357788,
      "rewards/rejected": -4.515096664428711,
      "step": 187
    },
    {
      "epoch": 0.052565357192786245,
      "grad_norm": 66.58807618416985,
      "learning_rate": 1.0926199633097154e-08,
      "logits/chosen": -1.131943941116333,
      "logits/rejected": -1.2059730291366577,
      "logps/chosen": -1.1455706357955933,
      "logps/rejected": -1.936645746231079,
      "loss": 0.7375,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.863926649093628,
      "rewards/margins": 1.9776880741119385,
      "rewards/rejected": -4.841614723205566,
      "step": 188
    },
    {
      "epoch": 0.05284496015657766,
      "grad_norm": 36.45672997200926,
      "learning_rate": 9.186408276168011e-09,
      "logits/chosen": -0.9764798879623413,
      "logits/rejected": -1.0748597383499146,
      "logps/chosen": -1.0145312547683716,
      "logps/rejected": -2.6933655738830566,
      "loss": 1.0416,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -2.5363283157348633,
      "rewards/margins": 4.197086334228516,
      "rewards/rejected": -6.733414649963379,
      "step": 189
    },
    {
      "epoch": 0.053124563120369075,
      "grad_norm": 23.88892255070114,
      "learning_rate": 7.59612349389599e-09,
      "logits/chosen": -0.809168815612793,
      "logits/rejected": -1.1629016399383545,
      "logps/chosen": -0.9837249517440796,
      "logps/rejected": -1.9994207620620728,
      "loss": 0.603,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.4593124389648438,
      "rewards/margins": 2.5392391681671143,
      "rewards/rejected": -4.998551845550537,
      "step": 190
    },
    {
      "epoch": 0.053404166084160494,
      "grad_norm": 38.039713749484235,
      "learning_rate": 6.15582970243117e-09,
      "logits/chosen": -1.061342716217041,
      "logits/rejected": -1.2929720878601074,
      "logps/chosen": -1.4455804824829102,
      "logps/rejected": -1.7912752628326416,
      "loss": 1.2204,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -3.6139512062072754,
      "rewards/margins": 0.8642373085021973,
      "rewards/rejected": -4.478188514709473,
      "step": 191
    },
    {
      "epoch": 0.053683769047951906,
      "grad_norm": 33.26330215420443,
      "learning_rate": 4.865965629214819e-09,
      "logits/chosen": -0.7917881608009338,
      "logits/rejected": -1.4771734476089478,
      "logps/chosen": -1.3420734405517578,
      "logps/rejected": -2.956531047821045,
      "loss": 1.2029,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.3551833629608154,
      "rewards/margins": 4.036144256591797,
      "rewards/rejected": -7.391327381134033,
      "step": 192
    },
    {
      "epoch": 0.053963372011743324,
      "grad_norm": 50.418507707166555,
      "learning_rate": 3.7269241793390084e-09,
      "logits/chosen": -0.7339926362037659,
      "logits/rejected": -0.8001390695571899,
      "logps/chosen": -1.375474214553833,
      "logps/rejected": -1.5513949394226074,
      "loss": 1.4448,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.438685655593872,
      "rewards/margins": 0.43980175256729126,
      "rewards/rejected": -3.8784873485565186,
      "step": 193
    },
    {
      "epoch": 0.05424297497553474,
      "grad_norm": 46.99311043278277,
      "learning_rate": 2.739052315863355e-09,
      "logits/chosen": -1.1927969455718994,
      "logits/rejected": -1.1135954856872559,
      "logps/chosen": -1.4356844425201416,
      "logps/rejected": -1.4172120094299316,
      "loss": 1.2511,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -3.5892109870910645,
      "rewards/margins": -0.046180903911590576,
      "rewards/rejected": -3.54302978515625,
      "step": 194
    },
    {
      "epoch": 0.054522577939326154,
      "grad_norm": 50.89269657493953,
      "learning_rate": 1.9026509541272273e-09,
      "logits/chosen": -1.0880765914916992,
      "logits/rejected": -1.1170682907104492,
      "logps/chosen": -1.426069974899292,
      "logps/rejected": -2.2877893447875977,
      "loss": 1.1481,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.5651750564575195,
      "rewards/margins": 2.1542983055114746,
      "rewards/rejected": -5.719472885131836,
      "step": 195
    },
    {
      "epoch": 0.05480218090311757,
      "grad_norm": 119.28075830322265,
      "learning_rate": 1.217974870087901e-09,
      "logits/chosen": -1.1547623872756958,
      "logits/rejected": -1.3626446723937988,
      "logps/chosen": -1.1919413805007935,
      "logps/rejected": -2.2724056243896484,
      "loss": 1.446,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.979853391647339,
      "rewards/margins": 2.7011606693267822,
      "rewards/rejected": -5.681014060974121,
      "step": 196
    },
    {
      "epoch": 0.05508178386690899,
      "grad_norm": 48.79918657535743,
      "learning_rate": 6.852326227130833e-10,
      "logits/chosen": -1.2602670192718506,
      "logits/rejected": -1.142798662185669,
      "logps/chosen": -1.5842217206954956,
      "logps/rejected": -1.682457685470581,
      "loss": 0.7233,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -3.9605541229248047,
      "rewards/margins": 0.2455899715423584,
      "rewards/rejected": -4.206143856048584,
      "step": 197
    },
    {
      "epoch": 0.0553613868307004,
      "grad_norm": 20.938919943310346,
      "learning_rate": 3.0458649045211894e-10,
      "logits/chosen": -0.6655712127685547,
      "logits/rejected": -0.7740037441253662,
      "logps/chosen": -1.6575719118118286,
      "logps/rejected": -1.8176357746124268,
      "loss": 1.2588,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -4.143929958343506,
      "rewards/margins": 0.4001593291759491,
      "rewards/rejected": -4.544089317321777,
      "step": 198
    },
    {
      "epoch": 0.05564098979449182,
      "grad_norm": 25.414963350937853,
      "learning_rate": 7.615242180436521e-11,
      "logits/chosen": -1.2654355764389038,
      "logits/rejected": -1.2140486240386963,
      "logps/chosen": -1.4302688837051392,
      "logps/rejected": -2.340785026550293,
      "loss": 0.8395,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -3.575672149658203,
      "rewards/margins": 2.276289939880371,
      "rewards/rejected": -5.851962089538574,
      "step": 199
    },
    {
      "epoch": 0.05592059275828324,
      "grad_norm": 82.36002044117453,
      "learning_rate": 0.0,
      "logits/chosen": -1.2721612453460693,
      "logits/rejected": -0.8177624940872192,
      "logps/chosen": -1.314448595046997,
      "logps/rejected": -1.1094343662261963,
      "loss": 1.4672,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -3.286121368408203,
      "rewards/margins": -0.5125355124473572,
      "rewards/rejected": -2.773585796356201,
      "step": 200
    },
    {
      "epoch": 0.05592059275828324,
      "eval_logits/chosen": -1.2117185592651367,
      "eval_logits/rejected": -1.3683098554611206,
      "eval_logps/chosen": -1.3815547227859497,
      "eval_logps/rejected": -2.264155149459839,
      "eval_loss": 0.9751168489456177,
      "eval_rewards/accuracies": 0.7612612843513489,
      "eval_rewards/chosen": -3.4538867473602295,
      "eval_rewards/margins": 2.206501007080078,
      "eval_rewards/rejected": -5.66038703918457,
      "eval_runtime": 344.9671,
      "eval_samples_per_second": 3.85,
      "eval_steps_per_second": 0.322,
      "step": 200
    },
    {
      "epoch": 0.05592059275828324,
      "step": 200,
      "total_flos": 0.0,
      "train_loss": 1.163502004146576,
      "train_runtime": 4144.9448,
      "train_samples_per_second": 0.579,
      "train_steps_per_second": 0.048
    }
  ],
  "logging_steps": 1,
  "max_steps": 200,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 10000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": false,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}