{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9994666666666666,
  "eval_steps": 500,
  "global_step": 937,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 5.319148936170213e-08,
      "logits/chosen": -0.1885092854499817,
      "logits/rejected": -0.3158565163612366,
      "logps/chosen": -579.0175170898438,
      "logps/rejected": -485.2366638183594,
      "loss": 0.2285,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "learning_rate": 5.319148936170213e-07,
      "logits/chosen": -0.1450551152229309,
      "logits/rejected": -0.07828879356384277,
      "logps/chosen": -490.4548034667969,
      "logps/rejected": -480.7243347167969,
      "loss": 0.21,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -5.791801231680438e-05,
      "rewards/margins": 4.912323129246943e-05,
      "rewards/rejected": -0.00010704125452321023,
      "step": 10
    },
    {
      "epoch": 0.02,
      "learning_rate": 1.0638297872340427e-06,
      "logits/chosen": -0.17377130687236786,
      "logits/rejected": -0.2042241394519806,
      "logps/chosen": -491.5914001464844,
      "logps/rejected": -509.80047607421875,
      "loss": 0.2024,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": 2.5502964490442537e-05,
      "rewards/margins": 0.00020013886387459934,
      "rewards/rejected": -0.00017463587573729455,
      "step": 20
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.595744680851064e-06,
      "logits/chosen": -0.13665731251239777,
      "logits/rejected": -0.13284753262996674,
      "logps/chosen": -530.023681640625,
      "logps/rejected": -542.8099975585938,
      "loss": 0.2154,
      "rewards/accuracies": 0.4312500059604645,
      "rewards/chosen": -0.0006009428761899471,
      "rewards/margins": 7.30833999114111e-05,
      "rewards/rejected": -0.0006740263779647648,
      "step": 30
    },
    {
      "epoch": 0.04,
      "learning_rate": 2.1276595744680853e-06,
      "logits/chosen": -0.10384176671504974,
      "logits/rejected": -0.09949521720409393,
      "logps/chosen": -578.4327392578125,
      "logps/rejected": -594.6671752929688,
      "loss": 0.2087,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.0006206859834492207,
      "rewards/margins": 0.0007750510121695697,
      "rewards/rejected": -0.0013957370538264513,
      "step": 40
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.6595744680851065e-06,
      "logits/chosen": -0.10368921607732773,
      "logits/rejected": -0.13414844870567322,
      "logps/chosen": -511.35809326171875,
      "logps/rejected": -534.3345947265625,
      "loss": 0.21,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.0013925038510933518,
      "rewards/margins": 0.0007838442688807845,
      "rewards/rejected": -0.00217634835280478,
      "step": 50
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.191489361702128e-06,
      "logits/chosen": -0.22053027153015137,
      "logits/rejected": -0.08493414521217346,
      "logps/chosen": -547.6785278320312,
      "logps/rejected": -560.6593627929688,
      "loss": 0.2124,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.002587032737210393,
      "rewards/margins": 0.0012574782595038414,
      "rewards/rejected": -0.003844511229544878,
      "step": 60
    },
    {
      "epoch": 0.07,
      "learning_rate": 3.723404255319149e-06,
      "logits/chosen": -0.11390117555856705,
      "logits/rejected": -0.09883277863264084,
      "logps/chosen": -533.27978515625,
      "logps/rejected": -538.4425048828125,
      "loss": 0.2137,
      "rewards/accuracies": 0.4312500059604645,
      "rewards/chosen": -0.005767723545432091,
      "rewards/margins": 0.0026725970674306154,
      "rewards/rejected": -0.008440319448709488,
      "step": 70
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.255319148936171e-06,
      "logits/chosen": -0.1157851368188858,
      "logits/rejected": -0.17272169888019562,
      "logps/chosen": -507.7423400878906,
      "logps/rejected": -523.6263427734375,
      "loss": 0.2106,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.00815755594521761,
      "rewards/margins": 0.0021632923744618893,
      "rewards/rejected": -0.010320847854018211,
      "step": 80
    },
    {
      "epoch": 0.1,
      "learning_rate": 4.787234042553192e-06,
      "logits/chosen": -0.21070297062397003,
      "logits/rejected": -0.20428261160850525,
      "logps/chosen": -488.6622009277344,
      "logps/rejected": -501.21856689453125,
      "loss": 0.2103,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.011966566555202007,
      "rewards/margins": 0.002781093353405595,
      "rewards/rejected": -0.014747659675776958,
      "step": 90
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.999375059004058e-06,
      "logits/chosen": -0.12732869386672974,
      "logits/rejected": -0.2185029536485672,
      "logps/chosen": -523.1364135742188,
      "logps/rejected": -521.9183349609375,
      "loss": 0.2006,
      "rewards/accuracies": 0.41874998807907104,
      "rewards/chosen": -0.01658087968826294,
      "rewards/margins": 0.0014435439370572567,
      "rewards/rejected": -0.018024425953626633,
      "step": 100
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.9955571065548795e-06,
      "logits/chosen": -0.23100972175598145,
      "logits/rejected": -0.26281121373176575,
      "logps/chosen": -547.6220092773438,
      "logps/rejected": -568.766845703125,
      "loss": 0.202,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.022633420303463936,
      "rewards/margins": 0.007052992470562458,
      "rewards/rejected": -0.02968641184270382,
      "step": 110
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.9882736864879e-06,
      "logits/chosen": -0.26259124279022217,
      "logits/rejected": -0.22674131393432617,
      "logps/chosen": -564.6571044921875,
      "logps/rejected": -574.2418212890625,
      "loss": 0.2018,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -0.03153581544756889,
      "rewards/margins": 0.011116450652480125,
      "rewards/rejected": -0.04265226796269417,
      "step": 120
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.977534912960124e-06,
      "logits/chosen": -0.19227923452854156,
      "logits/rejected": -0.21840712428092957,
      "logps/chosen": -534.7987060546875,
      "logps/rejected": -557.68994140625,
      "loss": 0.199,
      "rewards/accuracies": 0.4437499940395355,
      "rewards/chosen": -0.04380916431546211,
      "rewards/margins": 0.012184342369437218,
      "rewards/rejected": -0.05599350482225418,
      "step": 130
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.963355698422092e-06,
      "logits/chosen": -0.2127476930618286,
      "logits/rejected": -0.24109581112861633,
      "logps/chosen": -527.8475952148438,
      "logps/rejected": -546.1341552734375,
      "loss": 0.1976,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.039759356528520584,
      "rewards/margins": 0.020705662667751312,
      "rewards/rejected": -0.060465019196271896,
      "step": 140
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.945755732909625e-06,
      "logits/chosen": -0.29052820801734924,
      "logits/rejected": -0.18534401059150696,
      "logps/chosen": -568.7386474609375,
      "logps/rejected": -623.6467895507812,
      "loss": 0.203,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.05505699664354324,
      "rewards/margins": 0.02482803724706173,
      "rewards/rejected": -0.07988502830266953,
      "step": 150
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.924759456701167e-06,
      "logits/chosen": -0.21109052002429962,
      "logits/rejected": -0.23127368092536926,
      "logps/chosen": -509.3150329589844,
      "logps/rejected": -569.0074462890625,
      "loss": 0.1995,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -0.04530967399477959,
      "rewards/margins": 0.026868581771850586,
      "rewards/rejected": -0.07217825949192047,
      "step": 160
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.900396026378671e-06,
      "logits/chosen": -0.2945798635482788,
      "logits/rejected": -0.25223153829574585,
      "logps/chosen": -614.588134765625,
      "logps/rejected": -705.424072265625,
      "loss": 0.1997,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.07361334562301636,
      "rewards/margins": 0.043305329978466034,
      "rewards/rejected": -0.11691868305206299,
      "step": 170
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.872699274339169e-06,
      "logits/chosen": -0.2314704954624176,
      "logits/rejected": -0.11344774067401886,
      "logps/chosen": -570.4790649414062,
      "logps/rejected": -626.4537353515625,
      "loss": 0.203,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.06511107087135315,
      "rewards/margins": 0.03761782869696617,
      "rewards/rejected": -0.10272890329360962,
      "step": 180
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.8417076618132434e-06,
      "logits/chosen": -0.13913798332214355,
      "logits/rejected": -0.23164553940296173,
      "logps/chosen": -599.2550048828125,
      "logps/rejected": -628.2545776367188,
      "loss": 0.1984,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.06176919490098953,
      "rewards/margins": 0.029368698596954346,
      "rewards/rejected": -0.09113789349794388,
      "step": 190
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.807464225455655e-06,
      "logits/chosen": -0.1732599437236786,
      "logits/rejected": -0.24373570084571838,
      "logps/chosen": -551.7747802734375,
      "logps/rejected": -587.905517578125,
      "loss": 0.1991,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.060781996697187424,
      "rewards/margins": 0.024652186781167984,
      "rewards/rejected": -0.08543418347835541,
      "step": 200
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.770016517582283e-06,
      "logits/chosen": -0.19644469022750854,
      "logits/rejected": -0.19420596957206726,
      "logps/chosen": -596.8599853515625,
      "logps/rejected": -661.6636962890625,
      "loss": 0.1942,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.04881254583597183,
      "rewards/margins": 0.03198176622390747,
      "rewards/rejected": -0.0807943120598793,
      "step": 210
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.7294165401363616e-06,
      "logits/chosen": -0.2224278450012207,
      "logits/rejected": -0.10729773342609406,
      "logps/chosen": -604.28662109375,
      "logps/rejected": -651.0653076171875,
      "loss": 0.186,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.04510096088051796,
      "rewards/margins": 0.03683970868587494,
      "rewards/rejected": -0.0819406658411026,
      "step": 220
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.68572067247573e-06,
      "logits/chosen": -0.20905578136444092,
      "logits/rejected": -0.2025284320116043,
      "logps/chosen": -502.0655822753906,
      "logps/rejected": -572.6324462890625,
      "loss": 0.199,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.05115518718957901,
      "rewards/margins": 0.020948706194758415,
      "rewards/rejected": -0.07210389524698257,
      "step": 230
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.638989593081364e-06,
      "logits/chosen": -0.1577962040901184,
      "logits/rejected": -0.16388146579265594,
      "logps/chosen": -501.21551513671875,
      "logps/rejected": -542.8304443359375,
      "loss": 0.1941,
      "rewards/accuracies": 0.4312500059604645,
      "rewards/chosen": -0.03727288171648979,
      "rewards/margins": 0.03399407118558884,
      "rewards/rejected": -0.07126694917678833,
      "step": 240
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.5892881952959015e-06,
      "logits/chosen": -0.13874894380569458,
      "logits/rejected": -0.10486537218093872,
      "logps/chosen": -536.9546508789062,
      "logps/rejected": -576.2243041992188,
      "loss": 0.1903,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.03655445948243141,
      "rewards/margins": 0.023150647059082985,
      "rewards/rejected": -0.05970510095357895,
      "step": 250
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.536685497209182e-06,
      "logits/chosen": -0.15160498023033142,
      "logits/rejected": -0.20615887641906738,
      "logps/chosen": -536.8988647460938,
      "logps/rejected": -600.5662231445312,
      "loss": 0.1915,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.03945186734199524,
      "rewards/margins": 0.040184784680604935,
      "rewards/rejected": -0.07963664829730988,
      "step": 260
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.481254545815943e-06,
      "logits/chosen": -0.16373318433761597,
      "logits/rejected": -0.15171615779399872,
      "logps/chosen": -630.7620239257812,
      "logps/rejected": -654.4037475585938,
      "loss": 0.2068,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.05563617870211601,
      "rewards/margins": 0.019675999879837036,
      "rewards/rejected": -0.07531218230724335,
      "step": 270
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.42307231557875e-06,
      "logits/chosen": -0.06917186826467514,
      "logits/rejected": -0.1992654800415039,
      "logps/chosen": -583.5831909179688,
      "logps/rejected": -621.246826171875,
      "loss": 0.1911,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.03815799206495285,
      "rewards/margins": 0.02716698870062828,
      "rewards/rejected": -0.06532497704029083,
      "step": 280
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.3622196015370305e-06,
      "logits/chosen": -0.09893601387739182,
      "logits/rejected": -0.15837347507476807,
      "logps/chosen": -517.561279296875,
      "logps/rejected": -599.6585693359375,
      "loss": 0.1935,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.03791649267077446,
      "rewards/margins": 0.02659284509718418,
      "rewards/rejected": -0.0645093321800232,
      "step": 290
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.298780907110648e-06,
      "logits/chosen": -0.0635145753622055,
      "logits/rejected": -0.10447721183300018,
      "logps/chosen": -549.46875,
      "logps/rejected": -607.9141235351562,
      "loss": 0.1957,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.04532330110669136,
      "rewards/margins": 0.0257760938256979,
      "rewards/rejected": -0.07109940052032471,
      "step": 300
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.23284432675381e-06,
      "logits/chosen": -0.1379682570695877,
      "logits/rejected": -0.16269415616989136,
      "logps/chosen": -533.80029296875,
      "logps/rejected": -575.4193115234375,
      "loss": 0.1931,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.05671093985438347,
      "rewards/margins": 0.0316205732524395,
      "rewards/rejected": -0.08833150565624237,
      "step": 310
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.164501423622277e-06,
      "logits/chosen": -0.19988815486431122,
      "logits/rejected": -0.12660416960716248,
      "logps/chosen": -556.7050170898438,
      "logps/rejected": -533.2452392578125,
      "loss": 0.2061,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.0512329563498497,
      "rewards/margins": 0.006737883202731609,
      "rewards/rejected": -0.057970840483903885,
      "step": 320
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.0938471024237355e-06,
      "logits/chosen": -0.12714505195617676,
      "logits/rejected": -0.0678124725818634,
      "logps/chosen": -551.6757202148438,
      "logps/rejected": -634.5169677734375,
      "loss": 0.1845,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.04442024976015091,
      "rewards/margins": 0.053262293338775635,
      "rewards/rejected": -0.09768255054950714,
      "step": 330
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.020979477627907e-06,
      "logits/chosen": -0.08381222188472748,
      "logits/rejected": -0.1388164460659027,
      "logps/chosen": -567.6676635742188,
      "logps/rejected": -620.1175537109375,
      "loss": 0.1928,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.04053609445691109,
      "rewards/margins": 0.04673929885029793,
      "rewards/rejected": -0.08727538585662842,
      "step": 340
    },
    {
      "epoch": 0.37,
      "learning_rate": 3.9459997372194105e-06,
      "logits/chosen": -0.09087273478507996,
      "logits/rejected": -0.052247822284698486,
      "logps/chosen": -529.18408203125,
      "logps/rejected": -596.254638671875,
      "loss": 0.1969,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.05547628924250603,
      "rewards/margins": 0.04215295612812042,
      "rewards/rejected": -0.09762924164533615,
      "step": 350
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.869012002182573e-06,
      "logits/chosen": -0.2067844122648239,
      "logits/rejected": -0.17433646321296692,
      "logps/chosen": -554.9356689453125,
      "logps/rejected": -630.5103759765625,
      "loss": 0.1922,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.05108080059289932,
      "rewards/margins": 0.03945896774530411,
      "rewards/rejected": -0.09053976833820343,
      "step": 360
    },
    {
      "epoch": 0.39,
      "learning_rate": 3.7901231819133104e-06,
      "logits/chosen": -0.11504027992486954,
      "logits/rejected": -0.04383224993944168,
      "logps/chosen": -577.4075317382812,
      "logps/rejected": -607.4254150390625,
      "loss": 0.1934,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.044865719974040985,
      "rewards/margins": 0.025182927027344704,
      "rewards/rejected": -0.07004864513874054,
      "step": 370
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.709442825758875e-06,
      "logits/chosen": -0.16869018971920013,
      "logits/rejected": -0.1371726095676422,
      "logps/chosen": -584.6583251953125,
      "logps/rejected": -648.9109497070312,
      "loss": 0.2022,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.06359410285949707,
      "rewards/margins": 0.041747674345970154,
      "rewards/rejected": -0.10534176975488663,
      "step": 380
    },
    {
      "epoch": 0.42,
      "learning_rate": 3.6270829708916113e-06,
      "logits/chosen": -0.21822664141654968,
      "logits/rejected": -0.15734872221946716,
      "logps/chosen": -559.946533203125,
      "logps/rejected": -605.324951171875,
      "loss": 0.1974,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.051652587950229645,
      "rewards/margins": 0.02688014507293701,
      "rewards/rejected": -0.07853273302316666,
      "step": 390
    },
    {
      "epoch": 0.43,
      "learning_rate": 3.543157986727991e-06,
      "logits/chosen": -0.0996035560965538,
      "logits/rejected": -0.1632828712463379,
      "logps/chosen": -560.0746459960938,
      "logps/rejected": -612.9156494140625,
      "loss": 0.201,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.051237620413303375,
      "rewards/margins": 0.04104907065629959,
      "rewards/rejected": -0.09228668361902237,
      "step": 400
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4577844161089614e-06,
      "logits/chosen": -0.01575249433517456,
      "logits/rejected": -0.1881057471036911,
      "logps/chosen": -559.406494140625,
      "logps/rejected": -582.1732177734375,
      "loss": 0.2061,
      "rewards/accuracies": 0.41874998807907104,
      "rewards/chosen": -0.03435258939862251,
      "rewards/margins": 0.023811260238289833,
      "rewards/rejected": -0.05816385895013809,
      "step": 410
    },
    {
      "epoch": 0.45,
      "learning_rate": 3.3710808134621577e-06,
      "logits/chosen": -0.1044248715043068,
      "logits/rejected": -0.12559100985527039,
      "logps/chosen": -518.0560913085938,
      "logps/rejected": -559.5712890625,
      "loss": 0.2056,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.029239226132631302,
      "rewards/margins": 0.02849581278860569,
      "rewards/rejected": -0.05773504450917244,
      "step": 420
    },
    {
      "epoch": 0.46,
      "learning_rate": 3.2831675801707126e-06,
      "logits/chosen": -0.14103475213050842,
      "logits/rejected": -0.08099827915430069,
      "logps/chosen": -546.7091064453125,
      "logps/rejected": -608.6934814453125,
      "loss": 0.1944,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.028408538550138474,
      "rewards/margins": 0.034466955810785294,
      "rewards/rejected": -0.06287549436092377,
      "step": 430
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.194166797377289e-06,
      "logits/chosen": -0.1468835175037384,
      "logits/rejected": -0.10806012153625488,
      "logps/chosen": -539.1943969726562,
      "logps/rejected": -607.8096923828125,
      "loss": 0.1959,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.016944291070103645,
      "rewards/margins": 0.03214774280786514,
      "rewards/rejected": -0.04909203201532364,
      "step": 440
    },
    {
      "epoch": 0.48,
      "learning_rate": 3.104202056455501e-06,
      "logits/chosen": -0.07705841958522797,
      "logits/rejected": -0.13577719032764435,
      "logps/chosen": -534.7425537109375,
      "logps/rejected": -590.2757568359375,
      "loss": 0.1886,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.027219612151384354,
      "rewards/margins": 0.03623734414577484,
      "rewards/rejected": -0.0634569600224495,
      "step": 450
    },
    {
      "epoch": 0.49,
      "learning_rate": 3.013398287384144e-06,
      "logits/chosen": -0.05825149267911911,
      "logits/rejected": -0.0818571001291275,
      "logps/chosen": -504.8671875,
      "logps/rejected": -530.8888549804688,
      "loss": 0.1971,
      "rewards/accuracies": 0.45625001192092896,
      "rewards/chosen": -0.02120174467563629,
      "rewards/margins": 0.018600985407829285,
      "rewards/rejected": -0.039802733808755875,
      "step": 460
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.9218815852625717e-06,
      "logits/chosen": -0.1541839838027954,
      "logits/rejected": -0.06153281405568123,
      "logps/chosen": -549.5711669921875,
      "logps/rejected": -612.5526733398438,
      "loss": 0.1952,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.017424913123250008,
      "rewards/margins": 0.02850305661559105,
      "rewards/rejected": -0.04592796787619591,
      "step": 470
    },
    {
      "epoch": 0.51,
      "learning_rate": 2.829779035208113e-06,
      "logits/chosen": -0.03961920738220215,
      "logits/rejected": -0.1473100781440735,
      "logps/chosen": -557.7020874023438,
      "logps/rejected": -588.0806884765625,
      "loss": 0.1877,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.027860945090651512,
      "rewards/margins": 0.024425392970442772,
      "rewards/rejected": -0.052286338061094284,
      "step": 480
    },
    {
      "epoch": 0.52,
      "learning_rate": 2.737218535878705e-06,
      "logits/chosen": -0.029465090483427048,
      "logits/rejected": -0.08229364454746246,
      "logps/chosen": -511.9815368652344,
      "logps/rejected": -621.8477172851562,
      "loss": 0.1862,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.014062300324440002,
      "rewards/margins": 0.042990971356630325,
      "rewards/rejected": -0.05705326795578003,
      "step": 490
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.64432862186579e-06,
      "logits/chosen": -0.11332341283559799,
      "logits/rejected": -0.022313905879855156,
      "logps/chosen": -538.530517578125,
      "logps/rejected": -565.3148193359375,
      "loss": 0.1903,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.0199245847761631,
      "rewards/margins": 0.040730707347393036,
      "rewards/rejected": -0.06065528467297554,
      "step": 500
    },
    {
      "epoch": 0.54,
      "learning_rate": 2.551238285204126e-06,
      "logits/chosen": -0.17237767577171326,
      "logits/rejected": 0.029280701652169228,
      "logps/chosen": -522.3345947265625,
      "logps/rejected": -557.0701293945312,
      "loss": 0.185,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.02585068717598915,
      "rewards/margins": 0.03859715908765793,
      "rewards/rejected": -0.06444784253835678,
      "step": 510
    },
    {
      "epoch": 0.55,
      "learning_rate": 2.4580767962463688e-06,
      "logits/chosen": -0.1446155458688736,
      "logits/rejected": -0.15588733553886414,
      "logps/chosen": -495.2469177246094,
      "logps/rejected": -525.423095703125,
      "loss": 0.2006,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.03724539279937744,
      "rewards/margins": 0.0233670175075531,
      "rewards/rejected": -0.060612402856349945,
      "step": 520
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3649735241511546e-06,
      "logits/chosen": -0.10832977294921875,
      "logits/rejected": -0.17813685536384583,
      "logps/chosen": -487.40252685546875,
      "logps/rejected": -573.7322998046875,
      "loss": 0.1847,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.022226981818675995,
      "rewards/margins": 0.03627447038888931,
      "rewards/rejected": -0.05850145220756531,
      "step": 530
    },
    {
      "epoch": 0.58,
      "learning_rate": 2.2720577572339914e-06,
      "logits/chosen": -0.12618735432624817,
      "logits/rejected": -0.17089055478572845,
      "logps/chosen": -491.43927001953125,
      "logps/rejected": -632.1901245117188,
      "loss": 0.1815,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.0382881760597229,
      "rewards/margins": 0.06080668047070503,
      "rewards/rejected": -0.09909485280513763,
      "step": 540
    },
    {
      "epoch": 0.59,
      "learning_rate": 2.1794585234303995e-06,
      "logits/chosen": -0.16708405315876007,
      "logits/rejected": -0.11356528848409653,
      "logps/chosen": -579.8703002929688,
      "logps/rejected": -630.3308715820312,
      "loss": 0.2008,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.04018039256334305,
      "rewards/margins": 0.03480806201696396,
      "rewards/rejected": -0.074988454580307,
      "step": 550
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.0873044111206407e-06,
      "logits/chosen": -0.030877122655510902,
      "logits/rejected": -0.18640507757663727,
      "logps/chosen": -507.278564453125,
      "logps/rejected": -627.2957153320312,
      "loss": 0.1897,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.040919769555330276,
      "rewards/margins": 0.057074230164289474,
      "rewards/rejected": -0.09799399971961975,
      "step": 560
    },
    {
      "epoch": 0.61,
      "learning_rate": 1.9957233905648293e-06,
      "logits/chosen": -0.17021183669567108,
      "logits/rejected": -0.23673537373542786,
      "logps/chosen": -544.7388916015625,
      "logps/rejected": -587.5650024414062,
      "loss": 0.1941,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.04041619971394539,
      "rewards/margins": 0.024558167904615402,
      "rewards/rejected": -0.06497436761856079,
      "step": 570
    },
    {
      "epoch": 0.62,
      "learning_rate": 1.904842636196402e-06,
      "logits/chosen": -0.13388361036777496,
      "logits/rejected": -0.0614379458129406,
      "logps/chosen": -553.6912841796875,
      "logps/rejected": -629.6470336914062,
      "loss": 0.1903,
      "rewards/accuracies": 0.543749988079071,
      "rewards/chosen": -0.025933533906936646,
      "rewards/margins": 0.0378827229142189,
      "rewards/rejected": -0.06381625682115555,
      "step": 580
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.814788350020726e-06,
      "logits/chosen": -0.06043483689427376,
      "logits/rejected": -0.10723511874675751,
      "logps/chosen": -521.6217041015625,
      "logps/rejected": -643.6219482421875,
      "loss": 0.197,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.024261657148599625,
      "rewards/margins": 0.03998683765530586,
      "rewards/rejected": -0.06424849480390549,
      "step": 590
    },
    {
      "epoch": 0.64,
      "learning_rate": 1.725685586364051e-06,
      "logits/chosen": -0.1038782149553299,
      "logits/rejected": -0.047629911452531815,
      "logps/chosen": -535.3336181640625,
      "logps/rejected": -564.0821533203125,
      "loss": 0.1948,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.024827763438224792,
      "rewards/margins": 0.024268008768558502,
      "rewards/rejected": -0.049095772206783295,
      "step": 600
    },
    {
      "epoch": 0.65,
      "learning_rate": 1.6376580782162172e-06,
      "logits/chosen": -0.12502829730510712,
      "logits/rejected": -0.13951411843299866,
      "logps/chosen": -565.2417602539062,
      "logps/rejected": -653.1436157226562,
      "loss": 0.1921,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.027302825823426247,
      "rewards/margins": 0.03322170302271843,
      "rewards/rejected": -0.060524530708789825,
      "step": 610
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.550828065408227e-06,
      "logits/chosen": -0.15310420095920563,
      "logits/rejected": -0.035526055842638016,
      "logps/chosen": -510.84759521484375,
      "logps/rejected": -573.4019775390625,
      "loss": 0.1942,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.02462349459528923,
      "rewards/margins": 0.02571946755051613,
      "rewards/rejected": -0.05034296587109566,
      "step": 620
    },
    {
      "epoch": 0.67,
      "learning_rate": 1.4653161248633053e-06,
      "logits/chosen": -0.12701451778411865,
      "logits/rejected": -0.03017173707485199,
      "logps/chosen": -507.4302673339844,
      "logps/rejected": -554.05029296875,
      "loss": 0.1878,
      "rewards/accuracies": 0.48124998807907104,
      "rewards/chosen": -0.028983917087316513,
      "rewards/margins": 0.03194578364491463,
      "rewards/rejected": -0.06092969700694084,
      "step": 630
    },
    {
      "epoch": 0.68,
      "learning_rate": 1.381241003157162e-06,
      "logits/chosen": -0.1072702407836914,
      "logits/rejected": -0.12664569914340973,
      "logps/chosen": -511.8506774902344,
      "logps/rejected": -582.0699462890625,
      "loss": 0.199,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.034758515655994415,
      "rewards/margins": 0.027269739657640457,
      "rewards/rejected": -0.062028251588344574,
      "step": 640
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.298719451619979e-06,
      "logits/chosen": -0.05098239332437515,
      "logits/rejected": -0.1476130485534668,
      "logps/chosen": -521.5150146484375,
      "logps/rejected": -601.8914184570312,
      "loss": 0.1877,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.020487060770392418,
      "rewards/margins": 0.033522095531225204,
      "rewards/rejected": -0.054009150713682175,
      "step": 650
    },
    {
      "epoch": 0.7,
      "learning_rate": 1.2178660642091036e-06,
      "logits/chosen": -0.10045331716537476,
      "logits/rejected": -0.10961699485778809,
      "logps/chosen": -547.9536743164062,
      "logps/rejected": -613.6334228515625,
      "loss": 0.1825,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.02681289240717888,
      "rewards/margins": 0.05183552950620651,
      "rewards/rejected": -0.07864842563867569,
      "step": 660
    },
    {
      "epoch": 0.71,
      "learning_rate": 1.1387931183775821e-06,
      "logits/chosen": -0.08650709688663483,
      "logits/rejected": -0.17840668559074402,
      "logps/chosen": -531.9118041992188,
      "logps/rejected": -626.980224609375,
      "loss": 0.1847,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.019130777567625046,
      "rewards/margins": 0.05559687688946724,
      "rewards/rejected": -0.07472765445709229,
      "step": 670
    },
    {
      "epoch": 0.73,
      "learning_rate": 1.061610419159532e-06,
      "logits/chosen": -0.04611722379922867,
      "logits/rejected": -0.09121225029230118,
      "logps/chosen": -550.1459350585938,
      "logps/rejected": -607.9168090820312,
      "loss": 0.1874,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.029284963384270668,
      "rewards/margins": 0.05334455519914627,
      "rewards/rejected": -0.08262950927019119,
      "step": 680
    },
    {
      "epoch": 0.74,
      "learning_rate": 9.864251466888364e-07,
      "logits/chosen": -0.15967205166816711,
      "logits/rejected": -0.06603795289993286,
      "logps/chosen": -557.4961547851562,
      "logps/rejected": -607.64306640625,
      "loss": 0.1812,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.034804992377758026,
      "rewards/margins": 0.03645947948098183,
      "rewards/rejected": -0.07126447558403015,
      "step": 690
    },
    {
      "epoch": 0.75,
      "learning_rate": 9.133417073629288e-07,
      "logits/chosen": -0.15150725841522217,
      "logits/rejected": -0.11879472434520721,
      "logps/chosen": -504.88665771484375,
      "logps/rejected": -582.1925659179688,
      "loss": 0.192,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.03086964413523674,
      "rewards/margins": 0.04862387850880623,
      "rewards/rejected": -0.07949351519346237,
      "step": 700
    },
    {
      "epoch": 0.76,
      "learning_rate": 8.424615888583332e-07,
      "logits/chosen": -0.13878367841243744,
      "logits/rejected": -0.08197399228811264,
      "logps/chosen": -521.1176147460938,
      "logps/rejected": -626.109375,
      "loss": 0.1877,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.028367381542921066,
      "rewards/margins": 0.05848144367337227,
      "rewards/rejected": -0.08684882521629333,
      "step": 710
    },
    {
      "epoch": 0.77,
      "learning_rate": 7.738832191993092e-07,
      "logits/chosen": -0.11670303344726562,
      "logits/rejected": -0.1110248789191246,
      "logps/chosen": -546.02685546875,
      "logps/rejected": -633.3377075195312,
      "loss": 0.183,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.030107159167528152,
      "rewards/margins": 0.04191134124994278,
      "rewards/rejected": -0.07201849669218063,
      "step": 720
    },
    {
      "epoch": 0.78,
      "learning_rate": 7.077018300752917e-07,
      "logits/chosen": -0.20101483166217804,
      "logits/rejected": -0.09887303411960602,
      "logps/chosen": -515.279296875,
      "logps/rejected": -566.751220703125,
      "loss": 0.1959,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.02799845300614834,
      "rewards/margins": 0.039141736924648285,
      "rewards/rejected": -0.06714018434286118,
      "step": 730
    },
    {
      "epoch": 0.79,
      "learning_rate": 6.440093245969342e-07,
      "logits/chosen": -0.21685993671417236,
      "logits/rejected": -0.014155983924865723,
      "logps/chosen": -536.5148315429688,
      "logps/rejected": -555.0255126953125,
      "loss": 0.1909,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.042235199362039566,
      "rewards/margins": 0.033806830644607544,
      "rewards/rejected": -0.07604203373193741,
      "step": 740
    },
    {
      "epoch": 0.8,
      "learning_rate": 5.828941496744075e-07,
      "logits/chosen": -0.1709945797920227,
      "logits/rejected": -0.09519165009260178,
      "logps/chosen": -487.0716247558594,
      "logps/rejected": -570.6549072265625,
      "loss": 0.1891,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.03168477118015289,
      "rewards/margins": 0.05396095663309097,
      "rewards/rejected": -0.08564572036266327,
      "step": 750
    },
    {
      "epoch": 0.81,
      "learning_rate": 5.244411731951671e-07,
      "logits/chosen": -0.17727656662464142,
      "logits/rejected": -0.11151037365198135,
      "logps/chosen": -520.3614501953125,
      "logps/rejected": -573.2659301757812,
      "loss": 0.1921,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.024441594257950783,
      "rewards/margins": 0.03574614226818085,
      "rewards/rejected": -0.06018773466348648,
      "step": 760
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.6873156617173594e-07,
      "logits/chosen": -0.16788466274738312,
      "logits/rejected": -0.04893454164266586,
      "logps/chosen": -508.6934509277344,
      "logps/rejected": -575.983154296875,
      "loss": 0.1867,
      "rewards/accuracies": 0.4437499940395355,
      "rewards/chosen": -0.021922405809164047,
      "rewards/margins": 0.034182045608758926,
      "rewards/rejected": -0.05610444396734238,
      "step": 770
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.1584269002318653e-07,
      "logits/chosen": -0.07019881904125214,
      "logits/rejected": -0.18743988871574402,
      "logps/chosen": -485.97833251953125,
      "logps/rejected": -594.727783203125,
      "loss": 0.1866,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.03881939500570297,
      "rewards/margins": 0.05370662733912468,
      "rewards/rejected": -0.09252602607011795,
      "step": 780
    },
    {
      "epoch": 0.84,
      "learning_rate": 3.658479891468258e-07,
      "logits/chosen": -0.17362000048160553,
      "logits/rejected": -0.11806299537420273,
      "logps/chosen": -479.9105529785156,
      "logps/rejected": -610.3316040039062,
      "loss": 0.1924,
      "rewards/accuracies": 0.518750011920929,
      "rewards/chosen": -0.023625101894140244,
      "rewards/margins": 0.07324331998825073,
      "rewards/rejected": -0.09686841815710068,
      "step": 790
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.18816888929272e-07,
      "logits/chosen": -0.13044703006744385,
      "logits/rejected": -0.13462410867214203,
      "logps/chosen": -511.01275634765625,
      "logps/rejected": -576.4395141601562,
      "loss": 0.1845,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.024302612990140915,
      "rewards/margins": 0.04658300057053566,
      "rewards/rejected": -0.07088561356067657,
      "step": 800
    },
    {
      "epoch": 0.86,
      "learning_rate": 2.748146993385484e-07,
      "logits/chosen": -0.09310281276702881,
      "logits/rejected": -0.14107745885849,
      "logps/chosen": -542.4686279296875,
      "logps/rejected": -590.4827270507812,
      "loss": 0.1911,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.03655749931931496,
      "rewards/margins": 0.04121191054582596,
      "rewards/rejected": -0.07776941359043121,
      "step": 810
    },
    {
      "epoch": 0.87,
      "learning_rate": 2.3390252423108077e-07,
      "logits/chosen": -0.10764355957508087,
      "logits/rejected": -0.12145823240280151,
      "logps/chosen": -505.4794006347656,
      "logps/rejected": -602.5386352539062,
      "loss": 0.1825,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": -0.023508111014962196,
      "rewards/margins": 0.05395573377609253,
      "rewards/rejected": -0.07746385037899017,
      "step": 820
    },
    {
      "epoch": 0.89,
      "learning_rate": 1.961371764995243e-07,
      "logits/chosen": -0.10990991443395615,
      "logits/rejected": -0.13984012603759766,
      "logps/chosen": -488.1162109375,
      "logps/rejected": -533.455810546875,
      "loss": 0.1824,
      "rewards/accuracies": 0.4312500059604645,
      "rewards/chosen": -0.02799038216471672,
      "rewards/margins": 0.03639952093362808,
      "rewards/rejected": -0.0643898993730545,
      "step": 830
    },
    {
      "epoch": 0.9,
      "learning_rate": 1.61571099179261e-07,
      "logits/chosen": -0.1293899267911911,
      "logits/rejected": -0.13913944363594055,
      "logps/chosen": -565.6857299804688,
      "logps/rejected": -594.4491577148438,
      "loss": 0.1899,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.038701124489307404,
      "rewards/margins": 0.03532193973660469,
      "rewards/rejected": -0.07402306795120239,
      "step": 840
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.3025229262312367e-07,
      "logits/chosen": -0.11940865218639374,
      "logits/rejected": -0.14275464415550232,
      "logps/chosen": -517.4210815429688,
      "logps/rejected": -559.6951904296875,
      "loss": 0.1943,
      "rewards/accuracies": 0.4937500059604645,
      "rewards/chosen": -0.029175525531172752,
      "rewards/margins": 0.038891829550266266,
      "rewards/rejected": -0.06806735694408417,
      "step": 850
    },
    {
      "epoch": 0.92,
      "learning_rate": 1.0222424784546853e-07,
      "logits/chosen": -0.14586542546749115,
      "logits/rejected": -0.08942988514900208,
      "logps/chosen": -556.1111450195312,
      "logps/rejected": -648.3941650390625,
      "loss": 0.1964,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.03367298096418381,
      "rewards/margins": 0.05108407139778137,
      "rewards/rejected": -0.08475704491138458,
      "step": 860
    },
    {
      "epoch": 0.93,
      "learning_rate": 7.752588612816553e-08,
      "logits/chosen": -0.1201920285820961,
      "logits/rejected": -0.1033986434340477,
      "logps/chosen": -535.7073364257812,
      "logps/rejected": -629.0509033203125,
      "loss": 0.1942,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.03432587534189224,
      "rewards/margins": 0.05863683670759201,
      "rewards/rejected": -0.09296270459890366,
      "step": 870
    },
    {
      "epoch": 0.94,
      "learning_rate": 5.619150497236991e-08,
      "logits/chosen": -0.06793894618749619,
      "logits/rejected": -0.14243273437023163,
      "logps/chosen": -543.0166625976562,
      "logps/rejected": -573.83349609375,
      "loss": 0.1839,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.02393261529505253,
      "rewards/margins": 0.04055067151784897,
      "rewards/rejected": -0.06448328495025635,
      "step": 880
    },
    {
      "epoch": 0.95,
      "learning_rate": 3.825073047112743e-08,
      "logits/chosen": -0.09055639803409576,
      "logits/rejected": -0.09645718336105347,
      "logps/chosen": -589.1398315429688,
      "logps/rejected": -638.0357666015625,
      "loss": 0.2051,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.0366409607231617,
      "rewards/margins": 0.046200644224882126,
      "rewards/rejected": -0.08284159749746323,
      "step": 890
    },
    {
      "epoch": 0.96,
      "learning_rate": 2.372847616895685e-08,
      "logits/chosen": -0.24545852839946747,
      "logits/rejected": -0.13447916507720947,
      "logps/chosen": -500.57177734375,
      "logps/rejected": -562.6080322265625,
      "loss": 0.1894,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.04158453270792961,
      "rewards/margins": 0.03680362552404404,
      "rewards/rejected": -0.07838816195726395,
      "step": 900
    },
    {
      "epoch": 0.97,
      "learning_rate": 1.264490846553279e-08,
      "logits/chosen": -0.15378254652023315,
      "logits/rejected": -0.09510638564825058,
      "logps/chosen": -532.951416015625,
      "logps/rejected": -582.2918090820312,
      "loss": 0.1947,
      "rewards/accuracies": 0.5062500238418579,
      "rewards/chosen": -0.033251263201236725,
      "rewards/margins": 0.04195471107959747,
      "rewards/rejected": -0.0752059668302536,
      "step": 910
    },
    {
      "epoch": 0.98,
      "learning_rate": 5.015418611516165e-09,
      "logits/chosen": -0.06104772537946701,
      "logits/rejected": -0.19414536654949188,
      "logps/chosen": -515.5169067382812,
      "logps/rejected": -601.0435791015625,
      "loss": 0.1897,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.03219000995159149,
      "rewards/margins": 0.048967912793159485,
      "rewards/rejected": -0.08115791529417038,
      "step": 920
    },
    {
      "epoch": 0.99,
      "learning_rate": 8.506013354186993e-10,
      "logits/chosen": -0.08989392220973969,
      "logits/rejected": -0.16842840611934662,
      "logps/chosen": -527.9075927734375,
      "logps/rejected": -573.1229858398438,
      "loss": 0.1872,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.03477403521537781,
      "rewards/margins": 0.039979539811611176,
      "rewards/rejected": -0.07475356757640839,
      "step": 930
    },
    {
      "epoch": 1.0,
      "step": 937,
      "total_flos": 0.0,
      "train_loss": 0.08862862108611247,
      "train_runtime": 3263.0882,
      "train_samples_per_second": 9.194,
      "train_steps_per_second": 0.287
    }
  ],
  "logging_steps": 10,
  "max_steps": 937,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}