{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9981298423724285,
  "eval_steps": 400,
  "global_step": 467,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.010686615014694095,
      "grad_norm": 56.81669148706694,
      "learning_rate": 1.0638297872340425e-07,
      "logits/chosen": -1.0553786754608154,
      "logits/rejected": -0.9845818281173706,
      "logps/chosen": -0.7874154448509216,
      "logps/rejected": -0.7125983834266663,
      "loss": 3.9428,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": -7.874154567718506,
      "rewards/margins": -0.7481697797775269,
      "rewards/rejected": -7.125983238220215,
      "step": 5
    },
    {
      "epoch": 0.02137323002938819,
      "grad_norm": 90.64309336283279,
      "learning_rate": 2.127659574468085e-07,
      "logits/chosen": -0.9677240252494812,
      "logits/rejected": -0.9489688873291016,
      "logps/chosen": -0.8535294532775879,
      "logps/rejected": -1.0613572597503662,
      "loss": 3.6689,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -8.535294532775879,
      "rewards/margins": 2.0782790184020996,
      "rewards/rejected": -10.613574028015137,
      "step": 10
    },
    {
      "epoch": 0.03205984504408229,
      "grad_norm": 110.04849731267365,
      "learning_rate": 3.1914893617021275e-07,
      "logits/chosen": -0.9973424673080444,
      "logits/rejected": -0.9682930111885071,
      "logps/chosen": -0.8421398401260376,
      "logps/rejected": -0.9425029754638672,
      "loss": 3.9663,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -8.42139720916748,
      "rewards/margins": 1.0036319494247437,
      "rewards/rejected": -9.425029754638672,
      "step": 15
    },
    {
      "epoch": 0.04274646005877638,
      "grad_norm": 89.89735367257849,
      "learning_rate": 4.25531914893617e-07,
      "logits/chosen": -1.0042946338653564,
      "logits/rejected": -0.9698360562324524,
      "logps/chosen": -0.6765571236610413,
      "logps/rejected": -0.7212239503860474,
      "loss": 3.798,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -6.765571594238281,
      "rewards/margins": 0.44666871428489685,
      "rewards/rejected": -7.212240695953369,
      "step": 20
    },
    {
      "epoch": 0.053433075073470476,
      "grad_norm": 40.5523084584386,
      "learning_rate": 5.319148936170212e-07,
      "logits/chosen": -1.042263388633728,
      "logits/rejected": -1.020334005355835,
      "logps/chosen": -0.6275808215141296,
      "logps/rejected": -0.6092665195465088,
      "loss": 3.5027,
      "rewards/accuracies": 0.38749998807907104,
      "rewards/chosen": -6.275807857513428,
      "rewards/margins": -0.18314316868782043,
      "rewards/rejected": -6.092665195465088,
      "step": 25
    },
    {
      "epoch": 0.06411969008816458,
      "grad_norm": 82.97126019183185,
      "learning_rate": 6.382978723404255e-07,
      "logits/chosen": -1.0934265851974487,
      "logits/rejected": -1.0273423194885254,
      "logps/chosen": -0.4222745895385742,
      "logps/rejected": -0.4437929689884186,
      "loss": 3.2915,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -4.2227463722229,
      "rewards/margins": 0.21518389880657196,
      "rewards/rejected": -4.437930107116699,
      "step": 30
    },
    {
      "epoch": 0.07480630510285867,
      "grad_norm": 45.671980780198346,
      "learning_rate": 7.446808510638297e-07,
      "logits/chosen": -1.1021089553833008,
      "logits/rejected": -1.016642689704895,
      "logps/chosen": -0.40351754426956177,
      "logps/rejected": -0.4182681143283844,
      "loss": 3.0336,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -4.035175800323486,
      "rewards/margins": 0.14750559628009796,
      "rewards/rejected": -4.182681083679199,
      "step": 35
    },
    {
      "epoch": 0.08549292011755276,
      "grad_norm": 56.834561606796534,
      "learning_rate": 8.51063829787234e-07,
      "logits/chosen": -1.0844147205352783,
      "logits/rejected": -1.0052152872085571,
      "logps/chosen": -0.3840138018131256,
      "logps/rejected": -0.41540414094924927,
      "loss": 2.9873,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -3.8401379585266113,
      "rewards/margins": 0.3139037489891052,
      "rewards/rejected": -4.154041290283203,
      "step": 40
    },
    {
      "epoch": 0.09617953513224686,
      "grad_norm": 57.81609876406988,
      "learning_rate": 9.574468085106384e-07,
      "logits/chosen": -1.0846834182739258,
      "logits/rejected": -0.9838830828666687,
      "logps/chosen": -0.3058851957321167,
      "logps/rejected": -0.3447844386100769,
      "loss": 2.9473,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -3.058851957321167,
      "rewards/margins": 0.3889924883842468,
      "rewards/rejected": -3.4478442668914795,
      "step": 45
    },
    {
      "epoch": 0.10686615014694095,
      "grad_norm": 107.63293409046761,
      "learning_rate": 9.998741174712533e-07,
      "logits/chosen": -1.183288335800171,
      "logits/rejected": -1.0858242511749268,
      "logps/chosen": -0.3091445863246918,
      "logps/rejected": -0.3627066910266876,
      "loss": 2.9167,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -3.0914459228515625,
      "rewards/margins": 0.5356208682060242,
      "rewards/rejected": -3.6270668506622314,
      "step": 50
    },
    {
      "epoch": 0.11755276516163506,
      "grad_norm": 40.363889487849875,
      "learning_rate": 9.991050648838675e-07,
      "logits/chosen": -1.1698521375656128,
      "logits/rejected": -1.1601616144180298,
      "logps/chosen": -0.29625993967056274,
      "logps/rejected": -0.33069461584091187,
      "loss": 2.9262,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -2.962599515914917,
      "rewards/margins": 0.34434670209884644,
      "rewards/rejected": -3.306946277618408,
      "step": 55
    },
    {
      "epoch": 0.12823938017632916,
      "grad_norm": 49.68454575475429,
      "learning_rate": 9.97637968732563e-07,
      "logits/chosen": -1.2926113605499268,
      "logits/rejected": -1.2354081869125366,
      "logps/chosen": -0.34236031770706177,
      "logps/rejected": -0.370178759098053,
      "loss": 2.828,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -3.423603057861328,
      "rewards/margins": 0.2781844735145569,
      "rewards/rejected": -3.7017874717712402,
      "step": 60
    },
    {
      "epoch": 0.13892599519102325,
      "grad_norm": 64.97695318174799,
      "learning_rate": 9.954748808839674e-07,
      "logits/chosen": -1.2277723550796509,
      "logits/rejected": -1.1402448415756226,
      "logps/chosen": -0.33444857597351074,
      "logps/rejected": -0.38238364458084106,
      "loss": 2.8768,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -3.3444857597351074,
      "rewards/margins": 0.47935038805007935,
      "rewards/rejected": -3.8238365650177,
      "step": 65
    },
    {
      "epoch": 0.14961261020571734,
      "grad_norm": 42.426764348756556,
      "learning_rate": 9.926188266120295e-07,
      "logits/chosen": -1.2491295337677002,
      "logits/rejected": -1.2098782062530518,
      "logps/chosen": -0.2992488443851471,
      "logps/rejected": -0.34425443410873413,
      "loss": 2.8068,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -2.992488145828247,
      "rewards/margins": 0.45005664229393005,
      "rewards/rejected": -3.442544460296631,
      "step": 70
    },
    {
      "epoch": 0.16029922522041143,
      "grad_norm": 55.41018623107141,
      "learning_rate": 9.890738003669027e-07,
      "logits/chosen": -1.224936604499817,
      "logits/rejected": -1.1226717233657837,
      "logps/chosen": -0.3549841642379761,
      "logps/rejected": -0.42627596855163574,
      "loss": 2.6663,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -3.5498414039611816,
      "rewards/margins": 0.7129180431365967,
      "rewards/rejected": -4.262759685516357,
      "step": 75
    },
    {
      "epoch": 0.17098584023510552,
      "grad_norm": 67.19768319790357,
      "learning_rate": 9.848447601883433e-07,
      "logits/chosen": -1.22861647605896,
      "logits/rejected": -1.173981785774231,
      "logps/chosen": -0.35033124685287476,
      "logps/rejected": -0.3842136263847351,
      "loss": 2.6237,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -3.503312349319458,
      "rewards/margins": 0.3388238847255707,
      "rewards/rejected": -3.8421363830566406,
      "step": 80
    },
    {
      "epoch": 0.18167245524979964,
      "grad_norm": 56.06934270502333,
      "learning_rate": 9.799376207714444e-07,
      "logits/chosen": -1.1919827461242676,
      "logits/rejected": -1.145477056503296,
      "logps/chosen": -0.3422418534755707,
      "logps/rejected": -0.4096073508262634,
      "loss": 2.6001,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -3.4224185943603516,
      "rewards/margins": 0.6736553311347961,
      "rewards/rejected": -4.096073627471924,
      "step": 85
    },
    {
      "epoch": 0.19235907026449373,
      "grad_norm": 66.32485026269077,
      "learning_rate": 9.743592451943998e-07,
      "logits/chosen": -1.2551718950271606,
      "logits/rejected": -1.1607410907745361,
      "logps/chosen": -0.39799433946609497,
      "logps/rejected": -0.47051042318344116,
      "loss": 2.6013,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -3.9799437522888184,
      "rewards/margins": 0.7251602411270142,
      "rewards/rejected": -4.705103397369385,
      "step": 90
    },
    {
      "epoch": 0.20304568527918782,
      "grad_norm": 88.72645076406972,
      "learning_rate": 9.681174353198686e-07,
      "logits/chosen": -1.202864646911621,
      "logits/rejected": -1.0979254245758057,
      "logps/chosen": -0.40985170006752014,
      "logps/rejected": -0.4939080774784088,
      "loss": 2.47,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -4.098517417907715,
      "rewards/margins": 0.8405635952949524,
      "rewards/rejected": -4.939080238342285,
      "step": 95
    },
    {
      "epoch": 0.2137323002938819,
      "grad_norm": 78.62356168132955,
      "learning_rate": 9.612209208833646e-07,
      "logits/chosen": -1.210082769393921,
      "logits/rejected": -1.19521963596344,
      "logps/chosen": -0.4767809808254242,
      "logps/rejected": -0.6341355443000793,
      "loss": 2.4364,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -4.767809867858887,
      "rewards/margins": 1.5735454559326172,
      "rewards/rejected": -6.341355323791504,
      "step": 100
    },
    {
      "epoch": 0.224418915308576,
      "grad_norm": 182.17113716520097,
      "learning_rate": 9.536793472839324e-07,
      "logits/chosen": -1.318027138710022,
      "logits/rejected": -1.2231770753860474,
      "logps/chosen": -0.5009247064590454,
      "logps/rejected": -0.6060206890106201,
      "loss": 2.4925,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -5.009247779846191,
      "rewards/margins": 1.050958514213562,
      "rewards/rejected": -6.060206413269043,
      "step": 105
    },
    {
      "epoch": 0.2351055303232701,
      "grad_norm": 97.52898918968008,
      "learning_rate": 9.455032620941839e-07,
      "logits/chosen": -1.3112456798553467,
      "logits/rejected": -1.2372304201126099,
      "logps/chosen": -0.4988152086734772,
      "logps/rejected": -0.6063084602355957,
      "loss": 2.4096,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -4.988151550292969,
      "rewards/margins": 1.074933409690857,
      "rewards/rejected": -6.063085079193115,
      "step": 110
    },
    {
      "epoch": 0.2457921453379642,
      "grad_norm": 65.2666290162174,
      "learning_rate": 9.367041003085648e-07,
      "logits/chosen": -1.2346036434173584,
      "logits/rejected": -1.155144453048706,
      "logps/chosen": -0.46027320623397827,
      "logps/rejected": -0.539958655834198,
      "loss": 2.4476,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -4.602732181549072,
      "rewards/margins": 0.7968543767929077,
      "rewards/rejected": -5.3995866775512695,
      "step": 115
    },
    {
      "epoch": 0.2564787603526583,
      "grad_norm": 94.23205420362748,
      "learning_rate": 9.272941683504808e-07,
      "logits/chosen": -1.2661981582641602,
      "logits/rejected": -1.2356774806976318,
      "logps/chosen": -0.5204485058784485,
      "logps/rejected": -0.6347986459732056,
      "loss": 2.313,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -5.204484939575195,
      "rewards/margins": 1.1435019969940186,
      "rewards/rejected": -6.347987174987793,
      "step": 120
    },
    {
      "epoch": 0.2671653753673524,
      "grad_norm": 104.425486907994,
      "learning_rate": 9.172866268606513e-07,
      "logits/chosen": -1.4145832061767578,
      "logits/rejected": -1.3696868419647217,
      "logps/chosen": -0.5677964687347412,
      "logps/rejected": -0.7819775342941284,
      "loss": 2.0573,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -5.677964210510254,
      "rewards/margins": 2.1418113708496094,
      "rewards/rejected": -7.8197760581970215,
      "step": 125
    },
    {
      "epoch": 0.2778519903820465,
      "grad_norm": 118.15756723487034,
      "learning_rate": 9.066954722907638e-07,
      "logits/chosen": -1.2262728214263916,
      "logits/rejected": -1.1968516111373901,
      "logps/chosen": -0.5714520812034607,
      "logps/rejected": -0.7978314757347107,
      "loss": 2.229,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -5.7145209312438965,
      "rewards/margins": 2.2637939453125,
      "rewards/rejected": -7.9783148765563965,
      "step": 130
    },
    {
      "epoch": 0.2885386053967406,
      "grad_norm": 71.01619814345214,
      "learning_rate": 8.955355173281707e-07,
      "logits/chosen": -1.423404574394226,
      "logits/rejected": -1.3302305936813354,
      "logps/chosen": -0.6018441915512085,
      "logps/rejected": -0.7836464047431946,
      "loss": 1.9147,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -6.018441677093506,
      "rewards/margins": 1.8180221319198608,
      "rewards/rejected": -7.836463928222656,
      "step": 135
    },
    {
      "epoch": 0.2992252204114347,
      "grad_norm": 66.78115766091884,
      "learning_rate": 8.838223701790055e-07,
      "logits/chosen": -1.384155511856079,
      "logits/rejected": -1.3765170574188232,
      "logps/chosen": -0.6419236660003662,
      "logps/rejected": -0.8896235227584839,
      "loss": 1.9382,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -6.419236660003662,
      "rewards/margins": 2.476998805999756,
      "rewards/rejected": -8.896234512329102,
      "step": 140
    },
    {
      "epoch": 0.30991183542612877,
      "grad_norm": 102.33456775014218,
      "learning_rate": 8.71572412738697e-07,
      "logits/chosen": -1.3982570171356201,
      "logits/rejected": -1.3454999923706055,
      "logps/chosen": -0.6346415281295776,
      "logps/rejected": -0.8345499038696289,
      "loss": 1.8347,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -6.346415042877197,
      "rewards/margins": 1.9990848302841187,
      "rewards/rejected": -8.345499038696289,
      "step": 145
    },
    {
      "epoch": 0.32059845044082286,
      "grad_norm": 104.88848048977833,
      "learning_rate": 8.588027776804058e-07,
      "logits/chosen": -1.3800594806671143,
      "logits/rejected": -1.3424351215362549,
      "logps/chosen": -0.7959817051887512,
      "logps/rejected": -1.088423252105713,
      "loss": 1.9956,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -7.959817409515381,
      "rewards/margins": 2.9244155883789062,
      "rewards/rejected": -10.884233474731445,
      "step": 150
    },
    {
      "epoch": 0.33128506545551695,
      "grad_norm": 86.74090140921074,
      "learning_rate": 8.455313244934324e-07,
      "logits/chosen": -1.289271354675293,
      "logits/rejected": -1.221858263015747,
      "logps/chosen": -0.7765235900878906,
      "logps/rejected": -1.0151869058609009,
      "loss": 1.905,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -7.76523494720459,
      "rewards/margins": 2.386632204055786,
      "rewards/rejected": -10.15186882019043,
      "step": 155
    },
    {
      "epoch": 0.34197168047021104,
      "grad_norm": 103.52900771453592,
      "learning_rate": 8.317766145051057e-07,
      "logits/chosen": -1.3143701553344727,
      "logits/rejected": -1.2429687976837158,
      "logps/chosen": -0.8146079182624817,
      "logps/rejected": -1.076160192489624,
      "loss": 1.9347,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -8.146079063415527,
      "rewards/margins": 2.6155219078063965,
      "rewards/rejected": -10.761602401733398,
      "step": 160
    },
    {
      "epoch": 0.3526582954849052,
      "grad_norm": 91.78991869467103,
      "learning_rate": 8.175578849210894e-07,
      "logits/chosen": -1.277517557144165,
      "logits/rejected": -1.2467105388641357,
      "logps/chosen": -0.8164845705032349,
      "logps/rejected": -1.0702334642410278,
      "loss": 1.8253,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -8.16484546661377,
      "rewards/margins": 2.5374886989593506,
      "rewards/rejected": -10.7023344039917,
      "step": 165
    },
    {
      "epoch": 0.36334491049959927,
      "grad_norm": 146.18164794432158,
      "learning_rate": 8.028950219204099e-07,
      "logits/chosen": -1.4038187265396118,
      "logits/rejected": -1.32179856300354,
      "logps/chosen": -0.9452447891235352,
      "logps/rejected": -1.285444974899292,
      "loss": 1.8459,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -9.452447891235352,
      "rewards/margins": 3.4020028114318848,
      "rewards/rejected": -12.854451179504395,
      "step": 170
    },
    {
      "epoch": 0.37403152551429336,
      "grad_norm": 74.04178740704424,
      "learning_rate": 7.878085328428368e-07,
      "logits/chosen": -1.2986555099487305,
      "logits/rejected": -1.2864700555801392,
      "logps/chosen": -0.8545284271240234,
      "logps/rejected": -1.2237508296966553,
      "loss": 1.7393,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -8.545283317565918,
      "rewards/margins": 3.6922249794006348,
      "rewards/rejected": -12.237508773803711,
      "step": 175
    },
    {
      "epoch": 0.38471814052898745,
      "grad_norm": 146.93436823237963,
      "learning_rate": 7.723195175075135e-07,
      "logits/chosen": -1.4434401988983154,
      "logits/rejected": -1.3974043130874634,
      "logps/chosen": -1.084814190864563,
      "logps/rejected": -1.4313297271728516,
      "loss": 1.5783,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -10.84814167022705,
      "rewards/margins": 3.4651541709899902,
      "rewards/rejected": -14.3132963180542,
      "step": 180
    },
    {
      "epoch": 0.39540475554368154,
      "grad_norm": 300.3576490829542,
      "learning_rate": 7.564496387029531e-07,
      "logits/chosen": -1.331994891166687,
      "logits/rejected": -1.3323677778244019,
      "logps/chosen": -1.0099122524261475,
      "logps/rejected": -1.400909185409546,
      "loss": 1.6356,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -10.099123001098633,
      "rewards/margins": 3.9099678993225098,
      "rewards/rejected": -14.0090913772583,
      "step": 185
    },
    {
      "epoch": 0.40609137055837563,
      "grad_norm": 117.205754582783,
      "learning_rate": 7.402210918896689e-07,
      "logits/chosen": -1.3903909921646118,
      "logits/rejected": -1.395379662513733,
      "logps/chosen": -0.9584177136421204,
      "logps/rejected": -1.3904134035110474,
      "loss": 1.5868,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -9.584177017211914,
      "rewards/margins": 4.319957256317139,
      "rewards/rejected": -13.904134750366211,
      "step": 190
    },
    {
      "epoch": 0.4167779855730697,
      "grad_norm": 193.5248033898295,
      "learning_rate": 7.236565741578162e-07,
      "logits/chosen": -1.479113221168518,
      "logits/rejected": -1.431293249130249,
      "logps/chosen": -1.1905192136764526,
      "logps/rejected": -1.5327824354171753,
      "loss": 1.4974,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -11.905191421508789,
      "rewards/margins": 3.4226326942443848,
      "rewards/rejected": -15.327824592590332,
      "step": 195
    },
    {
      "epoch": 0.4274646005877638,
      "grad_norm": 72.90690697186486,
      "learning_rate": 7.067792524832603e-07,
      "logits/chosen": -1.464138388633728,
      "logits/rejected": -1.4378821849822998,
      "logps/chosen": -1.2057116031646729,
      "logps/rejected": -1.6115760803222656,
      "loss": 1.6772,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -12.05711555480957,
      "rewards/margins": 4.058643341064453,
      "rewards/rejected": -16.115758895874023,
      "step": 200
    },
    {
      "epoch": 0.4381512156024579,
      "grad_norm": 73.64255724761823,
      "learning_rate": 6.896127313264642e-07,
      "logits/chosen": -1.463689923286438,
      "logits/rejected": -1.4153610467910767,
      "logps/chosen": -0.9573764801025391,
      "logps/rejected": -1.4560494422912598,
      "loss": 1.5083,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -9.57376480102539,
      "rewards/margins": 4.986730098724365,
      "rewards/rejected": -14.560495376586914,
      "step": 205
    },
    {
      "epoch": 0.448837830617152,
      "grad_norm": 95.73515995991542,
      "learning_rate": 6.721810196195174e-07,
      "logits/chosen": -1.5166422128677368,
      "logits/rejected": -1.5037015676498413,
      "logps/chosen": -1.1732789278030396,
      "logps/rejected": -1.5742861032485962,
      "loss": 1.5533,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -11.732789993286133,
      "rewards/margins": 4.010071754455566,
      "rewards/rejected": -15.7428617477417,
      "step": 210
    },
    {
      "epoch": 0.45952444563184613,
      "grad_norm": 111.72733375525473,
      "learning_rate": 6.545084971874736e-07,
      "logits/chosen": -1.5549886226654053,
      "logits/rejected": -1.4790352582931519,
      "logps/chosen": -1.1950467824935913,
      "logps/rejected": -1.602651596069336,
      "loss": 1.5756,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -11.950468063354492,
      "rewards/margins": 4.076048851013184,
      "rewards/rejected": -16.02651596069336,
      "step": 215
    },
    {
      "epoch": 0.4702110606465402,
      "grad_norm": 95.02342077059286,
      "learning_rate": 6.3661988065096e-07,
      "logits/chosen": -1.483459711074829,
      "logits/rejected": -1.454347014427185,
      "logps/chosen": -1.2087767124176025,
      "logps/rejected": -1.5687811374664307,
      "loss": 1.434,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -12.087766647338867,
      "rewards/margins": 3.6000449657440186,
      "rewards/rejected": -15.687810897827148,
      "step": 220
    },
    {
      "epoch": 0.4808976756612343,
      "grad_norm": 80.04963040030957,
      "learning_rate": 6.185401888577487e-07,
      "logits/chosen": -1.4368016719818115,
      "logits/rejected": -1.4539368152618408,
      "logps/chosen": -1.1799414157867432,
      "logps/rejected": -1.7078040838241577,
      "loss": 1.2062,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -11.799413681030273,
      "rewards/margins": 5.278626918792725,
      "rewards/rejected": -17.078041076660156,
      "step": 225
    },
    {
      "epoch": 0.4915842906759284,
      "grad_norm": 90.27822366362969,
      "learning_rate": 6.002947078916364e-07,
      "logits/chosen": -1.5053470134735107,
      "logits/rejected": -1.4647208452224731,
      "logps/chosen": -1.3405472040176392,
      "logps/rejected": -1.8345695734024048,
      "loss": 1.1551,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -13.405473709106445,
      "rewards/margins": 4.9402241706848145,
      "rewards/rejected": -18.3456974029541,
      "step": 230
    },
    {
      "epoch": 0.5022709056906225,
      "grad_norm": 198.7780967148136,
      "learning_rate": 5.819089557075688e-07,
      "logits/chosen": -1.463410496711731,
      "logits/rejected": -1.4734344482421875,
      "logps/chosen": -1.4616682529449463,
      "logps/rejected": -1.8893461227416992,
      "loss": 1.5181,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -14.616681098937988,
      "rewards/margins": 4.276780128479004,
      "rewards/rejected": -18.893463134765625,
      "step": 235
    },
    {
      "epoch": 0.5129575207053166,
      "grad_norm": 92.23381750381444,
      "learning_rate": 5.634086464424742e-07,
      "logits/chosen": -1.5198612213134766,
      "logits/rejected": -1.5063666105270386,
      "logps/chosen": -1.4451749324798584,
      "logps/rejected": -2.1483402252197266,
      "loss": 1.3406,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -14.451748847961426,
      "rewards/margins": 7.031652927398682,
      "rewards/rejected": -21.483402252197266,
      "step": 240
    },
    {
      "epoch": 0.5236441357200107,
      "grad_norm": 101.7965922889438,
      "learning_rate": 5.448196544517167e-07,
      "logits/chosen": -1.4860137701034546,
      "logits/rejected": -1.4609707593917847,
      "logps/chosen": -1.2388508319854736,
      "logps/rejected": -1.7417356967926025,
      "loss": 1.2849,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -12.388508796691895,
      "rewards/margins": 5.0288496017456055,
      "rewards/rejected": -17.4173583984375,
      "step": 245
    },
    {
      "epoch": 0.5343307507347048,
      "grad_norm": 112.84840492502885,
      "learning_rate": 5.26167978121472e-07,
      "logits/chosen": -1.545151710510254,
      "logits/rejected": -1.48394775390625,
      "logps/chosen": -1.2914670705795288,
      "logps/rejected": -1.867653489112854,
      "loss": 1.226,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -12.9146728515625,
      "rewards/margins": 5.761862754821777,
      "rewards/rejected": -18.676532745361328,
      "step": 250
    },
    {
      "epoch": 0.5450173657493989,
      "grad_norm": 81.89729359110314,
      "learning_rate": 5.074797035076318e-07,
      "logits/chosen": -1.5118135213851929,
      "logits/rejected": -1.5217331647872925,
      "logps/chosen": -1.5564649105072021,
      "logps/rejected": -1.9826581478118896,
      "loss": 1.2119,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -15.564648628234863,
      "rewards/margins": 4.261934757232666,
      "rewards/rejected": -19.826580047607422,
      "step": 255
    },
    {
      "epoch": 0.555703980764093,
      "grad_norm": 97.54502724467275,
      "learning_rate": 4.887809678520975e-07,
      "logits/chosen": -1.5081006288528442,
      "logits/rejected": -1.4816151857376099,
      "logps/chosen": -1.337573766708374,
      "logps/rejected": -1.8502023220062256,
      "loss": 1.226,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -13.375738143920898,
      "rewards/margins": 5.126286029815674,
      "rewards/rejected": -18.502025604248047,
      "step": 260
    },
    {
      "epoch": 0.566390595778787,
      "grad_norm": 99.34071724726068,
      "learning_rate": 4.700979230274829e-07,
      "logits/chosen": -1.4894455671310425,
      "logits/rejected": -1.4260051250457764,
      "logps/chosen": -1.388643503189087,
      "logps/rejected": -1.9015836715698242,
      "loss": 1.1655,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -13.886438369750977,
      "rewards/margins": 5.129402160644531,
      "rewards/rejected": -19.015838623046875,
      "step": 265
    },
    {
      "epoch": 0.5770772107934812,
      "grad_norm": 150.30388856950947,
      "learning_rate": 4.514566989613559e-07,
      "logits/chosen": -1.5251694917678833,
      "logits/rejected": -1.510664701461792,
      "logps/chosen": -1.352906346321106,
      "logps/rejected": -1.9253631830215454,
      "loss": 1.2737,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -13.529065132141113,
      "rewards/margins": 5.724569320678711,
      "rewards/rejected": -19.253633499145508,
      "step": 270
    },
    {
      "epoch": 0.5877638258081752,
      "grad_norm": 106.62043935215682,
      "learning_rate": 4.328833670911724e-07,
      "logits/chosen": -1.6284313201904297,
      "logits/rejected": -1.5873377323150635,
      "logps/chosen": -1.4256664514541626,
      "logps/rejected": -2.05672025680542,
      "loss": 1.356,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -14.256665229797363,
      "rewards/margins": 6.310537338256836,
      "rewards/rejected": -20.567203521728516,
      "step": 275
    },
    {
      "epoch": 0.5984504408228694,
      "grad_norm": 84.26686987382098,
      "learning_rate": 4.144039039010124e-07,
      "logits/chosen": -1.4645414352416992,
      "logits/rejected": -1.4171415567398071,
      "logps/chosen": -1.4661189317703247,
      "logps/rejected": -1.9972292184829712,
      "loss": 1.1226,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -14.661189079284668,
      "rewards/margins": 5.311104774475098,
      "rewards/rejected": -19.972293853759766,
      "step": 280
    },
    {
      "epoch": 0.6091370558375635,
      "grad_norm": 111.79495099216508,
      "learning_rate": 3.960441545911204e-07,
      "logits/chosen": -1.580186128616333,
      "logits/rejected": -1.4852710962295532,
      "logps/chosen": -1.6370970010757446,
      "logps/rejected": -2.1605658531188965,
      "loss": 1.3708,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -16.370969772338867,
      "rewards/margins": 5.234689235687256,
      "rewards/rejected": -21.605661392211914,
      "step": 285
    },
    {
      "epoch": 0.6198236708522575,
      "grad_norm": 89.00502902448149,
      "learning_rate": 3.778297969310529e-07,
      "logits/chosen": -1.6441500186920166,
      "logits/rejected": -1.6167094707489014,
      "logps/chosen": -1.5200397968292236,
      "logps/rejected": -2.0216360092163086,
      "loss": 1.19,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -15.200396537780762,
      "rewards/margins": 5.01596212387085,
      "rewards/rejected": -20.216358184814453,
      "step": 290
    },
    {
      "epoch": 0.6305102858669517,
      "grad_norm": 101.58972547942741,
      "learning_rate": 3.5978630534699865e-07,
      "logits/chosen": -1.5900800228118896,
      "logits/rejected": -1.5477546453475952,
      "logps/chosen": -1.4291937351226807,
      "logps/rejected": -2.104898691177368,
      "loss": 1.0996,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -14.291936874389648,
      "rewards/margins": 6.757050514221191,
      "rewards/rejected": -21.04898452758789,
      "step": 295
    },
    {
      "epoch": 0.6411969008816457,
      "grad_norm": 114.7765696570134,
      "learning_rate": 3.4193891529348795e-07,
      "logits/chosen": -1.5715553760528564,
      "logits/rejected": -1.5747449398040771,
      "logps/chosen": -1.4967474937438965,
      "logps/rejected": -2.078972101211548,
      "loss": 1.0805,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -14.967473983764648,
      "rewards/margins": 5.8222455978393555,
      "rewards/rejected": -20.789722442626953,
      "step": 300
    },
    {
      "epoch": 0.6518835158963399,
      "grad_norm": 123.5878416310842,
      "learning_rate": 3.243125879593286e-07,
      "logits/chosen": -1.4123915433883667,
      "logits/rejected": -1.4187986850738525,
      "logps/chosen": -1.2821505069732666,
      "logps/rejected": -2.120354413986206,
      "loss": 1.1007,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -12.821507453918457,
      "rewards/margins": 8.382037162780762,
      "rewards/rejected": -21.203542709350586,
      "step": 305
    },
    {
      "epoch": 0.6625701309110339,
      "grad_norm": 94.61228245151283,
      "learning_rate": 3.069319753571269e-07,
      "logits/chosen": -1.5294448137283325,
      "logits/rejected": -1.5259336233139038,
      "logps/chosen": -1.5088483095169067,
      "logps/rejected": -2.108046531677246,
      "loss": 1.147,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -15.088482856750488,
      "rewards/margins": 5.991982460021973,
      "rewards/rejected": -21.080463409423828,
      "step": 310
    },
    {
      "epoch": 0.673256745925728,
      "grad_norm": 88.47117948255506,
      "learning_rate": 2.898213858452173e-07,
      "logits/chosen": -1.4750919342041016,
      "logits/rejected": -1.4806705713272095,
      "logps/chosen": -1.588282823562622,
      "logps/rejected": -2.2189738750457764,
      "loss": 1.0766,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -15.882827758789062,
      "rewards/margins": 6.306912899017334,
      "rewards/rejected": -22.189741134643555,
      "step": 315
    },
    {
      "epoch": 0.6839433609404221,
      "grad_norm": 125.62338182234869,
      "learning_rate": 2.730047501302266e-07,
      "logits/chosen": -1.6073205471038818,
      "logits/rejected": -1.538448691368103,
      "logps/chosen": -1.5898334980010986,
      "logps/rejected": -2.3282947540283203,
      "loss": 1.2473,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -15.898333549499512,
      "rewards/margins": 7.3846116065979,
      "rewards/rejected": -23.282947540283203,
      "step": 320
    },
    {
      "epoch": 0.6946299759551162,
      "grad_norm": 111.24354162328734,
      "learning_rate": 2.5650558779781635e-07,
      "logits/chosen": -1.5743417739868164,
      "logits/rejected": -1.566840410232544,
      "logps/chosen": -1.6232945919036865,
      "logps/rejected": -2.2983360290527344,
      "loss": 1.0508,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -16.232946395874023,
      "rewards/margins": 6.750412940979004,
      "rewards/rejected": -22.983360290527344,
      "step": 325
    },
    {
      "epoch": 0.7053165909698104,
      "grad_norm": 85.89351794884949,
      "learning_rate": 2.403469744184154e-07,
      "logits/chosen": -1.5692135095596313,
      "logits/rejected": -1.5725057125091553,
      "logps/chosen": -1.542747974395752,
      "logps/rejected": -2.168196678161621,
      "loss": 1.0903,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -15.42747974395752,
      "rewards/margins": 6.254487037658691,
      "rewards/rejected": -21.68196678161621,
      "step": 330
    },
    {
      "epoch": 0.7160032059845044,
      "grad_norm": 116.80439013939746,
      "learning_rate": 2.2455150927394878e-07,
      "logits/chosen": -1.55947744846344,
      "logits/rejected": -1.501947283744812,
      "logps/chosen": -1.5338480472564697,
      "logps/rejected": -2.0976195335388184,
      "loss": 1.0239,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -15.338479995727539,
      "rewards/margins": 5.637716770172119,
      "rewards/rejected": -20.976198196411133,
      "step": 335
    },
    {
      "epoch": 0.7266898209991985,
      "grad_norm": 78.12195704672156,
      "learning_rate": 2.0914128375069722e-07,
      "logits/chosen": -1.4831820726394653,
      "logits/rejected": -1.5086932182312012,
      "logps/chosen": -1.5225416421890259,
      "logps/rejected": -2.1293392181396484,
      "loss": 0.9575,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -15.22541618347168,
      "rewards/margins": 6.0679755210876465,
      "rewards/rejected": -21.293392181396484,
      "step": 340
    },
    {
      "epoch": 0.7373764360138926,
      "grad_norm": 91.10910799077911,
      "learning_rate": 1.9413785044249676e-07,
      "logits/chosen": -1.5832130908966064,
      "logits/rejected": -1.5328218936920166,
      "logps/chosen": -1.4637863636016846,
      "logps/rejected": -2.059370517730713,
      "loss": 1.0804,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -14.637863159179688,
      "rewards/margins": 5.95583963394165,
      "rewards/rejected": -20.593706130981445,
      "step": 345
    },
    {
      "epoch": 0.7480630510285867,
      "grad_norm": 120.29698335176921,
      "learning_rate": 1.7956219300748792e-07,
      "logits/chosen": -1.519425868988037,
      "logits/rejected": -1.5000859498977661,
      "logps/chosen": -1.5877223014831543,
      "logps/rejected": -2.4219236373901367,
      "loss": 1.282,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -15.877223014831543,
      "rewards/margins": 8.342010498046875,
      "rewards/rejected": -24.219234466552734,
      "step": 350
    },
    {
      "epoch": 0.7587496660432808,
      "grad_norm": 114.7323293702361,
      "learning_rate": 1.6543469682057104e-07,
      "logits/chosen": -1.5452375411987305,
      "logits/rejected": -1.477497935295105,
      "logps/chosen": -1.8073337078094482,
      "logps/rejected": -2.5182600021362305,
      "loss": 1.2039,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -18.073339462280273,
      "rewards/margins": 7.109262943267822,
      "rewards/rejected": -25.182600021362305,
      "step": 355
    },
    {
      "epoch": 0.7694362810579749,
      "grad_norm": 101.20181237745652,
      "learning_rate": 1.5177512046261666e-07,
      "logits/chosen": -1.5502760410308838,
      "logits/rejected": -1.5708996057510376,
      "logps/chosen": -1.6630090475082397,
      "logps/rejected": -2.226963520050049,
      "loss": 1.0971,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -16.63009262084961,
      "rewards/margins": 5.639544486999512,
      "rewards/rejected": -22.269634246826172,
      "step": 360
    },
    {
      "epoch": 0.7801228960726689,
      "grad_norm": 139.38000134354874,
      "learning_rate": 1.3860256808630427e-07,
      "logits/chosen": -1.5358211994171143,
      "logits/rejected": -1.5561330318450928,
      "logps/chosen": -1.6269845962524414,
      "logps/rejected": -2.3564658164978027,
      "loss": 1.1184,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -16.269845962524414,
      "rewards/margins": 7.294811248779297,
      "rewards/rejected": -23.564655303955078,
      "step": 365
    },
    {
      "epoch": 0.7908095110873631,
      "grad_norm": 104.011040382236,
      "learning_rate": 1.2593546269723647e-07,
      "logits/chosen": -1.5664985179901123,
      "logits/rejected": -1.4946684837341309,
      "logps/chosen": -1.5413891077041626,
      "logps/rejected": -2.3655648231506348,
      "loss": 1.0455,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -15.413891792297363,
      "rewards/margins": 8.241754531860352,
      "rewards/rejected": -23.6556453704834,
      "step": 370
    },
    {
      "epoch": 0.8014961261020572,
      "grad_norm": 98.02139267959178,
      "learning_rate": 1.1379152038770029e-07,
      "logits/chosen": -1.5228099822998047,
      "logits/rejected": -1.5087474584579468,
      "logps/chosen": -1.480344295501709,
      "logps/rejected": -2.425346851348877,
      "loss": 1.2231,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -14.803441047668457,
      "rewards/margins": 9.45002555847168,
      "rewards/rejected": -24.253467559814453,
      "step": 375
    },
    {
      "epoch": 0.8121827411167513,
      "grad_norm": 134.01729655272072,
      "learning_rate": 1.0218772555910954e-07,
      "logits/chosen": -1.497368574142456,
      "logits/rejected": -1.5044596195220947,
      "logps/chosen": -1.594871163368225,
      "logps/rejected": -2.4201245307922363,
      "loss": 1.2771,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -15.948710441589355,
      "rewards/margins": 8.252533912658691,
      "rewards/rejected": -24.201244354248047,
      "step": 380
    },
    {
      "epoch": 0.8228693561314454,
      "grad_norm": 74.68961856468854,
      "learning_rate": 9.114030716778432e-08,
      "logits/chosen": -1.551857352256775,
      "logits/rejected": -1.535230278968811,
      "logps/chosen": -1.6363334655761719,
      "logps/rejected": -2.4195470809936523,
      "loss": 1.0466,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -16.363332748413086,
      "rewards/margins": 7.832139015197754,
      "rewards/rejected": -24.195472717285156,
      "step": 385
    },
    {
      "epoch": 0.8335559711461394,
      "grad_norm": 107.5882104761143,
      "learning_rate": 8.066471602728803e-08,
      "logits/chosen": -1.5829932689666748,
      "logits/rejected": -1.5114977359771729,
      "logps/chosen": -1.5258924961090088,
      "logps/rejected": -2.1448683738708496,
      "loss": 1.0185,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -15.25892448425293,
      "rewards/margins": 6.189759254455566,
      "rewards/rejected": -21.448684692382812,
      "step": 390
    },
    {
      "epoch": 0.8442425861608336,
      "grad_norm": 102.80663837958856,
      "learning_rate": 7.077560319906694e-08,
      "logits/chosen": -1.4724600315093994,
      "logits/rejected": -1.4828989505767822,
      "logps/chosen": -1.5901472568511963,
      "logps/rejected": -2.1642448902130127,
      "loss": 1.1312,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -15.901473999023438,
      "rewards/margins": 5.740975379943848,
      "rewards/rejected": -21.642446517944336,
      "step": 395
    },
    {
      "epoch": 0.8549292011755276,
      "grad_norm": 101.28973665181952,
      "learning_rate": 6.148679950161672e-08,
      "logits/chosen": -1.561643123626709,
      "logits/rejected": -1.5329078435897827,
      "logps/chosen": -1.514060616493225,
      "logps/rejected": -2.152096748352051,
      "loss": 1.1687,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -15.140606880187988,
      "rewards/margins": 6.380361080169678,
      "rewards/rejected": -21.520967483520508,
      "step": 400
    },
    {
      "epoch": 0.8549292011755276,
      "eval_logits/chosen": -1.7677618265151978,
      "eval_logits/rejected": -1.773621916770935,
      "eval_logps/chosen": -1.601662278175354,
      "eval_logps/rejected": -2.307917356491089,
      "eval_loss": 1.0876977443695068,
      "eval_rewards/accuracies": 0.8870967626571655,
      "eval_rewards/chosen": -16.016620635986328,
      "eval_rewards/margins": 7.062554836273193,
      "eval_rewards/rejected": -23.07917594909668,
      "eval_runtime": 96.7009,
      "eval_samples_per_second": 20.279,
      "eval_steps_per_second": 0.641,
      "step": 400
    },
    {
      "epoch": 0.8656158161902218,
      "grad_norm": 133.15081807524635,
      "learning_rate": 5.2811296166831666e-08,
      "logits/chosen": -1.5344595909118652,
      "logits/rejected": -1.494829773902893,
      "logps/chosen": -1.7153148651123047,
      "logps/rejected": -2.2892117500305176,
      "loss": 1.2171,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -17.153148651123047,
      "rewards/margins": 5.7389678955078125,
      "rewards/rejected": -22.89211654663086,
      "step": 405
    },
    {
      "epoch": 0.8763024312049158,
      "grad_norm": 158.42453976320837,
      "learning_rate": 4.4761226670592066e-08,
      "logits/chosen": -1.5358643531799316,
      "logits/rejected": -1.4975647926330566,
      "logps/chosen": -1.5330779552459717,
      "logps/rejected": -2.2891881465911865,
      "loss": 1.1582,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -15.330778121948242,
      "rewards/margins": 7.561104774475098,
      "rewards/rejected": -22.89188575744629,
      "step": 410
    },
    {
      "epoch": 0.88698904621961,
      "grad_norm": 104.59131788649188,
      "learning_rate": 3.734784976300165e-08,
      "logits/chosen": -1.5346996784210205,
      "logits/rejected": -1.5254827737808228,
      "logps/chosen": -1.5777432918548584,
      "logps/rejected": -2.2442333698272705,
      "loss": 1.0981,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -15.777432441711426,
      "rewards/margins": 6.664898872375488,
      "rewards/rejected": -22.442333221435547,
      "step": 415
    },
    {
      "epoch": 0.897675661234304,
      "grad_norm": 95.79743506562838,
      "learning_rate": 3.058153372200695e-08,
      "logits/chosen": -1.5502920150756836,
      "logits/rejected": -1.5145201683044434,
      "logps/chosen": -1.432101845741272,
      "logps/rejected": -2.1306354999542236,
      "loss": 1.072,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -14.321019172668457,
      "rewards/margins": 6.985334873199463,
      "rewards/rejected": -21.306354522705078,
      "step": 420
    },
    {
      "epoch": 0.9083622762489981,
      "grad_norm": 91.78736413360674,
      "learning_rate": 2.4471741852423233e-08,
      "logits/chosen": -1.5086685419082642,
      "logits/rejected": -1.5284610986709595,
      "logps/chosen": -1.5403655767440796,
      "logps/rejected": -2.4292709827423096,
      "loss": 0.9504,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -15.403654098510742,
      "rewards/margins": 8.889054298400879,
      "rewards/rejected": -24.292707443237305,
      "step": 425
    },
    {
      "epoch": 0.9190488912636923,
      "grad_norm": 84.03123746989313,
      "learning_rate": 1.9027019250647036e-08,
      "logits/chosen": -1.6175905466079712,
      "logits/rejected": -1.6218225955963135,
      "logps/chosen": -1.5030295848846436,
      "logps/rejected": -2.3066420555114746,
      "loss": 1.0077,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -15.030294418334961,
      "rewards/margins": 8.036127090454102,
      "rewards/rejected": -23.066423416137695,
      "step": 430
    },
    {
      "epoch": 0.9297355062783863,
      "grad_norm": 117.932317769854,
      "learning_rate": 1.4254980853566246e-08,
      "logits/chosen": -1.5319820642471313,
      "logits/rejected": -1.5626695156097412,
      "logps/chosen": -1.5170115232467651,
      "logps/rejected": -2.136791229248047,
      "loss": 1.0213,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -15.170114517211914,
      "rewards/margins": 6.197798252105713,
      "rewards/rejected": -21.36791229248047,
      "step": 435
    },
    {
      "epoch": 0.9404221212930804,
      "grad_norm": 126.32296519295083,
      "learning_rate": 1.016230078838226e-08,
      "logits/chosen": -1.5235329866409302,
      "logits/rejected": -1.5282210111618042,
      "logps/chosen": -1.6138379573822021,
      "logps/rejected": -2.333554744720459,
      "loss": 1.203,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -16.13838005065918,
      "rewards/margins": 7.197165489196777,
      "rewards/rejected": -23.335546493530273,
      "step": 440
    },
    {
      "epoch": 0.9511087363077745,
      "grad_norm": 102.89001505284573,
      "learning_rate": 6.754703038239329e-09,
      "logits/chosen": -1.6163692474365234,
      "logits/rejected": -1.5660312175750732,
      "logps/chosen": -1.5261099338531494,
      "logps/rejected": -2.2487072944641113,
      "loss": 1.1582,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -15.261096000671387,
      "rewards/margins": 7.225976467132568,
      "rewards/rejected": -22.487075805664062,
      "step": 445
    },
    {
      "epoch": 0.9617953513224686,
      "grad_norm": 119.74095319029244,
      "learning_rate": 4.036953436716895e-09,
      "logits/chosen": -1.5374672412872314,
      "logits/rejected": -1.4871861934661865,
      "logps/chosen": -1.5755971670150757,
      "logps/rejected": -2.061048984527588,
      "loss": 1.073,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -15.75597095489502,
      "rewards/margins": 4.854517459869385,
      "rewards/rejected": -20.610488891601562,
      "step": 450
    },
    {
      "epoch": 0.9724819663371627,
      "grad_norm": 76.57788575531883,
      "learning_rate": 2.0128530023804656e-09,
      "logits/chosen": -1.591564416885376,
      "logits/rejected": -1.5509247779846191,
      "logps/chosen": -1.5886285305023193,
      "logps/rejected": -2.421255588531494,
      "loss": 0.9676,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -15.886283874511719,
      "rewards/margins": 8.326271057128906,
      "rewards/rejected": -24.212556838989258,
      "step": 455
    },
    {
      "epoch": 0.9831685813518568,
      "grad_norm": 109.6328495391863,
      "learning_rate": 6.852326227130833e-10,
      "logits/chosen": -1.5486748218536377,
      "logits/rejected": -1.549953818321228,
      "logps/chosen": -1.5361286401748657,
      "logps/rejected": -2.2728710174560547,
      "loss": 1.1484,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -15.361285209655762,
      "rewards/margins": 7.367424964904785,
      "rewards/rejected": -22.728710174560547,
      "step": 460
    },
    {
      "epoch": 0.9938551963665508,
      "grad_norm": 101.61409866072276,
      "learning_rate": 5.594909486328348e-11,
      "logits/chosen": -1.5434294939041138,
      "logits/rejected": -1.5330185890197754,
      "logps/chosen": -1.7000032663345337,
      "logps/rejected": -2.3660502433776855,
      "loss": 1.0908,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -17.000032424926758,
      "rewards/margins": 6.660471439361572,
      "rewards/rejected": -23.66050148010254,
      "step": 465
    },
    {
      "epoch": 0.9981298423724285,
      "step": 467,
      "total_flos": 0.0,
      "train_loss": 1.7411203522243133,
      "train_runtime": 12577.1262,
      "train_samples_per_second": 4.761,
      "train_steps_per_second": 0.037
    }
  ],
  "logging_steps": 5,
  "max_steps": 467,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 120,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}