{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9915492957746479,
  "eval_steps": 18,
  "global_step": 88,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.011267605633802818,
      "grad_norm": 13.436114311218262,
      "learning_rate": 8e-07,
      "log_odds_chosen": 0.3692830204963684,
      "log_odds_ratio": -0.6691731214523315,
      "logits/chosen": -2.073309898376465,
      "logits/rejected": -2.0528674125671387,
      "logps/chosen": -6.679821491241455,
      "logps/rejected": -7.049269676208496,
      "loss": 7.3222,
      "nll_loss": 7.255331993103027,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6679821610450745,
      "rewards/margins": 0.03694486618041992,
      "rewards/rejected": -0.7049269676208496,
      "step": 1
    },
    {
      "epoch": 0.022535211267605635,
      "grad_norm": 12.660109519958496,
      "learning_rate": 1.6e-06,
      "log_odds_chosen": 0.05790233612060547,
      "log_odds_ratio": -0.7999902963638306,
      "logits/chosen": -2.122316360473633,
      "logits/rejected": -2.1929054260253906,
      "logps/chosen": -6.760974884033203,
      "logps/rejected": -6.818158149719238,
      "loss": 6.7429,
      "nll_loss": 6.662899971008301,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6760974526405334,
      "rewards/margins": 0.005718342959880829,
      "rewards/rejected": -0.6818158626556396,
      "step": 2
    },
    {
      "epoch": 0.03380281690140845,
      "grad_norm": 15.815394401550293,
      "learning_rate": 2.4e-06,
      "log_odds_chosen": 0.5895991325378418,
      "log_odds_ratio": -0.5308127999305725,
      "logits/chosen": -2.167480945587158,
      "logits/rejected": -2.164055347442627,
      "logps/chosen": -6.8017683029174805,
      "logps/rejected": -7.390748977661133,
      "loss": 7.2921,
      "nll_loss": 7.239068031311035,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.680176854133606,
      "rewards/margins": 0.058898091316223145,
      "rewards/rejected": -0.7390749454498291,
      "step": 3
    },
    {
      "epoch": 0.04507042253521127,
      "grad_norm": 15.963722229003906,
      "learning_rate": 3.2e-06,
      "log_odds_chosen": -0.2698395252227783,
      "log_odds_ratio": -0.8873093128204346,
      "logits/chosen": -2.084567070007324,
      "logits/rejected": -2.0936269760131836,
      "logps/chosen": -6.373527526855469,
      "logps/rejected": -6.103606224060059,
      "loss": 7.1739,
      "nll_loss": 7.0851545333862305,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.637352705001831,
      "rewards/margins": -0.026992086321115494,
      "rewards/rejected": -0.6103606224060059,
      "step": 4
    },
    {
      "epoch": 0.056338028169014086,
      "grad_norm": 14.874364852905273,
      "learning_rate": 4e-06,
      "log_odds_chosen": -0.19485323131084442,
      "log_odds_ratio": -0.9221639633178711,
      "logits/chosen": -2.071868896484375,
      "logits/rejected": -2.113917827606201,
      "logps/chosen": -6.983762741088867,
      "logps/rejected": -6.790091037750244,
      "loss": 7.3473,
      "nll_loss": 7.25510835647583,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6983763575553894,
      "rewards/margins": -0.019367169588804245,
      "rewards/rejected": -0.6790091395378113,
      "step": 5
    },
    {
      "epoch": 0.0676056338028169,
      "grad_norm": 12.149578094482422,
      "learning_rate": 4.8e-06,
      "log_odds_chosen": 1.0678657293319702,
      "log_odds_ratio": -0.3254351019859314,
      "logits/chosen": -2.24808669090271,
      "logits/rejected": -2.239896774291992,
      "logps/chosen": -6.008720397949219,
      "logps/rejected": -7.073472023010254,
      "loss": 6.3764,
      "nll_loss": 6.343899250030518,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6008720993995667,
      "rewards/margins": 0.10647513717412949,
      "rewards/rejected": -0.7073472142219543,
      "step": 6
    },
    {
      "epoch": 0.07887323943661972,
      "grad_norm": 17.523408889770508,
      "learning_rate": 5.6e-06,
      "log_odds_chosen": -0.13575804233551025,
      "log_odds_ratio": -0.8457558751106262,
      "logits/chosen": -2.143648386001587,
      "logits/rejected": -2.1670777797698975,
      "logps/chosen": -6.814686298370361,
      "logps/rejected": -6.678918838500977,
      "loss": 7.1252,
      "nll_loss": 7.040646553039551,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.6814686059951782,
      "rewards/margins": -0.01357676088809967,
      "rewards/rejected": -0.6678918600082397,
      "step": 7
    },
    {
      "epoch": 0.09014084507042254,
      "grad_norm": 14.829681396484375,
      "learning_rate": 6.4e-06,
      "log_odds_chosen": -0.01214347779750824,
      "log_odds_ratio": -0.7939789295196533,
      "logits/chosen": -2.1688995361328125,
      "logits/rejected": -2.2231764793395996,
      "logps/chosen": -6.625088691711426,
      "logps/rejected": -6.613010406494141,
      "loss": 7.1343,
      "nll_loss": 7.054868221282959,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6625088453292847,
      "rewards/margins": -0.001207888126373291,
      "rewards/rejected": -0.6613010168075562,
      "step": 8
    },
    {
      "epoch": 0.10140845070422536,
      "grad_norm": 13.42191219329834,
      "learning_rate": 7.2e-06,
      "log_odds_chosen": -0.026338055729866028,
      "log_odds_ratio": -0.9790022969245911,
      "logits/chosen": -2.1700892448425293,
      "logits/rejected": -2.2076737880706787,
      "logps/chosen": -6.157310485839844,
      "logps/rejected": -6.1309003829956055,
      "loss": 6.7773,
      "nll_loss": 6.679416656494141,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.6157310605049133,
      "rewards/margins": -0.0026410408318042755,
      "rewards/rejected": -0.6130900382995605,
      "step": 9
    },
    {
      "epoch": 0.11267605633802817,
      "grad_norm": 13.783522605895996,
      "learning_rate": 8e-06,
      "log_odds_chosen": 0.06254176795482635,
      "log_odds_ratio": -0.7391232252120972,
      "logits/chosen": -2.1378564834594727,
      "logits/rejected": -2.195751667022705,
      "logps/chosen": -6.942321300506592,
      "logps/rejected": -7.00494384765625,
      "loss": 7.045,
      "nll_loss": 6.971048355102539,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.694232165813446,
      "rewards/margins": 0.006262235343456268,
      "rewards/rejected": -0.7004944086074829,
      "step": 10
    },
    {
      "epoch": 0.12394366197183099,
      "grad_norm": 16.540300369262695,
      "learning_rate": 7.897435897435898e-06,
      "log_odds_chosen": -0.39260566234588623,
      "log_odds_ratio": -0.9909158945083618,
      "logits/chosen": -2.095216751098633,
      "logits/rejected": -2.0956408977508545,
      "logps/chosen": -6.841823101043701,
      "logps/rejected": -6.4507737159729,
      "loss": 7.197,
      "nll_loss": 7.097899436950684,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.6841824054718018,
      "rewards/margins": -0.03910496458411217,
      "rewards/rejected": -0.6450773477554321,
      "step": 11
    },
    {
      "epoch": 0.1352112676056338,
      "grad_norm": 13.459568977355957,
      "learning_rate": 7.794871794871794e-06,
      "log_odds_chosen": 0.20161187648773193,
      "log_odds_ratio": -0.661760687828064,
      "logits/chosen": -2.1810107231140137,
      "logits/rejected": -2.2119851112365723,
      "logps/chosen": -5.996310234069824,
      "logps/rejected": -6.197461128234863,
      "loss": 6.3979,
      "nll_loss": 6.331758975982666,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5996310710906982,
      "rewards/margins": 0.020115099847316742,
      "rewards/rejected": -0.6197460889816284,
      "step": 12
    },
    {
      "epoch": 0.14647887323943662,
      "grad_norm": 11.37065601348877,
      "learning_rate": 7.692307692307692e-06,
      "log_odds_chosen": 0.2524498701095581,
      "log_odds_ratio": -0.6112524271011353,
      "logits/chosen": -2.2797930240631104,
      "logits/rejected": -2.2433085441589355,
      "logps/chosen": -5.814953327178955,
      "logps/rejected": -6.066402435302734,
      "loss": 6.0765,
      "nll_loss": 6.015379428863525,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5814952850341797,
      "rewards/margins": 0.02514498308300972,
      "rewards/rejected": -0.6066402792930603,
      "step": 13
    },
    {
      "epoch": 0.15774647887323945,
      "grad_norm": 12.900912284851074,
      "learning_rate": 7.589743589743589e-06,
      "log_odds_chosen": -0.4836970567703247,
      "log_odds_ratio": -1.0226668119430542,
      "logits/chosen": -2.215272903442383,
      "logits/rejected": -2.2382960319519043,
      "logps/chosen": -6.522824287414551,
      "logps/rejected": -6.040271759033203,
      "loss": 6.7429,
      "nll_loss": 6.640633583068848,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.6522824764251709,
      "rewards/margins": -0.0482553206384182,
      "rewards/rejected": -0.6040271520614624,
      "step": 14
    },
    {
      "epoch": 0.16901408450704225,
      "grad_norm": 11.437642097473145,
      "learning_rate": 7.487179487179487e-06,
      "log_odds_chosen": 0.15005001425743103,
      "log_odds_ratio": -0.6865947842597961,
      "logits/chosen": -2.3359432220458984,
      "logits/rejected": -2.382951021194458,
      "logps/chosen": -6.031307220458984,
      "logps/rejected": -6.181303024291992,
      "loss": 6.4206,
      "nll_loss": 6.3519721031188965,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.6031307578086853,
      "rewards/margins": 0.014999527484178543,
      "rewards/rejected": -0.6181303262710571,
      "step": 15
    },
    {
      "epoch": 0.18028169014084508,
      "grad_norm": 11.808122634887695,
      "learning_rate": 7.384615384615385e-06,
      "log_odds_chosen": -0.15265953540802002,
      "log_odds_ratio": -0.979753315448761,
      "logits/chosen": -2.309048652648926,
      "logits/rejected": -2.3273842334747314,
      "logps/chosen": -6.232041358947754,
      "logps/rejected": -6.080443859100342,
      "loss": 6.235,
      "nll_loss": 6.136982440948486,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.6232041120529175,
      "rewards/margins": -0.015159666538238525,
      "rewards/rejected": -0.608044445514679,
      "step": 16
    },
    {
      "epoch": 0.19154929577464788,
      "grad_norm": 13.229806900024414,
      "learning_rate": 7.282051282051281e-06,
      "log_odds_chosen": -0.13886889815330505,
      "log_odds_ratio": -0.8053479194641113,
      "logits/chosen": -2.339905261993408,
      "logits/rejected": -2.3555707931518555,
      "logps/chosen": -5.892672061920166,
      "logps/rejected": -5.755575180053711,
      "loss": 6.0915,
      "nll_loss": 6.010932445526123,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5892672538757324,
      "rewards/margins": -0.0137096606194973,
      "rewards/rejected": -0.5755575895309448,
      "step": 17
    },
    {
      "epoch": 0.2028169014084507,
      "grad_norm": 12.478164672851562,
      "learning_rate": 7.179487179487179e-06,
      "log_odds_chosen": -0.1651839315891266,
      "log_odds_ratio": -0.8343566656112671,
      "logits/chosen": -2.440002918243408,
      "logits/rejected": -2.462096691131592,
      "logps/chosen": -6.1440839767456055,
      "logps/rejected": -5.97899055480957,
      "loss": 6.1604,
      "nll_loss": 6.076998233795166,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.6144084930419922,
      "rewards/margins": -0.01650938391685486,
      "rewards/rejected": -0.5978990793228149,
      "step": 18
    },
    {
      "epoch": 0.2028169014084507,
      "eval_log_odds_chosen": 0.23532851040363312,
      "eval_log_odds_ratio": -0.7237306833267212,
      "eval_logits/chosen": -2.3824925422668457,
      "eval_logits/rejected": -2.3515429496765137,
      "eval_logps/chosen": -5.896577835083008,
      "eval_logps/rejected": -6.131969451904297,
      "eval_loss": 6.009965896606445,
      "eval_nll_loss": 5.93759298324585,
      "eval_rewards/accuracies": 0.625,
      "eval_rewards/chosen": -0.5896577835083008,
      "eval_rewards/margins": 0.02353917807340622,
      "eval_rewards/rejected": -0.6131969690322876,
      "eval_runtime": 5.3524,
      "eval_samples_per_second": 1.495,
      "eval_steps_per_second": 0.747,
      "step": 18
    },
    {
      "epoch": 0.2140845070422535,
      "grad_norm": 14.38143253326416,
      "learning_rate": 7.076923076923076e-06,
      "log_odds_chosen": 0.41149377822875977,
      "log_odds_ratio": -0.5679665207862854,
      "logits/chosen": -2.2283358573913574,
      "logits/rejected": -2.2696075439453125,
      "logps/chosen": -5.357198715209961,
      "logps/rejected": -5.767269611358643,
      "loss": 5.4298,
      "nll_loss": 5.373036861419678,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.5357198715209961,
      "rewards/margins": 0.04100704938173294,
      "rewards/rejected": -0.5767269730567932,
      "step": 19
    },
    {
      "epoch": 0.22535211267605634,
      "grad_norm": 13.76131534576416,
      "learning_rate": 6.974358974358974e-06,
      "log_odds_chosen": -0.2197912484407425,
      "log_odds_ratio": -0.8451388478279114,
      "logits/chosen": -2.3156001567840576,
      "logits/rejected": -2.3646183013916016,
      "logps/chosen": -5.359432697296143,
      "logps/rejected": -5.1403656005859375,
      "loss": 5.3198,
      "nll_loss": 5.235289096832275,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5359432697296143,
      "rewards/margins": -0.021906737238168716,
      "rewards/rejected": -0.5140365362167358,
      "step": 20
    },
    {
      "epoch": 0.23661971830985915,
      "grad_norm": 13.503231048583984,
      "learning_rate": 6.871794871794871e-06,
      "log_odds_chosen": -0.38248211145401,
      "log_odds_ratio": -1.001150369644165,
      "logits/chosen": -2.4095211029052734,
      "logits/rejected": -2.393695592880249,
      "logps/chosen": -5.506905555725098,
      "logps/rejected": -5.1283159255981445,
      "loss": 5.4066,
      "nll_loss": 5.306459903717041,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5506905913352966,
      "rewards/margins": -0.037858996540308,
      "rewards/rejected": -0.5128315687179565,
      "step": 21
    },
    {
      "epoch": 0.24788732394366197,
      "grad_norm": 11.429216384887695,
      "learning_rate": 6.769230769230769e-06,
      "log_odds_chosen": 0.3094485402107239,
      "log_odds_ratio": -0.5879026651382446,
      "logits/chosen": -2.545229434967041,
      "logits/rejected": -2.479078531265259,
      "logps/chosen": -4.822021484375,
      "logps/rejected": -5.128696918487549,
      "loss": 5.1873,
      "nll_loss": 5.128541469573975,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4822021722793579,
      "rewards/margins": 0.03066752851009369,
      "rewards/rejected": -0.5128697156906128,
      "step": 22
    },
    {
      "epoch": 0.2591549295774648,
      "grad_norm": 25.22710609436035,
      "learning_rate": 6.666666666666667e-06,
      "log_odds_chosen": -0.8079856634140015,
      "log_odds_ratio": -1.24208664894104,
      "logits/chosen": -2.232295036315918,
      "logits/rejected": -2.3469386100769043,
      "logps/chosen": -5.6021294593811035,
      "logps/rejected": -4.799851417541504,
      "loss": 5.6475,
      "nll_loss": 5.523254871368408,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.5602129697799683,
      "rewards/margins": -0.08022782951593399,
      "rewards/rejected": -0.4799851179122925,
      "step": 23
    },
    {
      "epoch": 0.2704225352112676,
      "grad_norm": 14.158093452453613,
      "learning_rate": 6.564102564102564e-06,
      "log_odds_chosen": -0.18114782869815826,
      "log_odds_ratio": -0.8821460008621216,
      "logits/chosen": -2.4146053791046143,
      "logits/rejected": -2.373107671737671,
      "logps/chosen": -5.3170366287231445,
      "logps/rejected": -5.139774799346924,
      "loss": 5.4074,
      "nll_loss": 5.319136619567871,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5317037105560303,
      "rewards/margins": -0.017726212739944458,
      "rewards/rejected": -0.5139775276184082,
      "step": 24
    },
    {
      "epoch": 0.28169014084507044,
      "grad_norm": 16.50591468811035,
      "learning_rate": 6.461538461538462e-06,
      "log_odds_chosen": -0.2912433445453644,
      "log_odds_ratio": -0.9506481885910034,
      "logits/chosen": -2.361240863800049,
      "logits/rejected": -2.369182586669922,
      "logps/chosen": -5.270952224731445,
      "logps/rejected": -4.982378959655762,
      "loss": 5.3692,
      "nll_loss": 5.274152755737305,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5270952582359314,
      "rewards/margins": -0.028857309371232986,
      "rewards/rejected": -0.4982379674911499,
      "step": 25
    },
    {
      "epoch": 0.29295774647887324,
      "grad_norm": 14.906917572021484,
      "learning_rate": 6.358974358974358e-06,
      "log_odds_chosen": -0.3039167523384094,
      "log_odds_ratio": -0.9283076524734497,
      "logits/chosen": -2.428100109100342,
      "logits/rejected": -2.4127683639526367,
      "logps/chosen": -4.490434169769287,
      "logps/rejected": -4.19234037399292,
      "loss": 4.7795,
      "nll_loss": 4.686664581298828,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4490433931350708,
      "rewards/margins": -0.02980935573577881,
      "rewards/rejected": -0.4192340672016144,
      "step": 26
    },
    {
      "epoch": 0.30422535211267604,
      "grad_norm": 12.076257705688477,
      "learning_rate": 6.256410256410256e-06,
      "log_odds_chosen": -0.006805986166000366,
      "log_odds_ratio": -0.8016737699508667,
      "logits/chosen": -2.520240068435669,
      "logits/rejected": -2.4479568004608154,
      "logps/chosen": -4.917964458465576,
      "logps/rejected": -4.9105448722839355,
      "loss": 4.8248,
      "nll_loss": 4.744668483734131,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.49179643392562866,
      "rewards/margins": -0.0007419846951961517,
      "rewards/rejected": -0.4910544455051422,
      "step": 27
    },
    {
      "epoch": 0.3154929577464789,
      "grad_norm": 13.027565002441406,
      "learning_rate": 6.153846153846154e-06,
      "log_odds_chosen": -0.18745584785938263,
      "log_odds_ratio": -0.8756581544876099,
      "logits/chosen": -2.5150811672210693,
      "logits/rejected": -2.483858108520508,
      "logps/chosen": -4.473248481750488,
      "logps/rejected": -4.290022373199463,
      "loss": 4.7801,
      "nll_loss": 4.692525386810303,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.44732481241226196,
      "rewards/margins": -0.018322546035051346,
      "rewards/rejected": -0.4290022850036621,
      "step": 28
    },
    {
      "epoch": 0.3267605633802817,
      "grad_norm": 11.076545715332031,
      "learning_rate": 6.051282051282051e-06,
      "log_odds_chosen": -0.3999772071838379,
      "log_odds_ratio": -0.9861010909080505,
      "logits/chosen": -2.4857661724090576,
      "logits/rejected": -2.4484825134277344,
      "logps/chosen": -5.028776168823242,
      "logps/rejected": -4.632839202880859,
      "loss": 5.1706,
      "nll_loss": 5.071954727172852,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.5028775930404663,
      "rewards/margins": -0.03959362953901291,
      "rewards/rejected": -0.4632839560508728,
      "step": 29
    },
    {
      "epoch": 0.3380281690140845,
      "grad_norm": 14.6617431640625,
      "learning_rate": 5.948717948717948e-06,
      "log_odds_chosen": -0.5616844892501831,
      "log_odds_ratio": -1.1660798788070679,
      "logits/chosen": -2.510787010192871,
      "logits/rejected": -2.4491658210754395,
      "logps/chosen": -4.740803241729736,
      "logps/rejected": -4.18714714050293,
      "loss": 4.4952,
      "nll_loss": 4.37855339050293,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.47408032417297363,
      "rewards/margins": -0.055365610867738724,
      "rewards/rejected": -0.418714702129364,
      "step": 30
    },
    {
      "epoch": 0.3492957746478873,
      "grad_norm": 12.136735916137695,
      "learning_rate": 5.8461538461538454e-06,
      "log_odds_chosen": -0.5842780470848083,
      "log_odds_ratio": -1.139350175857544,
      "logits/chosen": -2.523271083831787,
      "logits/rejected": -2.4937047958374023,
      "logps/chosen": -4.513538837432861,
      "logps/rejected": -3.9382987022399902,
      "loss": 4.4507,
      "nll_loss": 4.336762428283691,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.45135390758514404,
      "rewards/margins": -0.057523999363183975,
      "rewards/rejected": -0.393829882144928,
      "step": 31
    },
    {
      "epoch": 0.36056338028169016,
      "grad_norm": 13.099180221557617,
      "learning_rate": 5.743589743589743e-06,
      "log_odds_chosen": -0.37855735421180725,
      "log_odds_ratio": -0.9612523317337036,
      "logits/chosen": -2.5555570125579834,
      "logits/rejected": -2.4845659732818604,
      "logps/chosen": -4.688088417053223,
      "logps/rejected": -4.315810680389404,
      "loss": 4.7878,
      "nll_loss": 4.69166374206543,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.4688088893890381,
      "rewards/margins": -0.0372278057038784,
      "rewards/rejected": -0.4315810799598694,
      "step": 32
    },
    {
      "epoch": 0.37183098591549296,
      "grad_norm": 13.548160552978516,
      "learning_rate": 5.641025641025641e-06,
      "log_odds_chosen": -0.6679523587226868,
      "log_odds_ratio": -1.1635067462921143,
      "logits/chosen": -2.4850564002990723,
      "logits/rejected": -2.4568276405334473,
      "logps/chosen": -4.625258445739746,
      "logps/rejected": -3.966517925262451,
      "loss": 4.3309,
      "nll_loss": 4.214511871337891,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.4625258445739746,
      "rewards/margins": -0.06587404012680054,
      "rewards/rejected": -0.3966518044471741,
      "step": 33
    },
    {
      "epoch": 0.38309859154929576,
      "grad_norm": 11.71092700958252,
      "learning_rate": 5.538461538461538e-06,
      "log_odds_chosen": 0.24815137684345245,
      "log_odds_ratio": -0.642487645149231,
      "logits/chosen": -2.511573076248169,
      "logits/rejected": -2.466554880142212,
      "logps/chosen": -4.356619834899902,
      "logps/rejected": -4.603775501251221,
      "loss": 4.2485,
      "nll_loss": 4.18422794342041,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4356619715690613,
      "rewards/margins": 0.024715568870306015,
      "rewards/rejected": -0.4603775143623352,
      "step": 34
    },
    {
      "epoch": 0.39436619718309857,
      "grad_norm": 17.99422836303711,
      "learning_rate": 5.435897435897436e-06,
      "log_odds_chosen": -0.4279804825782776,
      "log_odds_ratio": -1.0195913314819336,
      "logits/chosen": -2.4478812217712402,
      "logits/rejected": -2.4185688495635986,
      "logps/chosen": -4.629607200622559,
      "logps/rejected": -4.212224960327148,
      "loss": 4.3901,
      "nll_loss": 4.288104057312012,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.46296072006225586,
      "rewards/margins": -0.041738200932741165,
      "rewards/rejected": -0.4212225377559662,
      "step": 35
    },
    {
      "epoch": 0.4056338028169014,
      "grad_norm": 26.7180118560791,
      "learning_rate": 5.333333333333333e-06,
      "log_odds_chosen": -0.49325498938560486,
      "log_odds_ratio": -1.1076762676239014,
      "logits/chosen": -2.384472608566284,
      "logits/rejected": -2.384478807449341,
      "logps/chosen": -4.394374370574951,
      "logps/rejected": -3.9007925987243652,
      "loss": 4.5105,
      "nll_loss": 4.399693489074707,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.4394374489784241,
      "rewards/margins": -0.04935820400714874,
      "rewards/rejected": -0.39007923007011414,
      "step": 36
    },
    {
      "epoch": 0.4056338028169014,
      "eval_log_odds_chosen": -0.021530766040086746,
      "eval_log_odds_ratio": -0.7920652627944946,
      "eval_logits/chosen": -2.437562942504883,
      "eval_logits/rejected": -2.4226341247558594,
      "eval_logps/chosen": -4.752222061157227,
      "eval_logps/rejected": -4.731179237365723,
      "eval_loss": 5.0522871017456055,
      "eval_nll_loss": 4.973080635070801,
      "eval_rewards/accuracies": 0.5,
      "eval_rewards/chosen": -0.47522225975990295,
      "eval_rewards/margins": -0.0021043792366981506,
      "eval_rewards/rejected": -0.4731178879737854,
      "eval_runtime": 5.3465,
      "eval_samples_per_second": 1.496,
      "eval_steps_per_second": 0.748,
      "step": 36
    },
    {
      "epoch": 0.4169014084507042,
      "grad_norm": 25.756877899169922,
      "learning_rate": 5.230769230769231e-06,
      "log_odds_chosen": -0.04049903154373169,
      "log_odds_ratio": -0.8640150427818298,
      "logits/chosen": -2.536158800125122,
      "logits/rejected": -2.481358051300049,
      "logps/chosen": -4.724617958068848,
      "logps/rejected": -4.686985492706299,
      "loss": 5.0137,
      "nll_loss": 4.927271366119385,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.4724617600440979,
      "rewards/margins": -0.0037632212042808533,
      "rewards/rejected": -0.46869853138923645,
      "step": 37
    },
    {
      "epoch": 0.428169014084507,
      "grad_norm": 15.886479377746582,
      "learning_rate": 5.128205128205129e-06,
      "log_odds_chosen": -0.37556567788124084,
      "log_odds_ratio": -0.9566004276275635,
      "logits/chosen": -2.4243788719177246,
      "logits/rejected": -2.399319887161255,
      "logps/chosen": -4.280425071716309,
      "logps/rejected": -3.9121856689453125,
      "loss": 4.3575,
      "nll_loss": 4.2618303298950195,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.428042471408844,
      "rewards/margins": -0.03682393580675125,
      "rewards/rejected": -0.3912185728549957,
      "step": 38
    },
    {
      "epoch": 0.4394366197183099,
      "grad_norm": 17.65472412109375,
      "learning_rate": 5.025641025641025e-06,
      "log_odds_chosen": 0.2230280339717865,
      "log_odds_ratio": -0.6618625521659851,
      "logits/chosen": -2.3576548099517822,
      "logits/rejected": -2.333857297897339,
      "logps/chosen": -3.9965877532958984,
      "logps/rejected": -4.214459419250488,
      "loss": 4.3671,
      "nll_loss": 4.300962448120117,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.399658739566803,
      "rewards/margins": 0.02178717404603958,
      "rewards/rejected": -0.42144593596458435,
      "step": 39
    },
    {
      "epoch": 0.4507042253521127,
      "grad_norm": 11.106135368347168,
      "learning_rate": 4.923076923076923e-06,
      "log_odds_chosen": 0.0009728074073791504,
      "log_odds_ratio": -0.7869582772254944,
      "logits/chosen": -2.4159560203552246,
      "logits/rejected": -2.4208855628967285,
      "logps/chosen": -3.9074864387512207,
      "logps/rejected": -3.903223991394043,
      "loss": 4.1296,
      "nll_loss": 4.050935745239258,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.39074867963790894,
      "rewards/margins": -0.0004262477159500122,
      "rewards/rejected": -0.39032241702079773,
      "step": 40
    },
    {
      "epoch": 0.4619718309859155,
      "grad_norm": 12.57712459564209,
      "learning_rate": 4.82051282051282e-06,
      "log_odds_chosen": -0.2918354868888855,
      "log_odds_ratio": -0.8778151869773865,
      "logits/chosen": -2.3547985553741455,
      "logits/rejected": -2.322421073913574,
      "logps/chosen": -4.1212158203125,
      "logps/rejected": -3.8306081295013428,
      "loss": 4.0642,
      "nll_loss": 3.9763693809509277,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.41212156414985657,
      "rewards/margins": -0.029060743749141693,
      "rewards/rejected": -0.3830608129501343,
      "step": 41
    },
    {
      "epoch": 0.4732394366197183,
      "grad_norm": 12.822919845581055,
      "learning_rate": 4.717948717948718e-06,
      "log_odds_chosen": -0.810235321521759,
      "log_odds_ratio": -1.2506808042526245,
      "logits/chosen": -2.2839574813842773,
      "logits/rejected": -2.288829803466797,
      "logps/chosen": -4.443515300750732,
      "logps/rejected": -3.658328056335449,
      "loss": 4.3262,
      "nll_loss": 4.201174259185791,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.44435155391693115,
      "rewards/margins": -0.07851871848106384,
      "rewards/rejected": -0.3658328056335449,
      "step": 42
    },
    {
      "epoch": 0.48450704225352115,
      "grad_norm": 12.537949562072754,
      "learning_rate": 4.6153846153846145e-06,
      "log_odds_chosen": -0.9749997854232788,
      "log_odds_ratio": -1.3443421125411987,
      "logits/chosen": -2.3435025215148926,
      "logits/rejected": -2.321125030517578,
      "logps/chosen": -4.51336145401001,
      "logps/rejected": -3.566725254058838,
      "loss": 4.5357,
      "nll_loss": 4.401299476623535,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.45133620500564575,
      "rewards/margins": -0.09466366469860077,
      "rewards/rejected": -0.3566725254058838,
      "step": 43
    },
    {
      "epoch": 0.49577464788732395,
      "grad_norm": 10.948065757751465,
      "learning_rate": 4.5128205128205124e-06,
      "log_odds_chosen": -0.7950465083122253,
      "log_odds_ratio": -1.22953200340271,
      "logits/chosen": -2.319894790649414,
      "logits/rejected": -2.3266992568969727,
      "logps/chosen": -4.4190673828125,
      "logps/rejected": -3.646463394165039,
      "loss": 4.4846,
      "nll_loss": 4.361649513244629,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.44190675020217896,
      "rewards/margins": -0.07726040482521057,
      "rewards/rejected": -0.3646463453769684,
      "step": 44
    },
    {
      "epoch": 0.5070422535211268,
      "grad_norm": 10.234577178955078,
      "learning_rate": 4.4102564102564104e-06,
      "log_odds_chosen": -0.03596124053001404,
      "log_odds_ratio": -0.7954179048538208,
      "logits/chosen": -2.3405230045318604,
      "logits/rejected": -2.3773298263549805,
      "logps/chosen": -3.984051465988159,
      "logps/rejected": -3.9488115310668945,
      "loss": 4.3053,
      "nll_loss": 4.225733280181885,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.39840513467788696,
      "rewards/margins": -0.003523964434862137,
      "rewards/rejected": -0.3948811888694763,
      "step": 45
    },
    {
      "epoch": 0.5183098591549296,
      "grad_norm": 11.955574035644531,
      "learning_rate": 4.3076923076923076e-06,
      "log_odds_chosen": -0.47261345386505127,
      "log_odds_ratio": -1.0186831951141357,
      "logits/chosen": -2.2264914512634277,
      "logits/rejected": -2.2124805450439453,
      "logps/chosen": -4.316086769104004,
      "logps/rejected": -3.8509907722473145,
      "loss": 4.2714,
      "nll_loss": 4.169527053833008,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.4316087067127228,
      "rewards/margins": -0.046509623527526855,
      "rewards/rejected": -0.3850990831851959,
      "step": 46
    },
    {
      "epoch": 0.5295774647887324,
      "grad_norm": 14.009207725524902,
      "learning_rate": 4.205128205128205e-06,
      "log_odds_chosen": -0.5015394687652588,
      "log_odds_ratio": -1.0697970390319824,
      "logits/chosen": -2.216539144515991,
      "logits/rejected": -2.236360549926758,
      "logps/chosen": -4.524508476257324,
      "logps/rejected": -4.033357620239258,
      "loss": 4.2457,
      "nll_loss": 4.138679504394531,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.45245084166526794,
      "rewards/margins": -0.04911504685878754,
      "rewards/rejected": -0.4033357799053192,
      "step": 47
    },
    {
      "epoch": 0.5408450704225352,
      "grad_norm": 11.495881080627441,
      "learning_rate": 4.102564102564102e-06,
      "log_odds_chosen": -0.0002641826868057251,
      "log_odds_ratio": -0.7958545088768005,
      "logits/chosen": -2.2024898529052734,
      "logits/rejected": -2.2096405029296875,
      "logps/chosen": -3.650709629058838,
      "logps/rejected": -3.6475670337677,
      "loss": 3.7327,
      "nll_loss": 3.653069019317627,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.3650709390640259,
      "rewards/margins": -0.0003142599016427994,
      "rewards/rejected": -0.36475670337677,
      "step": 48
    },
    {
      "epoch": 0.5521126760563381,
      "grad_norm": 10.599851608276367,
      "learning_rate": 4e-06,
      "log_odds_chosen": -0.40274348855018616,
      "log_odds_ratio": -0.9958802461624146,
      "logits/chosen": -2.1248817443847656,
      "logits/rejected": -2.1678757667541504,
      "logps/chosen": -3.9321091175079346,
      "logps/rejected": -3.5374057292938232,
      "loss": 3.7393,
      "nll_loss": 3.6396658420562744,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.39321091771125793,
      "rewards/margins": -0.03947034850716591,
      "rewards/rejected": -0.3537406027317047,
      "step": 49
    },
    {
      "epoch": 0.5633802816901409,
      "grad_norm": 14.113234519958496,
      "learning_rate": 3.897435897435897e-06,
      "log_odds_chosen": -0.15864649415016174,
      "log_odds_ratio": -0.8118338584899902,
      "logits/chosen": -2.182473659515381,
      "logits/rejected": -2.2479374408721924,
      "logps/chosen": -4.00465202331543,
      "logps/rejected": -3.8499021530151367,
      "loss": 3.984,
      "nll_loss": 3.902796745300293,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4004652500152588,
      "rewards/margins": -0.015475019812583923,
      "rewards/rejected": -0.38499021530151367,
      "step": 50
    },
    {
      "epoch": 0.5746478873239437,
      "grad_norm": 15.240550994873047,
      "learning_rate": 3.7948717948717945e-06,
      "log_odds_chosen": -0.3683556020259857,
      "log_odds_ratio": -1.0493247509002686,
      "logits/chosen": -2.142444133758545,
      "logits/rejected": -2.1910910606384277,
      "logps/chosen": -4.096914768218994,
      "logps/rejected": -3.7346315383911133,
      "loss": 3.9369,
      "nll_loss": 3.8319671154022217,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4096915125846863,
      "rewards/margins": -0.036228351294994354,
      "rewards/rejected": -0.37346315383911133,
      "step": 51
    },
    {
      "epoch": 0.5859154929577465,
      "grad_norm": 13.516912460327148,
      "learning_rate": 3.6923076923076925e-06,
      "log_odds_chosen": -0.14869263768196106,
      "log_odds_ratio": -0.8158061504364014,
      "logits/chosen": -2.1389424800872803,
      "logits/rejected": -2.1591155529022217,
      "logps/chosen": -3.914469003677368,
      "logps/rejected": -3.7656819820404053,
      "loss": 3.6203,
      "nll_loss": 3.538712978363037,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.39144688844680786,
      "rewards/margins": -0.014878695830702782,
      "rewards/rejected": -0.3765681982040405,
      "step": 52
    },
    {
      "epoch": 0.5971830985915493,
      "grad_norm": 12.95617961883545,
      "learning_rate": 3.5897435897435896e-06,
      "log_odds_chosen": -0.24592818319797516,
      "log_odds_ratio": -0.9381157159805298,
      "logits/chosen": -2.0287747383117676,
      "logits/rejected": -2.1300435066223145,
      "logps/chosen": -3.8074216842651367,
      "logps/rejected": -3.571408748626709,
      "loss": 3.6729,
      "nll_loss": 3.5790419578552246,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.3807421326637268,
      "rewards/margins": -0.023601267486810684,
      "rewards/rejected": -0.3571408987045288,
      "step": 53
    },
    {
      "epoch": 0.6084507042253521,
      "grad_norm": 11.833647727966309,
      "learning_rate": 3.487179487179487e-06,
      "log_odds_chosen": -0.5012875199317932,
      "log_odds_ratio": -1.0330029726028442,
      "logits/chosen": -2.141413688659668,
      "logits/rejected": -2.1845943927764893,
      "logps/chosen": -4.043262958526611,
      "logps/rejected": -3.5469398498535156,
      "loss": 3.9015,
      "nll_loss": 3.798215627670288,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.40432626008987427,
      "rewards/margins": -0.049632273614406586,
      "rewards/rejected": -0.3546940088272095,
      "step": 54
    },
    {
      "epoch": 0.6084507042253521,
      "eval_log_odds_chosen": -0.0070199985057115555,
      "eval_log_odds_ratio": -0.8004560470581055,
      "eval_logits/chosen": -2.2082390785217285,
      "eval_logits/rejected": -2.2092061042785645,
      "eval_logps/chosen": -4.265551567077637,
      "eval_logps/rejected": -4.259222030639648,
      "eval_loss": 4.50450325012207,
      "eval_nll_loss": 4.42445707321167,
      "eval_rewards/accuracies": 0.5,
      "eval_rewards/chosen": -0.42655515670776367,
      "eval_rewards/margins": -0.0006329789757728577,
      "eval_rewards/rejected": -0.4259222149848938,
      "eval_runtime": 5.3539,
      "eval_samples_per_second": 1.494,
      "eval_steps_per_second": 0.747,
      "step": 54
    },
    {
      "epoch": 0.6197183098591549,
      "grad_norm": 13.9421968460083,
      "learning_rate": 3.3846153846153843e-06,
      "log_odds_chosen": -0.17257481813430786,
      "log_odds_ratio": -0.8791506886482239,
      "logits/chosen": -2.223339080810547,
      "logits/rejected": -2.183260440826416,
      "logps/chosen": -4.611689567565918,
      "logps/rejected": -4.4401044845581055,
      "loss": 4.3277,
      "nll_loss": 4.239750862121582,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.46116903424263,
      "rewards/margins": -0.017158538103103638,
      "rewards/rejected": -0.444010466337204,
      "step": 55
    },
    {
      "epoch": 0.6309859154929578,
      "grad_norm": 11.743844032287598,
      "learning_rate": 3.282051282051282e-06,
      "log_odds_chosen": -0.662411630153656,
      "log_odds_ratio": -1.2490286827087402,
      "logits/chosen": -1.9998000860214233,
      "logits/rejected": -2.092249631881714,
      "logps/chosen": -3.955559730529785,
      "logps/rejected": -3.31760311126709,
      "loss": 3.5195,
      "nll_loss": 3.394559383392334,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.3955560028553009,
      "rewards/margins": -0.06379567086696625,
      "rewards/rejected": -0.33176031708717346,
      "step": 56
    },
    {
      "epoch": 0.6422535211267606,
      "grad_norm": 8.948927879333496,
      "learning_rate": 3.179487179487179e-06,
      "log_odds_chosen": -0.0703490823507309,
      "log_odds_ratio": -0.8050236105918884,
      "logits/chosen": -2.2200632095336914,
      "logits/rejected": -2.263523817062378,
      "logps/chosen": -3.7273356914520264,
      "logps/rejected": -3.654778480529785,
      "loss": 4.0303,
      "nll_loss": 3.9498226642608643,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.37273356318473816,
      "rewards/margins": -0.00725569948554039,
      "rewards/rejected": -0.36547785997390747,
      "step": 57
    },
    {
      "epoch": 0.6535211267605634,
      "grad_norm": 9.45151138305664,
      "learning_rate": 3.076923076923077e-06,
      "log_odds_chosen": -0.11648232489824295,
      "log_odds_ratio": -0.8264567852020264,
      "logits/chosen": -2.264216184616089,
      "logits/rejected": -2.270479202270508,
      "logps/chosen": -3.7268123626708984,
      "logps/rejected": -3.617461681365967,
      "loss": 4.3405,
      "nll_loss": 4.257853984832764,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.37268123030662537,
      "rewards/margins": -0.010935075581073761,
      "rewards/rejected": -0.3617461919784546,
      "step": 58
    },
    {
      "epoch": 0.6647887323943662,
      "grad_norm": 13.517334938049316,
      "learning_rate": 2.974358974358974e-06,
      "log_odds_chosen": -0.04971995949745178,
      "log_odds_ratio": -0.8288277387619019,
      "logits/chosen": -2.0212604999542236,
      "logits/rejected": -2.118274211883545,
      "logps/chosen": -3.4767675399780273,
      "logps/rejected": -3.431030035018921,
      "loss": 3.3954,
      "nll_loss": 3.312514066696167,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.34767675399780273,
      "rewards/margins": -0.004573751240968704,
      "rewards/rejected": -0.34310299158096313,
      "step": 59
    },
    {
      "epoch": 0.676056338028169,
      "grad_norm": 10.096856117248535,
      "learning_rate": 2.8717948717948717e-06,
      "log_odds_chosen": 0.0826490968465805,
      "log_odds_ratio": -0.8434144258499146,
      "logits/chosen": -2.0799660682678223,
      "logits/rejected": -2.1399242877960205,
      "logps/chosen": -3.7268831729888916,
      "logps/rejected": -3.8236966133117676,
      "loss": 3.7636,
      "nll_loss": 3.679297685623169,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.372688353061676,
      "rewards/margins": 0.009681336581707,
      "rewards/rejected": -0.38236966729164124,
      "step": 60
    },
    {
      "epoch": 0.6873239436619718,
      "grad_norm": 11.962920188903809,
      "learning_rate": 2.769230769230769e-06,
      "log_odds_chosen": 0.11713685840368271,
      "log_odds_ratio": -0.7304390072822571,
      "logits/chosen": -2.0528836250305176,
      "logits/rejected": -2.1740870475769043,
      "logps/chosen": -3.7169508934020996,
      "logps/rejected": -3.827744960784912,
      "loss": 3.7201,
      "nll_loss": 3.6470143795013428,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3716951012611389,
      "rewards/margins": 0.011079397052526474,
      "rewards/rejected": -0.3827744722366333,
      "step": 61
    },
    {
      "epoch": 0.6985915492957746,
      "grad_norm": 9.109440803527832,
      "learning_rate": 2.6666666666666664e-06,
      "log_odds_chosen": -0.16476285457611084,
      "log_odds_ratio": -0.902513325214386,
      "logits/chosen": -2.2049403190612793,
      "logits/rejected": -2.266833543777466,
      "logps/chosen": -3.761657476425171,
      "logps/rejected": -3.6014232635498047,
      "loss": 3.9372,
      "nll_loss": 3.846921443939209,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3761657476425171,
      "rewards/margins": -0.016023436561226845,
      "rewards/rejected": -0.3601423501968384,
      "step": 62
    },
    {
      "epoch": 0.7098591549295775,
      "grad_norm": 10.382341384887695,
      "learning_rate": 2.5641025641025644e-06,
      "log_odds_chosen": -0.4335900843143463,
      "log_odds_ratio": -1.0794651508331299,
      "logits/chosen": -2.0837039947509766,
      "logits/rejected": -2.1245696544647217,
      "logps/chosen": -3.4378552436828613,
      "logps/rejected": -3.0201077461242676,
      "loss": 3.6002,
      "nll_loss": 3.492260456085205,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.34378552436828613,
      "rewards/margins": -0.04177476093173027,
      "rewards/rejected": -0.30201077461242676,
      "step": 63
    },
    {
      "epoch": 0.7211267605633803,
      "grad_norm": 12.012513160705566,
      "learning_rate": 2.4615384615384615e-06,
      "log_odds_chosen": -0.21123048663139343,
      "log_odds_ratio": -0.8581590056419373,
      "logits/chosen": -2.011561155319214,
      "logits/rejected": -2.095082998275757,
      "logps/chosen": -3.706972599029541,
      "logps/rejected": -3.507680654525757,
      "loss": 3.5491,
      "nll_loss": 3.4632492065429688,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.3706972599029541,
      "rewards/margins": -0.01992921158671379,
      "rewards/rejected": -0.3507680296897888,
      "step": 64
    },
    {
      "epoch": 0.7323943661971831,
      "grad_norm": 10.67837142944336,
      "learning_rate": 2.358974358974359e-06,
      "log_odds_chosen": 0.16844214498996735,
      "log_odds_ratio": -0.6445605754852295,
      "logits/chosen": -2.1053593158721924,
      "logits/rejected": -2.14455509185791,
      "logps/chosen": -3.4388720989227295,
      "logps/rejected": -3.601630210876465,
      "loss": 3.4626,
      "nll_loss": 3.398159980773926,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.34388723969459534,
      "rewards/margins": 0.016275810077786446,
      "rewards/rejected": -0.36016303300857544,
      "step": 65
    },
    {
      "epoch": 0.7436619718309859,
      "grad_norm": 14.22414493560791,
      "learning_rate": 2.2564102564102562e-06,
      "log_odds_chosen": -0.837117075920105,
      "log_odds_ratio": -1.3559041023254395,
      "logits/chosen": -2.200282573699951,
      "logits/rejected": -2.2180466651916504,
      "logps/chosen": -4.45404052734375,
      "logps/rejected": -3.649200916290283,
      "loss": 4.4569,
      "nll_loss": 4.321296215057373,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.4454040825366974,
      "rewards/margins": -0.08048398792743683,
      "rewards/rejected": -0.36492010951042175,
      "step": 66
    },
    {
      "epoch": 0.7549295774647887,
      "grad_norm": 12.72874641418457,
      "learning_rate": 2.1538461538461538e-06,
      "log_odds_chosen": -0.9214501976966858,
      "log_odds_ratio": -1.3725779056549072,
      "logits/chosen": -2.1959099769592285,
      "logits/rejected": -2.1905465126037598,
      "logps/chosen": -4.213864803314209,
      "logps/rejected": -3.309765577316284,
      "loss": 4.0375,
      "nll_loss": 3.900239944458008,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.4213864803314209,
      "rewards/margins": -0.09040991961956024,
      "rewards/rejected": -0.33097654581069946,
      "step": 67
    },
    {
      "epoch": 0.7661971830985915,
      "grad_norm": 11.578482627868652,
      "learning_rate": 2.051282051282051e-06,
      "log_odds_chosen": -0.3243113160133362,
      "log_odds_ratio": -0.9195367097854614,
      "logits/chosen": -2.1235108375549316,
      "logits/rejected": -2.168004035949707,
      "logps/chosen": -3.7418808937072754,
      "logps/rejected": -3.425140142440796,
      "loss": 3.4577,
      "nll_loss": 3.365786075592041,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.374188095331192,
      "rewards/margins": -0.031674087047576904,
      "rewards/rejected": -0.3425140380859375,
      "step": 68
    },
    {
      "epoch": 0.7774647887323943,
      "grad_norm": 12.581503868103027,
      "learning_rate": 1.9487179487179485e-06,
      "log_odds_chosen": -0.03559496998786926,
      "log_odds_ratio": -0.8679975271224976,
      "logits/chosen": -1.9304691553115845,
      "logits/rejected": -2.009279727935791,
      "logps/chosen": -3.82916522026062,
      "logps/rejected": -3.7897675037384033,
      "loss": 3.6078,
      "nll_loss": 3.520955801010132,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.38291651010513306,
      "rewards/margins": -0.00393977016210556,
      "rewards/rejected": -0.3789767622947693,
      "step": 69
    },
    {
      "epoch": 0.7887323943661971,
      "grad_norm": 10.235701560974121,
      "learning_rate": 1.8461538461538462e-06,
      "log_odds_chosen": 0.7963358759880066,
      "log_odds_ratio": -0.4532206356525421,
      "logits/chosen": -2.097158908843994,
      "logits/rejected": -2.1584880352020264,
      "logps/chosen": -3.211224317550659,
      "logps/rejected": -3.9823241233825684,
      "loss": 3.4753,
      "nll_loss": 3.4299583435058594,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.321122407913208,
      "rewards/margins": 0.07711000740528107,
      "rewards/rejected": -0.39823243021965027,
      "step": 70
    },
    {
      "epoch": 0.8,
      "grad_norm": 10.014704704284668,
      "learning_rate": 1.7435897435897436e-06,
      "log_odds_chosen": -0.49982893466949463,
      "log_odds_ratio": -1.097215175628662,
      "logits/chosen": -2.100175380706787,
      "logits/rejected": -2.1770012378692627,
      "logps/chosen": -3.92124080657959,
      "logps/rejected": -3.4407193660736084,
      "loss": 3.9242,
      "nll_loss": 3.8145065307617188,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.39212408661842346,
      "rewards/margins": -0.04805214703083038,
      "rewards/rejected": -0.3440719544887543,
      "step": 71
    },
    {
      "epoch": 0.8112676056338028,
      "grad_norm": 11.128293991088867,
      "learning_rate": 1.641025641025641e-06,
      "log_odds_chosen": -0.03216097131371498,
      "log_odds_ratio": -0.7252599000930786,
      "logits/chosen": -1.9201287031173706,
      "logits/rejected": -2.0173075199127197,
      "logps/chosen": -3.436187744140625,
      "logps/rejected": -3.4034676551818848,
      "loss": 3.3486,
      "nll_loss": 3.2760744094848633,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3436187505722046,
      "rewards/margins": -0.0032719969749450684,
      "rewards/rejected": -0.3403467535972595,
      "step": 72
    },
    {
      "epoch": 0.8112676056338028,
      "eval_log_odds_chosen": 0.06351960450410843,
      "eval_log_odds_ratio": -0.7638184428215027,
      "eval_logits/chosen": -2.0969061851501465,
      "eval_logits/rejected": -2.098085880279541,
      "eval_logps/chosen": -3.975425958633423,
      "eval_logps/rejected": -4.037297248840332,
      "eval_loss": 4.211479187011719,
      "eval_nll_loss": 4.135097503662109,
      "eval_rewards/accuracies": 0.5,
      "eval_rewards/chosen": -0.3975425958633423,
      "eval_rewards/margins": 0.006187114864587784,
      "eval_rewards/rejected": -0.40372970700263977,
      "eval_runtime": 5.3655,
      "eval_samples_per_second": 1.491,
      "eval_steps_per_second": 0.746,
      "step": 72
    },
    {
      "epoch": 0.8225352112676056,
      "grad_norm": 13.299338340759277,
      "learning_rate": 1.5384615384615385e-06,
      "log_odds_chosen": -0.08484344184398651,
      "log_odds_ratio": -0.794756293296814,
      "logits/chosen": -1.9263019561767578,
      "logits/rejected": -1.9672499895095825,
      "logps/chosen": -3.4308979511260986,
      "logps/rejected": -3.347562789916992,
      "loss": 3.225,
      "nll_loss": 3.145517349243164,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3430897891521454,
      "rewards/margins": -0.008333522826433182,
      "rewards/rejected": -0.3347562849521637,
      "step": 73
    },
    {
      "epoch": 0.8338028169014085,
      "grad_norm": 10.876466751098633,
      "learning_rate": 1.4358974358974359e-06,
      "log_odds_chosen": -0.16019976139068604,
      "log_odds_ratio": -0.8210447430610657,
      "logits/chosen": -1.9499791860580444,
      "logits/rejected": -2.0576226711273193,
      "logps/chosen": -3.4298863410949707,
      "logps/rejected": -3.2793407440185547,
      "loss": 3.2026,
      "nll_loss": 3.1205201148986816,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.34298866987228394,
      "rewards/margins": -0.015054592862725258,
      "rewards/rejected": -0.32793405652046204,
      "step": 74
    },
    {
      "epoch": 0.8450704225352113,
      "grad_norm": 11.573190689086914,
      "learning_rate": 1.3333333333333332e-06,
      "log_odds_chosen": -0.4442010223865509,
      "log_odds_ratio": -0.9989137053489685,
      "logits/chosen": -2.0589771270751953,
      "logits/rejected": -2.151106595993042,
      "logps/chosen": -3.764126777648926,
      "logps/rejected": -3.340824604034424,
      "loss": 3.6574,
      "nll_loss": 3.557495594024658,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.37641265988349915,
      "rewards/margins": -0.042330194264650345,
      "rewards/rejected": -0.3340824842453003,
      "step": 75
    },
    {
      "epoch": 0.856338028169014,
      "grad_norm": 10.903554916381836,
      "learning_rate": 1.2307692307692308e-06,
      "log_odds_chosen": -0.5089474320411682,
      "log_odds_ratio": -1.0521657466888428,
      "logits/chosen": -2.061066150665283,
      "logits/rejected": -2.086852550506592,
      "logps/chosen": -3.958143711090088,
      "logps/rejected": -3.4646599292755127,
      "loss": 3.9915,
      "nll_loss": 3.886314868927002,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.39581435918807983,
      "rewards/margins": -0.049348361790180206,
      "rewards/rejected": -0.3464660048484802,
      "step": 76
    },
    {
      "epoch": 0.8676056338028169,
      "grad_norm": 13.457832336425781,
      "learning_rate": 1.1282051282051281e-06,
      "log_odds_chosen": 0.047474101185798645,
      "log_odds_ratio": -0.7185592651367188,
      "logits/chosen": -1.970052719116211,
      "logits/rejected": -2.0962562561035156,
      "logps/chosen": -3.3302972316741943,
      "logps/rejected": -3.3730051517486572,
      "loss": 3.4269,
      "nll_loss": 3.3550257682800293,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.33302971720695496,
      "rewards/margins": 0.004270805045962334,
      "rewards/rejected": -0.33730053901672363,
      "step": 77
    },
    {
      "epoch": 0.8788732394366198,
      "grad_norm": 13.770342826843262,
      "learning_rate": 1.0256410256410255e-06,
      "log_odds_chosen": 0.29137304425239563,
      "log_odds_ratio": -0.7174768447875977,
      "logits/chosen": -1.9147363901138306,
      "logits/rejected": -2.074281692504883,
      "logps/chosen": -3.6085116863250732,
      "logps/rejected": -3.9022023677825928,
      "loss": 3.6132,
      "nll_loss": 3.541477680206299,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3608511686325073,
      "rewards/margins": 0.02936907671391964,
      "rewards/rejected": -0.3902202248573303,
      "step": 78
    },
    {
      "epoch": 0.8901408450704226,
      "grad_norm": 11.069844245910645,
      "learning_rate": 9.230769230769231e-07,
      "log_odds_chosen": -0.5195303559303284,
      "log_odds_ratio": -1.1289513111114502,
      "logits/chosen": -2.013373851776123,
      "logits/rejected": -2.119136333465576,
      "logps/chosen": -4.341013431549072,
      "logps/rejected": -3.829310178756714,
      "loss": 4.1473,
      "nll_loss": 4.034441947937012,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.43410131335258484,
      "rewards/margins": -0.05117032676935196,
      "rewards/rejected": -0.3829309940338135,
      "step": 79
    },
    {
      "epoch": 0.9014084507042254,
      "grad_norm": 11.87746810913086,
      "learning_rate": 8.205128205128205e-07,
      "log_odds_chosen": -0.13725686073303223,
      "log_odds_ratio": -0.8136627674102783,
      "logits/chosen": -1.9058239459991455,
      "logits/rejected": -2.0132644176483154,
      "logps/chosen": -3.670649528503418,
      "logps/rejected": -3.545762062072754,
      "loss": 3.5501,
      "nll_loss": 3.46868634223938,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.3670649826526642,
      "rewards/margins": -0.01248876191675663,
      "rewards/rejected": -0.3545762300491333,
      "step": 80
    },
    {
      "epoch": 0.9126760563380282,
      "grad_norm": 14.643924713134766,
      "learning_rate": 7.179487179487179e-07,
      "log_odds_chosen": 0.27706030011177063,
      "log_odds_ratio": -0.6745563745498657,
      "logits/chosen": -1.9193668365478516,
      "logits/rejected": -2.0037789344787598,
      "logps/chosen": -3.511770725250244,
      "logps/rejected": -3.7823686599731445,
      "loss": 3.4836,
      "nll_loss": 3.416135311126709,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3511770963668823,
      "rewards/margins": 0.027059808373451233,
      "rewards/rejected": -0.37823688983917236,
      "step": 81
    },
    {
      "epoch": 0.923943661971831,
      "grad_norm": 11.816194534301758,
      "learning_rate": 6.153846153846154e-07,
      "log_odds_chosen": -0.5967518091201782,
      "log_odds_ratio": -1.1597626209259033,
      "logits/chosen": -2.0161638259887695,
      "logits/rejected": -2.067549228668213,
      "logps/chosen": -3.8716623783111572,
      "logps/rejected": -3.2700958251953125,
      "loss": 4.0015,
      "nll_loss": 3.8855390548706055,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.38716626167297363,
      "rewards/margins": -0.06015666574239731,
      "rewards/rejected": -0.3270095884799957,
      "step": 82
    },
    {
      "epoch": 0.9352112676056338,
      "grad_norm": 11.173247337341309,
      "learning_rate": 5.128205128205127e-07,
      "log_odds_chosen": -0.45643556118011475,
      "log_odds_ratio": -1.0668988227844238,
      "logits/chosen": -2.018862009048462,
      "logits/rejected": -2.0817437171936035,
      "logps/chosen": -4.08735466003418,
      "logps/rejected": -3.6331050395965576,
      "loss": 3.8471,
      "nll_loss": 3.740403890609741,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4087355136871338,
      "rewards/margins": -0.04542497545480728,
      "rewards/rejected": -0.3633105158805847,
      "step": 83
    },
    {
      "epoch": 0.9464788732394366,
      "grad_norm": 12.033004760742188,
      "learning_rate": 4.1025641025641024e-07,
      "log_odds_chosen": -0.16924908757209778,
      "log_odds_ratio": -0.8661594390869141,
      "logits/chosen": -1.8825433254241943,
      "logits/rejected": -1.9633598327636719,
      "logps/chosen": -3.092277765274048,
      "logps/rejected": -2.939509868621826,
      "loss": 2.9276,
      "nll_loss": 2.840937614440918,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3092277944087982,
      "rewards/margins": -0.015276828780770302,
      "rewards/rejected": -0.29395097494125366,
      "step": 84
    },
    {
      "epoch": 0.9577464788732394,
      "grad_norm": 11.308173179626465,
      "learning_rate": 3.076923076923077e-07,
      "log_odds_chosen": -0.5136163830757141,
      "log_odds_ratio": -1.1063024997711182,
      "logits/chosen": -1.9425265789031982,
      "logits/rejected": -2.0414481163024902,
      "logps/chosen": -3.515744924545288,
      "logps/rejected": -3.021228790283203,
      "loss": 3.4749,
      "nll_loss": 3.364316940307617,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.35157451033592224,
      "rewards/margins": -0.04945161193609238,
      "rewards/rejected": -0.30212289094924927,
      "step": 85
    },
    {
      "epoch": 0.9690140845070423,
      "grad_norm": 11.616045951843262,
      "learning_rate": 2.0512820512820512e-07,
      "log_odds_chosen": -0.21424178779125214,
      "log_odds_ratio": -0.8350873589515686,
      "logits/chosen": -1.8886396884918213,
      "logits/rejected": -1.9696400165557861,
      "logps/chosen": -3.475510358810425,
      "logps/rejected": -3.2696402072906494,
      "loss": 3.5142,
      "nll_loss": 3.430680990219116,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.34755104780197144,
      "rewards/margins": -0.020586999133229256,
      "rewards/rejected": -0.32696402072906494,
      "step": 86
    },
    {
      "epoch": 0.9802816901408451,
      "grad_norm": 11.651595115661621,
      "learning_rate": 1.0256410256410256e-07,
      "log_odds_chosen": 0.28179410099983215,
      "log_odds_ratio": -0.655161440372467,
      "logits/chosen": -1.94129478931427,
      "logits/rejected": -1.9500186443328857,
      "logps/chosen": -2.9875540733337402,
      "logps/rejected": -3.257594108581543,
      "loss": 2.9425,
      "nll_loss": 2.8769569396972656,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.298755407333374,
      "rewards/margins": 0.027004040777683258,
      "rewards/rejected": -0.3257594108581543,
      "step": 87
    },
    {
      "epoch": 0.9915492957746479,
      "grad_norm": 11.723467826843262,
      "learning_rate": 0.0,
      "log_odds_chosen": -0.272980272769928,
      "log_odds_ratio": -0.9368327856063843,
      "logits/chosen": -1.8292593955993652,
      "logits/rejected": -1.9691557884216309,
      "logps/chosen": -3.174346923828125,
      "logps/rejected": -2.9293620586395264,
      "loss": 2.9493,
      "nll_loss": 2.855593204498291,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.3174346685409546,
      "rewards/margins": -0.024498462677001953,
      "rewards/rejected": -0.292936235666275,
      "step": 88
    }
  ],
  "logging_steps": 1,
  "max_steps": 88,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}