{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.3215434083601286,
  "eval_steps": 100,
  "global_step": 400,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "grad_norm": 2.5493941690669297,
      "learning_rate": 4e-09,
      "logits/policy_chosen": -2.279445171356201,
      "logits/policy_rejected": -2.070641279220581,
      "logits/reference_chosen": -2.262451410293579,
      "logits/reference_rejected": -1.700181484222412,
      "logps/policy_chosen": -259.83294677734375,
      "logps/policy_rejected": -282.3813171386719,
      "logps/reference_chosen": -280.28363037109375,
      "logps/reference_rejected": -290.83868408203125,
      "loss": 0.6746,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.20450688898563385,
      "rewards/margins": 0.11993316560983658,
      "rewards/rejected": 0.08457372337579727,
      "step": 1,
      "tokens/chosen_response_len_mean": 528.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 444.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.0,
      "grad_norm": 2.2490732615654547,
      "learning_rate": 8e-09,
      "logits/policy_chosen": -2.745072364807129,
      "logits/policy_rejected": -2.607792854309082,
      "logits/reference_chosen": -2.701054096221924,
      "logits/reference_rejected": -2.529007911682129,
      "logps/policy_chosen": -147.3030242919922,
      "logps/policy_rejected": -197.99253845214844,
      "logps/reference_chosen": -156.72518920898438,
      "logps/reference_rejected": -206.69471740722656,
      "loss": 0.6863,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.09422144293785095,
      "rewards/margins": 0.0071996524930000305,
      "rewards/rejected": 0.08702179044485092,
      "step": 2,
      "tokens/chosen_response_len_mean": 298.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 277.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.0,
      "grad_norm": 2.2889093862107543,
      "learning_rate": 1.2e-08,
      "logits/policy_chosen": -2.882375478744507,
      "logits/policy_rejected": -2.992877960205078,
      "logits/reference_chosen": -2.748257637023926,
      "logits/reference_rejected": -2.9701337814331055,
      "logps/policy_chosen": -105.87107849121094,
      "logps/policy_rejected": -148.20022583007812,
      "logps/reference_chosen": -112.85089111328125,
      "logps/reference_rejected": -155.7706298828125,
      "loss": 0.6832,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.06979808956384659,
      "rewards/margins": -0.005905959755182266,
      "rewards/rejected": 0.07570404559373856,
      "step": 3,
      "tokens/chosen_response_len_mean": 239.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 254.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.0,
      "grad_norm": 2.2808153966501084,
      "learning_rate": 1.6e-08,
      "logits/policy_chosen": -2.9396884441375732,
      "logits/policy_rejected": -2.9384472370147705,
      "logits/reference_chosen": -2.9226865768432617,
      "logits/reference_rejected": -2.9309639930725098,
      "logps/policy_chosen": -123.63784790039062,
      "logps/policy_rejected": -180.25296020507812,
      "logps/reference_chosen": -129.93817138671875,
      "logps/reference_rejected": -184.21334838867188,
      "loss": 0.674,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.06300327926874161,
      "rewards/margins": 0.023399492725729942,
      "rewards/rejected": 0.039603784680366516,
      "step": 4,
      "tokens/chosen_response_len_mean": 298.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 297.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.0,
      "grad_norm": 2.2660235876486974,
      "learning_rate": 2e-08,
      "logits/policy_chosen": -3.221891164779663,
      "logits/policy_rejected": -2.8190152645111084,
      "logits/reference_chosen": -3.207385778427124,
      "logits/reference_rejected": -2.6983580589294434,
      "logps/policy_chosen": -179.60354614257812,
      "logps/policy_rejected": -206.77960205078125,
      "logps/reference_chosen": -189.15216064453125,
      "logps/reference_rejected": -209.81790161132812,
      "loss": 0.6778,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09548593312501907,
      "rewards/margins": 0.06510297954082489,
      "rewards/rejected": 0.03038296103477478,
      "step": 5,
      "tokens/chosen_response_len_mean": 440.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 232.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.0,
      "grad_norm": 2.0558578581082307,
      "learning_rate": 2.4e-08,
      "logits/policy_chosen": -2.4811384677886963,
      "logits/policy_rejected": -2.4067883491516113,
      "logits/reference_chosen": -2.330796003341675,
      "logits/reference_rejected": -2.4005446434020996,
      "logps/policy_chosen": -148.8319091796875,
      "logps/policy_rejected": -179.03482055664062,
      "logps/reference_chosen": -160.83175659179688,
      "logps/reference_rejected": -189.98275756835938,
      "loss": 0.6989,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.11999829858541489,
      "rewards/margins": 0.010519090108573437,
      "rewards/rejected": 0.10947921127080917,
      "step": 6,
      "tokens/chosen_response_len_mean": 289.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 309.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 2.5447256930451734,
      "learning_rate": 2.8e-08,
      "logits/policy_chosen": -2.2599525451660156,
      "logits/policy_rejected": -2.190964698791504,
      "logits/reference_chosen": -2.1052117347717285,
      "logits/reference_rejected": -2.134941577911377,
      "logps/policy_chosen": -19.338191986083984,
      "logps/policy_rejected": -38.73615646362305,
      "logps/reference_chosen": -20.665983200073242,
      "logps/reference_rejected": -40.615394592285156,
      "loss": 0.684,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01327789481729269,
      "rewards/margins": -0.005514495074748993,
      "rewards/rejected": 0.018792390823364258,
      "step": 7,
      "tokens/chosen_response_len_mean": 34.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 47.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 2.967658233666017,
      "learning_rate": 3.2e-08,
      "logits/policy_chosen": -2.4715514183044434,
      "logits/policy_rejected": -2.4913477897644043,
      "logits/reference_chosen": -2.421693801879883,
      "logits/reference_rejected": -2.4608089923858643,
      "logps/policy_chosen": -29.782058715820312,
      "logps/policy_rejected": -64.6925048828125,
      "logps/reference_chosen": -32.50696563720703,
      "logps/reference_rejected": -66.7137222290039,
      "loss": 0.6712,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.027249068021774292,
      "rewards/margins": 0.00703694112598896,
      "rewards/rejected": 0.020212125033140182,
      "step": 8,
      "tokens/chosen_response_len_mean": 40.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 48.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 3.4470069507233316,
      "learning_rate": 3.6e-08,
      "logits/policy_chosen": -2.7727508544921875,
      "logits/policy_rejected": -2.718796730041504,
      "logits/reference_chosen": -2.676337957382202,
      "logits/reference_rejected": -2.616457223892212,
      "logps/policy_chosen": -108.26776123046875,
      "logps/policy_rejected": -218.6373748779297,
      "logps/reference_chosen": -118.67279815673828,
      "logps/reference_rejected": -223.27325439453125,
      "loss": 0.6796,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.1040503978729248,
      "rewards/margins": 0.05769149586558342,
      "rewards/rejected": 0.046358898282051086,
      "step": 9,
      "tokens/chosen_response_len_mean": 252.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 222.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 2.304016636338041,
      "learning_rate": 4e-08,
      "logits/policy_chosen": -2.5566985607147217,
      "logits/policy_rejected": -2.5395240783691406,
      "logits/reference_chosen": -2.4825587272644043,
      "logits/reference_rejected": -2.498563051223755,
      "logps/policy_chosen": -168.69143676757812,
      "logps/policy_rejected": -287.2906494140625,
      "logps/reference_chosen": -181.2764129638672,
      "logps/reference_rejected": -299.7208557128906,
      "loss": 0.6885,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.1258496791124344,
      "rewards/margins": 0.001547420397400856,
      "rewards/rejected": 0.12430226802825928,
      "step": 10,
      "tokens/chosen_response_len_mean": 518.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 503.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 2.084799466815333,
      "learning_rate": 4.4e-08,
      "logits/policy_chosen": -2.7605199813842773,
      "logits/policy_rejected": -2.7186694145202637,
      "logits/reference_chosen": -2.7691569328308105,
      "logits/reference_rejected": -2.7842984199523926,
      "logps/policy_chosen": -145.88803100585938,
      "logps/policy_rejected": -219.43760681152344,
      "logps/reference_chosen": -155.90982055664062,
      "logps/reference_rejected": -227.8662109375,
      "loss": 0.6893,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.1002180427312851,
      "rewards/margins": 0.015932004898786545,
      "rewards/rejected": 0.08428603410720825,
      "step": 11,
      "tokens/chosen_response_len_mean": 604.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 448.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 2.7628214791467736,
      "learning_rate": 4.8e-08,
      "logits/policy_chosen": -2.9070513248443604,
      "logits/policy_rejected": -2.7778186798095703,
      "logits/reference_chosen": -2.8774163722991943,
      "logits/reference_rejected": -2.6446313858032227,
      "logps/policy_chosen": -101.27095794677734,
      "logps/policy_rejected": -136.12225341796875,
      "logps/reference_chosen": -108.19136047363281,
      "logps/reference_rejected": -140.3483123779297,
      "loss": 0.6673,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.06920397281646729,
      "rewards/margins": 0.026943188160657883,
      "rewards/rejected": 0.0422607846558094,
      "step": 12,
      "tokens/chosen_response_len_mean": 339.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 269.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 2.1795487992104348,
      "learning_rate": 5.1999999999999996e-08,
      "logits/policy_chosen": -2.4187445640563965,
      "logits/policy_rejected": -2.845353364944458,
      "logits/reference_chosen": -2.3343820571899414,
      "logits/reference_rejected": -2.5970327854156494,
      "logps/policy_chosen": -130.312255859375,
      "logps/policy_rejected": -73.26902770996094,
      "logps/reference_chosen": -134.85186767578125,
      "logps/reference_rejected": -73.45825958251953,
      "loss": 0.6772,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.045396123081445694,
      "rewards/margins": 0.04350382089614868,
      "rewards/rejected": 0.0018922993913292885,
      "step": 13,
      "tokens/chosen_response_len_mean": 177.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 101.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 2.7157502561420026,
      "learning_rate": 5.6e-08,
      "logits/policy_chosen": -3.211674928665161,
      "logits/policy_rejected": -3.0040595531463623,
      "logits/reference_chosen": -3.2025675773620605,
      "logits/reference_rejected": -3.0620803833007812,
      "logps/policy_chosen": -195.99533081054688,
      "logps/policy_rejected": -242.1226806640625,
      "logps/reference_chosen": -208.40536499023438,
      "logps/reference_rejected": -252.9475860595703,
      "loss": 0.6781,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.12410032004117966,
      "rewards/margins": 0.015851235017180443,
      "rewards/rejected": 0.10824908316135406,
      "step": 14,
      "tokens/chosen_response_len_mean": 543.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 408.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 3.50208846860952,
      "learning_rate": 6e-08,
      "logits/policy_chosen": -2.6727657318115234,
      "logits/policy_rejected": -2.5919432640075684,
      "logits/reference_chosen": -2.6382195949554443,
      "logits/reference_rejected": -2.417813777923584,
      "logps/policy_chosen": -170.3718719482422,
      "logps/policy_rejected": -223.43052673339844,
      "logps/reference_chosen": -191.32583618164062,
      "logps/reference_rejected": -242.33273315429688,
      "loss": 0.6686,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.209539532661438,
      "rewards/margins": 0.020517464727163315,
      "rewards/rejected": 0.18902206420898438,
      "step": 15,
      "tokens/chosen_response_len_mean": 431.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 403.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 1.6207728056732267,
      "learning_rate": 6.4e-08,
      "logits/policy_chosen": -2.954475164413452,
      "logits/policy_rejected": -2.8040027618408203,
      "logits/reference_chosen": -2.9069364070892334,
      "logits/reference_rejected": -2.8010973930358887,
      "logps/policy_chosen": -54.05670166015625,
      "logps/policy_rejected": -109.65800476074219,
      "logps/reference_chosen": -55.35580825805664,
      "logps/reference_rejected": -108.90874481201172,
      "loss": 0.6824,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.012991041876375675,
      "rewards/margins": 0.02048366516828537,
      "rewards/rejected": -0.007492623291909695,
      "step": 16,
      "tokens/chosen_response_len_mean": 89.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 96.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 2.409299627131989,
      "learning_rate": 6.8e-08,
      "logits/policy_chosen": -2.909968137741089,
      "logits/policy_rejected": -2.84816837310791,
      "logits/reference_chosen": -2.8810513019561768,
      "logits/reference_rejected": -2.6653146743774414,
      "logps/policy_chosen": -119.80767822265625,
      "logps/policy_rejected": -267.8531799316406,
      "logps/reference_chosen": -126.68125915527344,
      "logps/reference_rejected": -275.761474609375,
      "loss": 0.6995,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.06873579323291779,
      "rewards/margins": -0.010347230359911919,
      "rewards/rejected": 0.07908301800489426,
      "step": 17,
      "tokens/chosen_response_len_mean": 314.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 251.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 2.703021449933624,
      "learning_rate": 7.2e-08,
      "logits/policy_chosen": -2.8886208534240723,
      "logits/policy_rejected": -2.603667974472046,
      "logits/reference_chosen": -2.8919739723205566,
      "logits/reference_rejected": -2.336148262023926,
      "logps/policy_chosen": -144.8170166015625,
      "logps/policy_rejected": -144.6788330078125,
      "logps/reference_chosen": -164.3677215576172,
      "logps/reference_rejected": -156.78842163085938,
      "loss": 0.679,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1955069750547409,
      "rewards/margins": 0.07441116869449615,
      "rewards/rejected": 0.12109580636024475,
      "step": 18,
      "tokens/chosen_response_len_mean": 421.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 314.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 1.9691382131351773,
      "learning_rate": 7.599999999999999e-08,
      "logits/policy_chosen": -2.766261339187622,
      "logits/policy_rejected": -2.697242259979248,
      "logits/reference_chosen": -2.710810899734497,
      "logits/reference_rejected": -2.557218551635742,
      "logps/policy_chosen": -150.92379760742188,
      "logps/policy_rejected": -174.591064453125,
      "logps/reference_chosen": -160.51756286621094,
      "logps/reference_rejected": -185.50238037109375,
      "loss": 0.6851,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0959378108382225,
      "rewards/margins": -0.013175351545214653,
      "rewards/rejected": 0.1091131642460823,
      "step": 19,
      "tokens/chosen_response_len_mean": 263.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 253.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 2.1003754961271675,
      "learning_rate": 8e-08,
      "logits/policy_chosen": -2.8774542808532715,
      "logits/policy_rejected": -2.6342382431030273,
      "logits/reference_chosen": -2.742877244949341,
      "logits/reference_rejected": -2.4385361671447754,
      "logps/policy_chosen": -104.96978759765625,
      "logps/policy_rejected": -157.80657958984375,
      "logps/reference_chosen": -107.06401062011719,
      "logps/reference_rejected": -159.94366455078125,
      "loss": 0.6774,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.02094227820634842,
      "rewards/margins": -0.00042874296195805073,
      "rewards/rejected": 0.02137102000415325,
      "step": 20,
      "tokens/chosen_response_len_mean": 111.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 124.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 2.113555044323313,
      "learning_rate": 8.4e-08,
      "logits/policy_chosen": -3.1314477920532227,
      "logits/policy_rejected": -2.8576135635375977,
      "logits/reference_chosen": -3.1075565814971924,
      "logits/reference_rejected": -2.6684985160827637,
      "logps/policy_chosen": -121.34175109863281,
      "logps/policy_rejected": -152.86807250976562,
      "logps/reference_chosen": -127.80372619628906,
      "logps/reference_rejected": -158.0865478515625,
      "loss": 0.6994,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.06461974233388901,
      "rewards/margins": 0.012434981763362885,
      "rewards/rejected": 0.05218476057052612,
      "step": 21,
      "tokens/chosen_response_len_mean": 317.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 203.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 2.8572997186380196,
      "learning_rate": 8.8e-08,
      "logits/policy_chosen": -3.023026466369629,
      "logits/policy_rejected": -2.970913887023926,
      "logits/reference_chosen": -2.9523911476135254,
      "logits/reference_rejected": -2.8476057052612305,
      "logps/policy_chosen": -123.84272003173828,
      "logps/policy_rejected": -210.03341674804688,
      "logps/reference_chosen": -129.73898315429688,
      "logps/reference_rejected": -216.26409912109375,
      "loss": 0.672,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.05896264314651489,
      "rewards/margins": -0.0033443691208958626,
      "rewards/rejected": 0.06230701133608818,
      "step": 22,
      "tokens/chosen_response_len_mean": 241.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 224.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 2.4344792354730034,
      "learning_rate": 9.199999999999999e-08,
      "logits/policy_chosen": -2.45721435546875,
      "logits/policy_rejected": -2.4645514488220215,
      "logits/reference_chosen": -2.429760694503784,
      "logits/reference_rejected": -2.448065757751465,
      "logps/policy_chosen": -107.37543487548828,
      "logps/policy_rejected": -120.32754516601562,
      "logps/reference_chosen": -113.06903839111328,
      "logps/reference_rejected": -126.56575012207031,
      "loss": 0.6748,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.056935906410217285,
      "rewards/margins": -0.005446167662739754,
      "rewards/rejected": 0.06238207593560219,
      "step": 23,
      "tokens/chosen_response_len_mean": 314.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 315.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 3.5341818462448504,
      "learning_rate": 9.6e-08,
      "logits/policy_chosen": -2.5186548233032227,
      "logits/policy_rejected": -2.3946735858917236,
      "logits/reference_chosen": -2.5037264823913574,
      "logits/reference_rejected": -2.4204611778259277,
      "logps/policy_chosen": -71.1528549194336,
      "logps/policy_rejected": -96.15353393554688,
      "logps/reference_chosen": -74.45181274414062,
      "logps/reference_rejected": -93.63554382324219,
      "loss": 0.6593,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03298955038189888,
      "rewards/margins": 0.05816952511668205,
      "rewards/rejected": -0.025179967284202576,
      "step": 24,
      "tokens/chosen_response_len_mean": 247.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 155.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 2.6672688539385514,
      "learning_rate": 1e-07,
      "logits/policy_chosen": -3.052779197692871,
      "logits/policy_rejected": -2.9980740547180176,
      "logits/reference_chosen": -3.0356476306915283,
      "logits/reference_rejected": -2.9704558849334717,
      "logps/policy_chosen": -158.24560546875,
      "logps/policy_rejected": -246.05154418945312,
      "logps/reference_chosen": -166.8022003173828,
      "logps/reference_rejected": -252.77267456054688,
      "loss": 0.6801,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.08556582033634186,
      "rewards/margins": 0.01835446059703827,
      "rewards/rejected": 0.06721135973930359,
      "step": 25,
      "tokens/chosen_response_len_mean": 302.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 309.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 2.157863179356832,
      "learning_rate": 1.0399999999999999e-07,
      "logits/policy_chosen": -2.8244199752807617,
      "logits/policy_rejected": -2.8272688388824463,
      "logits/reference_chosen": -2.8120579719543457,
      "logits/reference_rejected": -2.81868839263916,
      "logps/policy_chosen": -188.75546264648438,
      "logps/policy_rejected": -245.15794372558594,
      "logps/reference_chosen": -201.83450317382812,
      "logps/reference_rejected": -256.3631591796875,
      "loss": 0.6824,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.13079024851322174,
      "rewards/margins": 0.018737995997071266,
      "rewards/rejected": 0.11205224692821503,
      "step": 26,
      "tokens/chosen_response_len_mean": 499.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 500.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 2.6785946054818295,
      "learning_rate": 1.0799999999999999e-07,
      "logits/policy_chosen": -3.041701078414917,
      "logits/policy_rejected": -3.0683014392852783,
      "logits/reference_chosen": -3.021995782852173,
      "logits/reference_rejected": -3.0257372856140137,
      "logps/policy_chosen": -102.09321594238281,
      "logps/policy_rejected": -163.72860717773438,
      "logps/reference_chosen": -109.2179946899414,
      "logps/reference_rejected": -167.15447998046875,
      "loss": 0.6755,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07124768942594528,
      "rewards/margins": 0.036988936364650726,
      "rewards/rejected": 0.03425874561071396,
      "step": 27,
      "tokens/chosen_response_len_mean": 243.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 237.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 3.0106278359730614,
      "learning_rate": 1.12e-07,
      "logits/policy_chosen": -3.058267116546631,
      "logits/policy_rejected": -3.0097880363464355,
      "logits/reference_chosen": -3.000871181488037,
      "logits/reference_rejected": -2.860621690750122,
      "logps/policy_chosen": -177.05154418945312,
      "logps/policy_rejected": -241.18173217773438,
      "logps/reference_chosen": -187.62625122070312,
      "logps/reference_rejected": -242.83314514160156,
      "loss": 0.6642,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.10574702173471451,
      "rewards/margins": 0.08923279494047165,
      "rewards/rejected": 0.01651422120630741,
      "step": 28,
      "tokens/chosen_response_len_mean": 391.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 363.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 1.9714081999119137,
      "learning_rate": 1.16e-07,
      "logits/policy_chosen": -2.7036781311035156,
      "logits/policy_rejected": -2.706456422805786,
      "logits/reference_chosen": -2.6821694374084473,
      "logits/reference_rejected": -2.70880126953125,
      "logps/policy_chosen": -50.51161193847656,
      "logps/policy_rejected": -97.51376342773438,
      "logps/reference_chosen": -53.98440170288086,
      "logps/reference_rejected": -100.45523834228516,
      "loss": 0.6839,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03472791612148285,
      "rewards/margins": 0.005313164554536343,
      "rewards/rejected": 0.029414750635623932,
      "step": 29,
      "tokens/chosen_response_len_mean": 246.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 215.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 3.3945073696477333,
      "learning_rate": 1.2e-07,
      "logits/policy_chosen": -2.679271936416626,
      "logits/policy_rejected": -2.747288227081299,
      "logits/reference_chosen": -2.648017406463623,
      "logits/reference_rejected": -2.542226552963257,
      "logps/policy_chosen": -181.09521484375,
      "logps/policy_rejected": -349.05694580078125,
      "logps/reference_chosen": -202.54962158203125,
      "logps/reference_rejected": -355.8513488769531,
      "loss": 0.6558,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21454395353794098,
      "rewards/margins": 0.14660006761550903,
      "rewards/rejected": 0.06794389337301254,
      "step": 30,
      "tokens/chosen_response_len_mean": 309.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 268.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 2.729481181496147,
      "learning_rate": 1.24e-07,
      "logits/policy_chosen": -3.1615757942199707,
      "logits/policy_rejected": -2.618534564971924,
      "logits/reference_chosen": -3.148972511291504,
      "logits/reference_rejected": -2.28782320022583,
      "logps/policy_chosen": -244.06692504882812,
      "logps/policy_rejected": -395.2162780761719,
      "logps/reference_chosen": -254.54090881347656,
      "logps/reference_rejected": -397.2469177246094,
      "loss": 0.6873,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10473988205194473,
      "rewards/margins": 0.08443356305360794,
      "rewards/rejected": 0.020306318998336792,
      "step": 31,
      "tokens/chosen_response_len_mean": 410.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 269.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 2.1927971055522133,
      "learning_rate": 1.28e-07,
      "logits/policy_chosen": -2.935199499130249,
      "logits/policy_rejected": -2.862617254257202,
      "logits/reference_chosen": -2.9165947437286377,
      "logits/reference_rejected": -2.809278964996338,
      "logps/policy_chosen": -156.64752197265625,
      "logps/policy_rejected": -224.47006225585938,
      "logps/reference_chosen": -176.1259765625,
      "logps/reference_rejected": -239.154296875,
      "loss": 0.6749,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.19478438794612885,
      "rewards/margins": 0.04794217646121979,
      "rewards/rejected": 0.14684221148490906,
      "step": 32,
      "tokens/chosen_response_len_mean": 545.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 527.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 2.6358653856327257,
      "learning_rate": 1.32e-07,
      "logits/policy_chosen": -2.5262393951416016,
      "logits/policy_rejected": -2.364823341369629,
      "logits/reference_chosen": -2.506678342819214,
      "logits/reference_rejected": -1.6685891151428223,
      "logps/policy_chosen": -104.21517181396484,
      "logps/policy_rejected": -126.63694763183594,
      "logps/reference_chosen": -114.83543395996094,
      "logps/reference_rejected": -136.51809692382812,
      "loss": 0.6837,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.10620267689228058,
      "rewards/margins": 0.007391175255179405,
      "rewards/rejected": 0.09881150722503662,
      "step": 33,
      "tokens/chosen_response_len_mean": 448.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 339.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 2.626247472893481,
      "learning_rate": 1.36e-07,
      "logits/policy_chosen": -3.0164244174957275,
      "logits/policy_rejected": -2.670468330383301,
      "logits/reference_chosen": -2.9480087757110596,
      "logits/reference_rejected": -2.077363967895508,
      "logps/policy_chosen": -144.32318115234375,
      "logps/policy_rejected": -178.09408569335938,
      "logps/reference_chosen": -154.3887939453125,
      "logps/reference_rejected": -182.9420623779297,
      "loss": 0.6748,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1006559506058693,
      "rewards/margins": 0.052176106721162796,
      "rewards/rejected": 0.0484798438847065,
      "step": 34,
      "tokens/chosen_response_len_mean": 407.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 217.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 2.296064376513374,
      "learning_rate": 1.4e-07,
      "logits/policy_chosen": -2.598402738571167,
      "logits/policy_rejected": -2.6598832607269287,
      "logits/reference_chosen": -2.553514003753662,
      "logits/reference_rejected": -2.6580069065093994,
      "logps/policy_chosen": -281.2328186035156,
      "logps/policy_rejected": -179.6129608154297,
      "logps/reference_chosen": -298.13836669921875,
      "logps/reference_rejected": -186.5122528076172,
      "loss": 0.6684,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.16905567049980164,
      "rewards/margins": 0.10006258636713028,
      "rewards/rejected": 0.06899307668209076,
      "step": 35,
      "tokens/chosen_response_len_mean": 377.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 248.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 2.898376793238501,
      "learning_rate": 1.44e-07,
      "logits/policy_chosen": -2.7883753776550293,
      "logits/policy_rejected": -2.6082558631896973,
      "logits/reference_chosen": -2.780714273452759,
      "logits/reference_rejected": -2.583864212036133,
      "logps/policy_chosen": -105.63471984863281,
      "logps/policy_rejected": -245.55908203125,
      "logps/reference_chosen": -110.00314331054688,
      "logps/reference_rejected": -244.04986572265625,
      "loss": 0.6715,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04368434101343155,
      "rewards/margins": 0.05877665430307388,
      "rewards/rejected": -0.015092315152287483,
      "step": 36,
      "tokens/chosen_response_len_mean": 183.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 284.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 2.6935110955386,
      "learning_rate": 1.4799999999999998e-07,
      "logits/policy_chosen": -3.0653228759765625,
      "logits/policy_rejected": -2.882253646850586,
      "logits/reference_chosen": -3.0485243797302246,
      "logits/reference_rejected": -2.6053519248962402,
      "logps/policy_chosen": -186.7164764404297,
      "logps/policy_rejected": -267.58123779296875,
      "logps/reference_chosen": -204.80267333984375,
      "logps/reference_rejected": -272.190185546875,
      "loss": 0.6697,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1808619499206543,
      "rewards/margins": 0.13477247953414917,
      "rewards/rejected": 0.046089474111795425,
      "step": 37,
      "tokens/chosen_response_len_mean": 433.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 321.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 2.597922955352179,
      "learning_rate": 1.5199999999999998e-07,
      "logits/policy_chosen": -2.699632167816162,
      "logits/policy_rejected": -2.7144150733947754,
      "logits/reference_chosen": -2.658318519592285,
      "logits/reference_rejected": -2.691765546798706,
      "logps/policy_chosen": -135.453369140625,
      "logps/policy_rejected": -242.96441650390625,
      "logps/reference_chosen": -146.15704345703125,
      "logps/reference_rejected": -248.74380493164062,
      "loss": 0.6585,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.10703684389591217,
      "rewards/margins": 0.049242909997701645,
      "rewards/rejected": 0.057793937623500824,
      "step": 38,
      "tokens/chosen_response_len_mean": 375.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 387.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 2.079049499920605,
      "learning_rate": 1.56e-07,
      "logits/policy_chosen": -3.077770233154297,
      "logits/policy_rejected": -3.098449945449829,
      "logits/reference_chosen": -3.0287766456604004,
      "logits/reference_rejected": -3.0804648399353027,
      "logps/policy_chosen": -12.474931716918945,
      "logps/policy_rejected": -25.898088455200195,
      "logps/reference_chosen": -12.915287971496582,
      "logps/reference_rejected": -27.526098251342773,
      "loss": 0.6793,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.00440356507897377,
      "rewards/margins": -0.011876537464559078,
      "rewards/rejected": 0.016280101612210274,
      "step": 39,
      "tokens/chosen_response_len_mean": 23.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 25.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 2.9270192927802197,
      "learning_rate": 1.6e-07,
      "logits/policy_chosen": -2.821016550064087,
      "logits/policy_rejected": -2.7184388637542725,
      "logits/reference_chosen": -2.5926756858825684,
      "logits/reference_rejected": -2.7159690856933594,
      "logps/policy_chosen": -115.9990005493164,
      "logps/policy_rejected": -150.73153686523438,
      "logps/reference_chosen": -120.78485107421875,
      "logps/reference_rejected": -157.8815460205078,
      "loss": 0.668,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.04785853251814842,
      "rewards/margins": -0.02364155277609825,
      "rewards/rejected": 0.07150008529424667,
      "step": 40,
      "tokens/chosen_response_len_mean": 179.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 168.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 2.807071363383845,
      "learning_rate": 1.64e-07,
      "logits/policy_chosen": -2.6167349815368652,
      "logits/policy_rejected": -2.5763139724731445,
      "logits/reference_chosen": -2.513126850128174,
      "logits/reference_rejected": -2.5481503009796143,
      "logps/policy_chosen": -213.66064453125,
      "logps/policy_rejected": -274.35003662109375,
      "logps/reference_chosen": -240.58090209960938,
      "logps/reference_rejected": -297.9310607910156,
      "loss": 0.6847,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.26920264959335327,
      "rewards/margins": 0.03339245170354843,
      "rewards/rejected": 0.23581019043922424,
      "step": 41,
      "tokens/chosen_response_len_mean": 468.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 442.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 3.1342954098759788,
      "learning_rate": 1.68e-07,
      "logits/policy_chosen": -2.98238205909729,
      "logits/policy_rejected": -2.6742758750915527,
      "logits/reference_chosen": -2.9786105155944824,
      "logits/reference_rejected": -2.5011706352233887,
      "logps/policy_chosen": -400.793701171875,
      "logps/policy_rejected": -471.21826171875,
      "logps/reference_chosen": -424.1178283691406,
      "logps/reference_rejected": -484.6050720214844,
      "loss": 0.6671,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.23324130475521088,
      "rewards/margins": 0.09937307238578796,
      "rewards/rejected": 0.1338682323694229,
      "step": 42,
      "tokens/chosen_response_len_mean": 593.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 479.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 1.7422493551589444,
      "learning_rate": 1.7199999999999998e-07,
      "logits/policy_chosen": -2.763662338256836,
      "logits/policy_rejected": -2.434535503387451,
      "logits/reference_chosen": -2.710179567337036,
      "logits/reference_rejected": -1.868179440498352,
      "logps/policy_chosen": -77.93124389648438,
      "logps/policy_rejected": -110.21360778808594,
      "logps/reference_chosen": -86.2520751953125,
      "logps/reference_rejected": -110.11550903320312,
      "loss": 0.679,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08320826292037964,
      "rewards/margins": 0.08418922126293182,
      "rewards/rejected": -0.0009809592738747597,
      "step": 43,
      "tokens/chosen_response_len_mean": 152.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 85.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 3.138281711904061,
      "learning_rate": 1.76e-07,
      "logits/policy_chosen": -2.8771750926971436,
      "logits/policy_rejected": -2.9175660610198975,
      "logits/reference_chosen": -2.8088254928588867,
      "logits/reference_rejected": -2.877462863922119,
      "logps/policy_chosen": -37.14703369140625,
      "logps/policy_rejected": -55.703121185302734,
      "logps/reference_chosen": -38.25192642211914,
      "logps/reference_rejected": -56.50023651123047,
      "loss": 0.686,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.011048920452594757,
      "rewards/margins": 0.0030777817592024803,
      "rewards/rejected": 0.007971138693392277,
      "step": 44,
      "tokens/chosen_response_len_mean": 66.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 70.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 3.284990775967774,
      "learning_rate": 1.8e-07,
      "logits/policy_chosen": -2.8581864833831787,
      "logits/policy_rejected": -2.766713857650757,
      "logits/reference_chosen": -2.8522939682006836,
      "logits/reference_rejected": -2.7666430473327637,
      "logps/policy_chosen": -165.7818603515625,
      "logps/policy_rejected": -241.84002685546875,
      "logps/reference_chosen": -181.072998046875,
      "logps/reference_rejected": -250.3277587890625,
      "loss": 0.6537,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.15291133522987366,
      "rewards/margins": 0.06803382933139801,
      "rewards/rejected": 0.08487750589847565,
      "step": 45,
      "tokens/chosen_response_len_mean": 560.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 473.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 2.463717582717747,
      "learning_rate": 1.8399999999999998e-07,
      "logits/policy_chosen": -2.900191068649292,
      "logits/policy_rejected": -2.9184107780456543,
      "logits/reference_chosen": -2.8673532009124756,
      "logits/reference_rejected": -2.892207145690918,
      "logps/policy_chosen": -51.21653747558594,
      "logps/policy_rejected": -107.85240936279297,
      "logps/reference_chosen": -54.76094436645508,
      "logps/reference_rejected": -117.80521392822266,
      "loss": 0.695,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.035444095730781555,
      "rewards/margins": -0.06408397108316422,
      "rewards/rejected": 0.09952806681394577,
      "step": 46,
      "tokens/chosen_response_len_mean": 109.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 108.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 1.9738986795582658,
      "learning_rate": 1.88e-07,
      "logits/policy_chosen": -2.2502527236938477,
      "logits/policy_rejected": -2.245668649673462,
      "logits/reference_chosen": -2.177823781967163,
      "logits/reference_rejected": -2.223409652709961,
      "logps/policy_chosen": -37.820743560791016,
      "logps/policy_rejected": -99.87039184570312,
      "logps/reference_chosen": -44.7762451171875,
      "logps/reference_rejected": -104.45420837402344,
      "loss": 0.6825,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.06955502927303314,
      "rewards/margins": 0.023716777563095093,
      "rewards/rejected": 0.04583824798464775,
      "step": 47,
      "tokens/chosen_response_len_mean": 93.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 110.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 2.8813658188320335,
      "learning_rate": 1.92e-07,
      "logits/policy_chosen": -2.6205151081085205,
      "logits/policy_rejected": -2.5673413276672363,
      "logits/reference_chosen": -2.5005288124084473,
      "logits/reference_rejected": -2.561748504638672,
      "logps/policy_chosen": -65.36790466308594,
      "logps/policy_rejected": -167.23562622070312,
      "logps/reference_chosen": -72.64520263671875,
      "logps/reference_rejected": -172.6849365234375,
      "loss": 0.6706,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.07277299463748932,
      "rewards/margins": 0.018280120566487312,
      "rewards/rejected": 0.054492875933647156,
      "step": 48,
      "tokens/chosen_response_len_mean": 199.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 209.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 2.862318181422901,
      "learning_rate": 1.96e-07,
      "logits/policy_chosen": -2.701411247253418,
      "logits/policy_rejected": -2.8611505031585693,
      "logits/reference_chosen": -2.2373385429382324,
      "logits/reference_rejected": -2.845200777053833,
      "logps/policy_chosen": -126.9096908569336,
      "logps/policy_rejected": -269.68243408203125,
      "logps/reference_chosen": -136.80958557128906,
      "logps/reference_rejected": -277.586181640625,
      "loss": 0.6796,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.09899893403053284,
      "rewards/margins": 0.01996154710650444,
      "rewards/rejected": 0.0790373906493187,
      "step": 49,
      "tokens/chosen_response_len_mean": 262.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 337.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 2.324007938345189,
      "learning_rate": 2e-07,
      "logits/policy_chosen": -2.2418417930603027,
      "logits/policy_rejected": -2.1697628498077393,
      "logits/reference_chosen": -2.142324924468994,
      "logits/reference_rejected": -2.1179537773132324,
      "logps/policy_chosen": -136.041259765625,
      "logps/policy_rejected": -266.0906677246094,
      "logps/reference_chosen": -151.6305694580078,
      "logps/reference_rejected": -273.1885986328125,
      "loss": 0.6677,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.15589313209056854,
      "rewards/margins": 0.08491363376379013,
      "rewards/rejected": 0.07097949832677841,
      "step": 50,
      "tokens/chosen_response_len_mean": 526.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 469.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 2.3504496124001744,
      "learning_rate": 2.0399999999999997e-07,
      "logits/policy_chosen": -2.899348497390747,
      "logits/policy_rejected": -2.8138720989227295,
      "logits/reference_chosen": -2.8722875118255615,
      "logits/reference_rejected": -2.8029227256774902,
      "logps/policy_chosen": -63.48197555541992,
      "logps/policy_rejected": -101.47474670410156,
      "logps/reference_chosen": -68.96565246582031,
      "logps/reference_rejected": -108.34456634521484,
      "loss": 0.6831,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.05483681336045265,
      "rewards/margins": -0.013861382380127907,
      "rewards/rejected": 0.06869819760322571,
      "step": 51,
      "tokens/chosen_response_len_mean": 212.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 181.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 2.4664067603984665,
      "learning_rate": 2.0799999999999998e-07,
      "logits/policy_chosen": -2.710914134979248,
      "logits/policy_rejected": -2.6489503383636475,
      "logits/reference_chosen": -2.6887543201446533,
      "logits/reference_rejected": -2.565074920654297,
      "logps/policy_chosen": -120.6403579711914,
      "logps/policy_rejected": -180.31842041015625,
      "logps/reference_chosen": -129.82321166992188,
      "logps/reference_rejected": -187.31112670898438,
      "loss": 0.6824,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0918285921216011,
      "rewards/margins": 0.021901624277234077,
      "rewards/rejected": 0.06992696225643158,
      "step": 52,
      "tokens/chosen_response_len_mean": 340.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 269.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 3.7734473565962108,
      "learning_rate": 2.12e-07,
      "logits/policy_chosen": -2.9076874256134033,
      "logits/policy_rejected": -2.8471598625183105,
      "logits/reference_chosen": -2.837496280670166,
      "logits/reference_rejected": -2.783947467803955,
      "logps/policy_chosen": -88.39242553710938,
      "logps/policy_rejected": -232.25318908691406,
      "logps/reference_chosen": -96.45068359375,
      "logps/reference_rejected": -237.35009765625,
      "loss": 0.6784,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.08058254420757294,
      "rewards/margins": 0.02961355820298195,
      "rewards/rejected": 0.05096898972988129,
      "step": 53,
      "tokens/chosen_response_len_mean": 195.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 194.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 2.8319761915445367,
      "learning_rate": 2.1599999999999998e-07,
      "logits/policy_chosen": -3.144252300262451,
      "logits/policy_rejected": -3.0141947269439697,
      "logits/reference_chosen": -3.116751194000244,
      "logits/reference_rejected": -2.8330681324005127,
      "logps/policy_chosen": -132.089111328125,
      "logps/policy_rejected": -225.1402587890625,
      "logps/reference_chosen": -136.70138549804688,
      "logps/reference_rejected": -224.64724731445312,
      "loss": 0.6737,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.046122774481773376,
      "rewards/margins": 0.05105280876159668,
      "rewards/rejected": -0.004930033814162016,
      "step": 54,
      "tokens/chosen_response_len_mean": 299.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 257.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 2.947847201514034,
      "learning_rate": 2.1999999999999998e-07,
      "logits/policy_chosen": -2.8887228965759277,
      "logits/policy_rejected": -2.68259596824646,
      "logits/reference_chosen": -2.864752769470215,
      "logits/reference_rejected": -2.628319263458252,
      "logps/policy_chosen": -135.53025817871094,
      "logps/policy_rejected": -133.9132843017578,
      "logps/reference_chosen": -145.9614715576172,
      "logps/reference_rejected": -137.02508544921875,
      "loss": 0.666,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10431211441755295,
      "rewards/margins": 0.07319390773773193,
      "rewards/rejected": 0.031118206679821014,
      "step": 55,
      "tokens/chosen_response_len_mean": 349.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 196.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 2.771906040366205,
      "learning_rate": 2.24e-07,
      "logits/policy_chosen": -2.231527328491211,
      "logits/policy_rejected": -2.0439558029174805,
      "logits/reference_chosen": -2.230081796646118,
      "logits/reference_rejected": -2.080432176589966,
      "logps/policy_chosen": -399.80303955078125,
      "logps/policy_rejected": -446.4791259765625,
      "logps/reference_chosen": -431.472412109375,
      "logps/reference_rejected": -470.2456970214844,
      "loss": 0.6709,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.31669357419013977,
      "rewards/margins": 0.07902783155441284,
      "rewards/rejected": 0.23766574263572693,
      "step": 56,
      "tokens/chosen_response_len_mean": 805.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 711.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 1.7428598517814506,
      "learning_rate": 2.28e-07,
      "logits/policy_chosen": -2.7684319019317627,
      "logits/policy_rejected": -2.71665096282959,
      "logits/reference_chosen": -2.747512102127075,
      "logits/reference_rejected": -2.690124034881592,
      "logps/policy_chosen": -85.4046630859375,
      "logps/policy_rejected": -111.96269226074219,
      "logps/reference_chosen": -96.96040344238281,
      "logps/reference_rejected": -121.33839416503906,
      "loss": 0.6888,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.11555740237236023,
      "rewards/margins": 0.021800296381115913,
      "rewards/rejected": 0.09375710785388947,
      "step": 57,
      "tokens/chosen_response_len_mean": 261.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 248.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 2.6040542171359817,
      "learning_rate": 2.32e-07,
      "logits/policy_chosen": -2.989000082015991,
      "logits/policy_rejected": -2.8631539344787598,
      "logits/reference_chosen": -2.9670627117156982,
      "logits/reference_rejected": -2.5129096508026123,
      "logps/policy_chosen": -87.29337310791016,
      "logps/policy_rejected": -119.82539367675781,
      "logps/reference_chosen": -95.52102661132812,
      "logps/reference_rejected": -122.27404022216797,
      "loss": 0.6657,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.08227652311325073,
      "rewards/margins": 0.05779005587100983,
      "rewards/rejected": 0.024486469104886055,
      "step": 58,
      "tokens/chosen_response_len_mean": 162.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 138.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 3.249420407772776,
      "learning_rate": 2.3599999999999997e-07,
      "logits/policy_chosen": -2.9996423721313477,
      "logits/policy_rejected": -2.974095582962036,
      "logits/reference_chosen": -2.964367151260376,
      "logits/reference_rejected": -2.953282356262207,
      "logps/policy_chosen": -56.97121810913086,
      "logps/policy_rejected": -141.20883178710938,
      "logps/reference_chosen": -58.637413024902344,
      "logps/reference_rejected": -143.08058166503906,
      "loss": 0.679,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0166619960218668,
      "rewards/margins": -0.0020555637311190367,
      "rewards/rejected": 0.018717560917139053,
      "step": 59,
      "tokens/chosen_response_len_mean": 121.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 127.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 3.503891824715986,
      "learning_rate": 2.4e-07,
      "logits/policy_chosen": -3.207751512527466,
      "logits/policy_rejected": -2.8613648414611816,
      "logits/reference_chosen": -3.1973516941070557,
      "logits/reference_rejected": -2.244464874267578,
      "logps/policy_chosen": -115.4853286743164,
      "logps/policy_rejected": -205.64161682128906,
      "logps/reference_chosen": -123.15602111816406,
      "logps/reference_rejected": -206.07244873046875,
      "loss": 0.6713,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0767069011926651,
      "rewards/margins": 0.07239863276481628,
      "rewards/rejected": 0.004308272153139114,
      "step": 60,
      "tokens/chosen_response_len_mean": 262.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 185.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 2.987682806191956,
      "learning_rate": 2.4399999999999996e-07,
      "logits/policy_chosen": -2.65006947517395,
      "logits/policy_rejected": -2.60097599029541,
      "logits/reference_chosen": -2.6217522621154785,
      "logits/reference_rejected": -2.507542610168457,
      "logps/policy_chosen": -184.2285614013672,
      "logps/policy_rejected": -184.65708923339844,
      "logps/reference_chosen": -215.3209228515625,
      "logps/reference_rejected": -187.43243408203125,
      "loss": 0.6592,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.31092363595962524,
      "rewards/margins": 0.2831703722476959,
      "rewards/rejected": 0.027753248810768127,
      "step": 61,
      "tokens/chosen_response_len_mean": 344.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 292.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 3.02187794129979,
      "learning_rate": 2.48e-07,
      "logits/policy_chosen": -3.102184534072876,
      "logits/policy_rejected": -3.0169076919555664,
      "logits/reference_chosen": -3.0980794429779053,
      "logits/reference_rejected": -3.06693434715271,
      "logps/policy_chosen": -180.20120239257812,
      "logps/policy_rejected": -290.76556396484375,
      "logps/reference_chosen": -186.42239379882812,
      "logps/reference_rejected": -299.5263977050781,
      "loss": 0.6663,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06221204251050949,
      "rewards/margins": -0.025396330282092094,
      "rewards/rejected": 0.08760837465524673,
      "step": 62,
      "tokens/chosen_response_len_mean": 628.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 573.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 2.2026851566699364,
      "learning_rate": 2.52e-07,
      "logits/policy_chosen": -2.9184327125549316,
      "logits/policy_rejected": -2.7862141132354736,
      "logits/reference_chosen": -2.9029111862182617,
      "logits/reference_rejected": -2.562068462371826,
      "logps/policy_chosen": -176.180908203125,
      "logps/policy_rejected": -224.06822204589844,
      "logps/reference_chosen": -191.051025390625,
      "logps/reference_rejected": -231.65249633789062,
      "loss": 0.682,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.14870098233222961,
      "rewards/margins": 0.07285812497138977,
      "rewards/rejected": 0.07584285736083984,
      "step": 63,
      "tokens/chosen_response_len_mean": 383.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 315.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 2.6923985748881183,
      "learning_rate": 2.56e-07,
      "logits/policy_chosen": -3.1094112396240234,
      "logits/policy_rejected": -3.059873104095459,
      "logits/reference_chosen": -3.0696709156036377,
      "logits/reference_rejected": -3.0277552604675293,
      "logps/policy_chosen": -109.50044250488281,
      "logps/policy_rejected": -227.6053009033203,
      "logps/reference_chosen": -114.45265197753906,
      "logps/reference_rejected": -232.44473266601562,
      "loss": 0.6689,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.04952218383550644,
      "rewards/margins": 0.0011278600431978703,
      "rewards/rejected": 0.04839432239532471,
      "step": 64,
      "tokens/chosen_response_len_mean": 172.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 173.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 2.3372174586928685,
      "learning_rate": 2.6e-07,
      "logits/policy_chosen": -2.9087581634521484,
      "logits/policy_rejected": -2.973167657852173,
      "logits/reference_chosen": -2.754256248474121,
      "logits/reference_rejected": -2.943002223968506,
      "logps/policy_chosen": -16.885406494140625,
      "logps/policy_rejected": -58.0799560546875,
      "logps/reference_chosen": -16.55327033996582,
      "logps/reference_rejected": -58.852867126464844,
      "loss": 0.6709,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0033213470596820116,
      "rewards/margins": -0.011050481349229813,
      "rewards/rejected": 0.007729134988039732,
      "step": 65,
      "tokens/chosen_response_len_mean": 25.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 37.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 1.8439881768995727,
      "learning_rate": 2.64e-07,
      "logits/policy_chosen": -3.2132272720336914,
      "logits/policy_rejected": -3.156252861022949,
      "logits/reference_chosen": -3.1969757080078125,
      "logits/reference_rejected": -3.008265733718872,
      "logps/policy_chosen": -89.50849914550781,
      "logps/policy_rejected": -105.58460998535156,
      "logps/reference_chosen": -96.482421875,
      "logps/reference_rejected": -111.33866119384766,
      "loss": 0.6855,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.06973925232887268,
      "rewards/margins": 0.0121987946331501,
      "rewards/rejected": 0.05754045769572258,
      "step": 66,
      "tokens/chosen_response_len_mean": 163.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 142.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 1.9679864280666897,
      "learning_rate": 2.68e-07,
      "logits/policy_chosen": -2.443567991256714,
      "logits/policy_rejected": -2.2462878227233887,
      "logits/reference_chosen": -2.3940353393554688,
      "logits/reference_rejected": -1.4334111213684082,
      "logps/policy_chosen": -133.6657257080078,
      "logps/policy_rejected": -126.65130615234375,
      "logps/reference_chosen": -143.77218627929688,
      "logps/reference_rejected": -130.96572875976562,
      "loss": 0.6801,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.10106448829174042,
      "rewards/margins": 0.057920388877391815,
      "rewards/rejected": 0.0431441031396389,
      "step": 67,
      "tokens/chosen_response_len_mean": 360.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 238.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 2.3444172227352817,
      "learning_rate": 2.72e-07,
      "logits/policy_chosen": -2.447854518890381,
      "logits/policy_rejected": -2.4554026126861572,
      "logits/reference_chosen": -2.4224636554718018,
      "logits/reference_rejected": -2.44649600982666,
      "logps/policy_chosen": -116.48789978027344,
      "logps/policy_rejected": -196.64881896972656,
      "logps/reference_chosen": -129.35415649414062,
      "logps/reference_rejected": -209.97300720214844,
      "loss": 0.6714,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.12866270542144775,
      "rewards/margins": -0.004579143598675728,
      "rewards/rejected": 0.13324186205863953,
      "step": 68,
      "tokens/chosen_response_len_mean": 627.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 597.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 1.9355500776493433,
      "learning_rate": 2.7600000000000004e-07,
      "logits/policy_chosen": -3.001335620880127,
      "logits/policy_rejected": -2.8170900344848633,
      "logits/reference_chosen": -2.9873673915863037,
      "logits/reference_rejected": -2.8246562480926514,
      "logps/policy_chosen": -122.1274642944336,
      "logps/policy_rejected": -174.14083862304688,
      "logps/reference_chosen": -129.572509765625,
      "logps/reference_rejected": -175.02163696289062,
      "loss": 0.6814,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07445055991411209,
      "rewards/margins": 0.06564265489578247,
      "rewards/rejected": 0.008807908743619919,
      "step": 69,
      "tokens/chosen_response_len_mean": 316.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 212.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 3.1880887553688613,
      "learning_rate": 2.8e-07,
      "logits/policy_chosen": -2.666834831237793,
      "logits/policy_rejected": -2.602752208709717,
      "logits/reference_chosen": -2.5575764179229736,
      "logits/reference_rejected": -2.594496965408325,
      "logps/policy_chosen": -144.6117401123047,
      "logps/policy_rejected": -323.0421447753906,
      "logps/reference_chosen": -156.17013549804688,
      "logps/reference_rejected": -333.77972412109375,
      "loss": 0.6725,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.11558418720960617,
      "rewards/margins": 0.00820855051279068,
      "rewards/rejected": 0.10737563669681549,
      "step": 70,
      "tokens/chosen_response_len_mean": 319.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 349.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 2.476324701775898,
      "learning_rate": 2.8399999999999995e-07,
      "logits/policy_chosen": -2.8258767127990723,
      "logits/policy_rejected": -2.808089017868042,
      "logits/reference_chosen": -2.8227195739746094,
      "logits/reference_rejected": -2.8310012817382812,
      "logps/policy_chosen": -141.63214111328125,
      "logps/policy_rejected": -178.0096893310547,
      "logps/reference_chosen": -151.22640991210938,
      "logps/reference_rejected": -182.14822387695312,
      "loss": 0.6621,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09594269841909409,
      "rewards/margins": 0.054557494819164276,
      "rewards/rejected": 0.04138519987463951,
      "step": 71,
      "tokens/chosen_response_len_mean": 486.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 302.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 2.699352211419919,
      "learning_rate": 2.88e-07,
      "logits/policy_chosen": -2.597538709640503,
      "logits/policy_rejected": -2.41672945022583,
      "logits/reference_chosen": -2.5760765075683594,
      "logits/reference_rejected": -1.9195555448532104,
      "logps/policy_chosen": -247.6851043701172,
      "logps/policy_rejected": -382.4797058105469,
      "logps/reference_chosen": -263.0463562011719,
      "logps/reference_rejected": -387.8023376464844,
      "loss": 0.6657,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.15361259877681732,
      "rewards/margins": 0.10038647800683975,
      "rewards/rejected": 0.05322612449526787,
      "step": 72,
      "tokens/chosen_response_len_mean": 411.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 344.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 2.5969468603738255,
      "learning_rate": 2.9199999999999997e-07,
      "logits/policy_chosen": -2.7702274322509766,
      "logits/policy_rejected": -2.5614566802978516,
      "logits/reference_chosen": -2.7665748596191406,
      "logits/reference_rejected": -2.3978798389434814,
      "logps/policy_chosen": -72.09918212890625,
      "logps/policy_rejected": -68.87893676757812,
      "logps/reference_chosen": -80.68479919433594,
      "logps/reference_rejected": -67.986328125,
      "loss": 0.6635,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.08585613965988159,
      "rewards/margins": 0.0947822630405426,
      "rewards/rejected": -0.008926120586693287,
      "step": 73,
      "tokens/chosen_response_len_mean": 205.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 90.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 2.8994112666814575,
      "learning_rate": 2.9599999999999995e-07,
      "logits/policy_chosen": -2.9491050243377686,
      "logits/policy_rejected": -2.9374337196350098,
      "logits/reference_chosen": -2.924247980117798,
      "logits/reference_rejected": -2.8732385635375977,
      "logps/policy_chosen": -163.5004425048828,
      "logps/policy_rejected": -186.22500610351562,
      "logps/reference_chosen": -180.18331909179688,
      "logps/reference_rejected": -202.66912841796875,
      "loss": 0.6673,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.16682888567447662,
      "rewards/margins": 0.00238775834441185,
      "rewards/rejected": 0.16444112360477448,
      "step": 74,
      "tokens/chosen_response_len_mean": 365.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 331.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 2.1648523975476306,
      "learning_rate": 3e-07,
      "logits/policy_chosen": -2.1604349613189697,
      "logits/policy_rejected": -2.168158769607544,
      "logits/reference_chosen": -2.1622910499572754,
      "logits/reference_rejected": -2.1353628635406494,
      "logps/policy_chosen": -88.18841552734375,
      "logps/policy_rejected": -119.30603790283203,
      "logps/reference_chosen": -93.11495971679688,
      "logps/reference_rejected": -123.05696868896484,
      "loss": 0.6881,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.04926542192697525,
      "rewards/margins": 0.011756086722016335,
      "rewards/rejected": 0.037509337067604065,
      "step": 75,
      "tokens/chosen_response_len_mean": 258.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 199.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 2.403809139260171,
      "learning_rate": 3.0399999999999997e-07,
      "logits/policy_chosen": -2.830843925476074,
      "logits/policy_rejected": -2.8032288551330566,
      "logits/reference_chosen": -2.8209171295166016,
      "logits/reference_rejected": -2.797828435897827,
      "logps/policy_chosen": -69.43246459960938,
      "logps/policy_rejected": -162.20603942871094,
      "logps/reference_chosen": -74.48234558105469,
      "logps/reference_rejected": -164.12376403808594,
      "loss": 0.6658,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05049885809421539,
      "rewards/margins": 0.03132178634405136,
      "rewards/rejected": 0.019177071750164032,
      "step": 76,
      "tokens/chosen_response_len_mean": 142.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 126.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 2.854272678012172,
      "learning_rate": 3.08e-07,
      "logits/policy_chosen": -3.0018627643585205,
      "logits/policy_rejected": -2.6590089797973633,
      "logits/reference_chosen": -2.995105028152466,
      "logits/reference_rejected": -2.385610580444336,
      "logps/policy_chosen": -85.27000427246094,
      "logps/policy_rejected": -132.92564392089844,
      "logps/reference_chosen": -90.3302001953125,
      "logps/reference_rejected": -134.35781860351562,
      "loss": 0.6608,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.05060192570090294,
      "rewards/margins": 0.03628016635775566,
      "rewards/rejected": 0.014321764931082726,
      "step": 77,
      "tokens/chosen_response_len_mean": 363.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 320.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 2.7505136356983133,
      "learning_rate": 3.12e-07,
      "logits/policy_chosen": -2.0338382720947266,
      "logits/policy_rejected": -2.276106357574463,
      "logits/reference_chosen": -2.0206451416015625,
      "logits/reference_rejected": -2.060913562774658,
      "logps/policy_chosen": -232.1044464111328,
      "logps/policy_rejected": -146.2607879638672,
      "logps/reference_chosen": -250.27952575683594,
      "logps/reference_rejected": -149.79270935058594,
      "loss": 0.6679,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.18175065517425537,
      "rewards/margins": 0.14643150568008423,
      "rewards/rejected": 0.03531915321946144,
      "step": 78,
      "tokens/chosen_response_len_mean": 521.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 307.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 2.4952987336105665,
      "learning_rate": 3.1599999999999997e-07,
      "logits/policy_chosen": -2.8599331378936768,
      "logits/policy_rejected": -2.8679845333099365,
      "logits/reference_chosen": -2.7079713344573975,
      "logits/reference_rejected": -2.4389915466308594,
      "logps/policy_chosen": -148.4790496826172,
      "logps/policy_rejected": -277.09405517578125,
      "logps/reference_chosen": -156.32412719726562,
      "logps/reference_rejected": -281.2509765625,
      "loss": 0.6709,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0784510001540184,
      "rewards/margins": 0.03688175603747368,
      "rewards/rejected": 0.041569244116544724,
      "step": 79,
      "tokens/chosen_response_len_mean": 276.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 336.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 2.3479129695327297,
      "learning_rate": 3.2e-07,
      "logits/policy_chosen": -1.8025144338607788,
      "logits/policy_rejected": -1.9184333086013794,
      "logits/reference_chosen": -1.7402411699295044,
      "logits/reference_rejected": -1.906538724899292,
      "logps/policy_chosen": -79.56036376953125,
      "logps/policy_rejected": -185.89906311035156,
      "logps/reference_chosen": -88.7896499633789,
      "logps/reference_rejected": -198.23004150390625,
      "loss": 0.6921,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.09229286015033722,
      "rewards/margins": -0.03101687878370285,
      "rewards/rejected": 0.12330973893404007,
      "step": 80,
      "tokens/chosen_response_len_mean": 432.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 410.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 1.4788277018262954,
      "learning_rate": 3.24e-07,
      "logits/policy_chosen": -2.7751684188842773,
      "logits/policy_rejected": -2.7188775539398193,
      "logits/reference_chosen": -2.774470567703247,
      "logits/reference_rejected": -2.3951942920684814,
      "logps/policy_chosen": -64.14806365966797,
      "logps/policy_rejected": -91.78968048095703,
      "logps/reference_chosen": -70.32818603515625,
      "logps/reference_rejected": -92.31802368164062,
      "loss": 0.6813,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06180126219987869,
      "rewards/margins": 0.05651785433292389,
      "rewards/rejected": 0.005283408332616091,
      "step": 81,
      "tokens/chosen_response_len_mean": 345.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 316.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.8935594664890503,
      "learning_rate": 3.28e-07,
      "logits/policy_chosen": -2.936018943786621,
      "logits/policy_rejected": -2.9313597679138184,
      "logits/reference_chosen": -2.9295578002929688,
      "logits/reference_rejected": -2.9169669151306152,
      "logps/policy_chosen": -141.27691650390625,
      "logps/policy_rejected": -264.863037109375,
      "logps/reference_chosen": -152.80865478515625,
      "logps/reference_rejected": -274.9104919433594,
      "loss": 0.6644,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11531716585159302,
      "rewards/margins": 0.014842674136161804,
      "rewards/rejected": 0.10047448426485062,
      "step": 82,
      "tokens/chosen_response_len_mean": 394.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 390.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.4649587565099678,
      "learning_rate": 3.32e-07,
      "logits/policy_chosen": -2.9249117374420166,
      "logits/policy_rejected": -2.9090662002563477,
      "logits/reference_chosen": -2.868709087371826,
      "logits/reference_rejected": -2.898679256439209,
      "logps/policy_chosen": -154.240234375,
      "logps/policy_rejected": -297.08160400390625,
      "logps/reference_chosen": -161.24609375,
      "logps/reference_rejected": -297.5753479003906,
      "loss": 0.6687,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07005855441093445,
      "rewards/margins": 0.06512109935283661,
      "rewards/rejected": 0.004937455989420414,
      "step": 83,
      "tokens/chosen_response_len_mean": 334.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 309.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.5355636198848175,
      "learning_rate": 3.36e-07,
      "logits/policy_chosen": -2.9488754272460938,
      "logits/policy_rejected": -2.9289844036102295,
      "logits/reference_chosen": -2.890922784805298,
      "logits/reference_rejected": -2.870727062225342,
      "logps/policy_chosen": -18.58037757873535,
      "logps/policy_rejected": -47.89413833618164,
      "logps/reference_chosen": -19.83108901977539,
      "logps/reference_rejected": -48.526878356933594,
      "loss": 0.6755,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.012507104314863682,
      "rewards/margins": 0.006179675459861755,
      "rewards/rejected": 0.006327427923679352,
      "step": 84,
      "tokens/chosen_response_len_mean": 27.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 32.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.5319776077967746,
      "learning_rate": 3.4000000000000003e-07,
      "logits/policy_chosen": -2.7499167919158936,
      "logits/policy_rejected": -2.690596103668213,
      "logits/reference_chosen": -2.73750638961792,
      "logits/reference_rejected": -2.6311354637145996,
      "logps/policy_chosen": -123.83195495605469,
      "logps/policy_rejected": -194.00137329101562,
      "logps/reference_chosen": -139.81263732910156,
      "logps/reference_rejected": -196.44784545898438,
      "loss": 0.6523,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.15980683267116547,
      "rewards/margins": 0.13534212112426758,
      "rewards/rejected": 0.024464718997478485,
      "step": 85,
      "tokens/chosen_response_len_mean": 606.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 519.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.007472503017741,
      "learning_rate": 3.4399999999999996e-07,
      "logits/policy_chosen": -2.9675540924072266,
      "logits/policy_rejected": -2.9613142013549805,
      "logits/reference_chosen": -2.9432828426361084,
      "logits/reference_rejected": -2.9374613761901855,
      "logps/policy_chosen": -12.885274887084961,
      "logps/policy_rejected": -34.69200134277344,
      "logps/reference_chosen": -13.591547012329102,
      "logps/reference_rejected": -35.15093994140625,
      "loss": 0.6775,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.007062721066176891,
      "rewards/margins": 0.002473349217325449,
      "rewards/rejected": 0.004589371383190155,
      "step": 86,
      "tokens/chosen_response_len_mean": 11.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 12.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.3366851120609398,
      "learning_rate": 3.4799999999999994e-07,
      "logits/policy_chosen": -2.510186195373535,
      "logits/policy_rejected": -2.3673720359802246,
      "logits/reference_chosen": -2.5208425521850586,
      "logits/reference_rejected": -2.307051658630371,
      "logps/policy_chosen": -103.87987518310547,
      "logps/policy_rejected": -118.88311767578125,
      "logps/reference_chosen": -114.24214172363281,
      "logps/reference_rejected": -118.43659973144531,
      "loss": 0.664,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.10362273454666138,
      "rewards/margins": 0.10808798670768738,
      "rewards/rejected": -0.004465251229703426,
      "step": 87,
      "tokens/chosen_response_len_mean": 216.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 106.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.4579695955138874,
      "learning_rate": 3.52e-07,
      "logits/policy_chosen": -2.9871153831481934,
      "logits/policy_rejected": -2.8764257431030273,
      "logits/reference_chosen": -2.9515204429626465,
      "logits/reference_rejected": -2.883835792541504,
      "logps/policy_chosen": -151.91807556152344,
      "logps/policy_rejected": -141.33587646484375,
      "logps/reference_chosen": -162.7379608154297,
      "logps/reference_rejected": -147.37442016601562,
      "loss": 0.6695,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10819891095161438,
      "rewards/margins": 0.04781348258256912,
      "rewards/rejected": 0.06038542836904526,
      "step": 88,
      "tokens/chosen_response_len_mean": 299.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 223.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.2124030442195703,
      "learning_rate": 3.5599999999999996e-07,
      "logits/policy_chosen": -3.097362995147705,
      "logits/policy_rejected": -2.821688652038574,
      "logits/reference_chosen": -3.0792651176452637,
      "logits/reference_rejected": -2.6497914791107178,
      "logps/policy_chosen": -106.85523986816406,
      "logps/policy_rejected": -142.70628356933594,
      "logps/reference_chosen": -114.98326873779297,
      "logps/reference_rejected": -145.11463928222656,
      "loss": 0.6769,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08128029853105545,
      "rewards/margins": 0.05719655007123947,
      "rewards/rejected": 0.02408374845981598,
      "step": 89,
      "tokens/chosen_response_len_mean": 288.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 241.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.726548509097342,
      "learning_rate": 3.6e-07,
      "logits/policy_chosen": -2.911972761154175,
      "logits/policy_rejected": -2.7545974254608154,
      "logits/reference_chosen": -2.8914873600006104,
      "logits/reference_rejected": -2.4686713218688965,
      "logps/policy_chosen": -90.4947509765625,
      "logps/policy_rejected": -144.89512634277344,
      "logps/reference_chosen": -100.72172546386719,
      "logps/reference_rejected": -144.84951782226562,
      "loss": 0.6578,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1022697389125824,
      "rewards/margins": 0.10272593796253204,
      "rewards/rejected": -0.0004562041722238064,
      "step": 90,
      "tokens/chosen_response_len_mean": 254.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 112.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.2900988215150644,
      "learning_rate": 3.64e-07,
      "logits/policy_chosen": -2.8383259773254395,
      "logits/policy_rejected": -2.824490547180176,
      "logits/reference_chosen": -2.8273422718048096,
      "logits/reference_rejected": -2.8102381229400635,
      "logps/policy_chosen": -132.3096466064453,
      "logps/policy_rejected": -264.598388671875,
      "logps/reference_chosen": -139.56736755371094,
      "logps/reference_rejected": -264.87725830078125,
      "loss": 0.6825,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.07257718592882156,
      "rewards/margins": 0.0697886273264885,
      "rewards/rejected": 0.002788563258945942,
      "step": 91,
      "tokens/chosen_response_len_mean": 395.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 397.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.8545865680492692,
      "learning_rate": 3.6799999999999996e-07,
      "logits/policy_chosen": -2.7706384658813477,
      "logits/policy_rejected": -2.5063834190368652,
      "logits/reference_chosen": -2.7504565715789795,
      "logits/reference_rejected": -1.929320216178894,
      "logps/policy_chosen": -126.52993774414062,
      "logps/policy_rejected": -134.23519897460938,
      "logps/reference_chosen": -138.69195556640625,
      "logps/reference_rejected": -138.32545471191406,
      "loss": 0.6536,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.12162025272846222,
      "rewards/margins": 0.08071767538785934,
      "rewards/rejected": 0.040902577340602875,
      "step": 92,
      "tokens/chosen_response_len_mean": 315.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 201.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.284769139553738,
      "learning_rate": 3.72e-07,
      "logits/policy_chosen": -2.872082233428955,
      "logits/policy_rejected": -2.6889710426330566,
      "logits/reference_chosen": -2.8072874546051025,
      "logits/reference_rejected": -2.4844841957092285,
      "logps/policy_chosen": -74.13471984863281,
      "logps/policy_rejected": -116.40203857421875,
      "logps/reference_chosen": -80.3674545288086,
      "logps/reference_rejected": -114.1785888671875,
      "loss": 0.6602,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06232735142111778,
      "rewards/margins": 0.08456189930438995,
      "rewards/rejected": -0.022234544157981873,
      "step": 93,
      "tokens/chosen_response_len_mean": 162.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 95.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 1.898688154588326,
      "learning_rate": 3.76e-07,
      "logits/policy_chosen": -2.469783306121826,
      "logits/policy_rejected": -2.3272128105163574,
      "logits/reference_chosen": -2.425753593444824,
      "logits/reference_rejected": -2.3546535968780518,
      "logps/policy_chosen": -158.62905883789062,
      "logps/policy_rejected": -291.726318359375,
      "logps/reference_chosen": -168.30911254882812,
      "logps/reference_rejected": -291.2076110839844,
      "loss": 0.6805,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09680052101612091,
      "rewards/margins": 0.10198744386434555,
      "rewards/rejected": -0.005186920985579491,
      "step": 94,
      "tokens/chosen_response_len_mean": 359.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 302.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 2.9949420570538363,
      "learning_rate": 3.7999999999999996e-07,
      "logits/policy_chosen": -2.8909621238708496,
      "logits/policy_rejected": -2.906135082244873,
      "logits/reference_chosen": -2.792299747467041,
      "logits/reference_rejected": -2.848114252090454,
      "logps/policy_chosen": -50.312339782714844,
      "logps/policy_rejected": -105.15270233154297,
      "logps/reference_chosen": -52.856605529785156,
      "logps/reference_rejected": -108.09819030761719,
      "loss": 0.6483,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02544267475605011,
      "rewards/margins": -0.004012217745184898,
      "rewards/rejected": 0.029454896226525307,
      "step": 95,
      "tokens/chosen_response_len_mean": 116.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 120.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 2.4241955868213405,
      "learning_rate": 3.84e-07,
      "logits/policy_chosen": -2.789196014404297,
      "logits/policy_rejected": -2.543649911880493,
      "logits/reference_chosen": -2.7888166904449463,
      "logits/reference_rejected": -2.5362277030944824,
      "logps/policy_chosen": -127.26165771484375,
      "logps/policy_rejected": -104.05661010742188,
      "logps/reference_chosen": -139.64187622070312,
      "logps/reference_rejected": -108.60116577148438,
      "loss": 0.6656,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.12380214780569077,
      "rewards/margins": 0.07835659384727478,
      "rewards/rejected": 0.04544556140899658,
      "step": 96,
      "tokens/chosen_response_len_mean": 381.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 272.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 3.4704291601036683,
      "learning_rate": 3.88e-07,
      "logits/policy_chosen": -2.9072158336639404,
      "logits/policy_rejected": -2.845478057861328,
      "logits/reference_chosen": -2.882009983062744,
      "logits/reference_rejected": -2.943215847015381,
      "logps/policy_chosen": -199.15065002441406,
      "logps/policy_rejected": -345.61346435546875,
      "logps/reference_chosen": -208.939697265625,
      "logps/reference_rejected": -351.9114990234375,
      "loss": 0.6532,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09789040684700012,
      "rewards/margins": 0.0349101684987545,
      "rewards/rejected": 0.06298023462295532,
      "step": 97,
      "tokens/chosen_response_len_mean": 443.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 334.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 2.413585232959013,
      "learning_rate": 3.92e-07,
      "logits/policy_chosen": -2.581110954284668,
      "logits/policy_rejected": -2.5821545124053955,
      "logits/reference_chosen": -2.6214118003845215,
      "logits/reference_rejected": -2.6232800483703613,
      "logps/policy_chosen": -66.4002914428711,
      "logps/policy_rejected": -92.362060546875,
      "logps/reference_chosen": -73.42770385742188,
      "logps/reference_rejected": -96.90080261230469,
      "loss": 0.6682,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07027411460876465,
      "rewards/margins": 0.02488674223423004,
      "rewards/rejected": 0.0453873835504055,
      "step": 98,
      "tokens/chosen_response_len_mean": 369.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 353.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 2.566883896226423,
      "learning_rate": 3.96e-07,
      "logits/policy_chosen": -3.13887357711792,
      "logits/policy_rejected": -3.1172709465026855,
      "logits/reference_chosen": -3.127457618713379,
      "logits/reference_rejected": -3.1038241386413574,
      "logps/policy_chosen": -131.76913452148438,
      "logps/policy_rejected": -274.8529052734375,
      "logps/reference_chosen": -142.9169158935547,
      "logps/reference_rejected": -277.9893798828125,
      "loss": 0.68,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.11147774755954742,
      "rewards/margins": 0.08011295646429062,
      "rewards/rejected": 0.031364794820547104,
      "step": 99,
      "tokens/chosen_response_len_mean": 291.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 296.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 2.7837569692907294,
      "learning_rate": 4e-07,
      "logits/policy_chosen": -3.046888589859009,
      "logits/policy_rejected": -2.9924631118774414,
      "logits/reference_chosen": -3.0203585624694824,
      "logits/reference_rejected": -2.9802844524383545,
      "logps/policy_chosen": -96.66795349121094,
      "logps/policy_rejected": -151.02667236328125,
      "logps/reference_chosen": -106.40546417236328,
      "logps/reference_rejected": -156.30604553222656,
      "loss": 0.6494,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.09737517684698105,
      "rewards/margins": 0.04458136856555939,
      "rewards/rejected": 0.05279380828142166,
      "step": 100,
      "tokens/chosen_response_len_mean": 246.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 213.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 2.5533090774871385,
      "learning_rate": 4.04e-07,
      "logits/policy_chosen": -3.0745935440063477,
      "logits/policy_rejected": -2.9924702644348145,
      "logits/reference_chosen": -3.0464026927948,
      "logits/reference_rejected": -2.9301576614379883,
      "logps/policy_chosen": -130.01454162597656,
      "logps/policy_rejected": -273.28997802734375,
      "logps/reference_chosen": -138.4945831298828,
      "logps/reference_rejected": -277.6722106933594,
      "loss": 0.6659,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08480045944452286,
      "rewards/margins": 0.04097801819443703,
      "rewards/rejected": 0.04382243752479553,
      "step": 101,
      "tokens/chosen_response_len_mean": 351.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 294.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 2.658053156359227,
      "learning_rate": 4.0799999999999995e-07,
      "logits/policy_chosen": -2.9965174198150635,
      "logits/policy_rejected": -2.756721019744873,
      "logits/reference_chosen": -3.0305070877075195,
      "logits/reference_rejected": -2.5150346755981445,
      "logps/policy_chosen": -150.61041259765625,
      "logps/policy_rejected": -145.7916717529297,
      "logps/reference_chosen": -169.15542602539062,
      "logps/reference_rejected": -147.83485412597656,
      "loss": 0.6388,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18545001745224,
      "rewards/margins": 0.16501817107200623,
      "rewards/rejected": 0.02043186128139496,
      "step": 102,
      "tokens/chosen_response_len_mean": 276.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 211.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 1.8527975992911931,
      "learning_rate": 4.12e-07,
      "logits/policy_chosen": -2.4530792236328125,
      "logits/policy_rejected": -2.4156646728515625,
      "logits/reference_chosen": -2.4479713439941406,
      "logits/reference_rejected": -2.3904917240142822,
      "logps/policy_chosen": -173.00030517578125,
      "logps/policy_rejected": -199.19415283203125,
      "logps/reference_chosen": -189.02093505859375,
      "logps/reference_rejected": -209.84201049804688,
      "loss": 0.6666,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16020633280277252,
      "rewards/margins": 0.05372770130634308,
      "rewards/rejected": 0.10647863149642944,
      "step": 103,
      "tokens/chosen_response_len_mean": 309.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 310.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 2.5555937206717187,
      "learning_rate": 4.1599999999999997e-07,
      "logits/policy_chosen": -3.0911600589752197,
      "logits/policy_rejected": -3.0214200019836426,
      "logits/reference_chosen": -3.089872360229492,
      "logits/reference_rejected": -2.882385730743408,
      "logps/policy_chosen": -153.61936950683594,
      "logps/policy_rejected": -227.18930053710938,
      "logps/reference_chosen": -176.565673828125,
      "logps/reference_rejected": -239.63970947265625,
      "loss": 0.665,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22946298122406006,
      "rewards/margins": 0.10495913028717041,
      "rewards/rejected": 0.12450385838747025,
      "step": 104,
      "tokens/chosen_response_len_mean": 368.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 301.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 2.0081558824498096,
      "learning_rate": 4.1999999999999995e-07,
      "logits/policy_chosen": -2.958939790725708,
      "logits/policy_rejected": -2.80196213722229,
      "logits/reference_chosen": -2.944668769836426,
      "logits/reference_rejected": -2.6392948627471924,
      "logps/policy_chosen": -37.521568298339844,
      "logps/policy_rejected": -74.32329559326172,
      "logps/reference_chosen": -41.80109786987305,
      "logps/reference_rejected": -72.32585906982422,
      "loss": 0.664,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0427953377366066,
      "rewards/margins": 0.06276971846818924,
      "rewards/rejected": -0.01997438445687294,
      "step": 105,
      "tokens/chosen_response_len_mean": 105.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 85.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 2.7526269849819696,
      "learning_rate": 4.24e-07,
      "logits/policy_chosen": -3.100095272064209,
      "logits/policy_rejected": -2.8839964866638184,
      "logits/reference_chosen": -3.0812511444091797,
      "logits/reference_rejected": -2.936655044555664,
      "logps/policy_chosen": -207.67698669433594,
      "logps/policy_rejected": -240.35739135742188,
      "logps/reference_chosen": -223.25807189941406,
      "logps/reference_rejected": -239.6046142578125,
      "loss": 0.6494,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.15581092238426208,
      "rewards/margins": 0.16333873569965363,
      "rewards/rejected": -0.007527794688940048,
      "step": 106,
      "tokens/chosen_response_len_mean": 392.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 203.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 1.9552389682760483,
      "learning_rate": 4.2799999999999997e-07,
      "logits/policy_chosen": -2.922211170196533,
      "logits/policy_rejected": -2.94032621383667,
      "logits/reference_chosen": -2.9038805961608887,
      "logits/reference_rejected": -2.9326558113098145,
      "logps/policy_chosen": -142.2540283203125,
      "logps/policy_rejected": -238.67227172851562,
      "logps/reference_chosen": -151.64273071289062,
      "logps/reference_rejected": -239.12753295898438,
      "loss": 0.6753,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09388697892427444,
      "rewards/margins": 0.08933433890342712,
      "rewards/rejected": 0.004552632570266724,
      "step": 107,
      "tokens/chosen_response_len_mean": 347.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 352.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 3.014699368900832,
      "learning_rate": 4.3199999999999995e-07,
      "logits/policy_chosen": -3.1775548458099365,
      "logits/policy_rejected": -3.03210186958313,
      "logits/reference_chosen": -3.1847496032714844,
      "logits/reference_rejected": -2.531522750854492,
      "logps/policy_chosen": -143.90025329589844,
      "logps/policy_rejected": -291.92388916015625,
      "logps/reference_chosen": -160.6278839111328,
      "logps/reference_rejected": -297.986572265625,
      "loss": 0.6498,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.16727623343467712,
      "rewards/margins": 0.10664979368448257,
      "rewards/rejected": 0.060626450926065445,
      "step": 108,
      "tokens/chosen_response_len_mean": 411.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 381.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 3.3247511691725236,
      "learning_rate": 4.36e-07,
      "logits/policy_chosen": -3.0070884227752686,
      "logits/policy_rejected": -3.006807565689087,
      "logits/reference_chosen": -2.9760210514068604,
      "logits/reference_rejected": -2.982029438018799,
      "logps/policy_chosen": -117.77191925048828,
      "logps/policy_rejected": -288.87689208984375,
      "logps/reference_chosen": -130.60389709472656,
      "logps/reference_rejected": -292.3191833496094,
      "loss": 0.6517,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.12831971049308777,
      "rewards/margins": 0.09389686584472656,
      "rewards/rejected": 0.034422844648361206,
      "step": 109,
      "tokens/chosen_response_len_mean": 318.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 302.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 3.158975765008515,
      "learning_rate": 4.3999999999999997e-07,
      "logits/policy_chosen": -2.50785756111145,
      "logits/policy_rejected": -2.504612684249878,
      "logits/reference_chosen": -2.5301709175109863,
      "logits/reference_rejected": -2.5272464752197266,
      "logps/policy_chosen": -136.28662109375,
      "logps/policy_rejected": -200.89352416992188,
      "logps/reference_chosen": -158.42433166503906,
      "logps/reference_rejected": -212.5179443359375,
      "loss": 0.638,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.22137700021266937,
      "rewards/margins": 0.10513283312320709,
      "rewards/rejected": 0.11624416708946228,
      "step": 110,
      "tokens/chosen_response_len_mean": 608.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 525.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 2.8278528577842095,
      "learning_rate": 4.44e-07,
      "logits/policy_chosen": -2.725069999694824,
      "logits/policy_rejected": -2.732088565826416,
      "logits/reference_chosen": -2.6485838890075684,
      "logits/reference_rejected": -2.76961612701416,
      "logps/policy_chosen": -178.09320068359375,
      "logps/policy_rejected": -237.05970764160156,
      "logps/reference_chosen": -192.22482299804688,
      "logps/reference_rejected": -248.2169647216797,
      "loss": 0.6253,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.14131632447242737,
      "rewards/margins": 0.029743948951363564,
      "rewards/rejected": 0.11157237738370895,
      "step": 111,
      "tokens/chosen_response_len_mean": 589.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 556.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 3.4191485129114714,
      "learning_rate": 4.48e-07,
      "logits/policy_chosen": -2.7914977073669434,
      "logits/policy_rejected": -2.804826259613037,
      "logits/reference_chosen": -2.7042627334594727,
      "logits/reference_rejected": -2.640076160430908,
      "logps/policy_chosen": -146.0531768798828,
      "logps/policy_rejected": -232.5332794189453,
      "logps/reference_chosen": -168.23060607910156,
      "logps/reference_rejected": -237.49801635742188,
      "loss": 0.6295,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22177428007125854,
      "rewards/margins": 0.172126904129982,
      "rewards/rejected": 0.049647387117147446,
      "step": 112,
      "tokens/chosen_response_len_mean": 438.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 396.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 2.331597318165628,
      "learning_rate": 4.5199999999999997e-07,
      "logits/policy_chosen": -3.1060757637023926,
      "logits/policy_rejected": -2.961545467376709,
      "logits/reference_chosen": -3.1052682399749756,
      "logits/reference_rejected": -2.833801746368408,
      "logps/policy_chosen": -127.66785430908203,
      "logps/policy_rejected": -196.163330078125,
      "logps/reference_chosen": -137.27505493164062,
      "logps/reference_rejected": -198.14578247070312,
      "loss": 0.6605,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09607198089361191,
      "rewards/margins": 0.07624746859073639,
      "rewards/rejected": 0.01982451230287552,
      "step": 113,
      "tokens/chosen_response_len_mean": 314.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 291.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 2.750538166446364,
      "learning_rate": 4.56e-07,
      "logits/policy_chosen": -2.7500100135803223,
      "logits/policy_rejected": -2.681910514831543,
      "logits/reference_chosen": -2.789179563522339,
      "logits/reference_rejected": -2.3359477519989014,
      "logps/policy_chosen": -85.98888397216797,
      "logps/policy_rejected": -100.74612426757812,
      "logps/reference_chosen": -98.14977264404297,
      "logps/reference_rejected": -102.4841537475586,
      "loss": 0.6408,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.12160897254943848,
      "rewards/margins": 0.10422859340906143,
      "rewards/rejected": 0.017380371689796448,
      "step": 114,
      "tokens/chosen_response_len_mean": 256.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 223.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 2.5440555248438996,
      "learning_rate": 4.6e-07,
      "logits/policy_chosen": -2.971923351287842,
      "logits/policy_rejected": -2.989212989807129,
      "logits/reference_chosen": -2.9268057346343994,
      "logits/reference_rejected": -2.919334650039673,
      "logps/policy_chosen": -98.5408706665039,
      "logps/policy_rejected": -166.144775390625,
      "logps/reference_chosen": -109.1729965209961,
      "logps/reference_rejected": -167.0085906982422,
      "loss": 0.6505,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10632127523422241,
      "rewards/margins": 0.09768319875001907,
      "rewards/rejected": 0.008638077415525913,
      "step": 115,
      "tokens/chosen_response_len_mean": 229.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 254.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 2.4244019154261474,
      "learning_rate": 4.64e-07,
      "logits/policy_chosen": -2.8886287212371826,
      "logits/policy_rejected": -2.922623872756958,
      "logits/reference_chosen": -2.8570451736450195,
      "logits/reference_rejected": -2.888643741607666,
      "logps/policy_chosen": -113.93293762207031,
      "logps/policy_rejected": -225.3817596435547,
      "logps/reference_chosen": -120.10133361816406,
      "logps/reference_rejected": -223.7104034423828,
      "loss": 0.6579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06168397143483162,
      "rewards/margins": 0.0783977136015892,
      "rewards/rejected": -0.016713742166757584,
      "step": 116,
      "tokens/chosen_response_len_mean": 330.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 318.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 2.400248188855275,
      "learning_rate": 4.68e-07,
      "logits/policy_chosen": -2.8977608680725098,
      "logits/policy_rejected": -2.9003448486328125,
      "logits/reference_chosen": -2.8988900184631348,
      "logits/reference_rejected": -2.9036717414855957,
      "logps/policy_chosen": -74.31845092773438,
      "logps/policy_rejected": -92.14234924316406,
      "logps/reference_chosen": -78.81231689453125,
      "logps/reference_rejected": -93.07196807861328,
      "loss": 0.6588,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04493863508105278,
      "rewards/margins": 0.035642486065626144,
      "rewards/rejected": 0.00929614994674921,
      "step": 117,
      "tokens/chosen_response_len_mean": 275.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 274.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 2.4323431386681595,
      "learning_rate": 4.7199999999999994e-07,
      "logits/policy_chosen": -2.6670236587524414,
      "logits/policy_rejected": -2.518751621246338,
      "logits/reference_chosen": -2.666407585144043,
      "logits/reference_rejected": -2.5256845951080322,
      "logps/policy_chosen": -92.44061279296875,
      "logps/policy_rejected": -151.3938446044922,
      "logps/reference_chosen": -100.54401397705078,
      "logps/reference_rejected": -153.23507690429688,
      "loss": 0.6474,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08103407174348831,
      "rewards/margins": 0.06262162327766418,
      "rewards/rejected": 0.01841244101524353,
      "step": 118,
      "tokens/chosen_response_len_mean": 302.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 332.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 2.9673553403256707,
      "learning_rate": 4.76e-07,
      "logits/policy_chosen": -2.9274580478668213,
      "logits/policy_rejected": -2.8594419956207275,
      "logits/reference_chosen": -2.9505772590637207,
      "logits/reference_rejected": -2.850255012512207,
      "logps/policy_chosen": -169.86106872558594,
      "logps/policy_rejected": -316.6098327636719,
      "logps/reference_chosen": -191.77130126953125,
      "logps/reference_rejected": -317.0496520996094,
      "loss": 0.6273,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21910235285758972,
      "rewards/margins": 0.21470384299755096,
      "rewards/rejected": 0.004398513585329056,
      "step": 119,
      "tokens/chosen_response_len_mean": 431.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 374.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 2.3505092556042304,
      "learning_rate": 4.8e-07,
      "logits/policy_chosen": -2.814655303955078,
      "logits/policy_rejected": -2.5319275856018066,
      "logits/reference_chosen": -2.8454208374023438,
      "logits/reference_rejected": -2.315487861633301,
      "logps/policy_chosen": -50.07554626464844,
      "logps/policy_rejected": -110.84510040283203,
      "logps/reference_chosen": -60.05486297607422,
      "logps/reference_rejected": -106.57113647460938,
      "loss": 0.6368,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09979310631752014,
      "rewards/margins": 0.14253273606300354,
      "rewards/rejected": -0.0427396334707737,
      "step": 120,
      "tokens/chosen_response_len_mean": 267.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 187.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 2.8274965378312586,
      "learning_rate": 4.839999999999999e-07,
      "logits/policy_chosen": -2.942972421646118,
      "logits/policy_rejected": -2.8031089305877686,
      "logits/reference_chosen": -2.961139678955078,
      "logits/reference_rejected": -2.5472524166107178,
      "logps/policy_chosen": -130.38365173339844,
      "logps/policy_rejected": -191.11968994140625,
      "logps/reference_chosen": -159.17897033691406,
      "logps/reference_rejected": -199.572021484375,
      "loss": 0.6282,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2879532277584076,
      "rewards/margins": 0.2034299373626709,
      "rewards/rejected": 0.0845232680439949,
      "step": 121,
      "tokens/chosen_response_len_mean": 420.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 307.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 2.886588252399639,
      "learning_rate": 4.879999999999999e-07,
      "logits/policy_chosen": -2.853999376296997,
      "logits/policy_rejected": -2.480708122253418,
      "logits/reference_chosen": -2.710318088531494,
      "logits/reference_rejected": -1.9466197490692139,
      "logps/policy_chosen": -167.15240478515625,
      "logps/policy_rejected": -201.3469696044922,
      "logps/reference_chosen": -181.32119750976562,
      "logps/reference_rejected": -195.66384887695312,
      "loss": 0.6126,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.14168795943260193,
      "rewards/margins": 0.1985190212726593,
      "rewards/rejected": -0.056831058114767075,
      "step": 122,
      "tokens/chosen_response_len_mean": 326.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 201.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 2.826829476556088,
      "learning_rate": 4.92e-07,
      "logits/policy_chosen": -2.465135335922241,
      "logits/policy_rejected": -2.4350569248199463,
      "logits/reference_chosen": -2.4818878173828125,
      "logits/reference_rejected": -2.3954761028289795,
      "logps/policy_chosen": -100.21925354003906,
      "logps/policy_rejected": -125.87073516845703,
      "logps/reference_chosen": -126.35533905029297,
      "logps/reference_rejected": -144.03236389160156,
      "loss": 0.6328,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.26136085391044617,
      "rewards/margins": 0.07974465936422348,
      "rewards/rejected": 0.1816161870956421,
      "step": 123,
      "tokens/chosen_response_len_mean": 426.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 408.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 2.5289041457286774,
      "learning_rate": 4.96e-07,
      "logits/policy_chosen": -2.6869404315948486,
      "logits/policy_rejected": -2.773239850997925,
      "logits/reference_chosen": -2.653362512588501,
      "logits/reference_rejected": -2.5821008682250977,
      "logps/policy_chosen": -71.6850357055664,
      "logps/policy_rejected": -108.95183563232422,
      "logps/reference_chosen": -77.52404022216797,
      "logps/reference_rejected": -107.73311614990234,
      "loss": 0.6376,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05839002877473831,
      "rewards/margins": 0.07057718932628632,
      "rewards/rejected": -0.012187155894935131,
      "step": 124,
      "tokens/chosen_response_len_mean": 229.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 211.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 3.7452560936185804,
      "learning_rate": 5e-07,
      "logits/policy_chosen": -2.8474879264831543,
      "logits/policy_rejected": -2.77992844581604,
      "logits/reference_chosen": -2.8415565490722656,
      "logits/reference_rejected": -2.658388376235962,
      "logps/policy_chosen": -85.31348419189453,
      "logps/policy_rejected": -100.18048095703125,
      "logps/reference_chosen": -91.71619415283203,
      "logps/reference_rejected": -100.39627838134766,
      "loss": 0.6215,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0640270784497261,
      "rewards/margins": 0.06186922267079353,
      "rewards/rejected": 0.00215785950422287,
      "step": 125,
      "tokens/chosen_response_len_mean": 91.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 71.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 2.791877501388191,
      "learning_rate": 4.999990147435196e-07,
      "logits/policy_chosen": -2.772634506225586,
      "logits/policy_rejected": -2.7663345336914062,
      "logits/reference_chosen": -2.8182811737060547,
      "logits/reference_rejected": -2.821171760559082,
      "logps/policy_chosen": -114.60906982421875,
      "logps/policy_rejected": -122.4944839477539,
      "logps/reference_chosen": -141.17056274414062,
      "logps/reference_rejected": -135.64431762695312,
      "loss": 0.63,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2656148672103882,
      "rewards/margins": 0.13411659002304077,
      "rewards/rejected": 0.1314982771873474,
      "step": 126,
      "tokens/chosen_response_len_mean": 458.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 332.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 2.692910890812567,
      "learning_rate": 4.99996058981844e-07,
      "logits/policy_chosen": -2.5161116123199463,
      "logits/policy_rejected": -2.3703317642211914,
      "logits/reference_chosen": -2.4154701232910156,
      "logits/reference_rejected": -2.4023566246032715,
      "logps/policy_chosen": -72.83512115478516,
      "logps/policy_rejected": -136.28607177734375,
      "logps/reference_chosen": -82.63104248046875,
      "logps/reference_rejected": -140.04986572265625,
      "loss": 0.6369,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09795919060707092,
      "rewards/margins": 0.06032119691371918,
      "rewards/rejected": 0.037637993693351746,
      "step": 127,
      "tokens/chosen_response_len_mean": 416.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 410.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 2.859419025019686,
      "learning_rate": 4.999911327382711e-07,
      "logits/policy_chosen": -2.0673351287841797,
      "logits/policy_rejected": -1.9885450601577759,
      "logits/reference_chosen": -2.1499083042144775,
      "logits/reference_rejected": -1.6471858024597168,
      "logps/policy_chosen": -57.06989669799805,
      "logps/policy_rejected": -87.03865814208984,
      "logps/reference_chosen": -69.13873291015625,
      "logps/reference_rejected": -83.46507263183594,
      "loss": 0.5998,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.12068840861320496,
      "rewards/margins": 0.15642426908016205,
      "rewards/rejected": -0.035735856741666794,
      "step": 128,
      "tokens/chosen_response_len_mean": 343.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 194.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 2.8759886658807288,
      "learning_rate": 4.999842360516294e-07,
      "logits/policy_chosen": -3.1045093536376953,
      "logits/policy_rejected": -3.1223738193511963,
      "logits/reference_chosen": -3.09148907661438,
      "logits/reference_rejected": -3.091776132583618,
      "logps/policy_chosen": -40.253719329833984,
      "logps/policy_rejected": -93.64241027832031,
      "logps/reference_chosen": -43.66665267944336,
      "logps/reference_rejected": -92.32077026367188,
      "loss": 0.5883,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03412933275103569,
      "rewards/margins": 0.047345712780952454,
      "rewards/rejected": -0.01321637723594904,
      "step": 129,
      "tokens/chosen_response_len_mean": 134.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 142.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 2.396608033911008,
      "learning_rate": 4.999753689762792e-07,
      "logits/policy_chosen": -2.7669222354888916,
      "logits/policy_rejected": -3.0244264602661133,
      "logits/reference_chosen": -2.7182681560516357,
      "logits/reference_rejected": -3.016852855682373,
      "logps/policy_chosen": -49.25522994995117,
      "logps/policy_rejected": -209.12026977539062,
      "logps/reference_chosen": -58.14155197143555,
      "logps/reference_rejected": -213.75338745117188,
      "loss": 0.6532,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08886321634054184,
      "rewards/margins": 0.04253204166889191,
      "rewards/rejected": 0.04633117839694023,
      "step": 130,
      "tokens/chosen_response_len_mean": 140.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 397.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 2.8338085411833127,
      "learning_rate": 4.999645315821111e-07,
      "logits/policy_chosen": -2.177436590194702,
      "logits/policy_rejected": -2.1923160552978516,
      "logits/reference_chosen": -2.23976993560791,
      "logits/reference_rejected": -2.227308511734009,
      "logps/policy_chosen": -92.42753601074219,
      "logps/policy_rejected": -104.93522644042969,
      "logps/reference_chosen": -111.59609985351562,
      "logps/reference_rejected": -120.66436004638672,
      "loss": 0.6227,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19168560206890106,
      "rewards/margins": 0.03439423814415932,
      "rewards/rejected": 0.15729136765003204,
      "step": 131,
      "tokens/chosen_response_len_mean": 428.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 393.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 2.537615272647266,
      "learning_rate": 4.99951723954546e-07,
      "logits/policy_chosen": -2.6528267860412598,
      "logits/policy_rejected": -2.7896065711975098,
      "logits/reference_chosen": -2.3186991214752197,
      "logits/reference_rejected": -2.7976412773132324,
      "logps/policy_chosen": -35.68031311035156,
      "logps/policy_rejected": -72.86119079589844,
      "logps/reference_chosen": -42.128787994384766,
      "logps/reference_rejected": -76.47801208496094,
      "loss": 0.6365,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06448473036289215,
      "rewards/margins": 0.028316427022218704,
      "rewards/rejected": 0.03616830334067345,
      "step": 132,
      "tokens/chosen_response_len_mean": 226.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 273.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 2.952022770646515,
      "learning_rate": 4.999369461945343e-07,
      "logits/policy_chosen": -3.1463866233825684,
      "logits/policy_rejected": -2.8932509422302246,
      "logits/reference_chosen": -3.141604423522949,
      "logits/reference_rejected": -2.444639205932617,
      "logps/policy_chosen": -83.71105194091797,
      "logps/policy_rejected": -113.13372802734375,
      "logps/reference_chosen": -94.0230941772461,
      "logps/reference_rejected": -108.04383850097656,
      "loss": 0.5892,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10312040150165558,
      "rewards/margins": 0.15401925146579742,
      "rewards/rejected": -0.05089886486530304,
      "step": 133,
      "tokens/chosen_response_len_mean": 249.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 178.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 3.540843119893174,
      "learning_rate": 4.999201984185552e-07,
      "logits/policy_chosen": -3.0958125591278076,
      "logits/policy_rejected": -3.042353868484497,
      "logits/reference_chosen": -3.133866310119629,
      "logits/reference_rejected": -3.0950450897216797,
      "logps/policy_chosen": -236.39605712890625,
      "logps/policy_rejected": -330.2846374511719,
      "logps/reference_chosen": -259.0321350097656,
      "logps/reference_rejected": -336.391357421875,
      "loss": 0.5663,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22636055946350098,
      "rewards/margins": 0.16529357433319092,
      "rewards/rejected": 0.06106698513031006,
      "step": 134,
      "tokens/chosen_response_len_mean": 717.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 639.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 2.981444512825392,
      "learning_rate": 4.999014807586154e-07,
      "logits/policy_chosen": -2.3856396675109863,
      "logits/policy_rejected": -2.1970813274383545,
      "logits/reference_chosen": -2.4693095684051514,
      "logits/reference_rejected": -1.5482537746429443,
      "logps/policy_chosen": -142.9355926513672,
      "logps/policy_rejected": -189.60543823242188,
      "logps/reference_chosen": -164.72271728515625,
      "logps/reference_rejected": -189.248046875,
      "loss": 0.6065,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2178712785243988,
      "rewards/margins": 0.22144514322280884,
      "rewards/rejected": -0.003573855385184288,
      "step": 135,
      "tokens/chosen_response_len_mean": 339.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 276.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 2.4024504350993654,
      "learning_rate": 4.998807933622484e-07,
      "logits/policy_chosen": -3.1550679206848145,
      "logits/policy_rejected": -3.1557650566101074,
      "logits/reference_chosen": -3.1761221885681152,
      "logits/reference_rejected": -3.135535955429077,
      "logps/policy_chosen": -118.20024871826172,
      "logps/policy_rejected": -264.43560791015625,
      "logps/reference_chosen": -130.25527954101562,
      "logps/reference_rejected": -264.16741943359375,
      "loss": 0.6336,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.12055033445358276,
      "rewards/margins": 0.12323223054409027,
      "rewards/rejected": -0.002681899815797806,
      "step": 136,
      "tokens/chosen_response_len_mean": 354.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 386.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 2.8382305196380684,
      "learning_rate": 4.998581363925135e-07,
      "logits/policy_chosen": -2.2047266960144043,
      "logits/policy_rejected": -2.433465003967285,
      "logits/reference_chosen": -2.275531530380249,
      "logits/reference_rejected": -2.245540142059326,
      "logps/policy_chosen": -378.1270751953125,
      "logps/policy_rejected": -344.7900085449219,
      "logps/reference_chosen": -416.6529541015625,
      "logps/reference_rejected": -347.785888671875,
      "loss": 0.5804,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3852587938308716,
      "rewards/margins": 0.3552997410297394,
      "rewards/rejected": 0.02995903044939041,
      "step": 137,
      "tokens/chosen_response_len_mean": 584.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 427.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 2.1385240477591636,
      "learning_rate": 4.998335100279941e-07,
      "logits/policy_chosen": -2.6462714672088623,
      "logits/policy_rejected": -2.658045768737793,
      "logits/reference_chosen": -2.729498863220215,
      "logits/reference_rejected": -2.694502830505371,
      "logps/policy_chosen": -93.70889282226562,
      "logps/policy_rejected": -182.60643005371094,
      "logps/reference_chosen": -114.53910827636719,
      "logps/reference_rejected": -185.69065856933594,
      "loss": 0.6218,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2083020955324173,
      "rewards/margins": 0.17745977640151978,
      "rewards/rejected": 0.030842319130897522,
      "step": 138,
      "tokens/chosen_response_len_mean": 282.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 247.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 2.116685308038921,
      "learning_rate": 4.998069144627964e-07,
      "logits/policy_chosen": -2.870461940765381,
      "logits/policy_rejected": -2.6627469062805176,
      "logits/reference_chosen": -2.924255132675171,
      "logits/reference_rejected": -2.6669623851776123,
      "logps/policy_chosen": -149.49957275390625,
      "logps/policy_rejected": -133.25009155273438,
      "logps/reference_chosen": -191.32994079589844,
      "logps/reference_rejected": -157.3734588623047,
      "loss": 0.6358,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4183034896850586,
      "rewards/margins": 0.1770700067281723,
      "rewards/rejected": 0.2412334829568863,
      "step": 139,
      "tokens/chosen_response_len_mean": 457.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 308.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 2.5870338720368387,
      "learning_rate": 4.99778349906548e-07,
      "logits/policy_chosen": -2.9523699283599854,
      "logits/policy_rejected": -2.9597270488739014,
      "logits/reference_chosen": -2.9815268516540527,
      "logits/reference_rejected": -2.9721755981445312,
      "logps/policy_chosen": -45.849361419677734,
      "logps/policy_rejected": -106.37723541259766,
      "logps/reference_chosen": -56.053375244140625,
      "logps/reference_rejected": -102.900390625,
      "loss": 0.6334,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10204015672206879,
      "rewards/margins": 0.13680851459503174,
      "rewards/rejected": -0.03476836904883385,
      "step": 140,
      "tokens/chosen_response_len_mean": 174.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 184.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 3.1171856643556626,
      "learning_rate": 4.997478165843964e-07,
      "logits/policy_chosen": -2.8171520233154297,
      "logits/policy_rejected": -2.8751494884490967,
      "logits/reference_chosen": -2.9145286083221436,
      "logits/reference_rejected": -2.9242196083068848,
      "logps/policy_chosen": -170.06459045410156,
      "logps/policy_rejected": -296.7114562988281,
      "logps/reference_chosen": -197.37327575683594,
      "logps/reference_rejected": -302.43438720703125,
      "loss": 0.5862,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.27308669686317444,
      "rewards/margins": 0.21585765480995178,
      "rewards/rejected": 0.057229042053222656,
      "step": 141,
      "tokens/chosen_response_len_mean": 416.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 422.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 3.0084655845363835,
      "learning_rate": 4.997153147370065e-07,
      "logits/policy_chosen": -2.85317325592041,
      "logits/policy_rejected": -2.8581879138946533,
      "logits/reference_chosen": -2.7449471950531006,
      "logits/reference_rejected": -2.9342892169952393,
      "logps/policy_chosen": -95.86734008789062,
      "logps/policy_rejected": -255.03668212890625,
      "logps/reference_chosen": -114.49011993408203,
      "logps/reference_rejected": -234.34010314941406,
      "loss": 0.5691,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18622779846191406,
      "rewards/margins": 0.39319345355033875,
      "rewards/rejected": -0.20696566998958588,
      "step": 142,
      "tokens/chosen_response_len_mean": 348.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 233.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 2.8051570403665753,
      "learning_rate": 4.996808446205598e-07,
      "logits/policy_chosen": -2.9738688468933105,
      "logits/policy_rejected": -2.8889176845550537,
      "logits/reference_chosen": -3.072221517562866,
      "logits/reference_rejected": -2.744046449661255,
      "logps/policy_chosen": -106.87481689453125,
      "logps/policy_rejected": -214.15478515625,
      "logps/reference_chosen": -126.41868591308594,
      "logps/reference_rejected": -216.4077606201172,
      "loss": 0.6244,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19543880224227905,
      "rewards/margins": 0.17290906608104706,
      "rewards/rejected": 0.022529736161231995,
      "step": 143,
      "tokens/chosen_response_len_mean": 340.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 310.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 3.1099351541314864,
      "learning_rate": 4.996444065067515e-07,
      "logits/policy_chosen": -2.849621534347534,
      "logits/policy_rejected": -2.8288486003875732,
      "logits/reference_chosen": -2.9862475395202637,
      "logits/reference_rejected": -2.9479100704193115,
      "logps/policy_chosen": -331.7215576171875,
      "logps/policy_rejected": -571.941650390625,
      "logps/reference_chosen": -382.40533447265625,
      "logps/reference_rejected": -549.6744995117188,
      "loss": 0.5803,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5068381428718567,
      "rewards/margins": 0.7295098304748535,
      "rewards/rejected": -0.2226717472076416,
      "step": 144,
      "tokens/chosen_response_len_mean": 687.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 550.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 2.712783580670268,
      "learning_rate": 4.996060006827887e-07,
      "logits/policy_chosen": -2.882990837097168,
      "logits/policy_rejected": -2.8665995597839355,
      "logits/reference_chosen": -2.9348299503326416,
      "logits/reference_rejected": -2.9280238151550293,
      "logps/policy_chosen": -75.93386840820312,
      "logps/policy_rejected": -159.058837890625,
      "logps/reference_chosen": -91.52436828613281,
      "logps/reference_rejected": -146.97247314453125,
      "loss": 0.5965,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.15590497851371765,
      "rewards/margins": 0.2767685055732727,
      "rewards/rejected": -0.12086354941129684,
      "step": 145,
      "tokens/chosen_response_len_mean": 248.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 191.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 2.364187725655967,
      "learning_rate": 4.995656274513881e-07,
      "logits/policy_chosen": -3.0416619777679443,
      "logits/policy_rejected": -2.995178699493408,
      "logits/reference_chosen": -3.053309440612793,
      "logits/reference_rejected": -3.0008325576782227,
      "logps/policy_chosen": -90.75337982177734,
      "logps/policy_rejected": -127.47672271728516,
      "logps/reference_chosen": -102.95506286621094,
      "logps/reference_rejected": -132.29486083984375,
      "loss": 0.6275,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1220167949795723,
      "rewards/margins": 0.07383548468351364,
      "rewards/rejected": 0.04818130284547806,
      "step": 146,
      "tokens/chosen_response_len_mean": 403.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 352.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 3.163944044368287,
      "learning_rate": 4.995232871307736e-07,
      "logits/policy_chosen": -3.0712456703186035,
      "logits/policy_rejected": -2.916374921798706,
      "logits/reference_chosen": -3.115262746810913,
      "logits/reference_rejected": -2.6205530166625977,
      "logps/policy_chosen": -104.03935241699219,
      "logps/policy_rejected": -156.13577270507812,
      "logps/reference_chosen": -121.57315063476562,
      "logps/reference_rejected": -153.22605895996094,
      "loss": 0.583,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17533791065216064,
      "rewards/margins": 0.2044350802898407,
      "rewards/rejected": -0.029097169637680054,
      "step": 147,
      "tokens/chosen_response_len_mean": 273.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 197.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 2.8981364214624588,
      "learning_rate": 4.994789800546737e-07,
      "logits/policy_chosen": -2.519289970397949,
      "logits/policy_rejected": -2.3818559646606445,
      "logits/reference_chosen": -2.5594940185546875,
      "logits/reference_rejected": -2.2251200675964355,
      "logps/policy_chosen": -64.7489242553711,
      "logps/policy_rejected": -90.21540832519531,
      "logps/reference_chosen": -80.6169204711914,
      "logps/reference_rejected": -92.92266845703125,
      "loss": 0.5887,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.15867996215820312,
      "rewards/margins": 0.13160738348960876,
      "rewards/rejected": 0.027072593569755554,
      "step": 148,
      "tokens/chosen_response_len_mean": 252.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 195.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 2.7118945452462073,
      "learning_rate": 4.994327065723192e-07,
      "logits/policy_chosen": -2.6516730785369873,
      "logits/policy_rejected": -2.5235743522644043,
      "logits/reference_chosen": -2.629504442214966,
      "logits/reference_rejected": -2.4736945629119873,
      "logps/policy_chosen": -80.69113159179688,
      "logps/policy_rejected": -134.16526794433594,
      "logps/reference_chosen": -103.33537292480469,
      "logps/reference_rejected": -144.43687438964844,
      "loss": 0.5924,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2264423370361328,
      "rewards/margins": 0.12372627854347229,
      "rewards/rejected": 0.10271605104207993,
      "step": 149,
      "tokens/chosen_response_len_mean": 244.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 244.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 2.5381060398035205,
      "learning_rate": 4.9938446704844e-07,
      "logits/policy_chosen": -2.8819587230682373,
      "logits/policy_rejected": -2.9721384048461914,
      "logits/reference_chosen": -2.7422800064086914,
      "logits/reference_rejected": -2.9950976371765137,
      "logps/policy_chosen": -57.52130126953125,
      "logps/policy_rejected": -120.23619079589844,
      "logps/reference_chosen": -59.788352966308594,
      "logps/reference_rejected": -115.03448486328125,
      "loss": 0.619,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.022670526057481766,
      "rewards/margins": 0.07468754798173904,
      "rewards/rejected": -0.05201702564954758,
      "step": 150,
      "tokens/chosen_response_len_mean": 145.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 157.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 2.8034120172472052,
      "learning_rate": 4.993342618632626e-07,
      "logits/policy_chosen": -3.0852291584014893,
      "logits/policy_rejected": -2.92305850982666,
      "logits/reference_chosen": -3.162454128265381,
      "logits/reference_rejected": -2.4117138385772705,
      "logps/policy_chosen": -101.7428207397461,
      "logps/policy_rejected": -242.9844207763672,
      "logps/reference_chosen": -119.85108184814453,
      "logps/reference_rejected": -215.4344940185547,
      "loss": 0.5702,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18108254671096802,
      "rewards/margins": 0.45658186078071594,
      "rewards/rejected": -0.2754993140697479,
      "step": 151,
      "tokens/chosen_response_len_mean": 318.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 234.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 2.9459543515337776,
      "learning_rate": 4.992820914125069e-07,
      "logits/policy_chosen": -2.8646183013916016,
      "logits/policy_rejected": -2.754851818084717,
      "logits/reference_chosen": -3.0292770862579346,
      "logits/reference_rejected": -2.5228896141052246,
      "logps/policy_chosen": -150.56875610351562,
      "logps/policy_rejected": -277.60125732421875,
      "logps/reference_chosen": -179.77772521972656,
      "logps/reference_rejected": -275.3694152832031,
      "loss": 0.586,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2920897901058197,
      "rewards/margins": 0.31440818309783936,
      "rewards/rejected": -0.022318381816148758,
      "step": 152,
      "tokens/chosen_response_len_mean": 543.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 491.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 2.9972427731212954,
      "learning_rate": 4.99227956107383e-07,
      "logits/policy_chosen": -2.133497714996338,
      "logits/policy_rejected": -2.275707721710205,
      "logits/reference_chosen": -2.4100184440612793,
      "logits/reference_rejected": -2.27435040473938,
      "logps/policy_chosen": -149.89268493652344,
      "logps/policy_rejected": -225.88043212890625,
      "logps/reference_chosen": -195.48631286621094,
      "logps/reference_rejected": -232.33531188964844,
      "loss": 0.5712,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4559362828731537,
      "rewards/margins": 0.3913875222206116,
      "rewards/rejected": 0.06454875320196152,
      "step": 153,
      "tokens/chosen_response_len_mean": 551.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 501.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 2.6455429598648053,
      "learning_rate": 4.991718563745882e-07,
      "logits/policy_chosen": -3.022681951522827,
      "logits/policy_rejected": -3.033928155899048,
      "logits/reference_chosen": -3.0507426261901855,
      "logits/reference_rejected": -3.0070152282714844,
      "logps/policy_chosen": -17.34947395324707,
      "logps/policy_rejected": -55.760353088378906,
      "logps/reference_chosen": -20.67547607421875,
      "logps/reference_rejected": -50.72275924682617,
      "loss": 0.5857,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.03326002135872841,
      "rewards/margins": 0.08363597095012665,
      "rewards/rejected": -0.05037594214081764,
      "step": 154,
      "tokens/chosen_response_len_mean": 28.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 28.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 2.188503580126624,
      "learning_rate": 4.991137926563035e-07,
      "logits/policy_chosen": -2.8026483058929443,
      "logits/policy_rejected": -2.840829372406006,
      "logits/reference_chosen": -2.866769790649414,
      "logits/reference_rejected": -2.868910789489746,
      "logps/policy_chosen": -45.941871643066406,
      "logps/policy_rejected": -90.47376251220703,
      "logps/reference_chosen": -53.47924041748047,
      "logps/reference_rejected": -84.99751281738281,
      "loss": 0.6102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.07537368685007095,
      "rewards/margins": 0.1301361620426178,
      "rewards/rejected": -0.054762475192546844,
      "step": 155,
      "tokens/chosen_response_len_mean": 173.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 133.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.5667127080808974,
      "learning_rate": 4.990537654101903e-07,
      "logits/policy_chosen": -2.980593681335449,
      "logits/policy_rejected": -3.0731048583984375,
      "logits/reference_chosen": -3.066528797149658,
      "logits/reference_rejected": -3.1092565059661865,
      "logps/policy_chosen": -92.3929672241211,
      "logps/policy_rejected": -223.2591552734375,
      "logps/reference_chosen": -117.49671936035156,
      "logps/reference_rejected": -224.93527221679688,
      "loss": 0.5677,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.25103750824928284,
      "rewards/margins": 0.2342762053012848,
      "rewards/rejected": 0.016761288046836853,
      "step": 156,
      "tokens/chosen_response_len_mean": 291.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 321.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.742556033679143,
      "learning_rate": 4.989917751093862e-07,
      "logits/policy_chosen": -2.9195590019226074,
      "logits/policy_rejected": -2.7180957794189453,
      "logits/reference_chosen": -3.0530753135681152,
      "logits/reference_rejected": -2.1471331119537354,
      "logps/policy_chosen": -86.05413818359375,
      "logps/policy_rejected": -171.83856201171875,
      "logps/reference_chosen": -108.33873748779297,
      "logps/reference_rejected": -158.24143981933594,
      "loss": 0.5624,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22284595668315887,
      "rewards/margins": 0.35881713032722473,
      "rewards/rejected": -0.13597117364406586,
      "step": 157,
      "tokens/chosen_response_len_mean": 300.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 193.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.320383809339318,
      "learning_rate": 4.989278222425021e-07,
      "logits/policy_chosen": -2.9507193565368652,
      "logits/policy_rejected": -3.0587496757507324,
      "logits/reference_chosen": -3.0194993019104004,
      "logits/reference_rejected": -3.0736992359161377,
      "logps/policy_chosen": -103.41101837158203,
      "logps/policy_rejected": -312.6053466796875,
      "logps/reference_chosen": -126.41258239746094,
      "logps/reference_rejected": -292.84515380859375,
      "loss": 0.5961,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2300155907869339,
      "rewards/margins": 0.4276174306869507,
      "rewards/rejected": -0.19760185480117798,
      "step": 158,
      "tokens/chosen_response_len_mean": 338.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 361.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.516259723460725,
      "learning_rate": 4.988619073136179e-07,
      "logits/policy_chosen": -2.578339099884033,
      "logits/policy_rejected": -2.6243929862976074,
      "logits/reference_chosen": -2.651563882827759,
      "logits/reference_rejected": -2.7075228691101074,
      "logps/policy_chosen": -60.65949249267578,
      "logps/policy_rejected": -124.75245666503906,
      "logps/reference_chosen": -77.07334899902344,
      "logps/reference_rejected": -121.59002685546875,
      "loss": 0.5969,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16413861513137817,
      "rewards/margins": 0.19576290249824524,
      "rewards/rejected": -0.031624287366867065,
      "step": 159,
      "tokens/chosen_response_len_mean": 344.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 347.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.715123355655463,
      "learning_rate": 4.987940308422783e-07,
      "logits/policy_chosen": -2.0114169120788574,
      "logits/policy_rejected": -2.0045886039733887,
      "logits/reference_chosen": -2.092721700668335,
      "logits/reference_rejected": -1.7612866163253784,
      "logps/policy_chosen": -84.94512176513672,
      "logps/policy_rejected": -133.75323486328125,
      "logps/reference_chosen": -107.3111801147461,
      "logps/reference_rejected": -128.45538330078125,
      "loss": 0.5549,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22366058826446533,
      "rewards/margins": 0.27663907408714294,
      "rewards/rejected": -0.05297844856977463,
      "step": 160,
      "tokens/chosen_response_len_mean": 425.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 374.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.6648898364230735,
      "learning_rate": 4.987241933634892e-07,
      "logits/policy_chosen": -2.6987576484680176,
      "logits/policy_rejected": -2.7210419178009033,
      "logits/reference_chosen": -2.804107189178467,
      "logits/reference_rejected": -2.7485485076904297,
      "logps/policy_chosen": -101.68658447265625,
      "logps/policy_rejected": -234.5952606201172,
      "logps/reference_chosen": -123.14704132080078,
      "logps/reference_rejected": -228.36183166503906,
      "loss": 0.5769,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21460454165935516,
      "rewards/margins": 0.276938796043396,
      "rewards/rejected": -0.06233424320816994,
      "step": 161,
      "tokens/chosen_response_len_mean": 409.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 407.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.82572590776649,
      "learning_rate": 4.986523954277134e-07,
      "logits/policy_chosen": -2.5539088249206543,
      "logits/policy_rejected": -2.5690407752990723,
      "logits/reference_chosen": -2.634115695953369,
      "logits/reference_rejected": -2.615056037902832,
      "logps/policy_chosen": -37.4378662109375,
      "logps/policy_rejected": -93.33854675292969,
      "logps/reference_chosen": -45.8214225769043,
      "logps/reference_rejected": -89.81768798828125,
      "loss": 0.5852,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.08383554965257645,
      "rewards/margins": 0.11904413998126984,
      "rewards/rejected": -0.03520859405398369,
      "step": 162,
      "tokens/chosen_response_len_mean": 228.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 240.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.7170854054640863,
      "learning_rate": 4.985786376008658e-07,
      "logits/policy_chosen": -2.820416212081909,
      "logits/policy_rejected": -2.79477858543396,
      "logits/reference_chosen": -2.9494552612304688,
      "logits/reference_rejected": -2.837876081466675,
      "logps/policy_chosen": -71.54603576660156,
      "logps/policy_rejected": -156.01953125,
      "logps/reference_chosen": -90.15393829345703,
      "logps/reference_rejected": -154.65646362304688,
      "loss": 0.5657,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1860790103673935,
      "rewards/margins": 0.1997096687555313,
      "rewards/rejected": -0.01363067515194416,
      "step": 163,
      "tokens/chosen_response_len_mean": 302.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 296.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.3740765633517396,
      "learning_rate": 4.985029204643094e-07,
      "logits/policy_chosen": -2.747786045074463,
      "logits/policy_rejected": -2.7663800716400146,
      "logits/reference_chosen": -2.8246467113494873,
      "logits/reference_rejected": -2.6436727046966553,
      "logps/policy_chosen": -58.324737548828125,
      "logps/policy_rejected": -223.62765502929688,
      "logps/reference_chosen": -85.9714126586914,
      "logps/reference_rejected": -203.6089630126953,
      "loss": 0.6018,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.27646681666374207,
      "rewards/margins": 0.4766535758972168,
      "rewards/rejected": -0.20018675923347473,
      "step": 164,
      "tokens/chosen_response_len_mean": 223.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 216.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.188806416183682,
      "learning_rate": 4.984252446148508e-07,
      "logits/policy_chosen": -2.701019287109375,
      "logits/policy_rejected": -2.729661226272583,
      "logits/reference_chosen": -2.8306493759155273,
      "logits/reference_rejected": -2.8457367420196533,
      "logps/policy_chosen": -154.78518676757812,
      "logps/policy_rejected": -214.3829345703125,
      "logps/reference_chosen": -194.75637817382812,
      "logps/reference_rejected": -246.5928497314453,
      "loss": 0.6206,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.39971184730529785,
      "rewards/margins": 0.07761266827583313,
      "rewards/rejected": 0.3220991790294647,
      "step": 165,
      "tokens/chosen_response_len_mean": 429.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 445.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.8889287147059104,
      "learning_rate": 4.983456106647348e-07,
      "logits/policy_chosen": -2.861962080001831,
      "logits/policy_rejected": -2.8529515266418457,
      "logits/reference_chosen": -3.064021587371826,
      "logits/reference_rejected": -2.927248001098633,
      "logps/policy_chosen": -146.7970428466797,
      "logps/policy_rejected": -283.18389892578125,
      "logps/reference_chosen": -181.90396118164062,
      "logps/reference_rejected": -277.1955871582031,
      "loss": 0.5642,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.35106930136680603,
      "rewards/margins": 0.4109525978565216,
      "rewards/rejected": -0.05988328531384468,
      "step": 166,
      "tokens/chosen_response_len_mean": 468.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 421.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.9841019270826292,
      "learning_rate": 4.982640192416404e-07,
      "logits/policy_chosen": -2.6893157958984375,
      "logits/policy_rejected": -2.681230068206787,
      "logits/reference_chosen": -2.948234796524048,
      "logits/reference_rejected": -2.248135566711426,
      "logps/policy_chosen": -122.96820831298828,
      "logps/policy_rejected": -197.66404724121094,
      "logps/reference_chosen": -155.029541015625,
      "logps/reference_rejected": -194.3126678466797,
      "loss": 0.5642,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.32061344385147095,
      "rewards/margins": 0.35412728786468506,
      "rewards/rejected": -0.0335138775408268,
      "step": 167,
      "tokens/chosen_response_len_mean": 452.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 366.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 2.766687221527344,
      "learning_rate": 4.981804709886755e-07,
      "logits/policy_chosen": -2.4212305545806885,
      "logits/policy_rejected": -2.4506514072418213,
      "logits/reference_chosen": -2.6806509494781494,
      "logits/reference_rejected": -2.608292579650879,
      "logps/policy_chosen": -60.471797943115234,
      "logps/policy_rejected": -120.82171630859375,
      "logps/reference_chosen": -92.31079864501953,
      "logps/reference_rejected": -130.82656860351562,
      "loss": 0.54,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.31838998198509216,
      "rewards/margins": 0.21834133565425873,
      "rewards/rejected": 0.10004864633083344,
      "step": 168,
      "tokens/chosen_response_len_mean": 324.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 310.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 2.1828651372724637,
      "learning_rate": 4.980949665643719e-07,
      "logits/policy_chosen": -2.7441906929016113,
      "logits/policy_rejected": -2.7218408584594727,
      "logits/reference_chosen": -2.8971827030181885,
      "logits/reference_rejected": -2.860405445098877,
      "logps/policy_chosen": -92.7531509399414,
      "logps/policy_rejected": -210.39797973632812,
      "logps/reference_chosen": -123.75934600830078,
      "logps/reference_rejected": -200.2626190185547,
      "loss": 0.5755,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3100619614124298,
      "rewards/margins": 0.41141554713249207,
      "rewards/rejected": -0.10135360062122345,
      "step": 169,
      "tokens/chosen_response_len_mean": 517.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 465.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 2.9989447327164425,
      "learning_rate": 4.980075066426795e-07,
      "logits/policy_chosen": -1.6698166131973267,
      "logits/policy_rejected": -1.7434771060943604,
      "logits/reference_chosen": -1.803000569343567,
      "logits/reference_rejected": -1.8658090829849243,
      "logps/policy_chosen": -61.05796813964844,
      "logps/policy_rejected": -89.74996948242188,
      "logps/reference_chosen": -77.35304260253906,
      "logps/reference_rejected": -98.12295532226562,
      "loss": 0.5561,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1629507839679718,
      "rewards/margins": 0.07922087609767914,
      "rewards/rejected": 0.08372990787029266,
      "step": 170,
      "tokens/chosen_response_len_mean": 243.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 245.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 2.5554431416025243,
      "learning_rate": 4.979180919129624e-07,
      "logits/policy_chosen": -2.85784912109375,
      "logits/policy_rejected": -2.838977098464966,
      "logits/reference_chosen": -2.8757684230804443,
      "logits/reference_rejected": -2.8608899116516113,
      "logps/policy_chosen": -53.759185791015625,
      "logps/policy_rejected": -90.618408203125,
      "logps/reference_chosen": -62.647789001464844,
      "logps/reference_rejected": -86.81078338623047,
      "loss": 0.5702,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.08888600766658783,
      "rewards/margins": 0.1269623190164566,
      "rewards/rejected": -0.03807631880044937,
      "step": 171,
      "tokens/chosen_response_len_mean": 207.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 209.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 2.7869612955855776,
      "learning_rate": 4.978267230799918e-07,
      "logits/policy_chosen": -2.0658278465270996,
      "logits/policy_rejected": -2.2496557235717773,
      "logits/reference_chosen": -2.1524839401245117,
      "logits/reference_rejected": -2.3013737201690674,
      "logps/policy_chosen": -86.49990844726562,
      "logps/policy_rejected": -197.741943359375,
      "logps/reference_chosen": -109.14804077148438,
      "logps/reference_rejected": -182.33616638183594,
      "loss": 0.5549,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22648128867149353,
      "rewards/margins": 0.380539208650589,
      "rewards/rejected": -0.15405790507793427,
      "step": 172,
      "tokens/chosen_response_len_mean": 443.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 439.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 2.9395952260087506,
      "learning_rate": 4.977334008639417e-07,
      "logits/policy_chosen": -2.605189800262451,
      "logits/policy_rejected": -2.5856776237487793,
      "logits/reference_chosen": -2.778407335281372,
      "logits/reference_rejected": -2.793548583984375,
      "logps/policy_chosen": -112.27513122558594,
      "logps/policy_rejected": -266.2890625,
      "logps/reference_chosen": -140.7950439453125,
      "logps/reference_rejected": -241.72708129882812,
      "loss": 0.5051,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.28519904613494873,
      "rewards/margins": 0.5308188796043396,
      "rewards/rejected": -0.24561986327171326,
      "step": 173,
      "tokens/chosen_response_len_mean": 462.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 293.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 2.7423154795634153,
      "learning_rate": 4.976381260003826e-07,
      "logits/policy_chosen": -2.7179572582244873,
      "logits/policy_rejected": -2.756556510925293,
      "logits/reference_chosen": -2.8246583938598633,
      "logits/reference_rejected": -2.793865203857422,
      "logps/policy_chosen": -80.221923828125,
      "logps/policy_rejected": -204.06829833984375,
      "logps/reference_chosen": -96.3402328491211,
      "logps/reference_rejected": -181.1507568359375,
      "loss": 0.5916,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16118302941322327,
      "rewards/margins": 0.3903582692146301,
      "rewards/rejected": -0.22917523980140686,
      "step": 174,
      "tokens/chosen_response_len_mean": 307.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 307.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 2.8390846969584946,
      "learning_rate": 4.97540899240276e-07,
      "logits/policy_chosen": -2.6149544715881348,
      "logits/policy_rejected": -2.7186009883880615,
      "logits/reference_chosen": -2.755988597869873,
      "logits/reference_rejected": -2.8059897422790527,
      "logps/policy_chosen": -83.78113555908203,
      "logps/policy_rejected": -158.60336303710938,
      "logps/reference_chosen": -105.53045654296875,
      "logps/reference_rejected": -161.6735076904297,
      "loss": 0.5707,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2174932360649109,
      "rewards/margins": 0.18679174780845642,
      "rewards/rejected": 0.030701473355293274,
      "step": 175,
      "tokens/chosen_response_len_mean": 227.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 221.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 1.928131402745428,
      "learning_rate": 4.974417213499681e-07,
      "logits/policy_chosen": -2.97554874420166,
      "logits/policy_rejected": -3.0169177055358887,
      "logits/reference_chosen": -3.1351237297058105,
      "logits/reference_rejected": -3.054442882537842,
      "logps/policy_chosen": -155.18731689453125,
      "logps/policy_rejected": -306.79449462890625,
      "logps/reference_chosen": -198.6287841796875,
      "logps/reference_rejected": -296.349609375,
      "loss": 0.5884,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4344147741794586,
      "rewards/margins": 0.5388633608818054,
      "rewards/rejected": -0.1044485792517662,
      "step": 176,
      "tokens/chosen_response_len_mean": 399.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 380.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 2.5891349823773604,
      "learning_rate": 4.973405931111844e-07,
      "logits/policy_chosen": -2.475595474243164,
      "logits/policy_rejected": -2.5474050045013428,
      "logits/reference_chosen": -2.574971914291382,
      "logits/reference_rejected": -2.7303850650787354,
      "logps/policy_chosen": -60.937068939208984,
      "logps/policy_rejected": -196.1728515625,
      "logps/reference_chosen": -93.98703002929688,
      "logps/reference_rejected": -193.26068115234375,
      "loss": 0.5721,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.330499529838562,
      "rewards/margins": 0.3596211075782776,
      "rewards/rejected": -0.029121577739715576,
      "step": 177,
      "tokens/chosen_response_len_mean": 277.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 290.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 3.186008328730442,
      "learning_rate": 4.972375153210227e-07,
      "logits/policy_chosen": -2.73168683052063,
      "logits/policy_rejected": -2.80218505859375,
      "logits/reference_chosen": -2.8586461544036865,
      "logits/reference_rejected": -2.856407642364502,
      "logps/policy_chosen": -41.751312255859375,
      "logps/policy_rejected": -121.0931396484375,
      "logps/reference_chosen": -52.108253479003906,
      "logps/reference_rejected": -104.97093200683594,
      "loss": 0.5198,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1035693883895874,
      "rewards/margins": 0.26479148864746094,
      "rewards/rejected": -0.16122210025787354,
      "step": 178,
      "tokens/chosen_response_len_mean": 131.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 131.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 2.4038166772062186,
      "learning_rate": 4.971324887919477e-07,
      "logits/policy_chosen": -2.768160820007324,
      "logits/policy_rejected": -2.2585337162017822,
      "logits/reference_chosen": -2.8858625888824463,
      "logits/reference_rejected": -2.4065427780151367,
      "logps/policy_chosen": -67.46320343017578,
      "logps/policy_rejected": -281.3670654296875,
      "logps/reference_chosen": -94.11170196533203,
      "logps/reference_rejected": -263.22698974609375,
      "loss": 0.5596,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.26648497581481934,
      "rewards/margins": 0.4478859305381775,
      "rewards/rejected": -0.18140093982219696,
      "step": 179,
      "tokens/chosen_response_len_mean": 239.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 333.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 2.530913530757402,
      "learning_rate": 4.970255143517838e-07,
      "logits/policy_chosen": -2.6580893993377686,
      "logits/policy_rejected": -2.717331886291504,
      "logits/reference_chosen": -2.90189790725708,
      "logits/reference_rejected": -2.8899142742156982,
      "logps/policy_chosen": -129.58033752441406,
      "logps/policy_rejected": -165.34884643554688,
      "logps/reference_chosen": -179.34756469726562,
      "logps/reference_rejected": -170.3519287109375,
      "loss": 0.5522,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4976721405982971,
      "rewards/margins": 0.4476412534713745,
      "rewards/rejected": 0.050030872225761414,
      "step": 180,
      "tokens/chosen_response_len_mean": 500.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 318.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 2.633909183828016,
      "learning_rate": 4.969165928437091e-07,
      "logits/policy_chosen": -2.570159912109375,
      "logits/policy_rejected": -2.5334911346435547,
      "logits/reference_chosen": -2.760227680206299,
      "logits/reference_rejected": -2.3865246772766113,
      "logps/policy_chosen": -55.27775955200195,
      "logps/policy_rejected": -119.49053955078125,
      "logps/reference_chosen": -72.7205581665039,
      "logps/reference_rejected": -111.53352355957031,
      "loss": 0.5199,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17442797124385834,
      "rewards/margins": 0.2539982497692108,
      "rewards/rejected": -0.07957027852535248,
      "step": 181,
      "tokens/chosen_response_len_mean": 307.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 267.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 2.3888599137710167,
      "learning_rate": 4.968057251262486e-07,
      "logits/policy_chosen": -2.6059675216674805,
      "logits/policy_rejected": -2.395353078842163,
      "logits/reference_chosen": -2.9607644081115723,
      "logits/reference_rejected": -1.8757669925689697,
      "logps/policy_chosen": -74.94976043701172,
      "logps/policy_rejected": -134.52859497070312,
      "logps/reference_chosen": -111.65763092041016,
      "logps/reference_rejected": -96.27473449707031,
      "loss": 0.5536,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.36707866191864014,
      "rewards/margins": 0.7496173977851868,
      "rewards/rejected": -0.38253867626190186,
      "step": 182,
      "tokens/chosen_response_len_mean": 351.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 148.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 2.826021624132104,
      "learning_rate": 4.966929120732675e-07,
      "logits/policy_chosen": -2.3233590126037598,
      "logits/policy_rejected": -2.4008846282958984,
      "logits/reference_chosen": -2.5622141361236572,
      "logits/reference_rejected": -2.381047248840332,
      "logps/policy_chosen": -185.09976196289062,
      "logps/policy_rejected": -336.95941162109375,
      "logps/reference_chosen": -236.7534637451172,
      "logps/reference_rejected": -320.8805847167969,
      "loss": 0.5087,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.516537070274353,
      "rewards/margins": 0.6773253679275513,
      "rewards/rejected": -0.16078826785087585,
      "step": 183,
      "tokens/chosen_response_len_mean": 441.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 384.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 2.26864738127188,
      "learning_rate": 4.965781545739638e-07,
      "logits/policy_chosen": -2.7422099113464355,
      "logits/policy_rejected": -2.7613723278045654,
      "logits/reference_chosen": -2.9637043476104736,
      "logits/reference_rejected": -2.921086549758911,
      "logps/policy_chosen": -45.85551452636719,
      "logps/policy_rejected": -118.8583984375,
      "logps/reference_chosen": -65.04806518554688,
      "logps/reference_rejected": -119.62251281738281,
      "loss": 0.5676,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.191925510764122,
      "rewards/margins": 0.1842842549085617,
      "rewards/rejected": 0.007641255855560303,
      "step": 184,
      "tokens/chosen_response_len_mean": 266.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 242.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 2.8182732561665023,
      "learning_rate": 4.964614535328625e-07,
      "logits/policy_chosen": -2.523998975753784,
      "logits/policy_rejected": -2.4280190467834473,
      "logits/reference_chosen": -2.717611789703369,
      "logits/reference_rejected": -2.5818982124328613,
      "logps/policy_chosen": -250.7300262451172,
      "logps/policy_rejected": -131.51202392578125,
      "logps/reference_chosen": -304.0659484863281,
      "logps/reference_rejected": -136.93946838378906,
      "loss": 0.5284,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.533359169960022,
      "rewards/margins": 0.47908464074134827,
      "rewards/rejected": 0.0542745515704155,
      "step": 185,
      "tokens/chosen_response_len_mean": 653.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 300.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 2.658487252908494,
      "learning_rate": 4.963428098698069e-07,
      "logits/policy_chosen": -2.8848061561584473,
      "logits/policy_rejected": -2.887396812438965,
      "logits/reference_chosen": -3.0770082473754883,
      "logits/reference_rejected": -3.012545347213745,
      "logps/policy_chosen": -26.844032287597656,
      "logps/policy_rejected": -64.93582916259766,
      "logps/reference_chosen": -37.33853530883789,
      "logps/reference_rejected": -57.781166076660156,
      "loss": 0.5617,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10494504123926163,
      "rewards/margins": 0.17649173736572266,
      "rewards/rejected": -0.07154669612646103,
      "step": 186,
      "tokens/chosen_response_len_mean": 101.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 99.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 3.192921534395214,
      "learning_rate": 4.962222245199527e-07,
      "logits/policy_chosen": -2.5711615085601807,
      "logits/policy_rejected": -2.556881904602051,
      "logits/reference_chosen": -2.9153711795806885,
      "logits/reference_rejected": -2.4689133167266846,
      "logps/policy_chosen": -79.51646423339844,
      "logps/policy_rejected": -212.90431213378906,
      "logps/reference_chosen": -108.06434631347656,
      "logps/reference_rejected": -189.78848266601562,
      "loss": 0.5117,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.285478800535202,
      "rewards/margins": 0.5166370868682861,
      "rewards/rejected": -0.2311583161354065,
      "step": 187,
      "tokens/chosen_response_len_mean": 368.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 330.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 3.185300778929063,
      "learning_rate": 4.960996984337598e-07,
      "logits/policy_chosen": -2.322800397872925,
      "logits/policy_rejected": -2.180086851119995,
      "logits/reference_chosen": -2.647416830062866,
      "logits/reference_rejected": -2.404383420944214,
      "logps/policy_chosen": -88.6377944946289,
      "logps/policy_rejected": -154.1780242919922,
      "logps/reference_chosen": -146.73565673828125,
      "logps/reference_rejected": -158.4337921142578,
      "loss": 0.4879,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5809786319732666,
      "rewards/margins": 0.5384209156036377,
      "rewards/rejected": 0.042557742446660995,
      "step": 188,
      "tokens/chosen_response_len_mean": 496.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 375.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 3.018235573859185,
      "learning_rate": 4.959752325769852e-07,
      "logits/policy_chosen": -2.469785213470459,
      "logits/policy_rejected": -2.580235481262207,
      "logits/reference_chosen": -2.680108070373535,
      "logits/reference_rejected": -2.8124053478240967,
      "logps/policy_chosen": -57.80915069580078,
      "logps/policy_rejected": -210.04135131835938,
      "logps/reference_chosen": -90.7098617553711,
      "logps/reference_rejected": -191.3103485107422,
      "loss": 0.5062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.329007089138031,
      "rewards/margins": 0.5163170099258423,
      "rewards/rejected": -0.18730990588665009,
      "step": 189,
      "tokens/chosen_response_len_mean": 332.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 348.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 3.1505949518170437,
      "learning_rate": 4.958488279306753e-07,
      "logits/policy_chosen": -2.3762245178222656,
      "logits/policy_rejected": -2.466752767562866,
      "logits/reference_chosen": -2.847898483276367,
      "logits/reference_rejected": -2.2204389572143555,
      "logps/policy_chosen": -104.57975006103516,
      "logps/policy_rejected": -250.35414123535156,
      "logps/reference_chosen": -159.56100463867188,
      "logps/reference_rejected": -225.86146545410156,
      "loss": 0.522,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5498126149177551,
      "rewards/margins": 0.7947396039962769,
      "rewards/rejected": -0.24492698907852173,
      "step": 190,
      "tokens/chosen_response_len_mean": 512.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 435.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 3.056135297935867,
      "learning_rate": 4.957204854911578e-07,
      "logits/policy_chosen": -2.485416889190674,
      "logits/policy_rejected": -2.3778135776519775,
      "logits/reference_chosen": -2.856623411178589,
      "logits/reference_rejected": -2.3928959369659424,
      "logps/policy_chosen": -83.4710693359375,
      "logps/policy_rejected": -267.17791748046875,
      "logps/reference_chosen": -124.20071411132812,
      "logps/reference_rejected": -209.21670532226562,
      "loss": 0.5036,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.40729638934135437,
      "rewards/margins": 0.9869081377983093,
      "rewards/rejected": -0.5796117782592773,
      "step": 191,
      "tokens/chosen_response_len_mean": 528.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 410.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 2.503765131980355,
      "learning_rate": 4.955902062700349e-07,
      "logits/policy_chosen": -2.3622069358825684,
      "logits/policy_rejected": -2.4439258575439453,
      "logits/reference_chosen": -2.7395248413085938,
      "logits/reference_rejected": -2.6338257789611816,
      "logps/policy_chosen": -76.7771224975586,
      "logps/policy_rejected": -262.30841064453125,
      "logps/reference_chosen": -117.23094940185547,
      "logps/reference_rejected": -224.7442626953125,
      "loss": 0.5445,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.40453827381134033,
      "rewards/margins": 0.7801797986030579,
      "rewards/rejected": -0.37564149498939514,
      "step": 192,
      "tokens/chosen_response_len_mean": 359.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 367.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.16,
      "grad_norm": 2.9703660280881734,
      "learning_rate": 4.954579912941737e-07,
      "logits/policy_chosen": -2.622279644012451,
      "logits/policy_rejected": -2.6470251083374023,
      "logits/reference_chosen": -2.853696823120117,
      "logits/reference_rejected": -2.840925931930542,
      "logps/policy_chosen": -38.201744079589844,
      "logps/policy_rejected": -138.85470581054688,
      "logps/reference_chosen": -63.35798645019531,
      "logps/reference_rejected": -113.46359252929688,
      "loss": 0.4864,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2515623867511749,
      "rewards/margins": 0.5054734349250793,
      "rewards/rejected": -0.25391101837158203,
      "step": 193,
      "tokens/chosen_response_len_mean": 157.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 169.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.16,
      "grad_norm": 2.9246186619159347,
      "learning_rate": 4.953238416057e-07,
      "logits/policy_chosen": -2.605665683746338,
      "logits/policy_rejected": -2.5566539764404297,
      "logits/reference_chosen": -2.9360122680664062,
      "logits/reference_rejected": -2.068471908569336,
      "logps/policy_chosen": -84.89468383789062,
      "logps/policy_rejected": -218.09048461914062,
      "logps/reference_chosen": -133.76991271972656,
      "logps/reference_rejected": -183.8456573486328,
      "loss": 0.4881,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.48875218629837036,
      "rewards/margins": 0.8312005996704102,
      "rewards/rejected": -0.3424483835697174,
      "step": 194,
      "tokens/chosen_response_len_mean": 360.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 238.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.16,
      "grad_norm": 2.80153859644754,
      "learning_rate": 4.951877582619881e-07,
      "logits/policy_chosen": -2.8731517791748047,
      "logits/policy_rejected": -2.975919246673584,
      "logits/reference_chosen": -3.0668981075286865,
      "logits/reference_rejected": -3.1014294624328613,
      "logps/policy_chosen": -24.165613174438477,
      "logps/policy_rejected": -61.2584342956543,
      "logps/reference_chosen": -23.939205169677734,
      "logps/reference_rejected": -43.215232849121094,
      "loss": 0.5741,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.002264055423438549,
      "rewards/margins": 0.1781679093837738,
      "rewards/rejected": -0.18043196201324463,
      "step": 195,
      "tokens/chosen_response_len_mean": 62.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 63.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.16,
      "grad_norm": 2.7527879974178955,
      "learning_rate": 4.950497423356542e-07,
      "logits/policy_chosen": -2.9940450191497803,
      "logits/policy_rejected": -2.710885763168335,
      "logits/reference_chosen": -3.169459581375122,
      "logits/reference_rejected": -2.7128515243530273,
      "logps/policy_chosen": -120.05692291259766,
      "logps/policy_rejected": -105.24632263183594,
      "logps/reference_chosen": -160.91629028320312,
      "logps/reference_rejected": -78.08905029296875,
      "loss": 0.5115,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4085935354232788,
      "rewards/margins": 0.6801662445068359,
      "rewards/rejected": -0.27157267928123474,
      "step": 196,
      "tokens/chosen_response_len_mean": 332.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 53.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.16,
      "grad_norm": 2.473954328567208,
      "learning_rate": 4.949097949145471e-07,
      "logits/policy_chosen": -2.994095802307129,
      "logits/policy_rejected": -2.9911117553710938,
      "logits/reference_chosen": -3.0934770107269287,
      "logits/reference_rejected": -3.05942440032959,
      "logps/policy_chosen": -54.1805419921875,
      "logps/policy_rejected": -158.96043395996094,
      "logps/reference_chosen": -74.52861785888672,
      "logps/reference_rejected": -128.6375732421875,
      "loss": 0.4544,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2034807801246643,
      "rewards/margins": 0.5067094564437866,
      "rewards/rejected": -0.3032286465167999,
      "step": 197,
      "tokens/chosen_response_len_mean": 182.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 165.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.16,
      "grad_norm": 2.6028329307586944,
      "learning_rate": 4.947679171017394e-07,
      "logits/policy_chosen": -2.7241291999816895,
      "logits/policy_rejected": -2.8773388862609863,
      "logits/reference_chosen": -2.9862842559814453,
      "logits/reference_rejected": -2.9953157901763916,
      "logps/policy_chosen": -56.624752044677734,
      "logps/policy_rejected": -255.8826141357422,
      "logps/reference_chosen": -101.90193176269531,
      "logps/reference_rejected": -203.60231018066406,
      "loss": 0.491,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4527718722820282,
      "rewards/margins": 0.9755749106407166,
      "rewards/rejected": -0.5228029489517212,
      "step": 198,
      "tokens/chosen_response_len_mean": 225.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 217.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.16,
      "grad_norm": 2.8264702439738474,
      "learning_rate": 4.946241100155197e-07,
      "logits/policy_chosen": -2.6785213947296143,
      "logits/policy_rejected": -2.800018310546875,
      "logits/reference_chosen": -2.9413294792175293,
      "logits/reference_rejected": -2.913431406021118,
      "logps/policy_chosen": -38.42972183227539,
      "logps/policy_rejected": -164.37850952148438,
      "logps/reference_chosen": -65.64104461669922,
      "logps/reference_rejected": -128.12234497070312,
      "loss": 0.4973,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.27211323380470276,
      "rewards/margins": 0.6346749067306519,
      "rewards/rejected": -0.3625616431236267,
      "step": 199,
      "tokens/chosen_response_len_mean": 312.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 278.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.16,
      "grad_norm": 3.0030443321611644,
      "learning_rate": 4.944783747893825e-07,
      "logits/policy_chosen": -2.727503538131714,
      "logits/policy_rejected": -2.6386661529541016,
      "logits/reference_chosen": -2.9862632751464844,
      "logits/reference_rejected": -2.1630351543426514,
      "logps/policy_chosen": -84.61045837402344,
      "logps/policy_rejected": -191.14212036132812,
      "logps/reference_chosen": -125.34469604492188,
      "logps/reference_rejected": -150.1243896484375,
      "loss": 0.4714,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4073423147201538,
      "rewards/margins": 0.8175195455551147,
      "rewards/rejected": -0.41017720103263855,
      "step": 200,
      "tokens/chosen_response_len_mean": 264.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 164.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.16,
      "grad_norm": 2.7775892532799955,
      "learning_rate": 4.943307125720206e-07,
      "logits/policy_chosen": -2.4299516677856445,
      "logits/policy_rejected": -2.5677711963653564,
      "logits/reference_chosen": -2.907986640930176,
      "logits/reference_rejected": -2.6046524047851562,
      "logps/policy_chosen": -107.82955169677734,
      "logps/policy_rejected": -325.7413635253906,
      "logps/reference_chosen": -176.93951416015625,
      "logps/reference_rejected": -240.94747924804688,
      "loss": 0.425,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6910995841026306,
      "rewards/margins": 1.5390385389328003,
      "rewards/rejected": -0.8479388952255249,
      "step": 201,
      "tokens/chosen_response_len_mean": 608.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 298.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.16,
      "grad_norm": 2.8271865552266267,
      "learning_rate": 4.941811245273153e-07,
      "logits/policy_chosen": -2.6319420337677,
      "logits/policy_rejected": -2.5893003940582275,
      "logits/reference_chosen": -3.000640392303467,
      "logits/reference_rejected": -2.3574352264404297,
      "logps/policy_chosen": -64.42079162597656,
      "logps/policy_rejected": -202.20693969726562,
      "logps/reference_chosen": -102.7632064819336,
      "logps/reference_rejected": -181.3625030517578,
      "loss": 0.3885,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3834240138530731,
      "rewards/margins": 0.5918683409690857,
      "rewards/rejected": -0.20844432711601257,
      "step": 202,
      "tokens/chosen_response_len_mean": 263.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 237.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.16,
      "grad_norm": 2.804030062412289,
      "learning_rate": 4.940296118343273e-07,
      "logits/policy_chosen": -2.741957187652588,
      "logits/policy_rejected": -2.7859504222869873,
      "logits/reference_chosen": -2.956801414489746,
      "logits/reference_rejected": -2.9367804527282715,
      "logps/policy_chosen": -72.28400421142578,
      "logps/policy_rejected": -123.73040771484375,
      "logps/reference_chosen": -100.23980712890625,
      "logps/reference_rejected": -129.1441650390625,
      "loss": 0.5117,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2795579731464386,
      "rewards/margins": 0.22542047500610352,
      "rewards/rejected": 0.05413751304149628,
      "step": 203,
      "tokens/chosen_response_len_mean": 226.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 221.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.16,
      "grad_norm": 3.1378106559973666,
      "learning_rate": 4.938761756872874e-07,
      "logits/policy_chosen": -2.6754446029663086,
      "logits/policy_rejected": -2.5880160331726074,
      "logits/reference_chosen": -3.173189163208008,
      "logits/reference_rejected": -2.022594451904297,
      "logps/policy_chosen": -59.6561393737793,
      "logps/policy_rejected": -180.80789184570312,
      "logps/reference_chosen": -116.20298767089844,
      "logps/reference_rejected": -130.52818298339844,
      "loss": 0.4056,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5654685497283936,
      "rewards/margins": 1.0682657957077026,
      "rewards/rejected": -0.5027973055839539,
      "step": 204,
      "tokens/chosen_response_len_mean": 284.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 165.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.16,
      "grad_norm": 3.1932802336332298,
      "learning_rate": 4.937208172955876e-07,
      "logits/policy_chosen": -2.549724578857422,
      "logits/policy_rejected": -2.6621100902557373,
      "logits/reference_chosen": -2.8741047382354736,
      "logits/reference_rejected": -2.9152588844299316,
      "logps/policy_chosen": -46.50154113769531,
      "logps/policy_rejected": -164.41668701171875,
      "logps/reference_chosen": -80.38411712646484,
      "logps/reference_rejected": -138.61923217773438,
      "loss": 0.47,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3388257622718811,
      "rewards/margins": 0.5968001484870911,
      "rewards/rejected": -0.25797438621520996,
      "step": 205,
      "tokens/chosen_response_len_mean": 209.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 216.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.17,
      "grad_norm": 3.3325761570853016,
      "learning_rate": 4.935635378837702e-07,
      "logits/policy_chosen": -2.4038610458374023,
      "logits/policy_rejected": -2.402017831802368,
      "logits/reference_chosen": -2.797759771347046,
      "logits/reference_rejected": -2.5829825401306152,
      "logps/policy_chosen": -44.91532516479492,
      "logps/policy_rejected": -160.6805419921875,
      "logps/reference_chosen": -74.13213348388672,
      "logps/reference_rejected": -132.3360595703125,
      "loss": 0.4302,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2921680212020874,
      "rewards/margins": 0.5756129026412964,
      "rewards/rejected": -0.2834448516368866,
      "step": 206,
      "tokens/chosen_response_len_mean": 232.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 251.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.17,
      "grad_norm": 2.7368863153844414,
      "learning_rate": 4.934043386915202e-07,
      "logits/policy_chosen": -2.5967376232147217,
      "logits/policy_rejected": -2.661456823348999,
      "logits/reference_chosen": -2.750797748565674,
      "logits/reference_rejected": -2.847586154937744,
      "logps/policy_chosen": -31.504817962646484,
      "logps/policy_rejected": -81.93124389648438,
      "logps/reference_chosen": -47.813560485839844,
      "logps/reference_rejected": -76.41636657714844,
      "loss": 0.4935,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16308742761611938,
      "rewards/margins": 0.21823616325855255,
      "rewards/rejected": -0.05514874309301376,
      "step": 207,
      "tokens/chosen_response_len_mean": 135.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 144.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.17,
      "grad_norm": 3.2289580903320956,
      "learning_rate": 4.932432209736538e-07,
      "logits/policy_chosen": -2.779569149017334,
      "logits/policy_rejected": -2.719834089279175,
      "logits/reference_chosen": -3.0510177612304688,
      "logits/reference_rejected": -2.049421787261963,
      "logps/policy_chosen": -39.49280548095703,
      "logps/policy_rejected": -97.85432434082031,
      "logps/reference_chosen": -61.321197509765625,
      "logps/reference_rejected": -68.41065979003906,
      "loss": 0.4074,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21828393638134003,
      "rewards/margins": 0.5127205848693848,
      "rewards/rejected": -0.29443657398223877,
      "step": 208,
      "tokens/chosen_response_len_mean": 127.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 47.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.17,
      "grad_norm": 3.6234390581200833,
      "learning_rate": 4.93080186000109e-07,
      "logits/policy_chosen": -2.0586435794830322,
      "logits/policy_rejected": -2.2156870365142822,
      "logits/reference_chosen": -2.455575466156006,
      "logits/reference_rejected": -2.4480810165405273,
      "logps/policy_chosen": -179.0718994140625,
      "logps/policy_rejected": -132.7694854736328,
      "logps/reference_chosen": -197.98556518554688,
      "logps/reference_rejected": -119.65494537353516,
      "loss": 0.4632,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18913674354553223,
      "rewards/margins": 0.32028234004974365,
      "rewards/rejected": -0.13114558160305023,
      "step": 209,
      "tokens/chosen_response_len_mean": 428.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 281.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.17,
      "grad_norm": 3.033273237981518,
      "learning_rate": 4.92915235055936e-07,
      "logits/policy_chosen": -2.756727933883667,
      "logits/policy_rejected": -2.7047367095947266,
      "logits/reference_chosen": -3.014847755432129,
      "logits/reference_rejected": -2.7174530029296875,
      "logps/policy_chosen": -55.59083938598633,
      "logps/policy_rejected": -117.91790008544922,
      "logps/reference_chosen": -91.90667724609375,
      "logps/reference_rejected": -112.02029418945312,
      "loss": 0.4147,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.36315834522247314,
      "rewards/margins": 0.4221343994140625,
      "rewards/rejected": -0.05897604674100876,
      "step": 210,
      "tokens/chosen_response_len_mean": 255.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 197.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.17,
      "grad_norm": 2.632232149269225,
      "learning_rate": 4.927483694412868e-07,
      "logits/policy_chosen": -2.6325759887695312,
      "logits/policy_rejected": -2.4173927307128906,
      "logits/reference_chosen": -2.4960875511169434,
      "logits/reference_rejected": -2.534782648086548,
      "logps/policy_chosen": -24.59671974182129,
      "logps/policy_rejected": -98.01979064941406,
      "logps/reference_chosen": -36.76423645019531,
      "logps/reference_rejected": -77.22918701171875,
      "loss": 0.4171,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.12167516350746155,
      "rewards/margins": 0.32958126068115234,
      "rewards/rejected": -0.2079060971736908,
      "step": 211,
      "tokens/chosen_response_len_mean": 114.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 95.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.17,
      "grad_norm": 2.6677296030035,
      "learning_rate": 4.925795904714048e-07,
      "logits/policy_chosen": -2.6325550079345703,
      "logits/policy_rejected": -2.7918920516967773,
      "logits/reference_chosen": -3.0179286003112793,
      "logits/reference_rejected": -2.9879252910614014,
      "logps/policy_chosen": -68.28926849365234,
      "logps/policy_rejected": -233.2633514404297,
      "logps/reference_chosen": -110.06537628173828,
      "logps/reference_rejected": -193.7027587890625,
      "loss": 0.3524,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.41776108741760254,
      "rewards/margins": 0.8133671283721924,
      "rewards/rejected": -0.39560604095458984,
      "step": 212,
      "tokens/chosen_response_len_mean": 270.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 280.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.17,
      "grad_norm": 3.4599652097860303,
      "learning_rate": 4.924088994766144e-07,
      "logits/policy_chosen": -2.2819485664367676,
      "logits/policy_rejected": -2.3868868350982666,
      "logits/reference_chosen": -2.75758695602417,
      "logits/reference_rejected": -2.2176506519317627,
      "logps/policy_chosen": -94.18050384521484,
      "logps/policy_rejected": -299.33758544921875,
      "logps/reference_chosen": -170.72991943359375,
      "logps/reference_rejected": -266.3306579589844,
      "loss": 0.4024,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.765494167804718,
      "rewards/margins": 1.0955636501312256,
      "rewards/rejected": -0.3300694227218628,
      "step": 213,
      "tokens/chosen_response_len_mean": 548.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 511.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.17,
      "grad_norm": 2.821041192679951,
      "learning_rate": 4.922362978023111e-07,
      "logits/policy_chosen": -2.624894618988037,
      "logits/policy_rejected": -2.621846914291382,
      "logits/reference_chosen": -3.108778476715088,
      "logits/reference_rejected": -3.036085367202759,
      "logps/policy_chosen": -128.86549377441406,
      "logps/policy_rejected": -284.0617370605469,
      "logps/reference_chosen": -219.0366973876953,
      "logps/reference_rejected": -296.7759094238281,
      "loss": 0.4239,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9017120003700256,
      "rewards/margins": 0.7745702266693115,
      "rewards/rejected": 0.12714174389839172,
      "step": 214,
      "tokens/chosen_response_len_mean": 571.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 543.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.17,
      "grad_norm": 3.0002891083329857,
      "learning_rate": 4.9206178680895e-07,
      "logits/policy_chosen": -1.7852914333343506,
      "logits/policy_rejected": -1.8987267017364502,
      "logits/reference_chosen": -1.9375059604644775,
      "logits/reference_rejected": -2.050326108932495,
      "logps/policy_chosen": -52.338626861572266,
      "logps/policy_rejected": -93.70428466796875,
      "logps/reference_chosen": -71.64360046386719,
      "logps/reference_rejected": -91.83578491210938,
      "loss": 0.4021,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19304978847503662,
      "rewards/margins": 0.2117348462343216,
      "rewards/rejected": -0.01868506893515587,
      "step": 215,
      "tokens/chosen_response_len_mean": 359.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 364.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.17,
      "grad_norm": 3.157069335949432,
      "learning_rate": 4.918853678720361e-07,
      "logits/policy_chosen": -2.6262574195861816,
      "logits/policy_rejected": -2.659903049468994,
      "logits/reference_chosen": -2.9374420642852783,
      "logits/reference_rejected": -2.76753568649292,
      "logps/policy_chosen": -38.7560920715332,
      "logps/policy_rejected": -142.21218872070312,
      "logps/reference_chosen": -69.5650634765625,
      "logps/reference_rejected": -97.97706604003906,
      "loss": 0.5087,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3080897629261017,
      "rewards/margins": 0.7504409551620483,
      "rewards/rejected": -0.44235122203826904,
      "step": 216,
      "tokens/chosen_response_len_mean": 248.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 202.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.17,
      "grad_norm": 2.8169264822166338,
      "learning_rate": 4.917070423821125e-07,
      "logits/policy_chosen": -2.4803824424743652,
      "logits/policy_rejected": -2.511946439743042,
      "logits/reference_chosen": -2.749699592590332,
      "logits/reference_rejected": -2.704967975616455,
      "logps/policy_chosen": -83.62620544433594,
      "logps/policy_rejected": -296.3905029296875,
      "logps/reference_chosen": -138.813720703125,
      "logps/reference_rejected": -214.48797607421875,
      "loss": 0.3658,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5518751740455627,
      "rewards/margins": 1.3709005117416382,
      "rewards/rejected": -0.8190253376960754,
      "step": 217,
      "tokens/chosen_response_len_mean": 493.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 406.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.18,
      "grad_norm": 2.7162895030534195,
      "learning_rate": 4.915268117447497e-07,
      "logits/policy_chosen": -2.778090476989746,
      "logits/policy_rejected": -2.7868547439575195,
      "logits/reference_chosen": -3.147437572479248,
      "logits/reference_rejected": -2.9738471508026123,
      "logps/policy_chosen": -58.98277282714844,
      "logps/policy_rejected": -193.54931640625,
      "logps/reference_chosen": -103.57978057861328,
      "logps/reference_rejected": -130.34754943847656,
      "loss": 0.4043,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4459700584411621,
      "rewards/margins": 1.077987790107727,
      "rewards/rejected": -0.6320176720619202,
      "step": 218,
      "tokens/chosen_response_len_mean": 233.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 185.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.18,
      "grad_norm": 3.6458168828683926,
      "learning_rate": 4.913446773805354e-07,
      "logits/policy_chosen": -2.333178997039795,
      "logits/policy_rejected": -2.3620800971984863,
      "logits/reference_chosen": -3.10478138923645,
      "logits/reference_rejected": -2.8959479331970215,
      "logps/policy_chosen": -90.79602813720703,
      "logps/policy_rejected": -286.6520690917969,
      "logps/reference_chosen": -163.10195922851562,
      "logps/reference_rejected": -245.46189880371094,
      "loss": 0.4028,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.723059356212616,
      "rewards/margins": 1.1349611282348633,
      "rewards/rejected": -0.4119018018245697,
      "step": 219,
      "tokens/chosen_response_len_mean": 469.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 442.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.18,
      "grad_norm": 3.233765475571934,
      "learning_rate": 4.911606407250617e-07,
      "logits/policy_chosen": -2.2644901275634766,
      "logits/policy_rejected": -2.4729533195495605,
      "logits/reference_chosen": -2.8000075817108154,
      "logits/reference_rejected": -2.8166446685791016,
      "logps/policy_chosen": -30.913768768310547,
      "logps/policy_rejected": -142.3131561279297,
      "logps/reference_chosen": -51.407657623291016,
      "logps/reference_rejected": -95.70589447021484,
      "loss": 0.4707,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2049388736486435,
      "rewards/margins": 0.6710115671157837,
      "rewards/rejected": -0.4660727083683014,
      "step": 220,
      "tokens/chosen_response_len_mean": 272.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 233.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.18,
      "grad_norm": 2.816944510019336,
      "learning_rate": 4.909747032289152e-07,
      "logits/policy_chosen": -1.9074110984802246,
      "logits/policy_rejected": -2.0913877487182617,
      "logits/reference_chosen": -2.375129461288452,
      "logits/reference_rejected": -2.261173725128174,
      "logps/policy_chosen": -131.19876098632812,
      "logps/policy_rejected": -256.4100646972656,
      "logps/reference_chosen": -193.19882202148438,
      "logps/reference_rejected": -205.95828247070312,
      "loss": 0.4362,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6200005412101746,
      "rewards/margins": 1.1245182752609253,
      "rewards/rejected": -0.5045177936553955,
      "step": 221,
      "tokens/chosen_response_len_mean": 388.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 280.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.18,
      "grad_norm": 3.102088864340547,
      "learning_rate": 4.90786866357665e-07,
      "logits/policy_chosen": -2.299764633178711,
      "logits/policy_rejected": -2.4341142177581787,
      "logits/reference_chosen": -3.09466290473938,
      "logits/reference_rejected": -2.8344757556915283,
      "logps/policy_chosen": -104.12731170654297,
      "logps/policy_rejected": -403.4098205566406,
      "logps/reference_chosen": -205.33453369140625,
      "logps/reference_rejected": -297.8888854980469,
      "loss": 0.4303,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0120720863342285,
      "rewards/margins": 2.067281484603882,
      "rewards/rejected": -1.0552091598510742,
      "step": 222,
      "tokens/chosen_response_len_mean": 487.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 350.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.18,
      "grad_norm": 2.6980157978791506,
      "learning_rate": 4.905971315918508e-07,
      "logits/policy_chosen": -2.2736315727233887,
      "logits/policy_rejected": -2.342026472091675,
      "logits/reference_chosen": -2.8474254608154297,
      "logits/reference_rejected": -2.5709283351898193,
      "logps/policy_chosen": -85.25898742675781,
      "logps/policy_rejected": -367.0793151855469,
      "logps/reference_chosen": -168.6109161376953,
      "logps/reference_rejected": -258.70306396484375,
      "loss": 0.3805,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8335192203521729,
      "rewards/margins": 1.9172817468643188,
      "rewards/rejected": -1.0837624073028564,
      "step": 223,
      "tokens/chosen_response_len_mean": 459.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 381.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.18,
      "grad_norm": 3.023484025148734,
      "learning_rate": 4.904055004269721e-07,
      "logits/policy_chosen": -2.454890251159668,
      "logits/policy_rejected": -2.5622761249542236,
      "logits/reference_chosen": -2.9128506183624268,
      "logits/reference_rejected": -2.8558335304260254,
      "logps/policy_chosen": -36.11943435668945,
      "logps/policy_rejected": -159.4558563232422,
      "logps/reference_chosen": -75.32926177978516,
      "logps/reference_rejected": -124.79375457763672,
      "loss": 0.3956,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.39209815859794617,
      "rewards/margins": 0.7387192845344543,
      "rewards/rejected": -0.3466210961341858,
      "step": 224,
      "tokens/chosen_response_len_mean": 181.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 169.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.18,
      "grad_norm": 2.879025667733998,
      "learning_rate": 4.902119743734755e-07,
      "logits/policy_chosen": -2.664403200149536,
      "logits/policy_rejected": -2.587775230407715,
      "logits/reference_chosen": -3.1453912258148193,
      "logits/reference_rejected": -2.816845417022705,
      "logps/policy_chosen": -49.29490661621094,
      "logps/policy_rejected": -94.93446350097656,
      "logps/reference_chosen": -71.66400146484375,
      "logps/reference_rejected": -87.74375915527344,
      "loss": 0.3954,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.22369101643562317,
      "rewards/margins": 0.2955980896949768,
      "rewards/rejected": -0.07190708816051483,
      "step": 225,
      "tokens/chosen_response_len_mean": 198.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 179.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.18,
      "grad_norm": 3.246164993624403,
      "learning_rate": 4.900165549567435e-07,
      "logits/policy_chosen": -2.3957905769348145,
      "logits/policy_rejected": -2.423283100128174,
      "logits/reference_chosen": -2.991285800933838,
      "logits/reference_rejected": -2.787295341491699,
      "logps/policy_chosen": -88.65385437011719,
      "logps/policy_rejected": -120.2602767944336,
      "logps/reference_chosen": -112.840087890625,
      "logps/reference_rejected": -75.72503662109375,
      "loss": 0.3168,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.24186235666275024,
      "rewards/margins": 0.6872147917747498,
      "rewards/rejected": -0.44535237550735474,
      "step": 226,
      "tokens/chosen_response_len_mean": 422.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 400.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.18,
      "grad_norm": 3.0799697537609085,
      "learning_rate": 4.898192437170821e-07,
      "logits/policy_chosen": -2.530318021774292,
      "logits/policy_rejected": -2.645883083343506,
      "logits/reference_chosen": -3.1499884128570557,
      "logits/reference_rejected": -2.5675787925720215,
      "logps/policy_chosen": -54.94190216064453,
      "logps/policy_rejected": -261.4326171875,
      "logps/reference_chosen": -134.1338348388672,
      "logps/reference_rejected": -190.62583923339844,
      "loss": 0.3325,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7919193506240845,
      "rewards/margins": 1.4999868869781494,
      "rewards/rejected": -0.7080676555633545,
      "step": 227,
      "tokens/chosen_response_len_mean": 255.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 204.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.18,
      "grad_norm": 2.659841913817482,
      "learning_rate": 4.896200422097086e-07,
      "logits/policy_chosen": -1.9145750999450684,
      "logits/policy_rejected": -2.0923149585723877,
      "logits/reference_chosen": -2.8488292694091797,
      "logits/reference_rejected": -2.7205710411071777,
      "logps/policy_chosen": -107.40567016601562,
      "logps/policy_rejected": -427.5948486328125,
      "logps/reference_chosen": -223.86032104492188,
      "logps/reference_rejected": -329.6167297363281,
      "loss": 0.1916,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1645466089248657,
      "rewards/margins": 2.1443278789520264,
      "rewards/rejected": -0.9797812700271606,
      "step": 228,
      "tokens/chosen_response_len_mean": 517.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 439.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.18,
      "grad_norm": 3.234230812286525,
      "learning_rate": 4.894189520047397e-07,
      "logits/policy_chosen": -2.2693915367126465,
      "logits/policy_rejected": -2.3958592414855957,
      "logits/reference_chosen": -3.0886495113372803,
      "logits/reference_rejected": -2.8636574745178223,
      "logps/policy_chosen": -51.258445739746094,
      "logps/policy_rejected": -378.75201416015625,
      "logps/reference_chosen": -118.88182830810547,
      "logps/reference_rejected": -246.41397094726562,
      "loss": 0.3282,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6762338280677795,
      "rewards/margins": 1.9996143579483032,
      "rewards/rejected": -1.3233805894851685,
      "step": 229,
      "tokens/chosen_response_len_mean": 271.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 266.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.18,
      "grad_norm": 3.457038560417765,
      "learning_rate": 4.892159746871788e-07,
      "logits/policy_chosen": -2.3888025283813477,
      "logits/policy_rejected": -2.411865234375,
      "logits/reference_chosen": -3.154548168182373,
      "logits/reference_rejected": -2.1464414596557617,
      "logps/policy_chosen": -57.5999870300293,
      "logps/policy_rejected": -173.8509063720703,
      "logps/reference_chosen": -131.91171264648438,
      "logps/reference_rejected": -113.2002182006836,
      "loss": 0.3683,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7431172728538513,
      "rewards/margins": 1.3496241569519043,
      "rewards/rejected": -0.6065068244934082,
      "step": 230,
      "tokens/chosen_response_len_mean": 285.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 94.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.19,
      "grad_norm": 3.453784172007788,
      "learning_rate": 4.890111118569036e-07,
      "logits/policy_chosen": -2.0798964500427246,
      "logits/policy_rejected": -2.2078676223754883,
      "logits/reference_chosen": -2.8997390270233154,
      "logits/reference_rejected": -2.4869751930236816,
      "logps/policy_chosen": -75.02999877929688,
      "logps/policy_rejected": -229.92823791503906,
      "logps/reference_chosen": -143.95648193359375,
      "logps/reference_rejected": -181.41172790527344,
      "loss": 0.327,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6892647743225098,
      "rewards/margins": 1.1744297742843628,
      "rewards/rejected": -0.485164999961853,
      "step": 231,
      "tokens/chosen_response_len_mean": 280.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 238.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.19,
      "grad_norm": 3.239627424882494,
      "learning_rate": 4.888043651286537e-07,
      "logits/policy_chosen": -2.5193958282470703,
      "logits/policy_rejected": -2.452767848968506,
      "logits/reference_chosen": -3.070887565612793,
      "logits/reference_rejected": -2.4067721366882324,
      "logps/policy_chosen": -44.861671447753906,
      "logps/policy_rejected": -245.7529296875,
      "logps/reference_chosen": -78.50507354736328,
      "logps/reference_rejected": -147.85055541992188,
      "loss": 0.3734,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.336434006690979,
      "rewards/margins": 1.315457820892334,
      "rewards/rejected": -0.979023814201355,
      "step": 232,
      "tokens/chosen_response_len_mean": 251.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 159.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.19,
      "grad_norm": 3.085937654519379,
      "learning_rate": 4.885957361320172e-07,
      "logits/policy_chosen": -1.9149789810180664,
      "logits/policy_rejected": -2.088989734649658,
      "logits/reference_chosen": -2.9695346355438232,
      "logits/reference_rejected": -2.6794815063476562,
      "logps/policy_chosen": -73.91265869140625,
      "logps/policy_rejected": -403.6729431152344,
      "logps/reference_chosen": -209.8995819091797,
      "logps/reference_rejected": -327.79486083984375,
      "loss": 0.2662,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.359869360923767,
      "rewards/margins": 2.118650197982788,
      "rewards/rejected": -0.7587809562683105,
      "step": 233,
      "tokens/chosen_response_len_mean": 403.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 360.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.19,
      "grad_norm": 2.7768750875070762,
      "learning_rate": 4.88385226511419e-07,
      "logits/policy_chosen": -1.917398452758789,
      "logits/policy_rejected": -2.089282989501953,
      "logits/reference_chosen": -3.025069236755371,
      "logits/reference_rejected": -2.1790575981140137,
      "logps/policy_chosen": -98.6364974975586,
      "logps/policy_rejected": -710.46142578125,
      "logps/reference_chosen": -247.2069854736328,
      "logps/reference_rejected": -409.91046142578125,
      "loss": 0.2487,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.485705018043518,
      "rewards/margins": 4.491214752197266,
      "rewards/rejected": -3.005509853363037,
      "step": 234,
      "tokens/chosen_response_len_mean": 516.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 393.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.19,
      "grad_norm": 2.6357031102373694,
      "learning_rate": 4.881728379261067e-07,
      "logits/policy_chosen": -2.5067009925842285,
      "logits/policy_rejected": -2.6151890754699707,
      "logits/reference_chosen": -2.879830837249756,
      "logits/reference_rejected": -2.289306879043579,
      "logps/policy_chosen": -28.403209686279297,
      "logps/policy_rejected": -94.16014099121094,
      "logps/reference_chosen": -50.78627014160156,
      "logps/reference_rejected": -61.18941116333008,
      "loss": 0.3383,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.22383059561252594,
      "rewards/margins": 0.553537905216217,
      "rewards/rejected": -0.3297072649002075,
      "step": 235,
      "tokens/chosen_response_len_mean": 130.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 70.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.19,
      "grad_norm": 2.5416162412883967,
      "learning_rate": 4.879585720501382e-07,
      "logits/policy_chosen": -2.079270362854004,
      "logits/policy_rejected": -2.1965620517730713,
      "logits/reference_chosen": -2.69492769241333,
      "logits/reference_rejected": -2.7009551525115967,
      "logps/policy_chosen": -47.36531448364258,
      "logps/policy_rejected": -210.86045837402344,
      "logps/reference_chosen": -109.4851303100586,
      "logps/reference_rejected": -157.570556640625,
      "loss": 0.4301,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6211981177330017,
      "rewards/margins": 1.1540971994400024,
      "rewards/rejected": -0.5328990817070007,
      "step": 236,
      "tokens/chosen_response_len_mean": 382.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 342.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.19,
      "grad_norm": 3.3640626360746606,
      "learning_rate": 4.877424305723682e-07,
      "logits/policy_chosen": -2.3050661087036133,
      "logits/policy_rejected": -2.362180709838867,
      "logits/reference_chosen": -2.8899943828582764,
      "logits/reference_rejected": -2.970350503921509,
      "logps/policy_chosen": -58.130821228027344,
      "logps/policy_rejected": -187.88409423828125,
      "logps/reference_chosen": -105.74044036865234,
      "logps/reference_rejected": -162.71900939941406,
      "loss": 0.3679,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4760962128639221,
      "rewards/margins": 0.7277471423149109,
      "rewards/rejected": -0.2516508996486664,
      "step": 237,
      "tokens/chosen_response_len_mean": 321.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 332.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.19,
      "grad_norm": 3.092634828507096,
      "learning_rate": 4.87524415196435e-07,
      "logits/policy_chosen": -1.7353723049163818,
      "logits/policy_rejected": -1.8133890628814697,
      "logits/reference_chosen": -2.299503803253174,
      "logits/reference_rejected": -1.7435356378555298,
      "logps/policy_chosen": -25.773319244384766,
      "logps/policy_rejected": -84.91059875488281,
      "logps/reference_chosen": -49.502906799316406,
      "logps/reference_rejected": -59.85688781738281,
      "loss": 0.4402,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.23729589581489563,
      "rewards/margins": 0.4878329634666443,
      "rewards/rejected": -0.25053703784942627,
      "step": 238,
      "tokens/chosen_response_len_mean": 180.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 131.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.19,
      "grad_norm": 2.30111282627728,
      "learning_rate": 4.873045276407472e-07,
      "logits/policy_chosen": -2.3006367683410645,
      "logits/policy_rejected": -2.373878240585327,
      "logits/reference_chosen": -3.0563242435455322,
      "logits/reference_rejected": -2.862274408340454,
      "logps/policy_chosen": -60.457550048828125,
      "logps/policy_rejected": -362.2609558105469,
      "logps/reference_chosen": -138.99404907226562,
      "logps/reference_rejected": -211.34681701660156,
      "loss": 0.263,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7853649258613586,
      "rewards/margins": 2.294506072998047,
      "rewards/rejected": -1.5091413259506226,
      "step": 239,
      "tokens/chosen_response_len_mean": 323.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 253.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.19,
      "grad_norm": 3.3647443577329184,
      "learning_rate": 4.870827696384697e-07,
      "logits/policy_chosen": -1.7883644104003906,
      "logits/policy_rejected": -1.9111793041229248,
      "logits/reference_chosen": -2.7533793449401855,
      "logits/reference_rejected": -2.8058459758758545,
      "logps/policy_chosen": -63.6192512512207,
      "logps/policy_rejected": -472.13812255859375,
      "logps/reference_chosen": -182.55494689941406,
      "logps/reference_rejected": -324.2348327636719,
      "loss": 0.4491,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.189356803894043,
      "rewards/margins": 2.6683895587921143,
      "rewards/rejected": -1.4790325164794922,
      "step": 240,
      "tokens/chosen_response_len_mean": 425.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 412.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.19,
      "grad_norm": 2.8850200026957507,
      "learning_rate": 4.868591429375109e-07,
      "logits/policy_chosen": -2.5384299755096436,
      "logits/policy_rejected": -2.5679569244384766,
      "logits/reference_chosen": -3.137737274169922,
      "logits/reference_rejected": -2.9568300247192383,
      "logps/policy_chosen": -52.663124084472656,
      "logps/policy_rejected": -150.413818359375,
      "logps/reference_chosen": -119.31211853027344,
      "logps/reference_rejected": -144.43154907226562,
      "loss": 0.3981,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6664900183677673,
      "rewards/margins": 0.7263126373291016,
      "rewards/rejected": -0.059822604060173035,
      "step": 241,
      "tokens/chosen_response_len_mean": 247.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 232.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.19,
      "grad_norm": 2.740311113779816,
      "learning_rate": 4.866336493005077e-07,
      "logits/policy_chosen": -2.3799917697906494,
      "logits/policy_rejected": -2.3980116844177246,
      "logits/reference_chosen": -2.8105556964874268,
      "logits/reference_rejected": -2.789691925048828,
      "logps/policy_chosen": -86.99842834472656,
      "logps/policy_rejected": -245.59523010253906,
      "logps/reference_chosen": -164.35971069335938,
      "logps/reference_rejected": -204.79449462890625,
      "loss": 0.3237,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7736127376556396,
      "rewards/margins": 1.1816203594207764,
      "rewards/rejected": -0.4080074727535248,
      "step": 242,
      "tokens/chosen_response_len_mean": 234.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 228.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.2,
      "grad_norm": 3.5286650522231846,
      "learning_rate": 4.86406290504813e-07,
      "logits/policy_chosen": -2.302743434906006,
      "logits/policy_rejected": -2.3800129890441895,
      "logits/reference_chosen": -3.100501537322998,
      "logits/reference_rejected": -2.502018690109253,
      "logps/policy_chosen": -56.89078140258789,
      "logps/policy_rejected": -293.7001953125,
      "logps/reference_chosen": -124.03472900390625,
      "logps/reference_rejected": -186.7720489501953,
      "loss": 0.262,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.671439528465271,
      "rewards/margins": 1.7407207489013672,
      "rewards/rejected": -1.0692812204360962,
      "step": 243,
      "tokens/chosen_response_len_mean": 347.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 276.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.2,
      "grad_norm": 3.590713146645003,
      "learning_rate": 4.861770683424803e-07,
      "logits/policy_chosen": -2.072890043258667,
      "logits/policy_rejected": -1.9427573680877686,
      "logits/reference_chosen": -2.6935362815856934,
      "logits/reference_rejected": -2.673877000808716,
      "logps/policy_chosen": -45.185890197753906,
      "logps/policy_rejected": -246.34381103515625,
      "logps/reference_chosen": -113.1274642944336,
      "logps/reference_rejected": -189.79290771484375,
      "loss": 0.365,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6794158220291138,
      "rewards/margins": 1.2449246644973755,
      "rewards/rejected": -0.5655089616775513,
      "step": 244,
      "tokens/chosen_response_len_mean": 282.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 369.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.2,
      "grad_norm": 2.610774413059978,
      "learning_rate": 4.859459846202507e-07,
      "logits/policy_chosen": -2.30108380317688,
      "logits/policy_rejected": -2.4250240325927734,
      "logits/reference_chosen": -3.1003952026367188,
      "logits/reference_rejected": -2.8910207748413086,
      "logps/policy_chosen": -113.18220520019531,
      "logps/policy_rejected": -571.716064453125,
      "logps/reference_chosen": -273.4241943359375,
      "logps/reference_rejected": -353.7633056640625,
      "loss": 0.3468,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.6024198532104492,
      "rewards/margins": 3.781947135925293,
      "rewards/rejected": -2.1795272827148438,
      "step": 245,
      "tokens/chosen_response_len_mean": 495.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 352.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.2,
      "grad_norm": 3.1327030609878674,
      "learning_rate": 4.857130411595381e-07,
      "logits/policy_chosen": -1.9849697351455688,
      "logits/policy_rejected": -2.2542386054992676,
      "logits/reference_chosen": -2.8174877166748047,
      "logits/reference_rejected": -2.6481382846832275,
      "logps/policy_chosen": -65.47968292236328,
      "logps/policy_rejected": -309.6473083496094,
      "logps/reference_chosen": -125.56417846679688,
      "logps/reference_rejected": -215.66690063476562,
      "loss": 0.2941,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6008449792861938,
      "rewards/margins": 1.5406490564346313,
      "rewards/rejected": -0.9398040771484375,
      "step": 246,
      "tokens/chosen_response_len_mean": 314.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 219.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.2,
      "grad_norm": 2.8949654648587106,
      "learning_rate": 4.854782397964148e-07,
      "logits/policy_chosen": -2.4038028717041016,
      "logits/policy_rejected": -2.535879373550415,
      "logits/reference_chosen": -3.110623836517334,
      "logits/reference_rejected": -2.892681360244751,
      "logps/policy_chosen": -42.45046615600586,
      "logps/policy_rejected": -223.6653594970703,
      "logps/reference_chosen": -108.52606201171875,
      "logps/reference_rejected": -122.96241760253906,
      "loss": 0.1793,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6607558727264404,
      "rewards/margins": 1.667785406112671,
      "rewards/rejected": -1.0070295333862305,
      "step": 247,
      "tokens/chosen_response_len_mean": 224.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 148.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.2,
      "grad_norm": 3.2467096251740397,
      "learning_rate": 4.852415823815975e-07,
      "logits/policy_chosen": -2.0340566635131836,
      "logits/policy_rejected": -2.007965087890625,
      "logits/reference_chosen": -2.2954137325286865,
      "logits/reference_rejected": -2.1136422157287598,
      "logps/policy_chosen": -48.099082946777344,
      "logps/policy_rejected": -218.53590393066406,
      "logps/reference_chosen": -83.52349090576172,
      "logps/reference_rejected": -118.22250366210938,
      "loss": 0.3545,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3542441725730896,
      "rewards/margins": 1.3573782444000244,
      "rewards/rejected": -1.0031341314315796,
      "step": 248,
      "tokens/chosen_response_len_mean": 159.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 128.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.2,
      "grad_norm": 2.4511654553576294,
      "learning_rate": 4.850030707804322e-07,
      "logits/policy_chosen": -2.421999454498291,
      "logits/policy_rejected": -2.4670658111572266,
      "logits/reference_chosen": -3.0133814811706543,
      "logits/reference_rejected": -2.3799855709075928,
      "logps/policy_chosen": -53.002376556396484,
      "logps/policy_rejected": -286.0945739746094,
      "logps/reference_chosen": -161.1949462890625,
      "logps/reference_rejected": -157.0173797607422,
      "loss": 0.3471,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.081925630569458,
      "rewards/margins": 2.372697353363037,
      "rewards/rejected": -1.2907719612121582,
      "step": 249,
      "tokens/chosen_response_len_mean": 254.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 189.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.2,
      "grad_norm": 3.2467420827995683,
      "learning_rate": 4.847627068728794e-07,
      "logits/policy_chosen": -2.1038780212402344,
      "logits/policy_rejected": -2.1949315071105957,
      "logits/reference_chosen": -3.1234359741210938,
      "logits/reference_rejected": -2.4281110763549805,
      "logps/policy_chosen": -33.008514404296875,
      "logps/policy_rejected": -167.2099151611328,
      "logps/reference_chosen": -75.09925842285156,
      "logps/reference_rejected": -86.3980484008789,
      "loss": 0.3442,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4209074079990387,
      "rewards/margins": 1.2290260791778564,
      "rewards/rejected": -0.8081187009811401,
      "step": 250,
      "tokens/chosen_response_len_mean": 105.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 71.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.2,
      "grad_norm": 2.2929712454735145,
      "learning_rate": 4.845204925535003e-07,
      "logits/policy_chosen": -1.7899479866027832,
      "logits/policy_rejected": -1.9353580474853516,
      "logits/reference_chosen": -2.701301336288452,
      "logits/reference_rejected": -2.7325639724731445,
      "logps/policy_chosen": -50.73257827758789,
      "logps/policy_rejected": -341.6130676269531,
      "logps/reference_chosen": -127.89730834960938,
      "logps/reference_rejected": -177.73052978515625,
      "loss": 0.1825,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7716473340988159,
      "rewards/margins": 2.4104726314544678,
      "rewards/rejected": -1.638825535774231,
      "step": 251,
      "tokens/chosen_response_len_mean": 542.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 416.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.2,
      "grad_norm": 2.718487080105349,
      "learning_rate": 4.842764297314406e-07,
      "logits/policy_chosen": -1.9718434810638428,
      "logits/policy_rejected": -2.002476692199707,
      "logits/reference_chosen": -2.571845293045044,
      "logits/reference_rejected": -2.6133267879486084,
      "logps/policy_chosen": -41.810699462890625,
      "logps/policy_rejected": -177.89076232910156,
      "logps/reference_chosen": -95.01820373535156,
      "logps/reference_rejected": -145.2048797607422,
      "loss": 0.2304,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5320751070976257,
      "rewards/margins": 0.8589338064193726,
      "rewards/rejected": -0.3268587589263916,
      "step": 252,
      "tokens/chosen_response_len_mean": 206.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 208.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.2,
      "grad_norm": 3.7180834260951325,
      "learning_rate": 4.840305203304159e-07,
      "logits/policy_chosen": -2.030735969543457,
      "logits/policy_rejected": -2.1171207427978516,
      "logits/reference_chosen": -2.9690611362457275,
      "logits/reference_rejected": -2.7168192863464355,
      "logps/policy_chosen": -41.81312561035156,
      "logps/policy_rejected": -201.77508544921875,
      "logps/reference_chosen": -100.09971618652344,
      "logps/reference_rejected": -119.85955047607422,
      "loss": 0.3607,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5828658938407898,
      "rewards/margins": 1.4020211696624756,
      "rewards/rejected": -0.8191554546356201,
      "step": 253,
      "tokens/chosen_response_len_mean": 273.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 156.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.2,
      "grad_norm": 2.6023063627206815,
      "learning_rate": 4.837827662886971e-07,
      "logits/policy_chosen": -2.1987662315368652,
      "logits/policy_rejected": -2.3438453674316406,
      "logits/reference_chosen": -3.212616443634033,
      "logits/reference_rejected": -3.013723134994507,
      "logps/policy_chosen": -59.346351623535156,
      "logps/policy_rejected": -576.636962890625,
      "logps/reference_chosen": -173.44329833984375,
      "logps/reference_rejected": -289.7512512207031,
      "loss": 0.2857,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1409695148468018,
      "rewards/margins": 4.009826183319092,
      "rewards/rejected": -2.86885666847229,
      "step": 254,
      "tokens/chosen_response_len_mean": 327.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 266.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.2,
      "grad_norm": 2.968765579985845,
      "learning_rate": 4.835331695590943e-07,
      "logits/policy_chosen": -2.362076759338379,
      "logits/policy_rejected": -2.3499341011047363,
      "logits/reference_chosen": -3.076436996459961,
      "logits/reference_rejected": -2.8915157318115234,
      "logps/policy_chosen": -24.552675247192383,
      "logps/policy_rejected": -53.35011291503906,
      "logps/reference_chosen": -56.392234802246094,
      "logps/reference_rejected": -45.72980499267578,
      "loss": 0.3399,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.31839558482170105,
      "rewards/margins": 0.39459866285324097,
      "rewards/rejected": -0.07620306313037872,
      "step": 255,
      "tokens/chosen_response_len_mean": 143.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 110.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.21,
      "grad_norm": 2.8370570975666873,
      "learning_rate": 4.832817321089418e-07,
      "logits/policy_chosen": -2.4410159587860107,
      "logits/policy_rejected": -2.4411826133728027,
      "logits/reference_chosen": -2.7271342277526855,
      "logits/reference_rejected": -2.732917308807373,
      "logps/policy_chosen": -31.23749351501465,
      "logps/policy_rejected": -98.78245544433594,
      "logps/reference_chosen": -43.81713104248047,
      "logps/reference_rejected": -72.64688873291016,
      "loss": 0.4067,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1257963329553604,
      "rewards/margins": 0.3871520459651947,
      "rewards/rejected": -0.2613556981086731,
      "step": 256,
      "tokens/chosen_response_len_mean": 45.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 46.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.21,
      "grad_norm": 2.7797560376214,
      "learning_rate": 4.830284559200827e-07,
      "logits/policy_chosen": -1.952653408050537,
      "logits/policy_rejected": -2.1609013080596924,
      "logits/reference_chosen": -3.012723922729492,
      "logits/reference_rejected": -2.739423990249634,
      "logps/policy_chosen": -57.349365234375,
      "logps/policy_rejected": -257.4586486816406,
      "logps/reference_chosen": -145.12765502929688,
      "logps/reference_rejected": -142.13612365722656,
      "loss": 0.2769,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8777830004692078,
      "rewards/margins": 2.03100848197937,
      "rewards/rejected": -1.153225302696228,
      "step": 257,
      "tokens/chosen_response_len_mean": 332.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 215.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.21,
      "grad_norm": 2.6056834165338865,
      "learning_rate": 4.827733429888531e-07,
      "logits/policy_chosen": -2.5333569049835205,
      "logits/policy_rejected": -2.632535934448242,
      "logits/reference_chosen": -3.0343687534332275,
      "logits/reference_rejected": -2.7431583404541016,
      "logps/policy_chosen": -51.503787994384766,
      "logps/policy_rejected": -205.5330810546875,
      "logps/reference_chosen": -112.60284423828125,
      "logps/reference_rejected": -109.25039672851562,
      "loss": 0.2703,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6109905242919922,
      "rewards/margins": 1.573817253112793,
      "rewards/rejected": -0.962826669216156,
      "step": 258,
      "tokens/chosen_response_len_mean": 209.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 112.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.21,
      "grad_norm": 2.7017642438408416,
      "learning_rate": 4.825163953260663e-07,
      "logits/policy_chosen": -2.209869146347046,
      "logits/policy_rejected": -2.233635902404785,
      "logits/reference_chosen": -3.0763533115386963,
      "logits/reference_rejected": -2.9993386268615723,
      "logps/policy_chosen": -72.18954467773438,
      "logps/policy_rejected": -188.30348205566406,
      "logps/reference_chosen": -163.45166015625,
      "logps/reference_rejected": -172.4595947265625,
      "loss": 0.2949,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9126211404800415,
      "rewards/margins": 1.0710599422454834,
      "rewards/rejected": -0.1584388017654419,
      "step": 259,
      "tokens/chosen_response_len_mean": 309.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 269.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.21,
      "grad_norm": 2.7221262346156205,
      "learning_rate": 4.82257614956997e-07,
      "logits/policy_chosen": -1.0971046686172485,
      "logits/policy_rejected": -1.240858793258667,
      "logits/reference_chosen": -2.244776964187622,
      "logits/reference_rejected": -1.5502854585647583,
      "logps/policy_chosen": -91.92208862304688,
      "logps/policy_rejected": -343.6435241699219,
      "logps/reference_chosen": -204.17926025390625,
      "logps/reference_rejected": -210.630615234375,
      "loss": 0.2794,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.122571587562561,
      "rewards/margins": 2.452700614929199,
      "rewards/rejected": -1.3301290273666382,
      "step": 260,
      "tokens/chosen_response_len_mean": 590.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 508.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.21,
      "grad_norm": 3.8028582614963775,
      "learning_rate": 4.819970039213655e-07,
      "logits/policy_chosen": -2.1693973541259766,
      "logits/policy_rejected": -2.2979447841644287,
      "logits/reference_chosen": -3.102466106414795,
      "logits/reference_rejected": -2.547724723815918,
      "logps/policy_chosen": -59.068519592285156,
      "logps/policy_rejected": -445.1409606933594,
      "logps/reference_chosen": -156.0995635986328,
      "logps/reference_rejected": -305.90802001953125,
      "loss": 0.3756,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9703104496002197,
      "rewards/margins": 2.362639904022217,
      "rewards/rejected": -1.3923293352127075,
      "step": 261,
      "tokens/chosen_response_len_mean": 229.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 209.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.21,
      "grad_norm": 3.239958730128674,
      "learning_rate": 4.817345642733218e-07,
      "logits/policy_chosen": -2.0956246852874756,
      "logits/policy_rejected": -2.182451009750366,
      "logits/reference_chosen": -3.003354072570801,
      "logits/reference_rejected": -2.506141424179077,
      "logps/policy_chosen": -42.89665603637695,
      "logps/policy_rejected": -128.68064880371094,
      "logps/reference_chosen": -79.88959503173828,
      "logps/reference_rejected": -72.20051574707031,
      "loss": 0.2773,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.36992937326431274,
      "rewards/margins": 0.9347306489944458,
      "rewards/rejected": -0.5648012757301331,
      "step": 262,
      "tokens/chosen_response_len_mean": 154.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 111.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.21,
      "grad_norm": 2.7801336555461136,
      "learning_rate": 4.814702980814284e-07,
      "logits/policy_chosen": -2.0152645111083984,
      "logits/policy_rejected": -2.148892879486084,
      "logits/reference_chosen": -3.0190117359161377,
      "logits/reference_rejected": -2.243680238723755,
      "logps/policy_chosen": -57.687110900878906,
      "logps/policy_rejected": -464.7051696777344,
      "logps/reference_chosen": -180.52468872070312,
      "logps/reference_rejected": -230.62245178222656,
      "loss": 0.1516,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2283756732940674,
      "rewards/margins": 3.5692026615142822,
      "rewards/rejected": -2.340827226638794,
      "step": 263,
      "tokens/chosen_response_len_mean": 365.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 254.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.21,
      "grad_norm": 2.66395215116379,
      "learning_rate": 4.812042074286453e-07,
      "logits/policy_chosen": -2.0517807006835938,
      "logits/policy_rejected": -2.1628949642181396,
      "logits/reference_chosen": -3.143606662750244,
      "logits/reference_rejected": -2.7492570877075195,
      "logps/policy_chosen": -114.81196594238281,
      "logps/policy_rejected": -593.2320556640625,
      "logps/reference_chosen": -350.7387390136719,
      "logps/reference_rejected": -264.07171630859375,
      "loss": 0.3429,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3592677116394043,
      "rewards/margins": 5.650871276855469,
      "rewards/rejected": -3.2916030883789062,
      "step": 264,
      "tokens/chosen_response_len_mean": 589.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 265.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.21,
      "grad_norm": 2.579532955538129,
      "learning_rate": 4.809362944123128e-07,
      "logits/policy_chosen": -2.0034396648406982,
      "logits/policy_rejected": -2.1076416969299316,
      "logits/reference_chosen": -2.9764785766601562,
      "logits/reference_rejected": -2.718217611312866,
      "logps/policy_chosen": -56.3231086730957,
      "logps/policy_rejected": -294.1711730957031,
      "logps/reference_chosen": -122.71052551269531,
      "logps/reference_rejected": -174.21401977539062,
      "loss": 0.3059,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6638741493225098,
      "rewards/margins": 1.8634452819824219,
      "rewards/rejected": -1.1995712518692017,
      "step": 265,
      "tokens/chosen_response_len_mean": 196.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 169.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.21,
      "grad_norm": 2.6877602739497,
      "learning_rate": 4.806665611441354e-07,
      "logits/policy_chosen": -2.2234950065612793,
      "logits/policy_rejected": -2.2985267639160156,
      "logits/reference_chosen": -3.1664934158325195,
      "logits/reference_rejected": -2.4548802375793457,
      "logps/policy_chosen": -53.821807861328125,
      "logps/policy_rejected": -135.544921875,
      "logps/reference_chosen": -155.88006591796875,
      "logps/reference_rejected": -98.04533386230469,
      "loss": 0.3466,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.020582675933838,
      "rewards/margins": 1.3955786228179932,
      "rewards/rejected": -0.3749959468841553,
      "step": 266,
      "tokens/chosen_response_len_mean": 369.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 267.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.21,
      "grad_norm": 2.691839396443011,
      "learning_rate": 4.803950097501643e-07,
      "logits/policy_chosen": -2.3689355850219727,
      "logits/policy_rejected": -2.406802177429199,
      "logits/reference_chosen": -3.110931158065796,
      "logits/reference_rejected": -2.629725694656372,
      "logps/policy_chosen": -61.78421401977539,
      "logps/policy_rejected": -293.3619079589844,
      "logps/reference_chosen": -205.92092895507812,
      "logps/reference_rejected": -133.5921630859375,
      "loss": 0.3107,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4413671493530273,
      "rewards/margins": 3.039064407348633,
      "rewards/rejected": -1.597697377204895,
      "step": 267,
      "tokens/chosen_response_len_mean": 334.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 111.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.22,
      "grad_norm": 4.332732574683176,
      "learning_rate": 4.80121642370782e-07,
      "logits/policy_chosen": -1.435086965560913,
      "logits/policy_rejected": -1.555901288986206,
      "logits/reference_chosen": -2.4242351055145264,
      "logits/reference_rejected": -2.2076950073242188,
      "logps/policy_chosen": -122.86267852783203,
      "logps/policy_rejected": -370.72705078125,
      "logps/reference_chosen": -256.2296142578125,
      "logps/reference_rejected": -221.7090301513672,
      "loss": 0.2893,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3336691856384277,
      "rewards/margins": 2.8238492012023926,
      "rewards/rejected": -1.4901798963546753,
      "step": 268,
      "tokens/chosen_response_len_mean": 474.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 229.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.22,
      "grad_norm": 3.679173265554199,
      "learning_rate": 4.798464611606842e-07,
      "logits/policy_chosen": -1.9900972843170166,
      "logits/policy_rejected": -0.7595923542976379,
      "logits/reference_chosen": -2.307075023651123,
      "logits/reference_rejected": -2.1424033641815186,
      "logps/policy_chosen": -39.70943832397461,
      "logps/policy_rejected": -101.0633544921875,
      "logps/reference_chosen": -61.08944320678711,
      "logps/reference_rejected": -77.70144653320312,
      "loss": 0.3654,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.21380005776882172,
      "rewards/margins": 0.44741907715797424,
      "rewards/rejected": -0.23361904919147491,
      "step": 269,
      "tokens/chosen_response_len_mean": 73.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 361.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.22,
      "grad_norm": 2.860165012641077,
      "learning_rate": 4.795694682888635e-07,
      "logits/policy_chosen": -1.8247418403625488,
      "logits/policy_rejected": -2.0296616554260254,
      "logits/reference_chosen": -2.9217374324798584,
      "logits/reference_rejected": -2.3353545665740967,
      "logps/policy_chosen": -60.13949966430664,
      "logps/policy_rejected": -374.595947265625,
      "logps/reference_chosen": -168.12413024902344,
      "logps/reference_rejected": -216.00479125976562,
      "loss": 0.1644,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0798462629318237,
      "rewards/margins": 2.665757656097412,
      "rewards/rejected": -1.5859116315841675,
      "step": 270,
      "tokens/chosen_response_len_mean": 366.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 259.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.22,
      "grad_norm": 2.8914101024190764,
      "learning_rate": 4.79290665938592e-07,
      "logits/policy_chosen": -2.393129587173462,
      "logits/policy_rejected": -2.532593250274658,
      "logits/reference_chosen": -3.0000436305999756,
      "logits/reference_rejected": -2.605870246887207,
      "logps/policy_chosen": -32.41455841064453,
      "logps/policy_rejected": -120.82582092285156,
      "logps/reference_chosen": -80.18354034423828,
      "logps/reference_rejected": -52.557701110839844,
      "loss": 0.2656,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4776897728443146,
      "rewards/margins": 1.1603708267211914,
      "rewards/rejected": -0.6826812028884888,
      "step": 271,
      "tokens/chosen_response_len_mean": 264.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 82.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.22,
      "grad_norm": 1.9537358053696072,
      "learning_rate": 4.790100563074044e-07,
      "logits/policy_chosen": -1.9999793767929077,
      "logits/policy_rejected": -2.0931291580200195,
      "logits/reference_chosen": -2.969376564025879,
      "logits/reference_rejected": -1.7754515409469604,
      "logps/policy_chosen": -88.11581420898438,
      "logps/policy_rejected": -398.7203369140625,
      "logps/reference_chosen": -281.721435546875,
      "logps/reference_rejected": -166.79583740234375,
      "loss": 0.3137,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.936056137084961,
      "rewards/margins": 4.255301475524902,
      "rewards/rejected": -2.3192451000213623,
      "step": 272,
      "tokens/chosen_response_len_mean": 415.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 262.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.22,
      "grad_norm": 2.7544872780237926,
      "learning_rate": 4.787276416070804e-07,
      "logits/policy_chosen": -2.0469934940338135,
      "logits/policy_rejected": -1.6853868961334229,
      "logits/reference_chosen": -2.1769094467163086,
      "logits/reference_rejected": -2.2352547645568848,
      "logps/policy_chosen": -33.57738494873047,
      "logps/policy_rejected": -220.69619750976562,
      "logps/reference_chosen": -67.25520324707031,
      "logps/reference_rejected": -119.21714782714844,
      "loss": 0.298,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3367781937122345,
      "rewards/margins": 1.351568579673767,
      "rewards/rejected": -1.014790415763855,
      "step": 273,
      "tokens/chosen_response_len_mean": 89.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 110.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.22,
      "grad_norm": 2.311066363535201,
      "learning_rate": 4.784434240636271e-07,
      "logits/policy_chosen": -1.7351186275482178,
      "logits/policy_rejected": -1.8542585372924805,
      "logits/reference_chosen": -3.0118846893310547,
      "logits/reference_rejected": -2.1115365028381348,
      "logps/policy_chosen": -82.1336898803711,
      "logps/policy_rejected": -439.58245849609375,
      "logps/reference_chosen": -232.67691040039062,
      "logps/reference_rejected": -256.78057861328125,
      "loss": 0.1728,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.50543212890625,
      "rewards/margins": 3.3334507942199707,
      "rewards/rejected": -1.8280186653137207,
      "step": 274,
      "tokens/chosen_response_len_mean": 609.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 465.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.22,
      "grad_norm": 2.9956815155001992,
      "learning_rate": 4.78157405917262e-07,
      "logits/policy_chosen": -1.8693939447402954,
      "logits/policy_rejected": -1.9391303062438965,
      "logits/reference_chosen": -2.969177484512329,
      "logits/reference_rejected": -2.4460105895996094,
      "logps/policy_chosen": -86.36649322509766,
      "logps/policy_rejected": -566.4467163085938,
      "logps/reference_chosen": -261.6522216796875,
      "logps/reference_rejected": -263.3449401855469,
      "loss": 0.2451,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7528573274612427,
      "rewards/margins": 4.783874988555908,
      "rewards/rejected": -3.031017780303955,
      "step": 275,
      "tokens/chosen_response_len_mean": 339.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 251.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.22,
      "grad_norm": 3.2803570213469144,
      "learning_rate": 4.778695894223952e-07,
      "logits/policy_chosen": -2.3628909587860107,
      "logits/policy_rejected": -2.3530797958374023,
      "logits/reference_chosen": -2.9668712615966797,
      "logits/reference_rejected": -2.966824531555176,
      "logps/policy_chosen": -54.248233795166016,
      "logps/policy_rejected": -327.43170166015625,
      "logps/reference_chosen": -146.84291076660156,
      "logps/reference_rejected": -196.168212890625,
      "loss": 0.3313,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9259467124938965,
      "rewards/margins": 2.238581418991089,
      "rewards/rejected": -1.3126345872879028,
      "step": 276,
      "tokens/chosen_response_len_mean": 289.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 287.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.22,
      "grad_norm": 3.3737296079999517,
      "learning_rate": 4.775799768476108e-07,
      "logits/policy_chosen": -1.623067855834961,
      "logits/policy_rejected": -1.66438627243042,
      "logits/reference_chosen": -2.50834321975708,
      "logits/reference_rejected": -2.3877673149108887,
      "logps/policy_chosen": -32.76015853881836,
      "logps/policy_rejected": -140.15492248535156,
      "logps/reference_chosen": -57.18016052246094,
      "logps/reference_rejected": -86.19461822509766,
      "loss": 0.2575,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.24420003592967987,
      "rewards/margins": 0.7838031053543091,
      "rewards/rejected": -0.5396031141281128,
      "step": 277,
      "tokens/chosen_response_len_mean": 176.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 170.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.22,
      "grad_norm": 9.199685663575346,
      "learning_rate": 4.772885704756505e-07,
      "logits/policy_chosen": -1.915112018585205,
      "logits/policy_rejected": -2.189150333404541,
      "logits/reference_chosen": -3.0652871131896973,
      "logits/reference_rejected": -2.8179969787597656,
      "logps/policy_chosen": -88.47130584716797,
      "logps/policy_rejected": -441.1356506347656,
      "logps/reference_chosen": -312.25732421875,
      "logps/reference_rejected": -313.57989501953125,
      "loss": 0.3432,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2378599643707275,
      "rewards/margins": 3.5134174823760986,
      "rewards/rejected": -1.2755577564239502,
      "step": 278,
      "tokens/chosen_response_len_mean": 416.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 258.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.22,
      "grad_norm": 2.568407460041794,
      "learning_rate": 4.769953726033942e-07,
      "logits/policy_chosen": -2.126152515411377,
      "logits/policy_rejected": -2.1976869106292725,
      "logits/reference_chosen": -2.918933391571045,
      "logits/reference_rejected": -2.9181439876556396,
      "logps/policy_chosen": -55.46541213989258,
      "logps/policy_rejected": -393.147705078125,
      "logps/reference_chosen": -175.48330688476562,
      "logps/reference_rejected": -261.3435363769531,
      "loss": 0.1778,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.200179100036621,
      "rewards/margins": 2.518220901489258,
      "rewards/rejected": -1.3180416822433472,
      "step": 279,
      "tokens/chosen_response_len_mean": 215.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 210.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.23,
      "grad_norm": 3.1302891031635984,
      "learning_rate": 4.767003855418429e-07,
      "logits/policy_chosen": -2.0603606700897217,
      "logits/policy_rejected": -2.1878933906555176,
      "logits/reference_chosen": -3.049166679382324,
      "logits/reference_rejected": -2.4211201667785645,
      "logps/policy_chosen": -70.95120239257812,
      "logps/policy_rejected": -435.4041442871094,
      "logps/reference_chosen": -203.48745727539062,
      "logps/reference_rejected": -209.47271728515625,
      "loss": 0.1805,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3253624439239502,
      "rewards/margins": 3.5846762657165527,
      "rewards/rejected": -2.2593138217926025,
      "step": 280,
      "tokens/chosen_response_len_mean": 273.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 158.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.23,
      "grad_norm": 3.0759421417192003,
      "learning_rate": 4.764036116160997e-07,
      "logits/policy_chosen": -1.9647924900054932,
      "logits/policy_rejected": -2.0003933906555176,
      "logits/reference_chosen": -3.1076905727386475,
      "logits/reference_rejected": -2.8670225143432617,
      "logps/policy_chosen": -52.48387145996094,
      "logps/policy_rejected": -527.5159912109375,
      "logps/reference_chosen": -158.17994689941406,
      "logps/reference_rejected": -246.14735412597656,
      "loss": 0.1958,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0569607019424438,
      "rewards/margins": 3.8706471920013428,
      "rewards/rejected": -2.8136863708496094,
      "step": 281,
      "tokens/chosen_response_len_mean": 214.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 191.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.23,
      "grad_norm": 2.7361755546386277,
      "learning_rate": 4.761050531653523e-07,
      "logits/policy_chosen": -1.5984692573547363,
      "logits/policy_rejected": -1.8361363410949707,
      "logits/reference_chosen": -2.7592179775238037,
      "logits/reference_rejected": -2.4084267616271973,
      "logps/policy_chosen": -80.7918930053711,
      "logps/policy_rejected": -345.93499755859375,
      "logps/reference_chosen": -228.01821899414062,
      "logps/reference_rejected": -187.31529235839844,
      "loss": 0.1742,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4722633361816406,
      "rewards/margins": 3.058460235595703,
      "rewards/rejected": -1.586196780204773,
      "step": 282,
      "tokens/chosen_response_len_mean": 634.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 348.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.23,
      "grad_norm": 3.522563546814339,
      "learning_rate": 4.758047125428537e-07,
      "logits/policy_chosen": -1.7348957061767578,
      "logits/policy_rejected": -2.081566333770752,
      "logits/reference_chosen": -2.989297389984131,
      "logits/reference_rejected": -2.2824132442474365,
      "logps/policy_chosen": -58.18089294433594,
      "logps/policy_rejected": -389.5596923828125,
      "logps/reference_chosen": -171.86676025390625,
      "logps/reference_rejected": -148.6519775390625,
      "loss": 0.2789,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1368587017059326,
      "rewards/margins": 3.54593563079834,
      "rewards/rejected": -2.409076690673828,
      "step": 283,
      "tokens/chosen_response_len_mean": 379.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 131.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.23,
      "grad_norm": 3.4857525347681495,
      "learning_rate": 4.7550259211590436e-07,
      "logits/policy_chosen": -2.155935764312744,
      "logits/policy_rejected": -2.212632894515991,
      "logits/reference_chosen": -3.005995750427246,
      "logits/reference_rejected": -2.9473025798797607,
      "logps/policy_chosen": -46.34202575683594,
      "logps/policy_rejected": -374.5079345703125,
      "logps/reference_chosen": -132.93014526367188,
      "logps/reference_rejected": -190.07769775390625,
      "loss": 0.2694,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8658812642097473,
      "rewards/margins": 2.710183620452881,
      "rewards/rejected": -1.8443021774291992,
      "step": 284,
      "tokens/chosen_response_len_mean": 127.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 132.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.23,
      "grad_norm": 2.146664751577214,
      "learning_rate": 4.751986942658332e-07,
      "logits/policy_chosen": -1.8320860862731934,
      "logits/policy_rejected": -1.9627416133880615,
      "logits/reference_chosen": -3.0087828636169434,
      "logits/reference_rejected": -2.798800230026245,
      "logps/policy_chosen": -99.31060028076172,
      "logps/policy_rejected": -590.6198120117188,
      "logps/reference_chosen": -292.62762451171875,
      "logps/reference_rejected": -301.8166198730469,
      "loss": 0.1743,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9331700801849365,
      "rewards/margins": 4.821201801300049,
      "rewards/rejected": -2.8880317211151123,
      "step": 285,
      "tokens/chosen_response_len_mean": 672.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 380.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.23,
      "grad_norm": 4.0068979390995665,
      "learning_rate": 4.748930213879787e-07,
      "logits/policy_chosen": -1.7841126918792725,
      "logits/policy_rejected": -1.616112232208252,
      "logits/reference_chosen": -2.8974661827087402,
      "logits/reference_rejected": -2.3146543502807617,
      "logps/policy_chosen": -83.1224365234375,
      "logps/policy_rejected": -509.5625305175781,
      "logps/reference_chosen": -248.04489135742188,
      "logps/reference_rejected": -299.60675048828125,
      "loss": 0.2229,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6492245197296143,
      "rewards/margins": 3.7487823963165283,
      "rewards/rejected": -2.099557638168335,
      "step": 286,
      "tokens/chosen_response_len_mean": 312.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 268.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.23,
      "grad_norm": 3.6884808286561905,
      "learning_rate": 4.745855758916704e-07,
      "logits/policy_chosen": -1.9492144584655762,
      "logits/policy_rejected": -2.23884916305542,
      "logits/reference_chosen": -2.644348382949829,
      "logits/reference_rejected": -2.677292823791504,
      "logps/policy_chosen": -167.10963439941406,
      "logps/policy_rejected": -408.90594482421875,
      "logps/reference_chosen": -245.54014587402344,
      "logps/reference_rejected": -228.08700561523438,
      "loss": 0.3101,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.784305214881897,
      "rewards/margins": 2.592494487762451,
      "rewards/rejected": -1.8081893920898438,
      "step": 287,
      "tokens/chosen_response_len_mean": 660.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 374.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.23,
      "grad_norm": 2.9811598150339806,
      "learning_rate": 4.7427636020020964e-07,
      "logits/policy_chosen": -1.6351268291473389,
      "logits/policy_rejected": -1.8411903381347656,
      "logits/reference_chosen": -3.084428310394287,
      "logits/reference_rejected": -2.840968132019043,
      "logps/policy_chosen": -109.2936019897461,
      "logps/policy_rejected": -708.4165649414062,
      "logps/reference_chosen": -367.9963073730469,
      "logps/reference_rejected": -379.46942138671875,
      "loss": 0.1152,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.587027072906494,
      "rewards/margins": 5.876498222351074,
      "rewards/rejected": -3.289471387863159,
      "step": 288,
      "tokens/chosen_response_len_mean": 711.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 495.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.23,
      "grad_norm": 4.220920032892386,
      "learning_rate": 4.7396537675085046e-07,
      "logits/policy_chosen": -1.8225438594818115,
      "logits/policy_rejected": -1.9494044780731201,
      "logits/reference_chosen": -2.518173933029175,
      "logits/reference_rejected": -2.711627721786499,
      "logps/policy_chosen": -47.87618637084961,
      "logps/policy_rejected": -247.03065490722656,
      "logps/reference_chosen": -69.38465881347656,
      "logps/reference_rejected": -130.92787170410156,
      "loss": 0.2582,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21508470177650452,
      "rewards/margins": 1.3761123418807983,
      "rewards/rejected": -1.1610276699066162,
      "step": 289,
      "tokens/chosen_response_len_mean": 285.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 109.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.23,
      "grad_norm": 3.2644411706471614,
      "learning_rate": 4.7365262799478065e-07,
      "logits/policy_chosen": -1.631312370300293,
      "logits/policy_rejected": -1.8853410482406616,
      "logits/reference_chosen": -2.863694906234741,
      "logits/reference_rejected": -2.2361578941345215,
      "logps/policy_chosen": -87.65226745605469,
      "logps/policy_rejected": -368.45916748046875,
      "logps/reference_chosen": -175.37078857421875,
      "logps/reference_rejected": -180.3101043701172,
      "loss": 0.3482,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8771853446960449,
      "rewards/margins": 2.758676052093506,
      "rewards/rejected": -1.881490707397461,
      "step": 290,
      "tokens/chosen_response_len_mean": 238.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 142.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.23,
      "grad_norm": 4.164740816732374,
      "learning_rate": 4.733381163971021e-07,
      "logits/policy_chosen": -2.0723226070404053,
      "logits/policy_rejected": -2.1896250247955322,
      "logits/reference_chosen": -2.9462733268737793,
      "logits/reference_rejected": -2.904141902923584,
      "logps/policy_chosen": -82.70597839355469,
      "logps/policy_rejected": -706.9203491210938,
      "logps/reference_chosen": -254.06658935546875,
      "logps/reference_rejected": -371.26043701171875,
      "loss": 0.2392,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7136059999465942,
      "rewards/margins": 5.070204734802246,
      "rewards/rejected": -3.3565988540649414,
      "step": 291,
      "tokens/chosen_response_len_mean": 440.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 360.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.23,
      "grad_norm": 2.591496235271577,
      "learning_rate": 4.7302184443681135e-07,
      "logits/policy_chosen": -1.7894690036773682,
      "logits/policy_rejected": -1.8064680099487305,
      "logits/reference_chosen": -2.837014675140381,
      "logits/reference_rejected": -2.7575623989105225,
      "logps/policy_chosen": -70.64473724365234,
      "logps/policy_rejected": -504.9518737792969,
      "logps/reference_chosen": -204.25482177734375,
      "logps/reference_rejected": -285.4273376464844,
      "loss": 0.1799,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3361008167266846,
      "rewards/margins": 3.531346321105957,
      "rewards/rejected": -2.1952455043792725,
      "step": 292,
      "tokens/chosen_response_len_mean": 480.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 399.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.24,
      "grad_norm": 3.3860324397417396,
      "learning_rate": 4.7270381460678064e-07,
      "logits/policy_chosen": -1.842910885810852,
      "logits/policy_rejected": -1.6502610445022583,
      "logits/reference_chosen": -2.704162120819092,
      "logits/reference_rejected": -2.680164098739624,
      "logps/policy_chosen": -49.48709487915039,
      "logps/policy_rejected": -214.38595581054688,
      "logps/reference_chosen": -82.64054107666016,
      "logps/reference_rejected": -121.0648193359375,
      "loss": 0.2579,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3315344452857971,
      "rewards/margins": 1.264745831489563,
      "rewards/rejected": -0.9332114458084106,
      "step": 293,
      "tokens/chosen_response_len_mean": 103.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 131.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.24,
      "grad_norm": 3.188225158781283,
      "learning_rate": 4.7238402941373743e-07,
      "logits/policy_chosen": -1.6677745580673218,
      "logits/policy_rejected": -1.6923861503601074,
      "logits/reference_chosen": -2.790350914001465,
      "logits/reference_rejected": -2.7842464447021484,
      "logps/policy_chosen": -69.78219604492188,
      "logps/policy_rejected": -374.1966552734375,
      "logps/reference_chosen": -223.26382446289062,
      "logps/reference_rejected": -258.4195861816406,
      "loss": 0.2856,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5348161458969116,
      "rewards/margins": 2.692586898803711,
      "rewards/rejected": -1.1577708721160889,
      "step": 294,
      "tokens/chosen_response_len_mean": 361.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 321.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.24,
      "grad_norm": 2.8300164573379623,
      "learning_rate": 4.7206249137824525e-07,
      "logits/policy_chosen": -1.2584807872772217,
      "logits/policy_rejected": -1.1313340663909912,
      "logits/reference_chosen": -2.185845136642456,
      "logits/reference_rejected": -1.8424296379089355,
      "logps/policy_chosen": -100.97215270996094,
      "logps/policy_rejected": -257.4559631347656,
      "logps/reference_chosen": -234.6266326904297,
      "logps/reference_rejected": -179.3731689453125,
      "loss": 0.3429,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3365448713302612,
      "rewards/margins": 2.117372512817383,
      "rewards/rejected": -0.7808278203010559,
      "step": 295,
      "tokens/chosen_response_len_mean": 382.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 457.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.24,
      "grad_norm": 3.830130538309461,
      "learning_rate": 4.717392030346835e-07,
      "logits/policy_chosen": -2.0780117511749268,
      "logits/policy_rejected": -2.133204698562622,
      "logits/reference_chosen": -2.9950385093688965,
      "logits/reference_rejected": -2.916698932647705,
      "logps/policy_chosen": -83.94184875488281,
      "logps/policy_rejected": -504.6825866699219,
      "logps/reference_chosen": -269.20159912109375,
      "logps/reference_rejected": -335.22650146484375,
      "loss": 0.2967,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.8525973558425903,
      "rewards/margins": 3.5471582412719727,
      "rewards/rejected": -1.6945607662200928,
      "step": 296,
      "tokens/chosen_response_len_mean": 454.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 379.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.24,
      "grad_norm": 6.893378090531771,
      "learning_rate": 4.714141669312277e-07,
      "logits/policy_chosen": -1.9810442924499512,
      "logits/policy_rejected": -2.0455687046051025,
      "logits/reference_chosen": -2.7563416957855225,
      "logits/reference_rejected": -2.617710590362549,
      "logps/policy_chosen": -73.21089935302734,
      "logps/policy_rejected": -340.0194091796875,
      "logps/reference_chosen": -221.56581115722656,
      "logps/reference_rejected": -196.06849670410156,
      "loss": 0.3247,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4835491180419922,
      "rewards/margins": 2.923058032989502,
      "rewards/rejected": -1.4395091533660889,
      "step": 297,
      "tokens/chosen_response_len_mean": 321.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 218.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.24,
      "grad_norm": 2.56574676261905,
      "learning_rate": 4.710873856298293e-07,
      "logits/policy_chosen": -1.895401954650879,
      "logits/policy_rejected": -1.9954261779785156,
      "logits/reference_chosen": -2.9610021114349365,
      "logits/reference_rejected": -2.7275171279907227,
      "logps/policy_chosen": -93.83104705810547,
      "logps/policy_rejected": -574.4371948242188,
      "logps/reference_chosen": -263.5850524902344,
      "logps/reference_rejected": -328.8235168457031,
      "loss": 0.1995,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6975396871566772,
      "rewards/margins": 4.1536760330200195,
      "rewards/rejected": -2.456136465072632,
      "step": 298,
      "tokens/chosen_response_len_mean": 543.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 433.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.24,
      "grad_norm": 2.62583819131534,
      "learning_rate": 4.7075886170619536e-07,
      "logits/policy_chosen": -1.3762860298156738,
      "logits/policy_rejected": -1.4836927652359009,
      "logits/reference_chosen": -2.8365345001220703,
      "logits/reference_rejected": -2.107300281524658,
      "logps/policy_chosen": -109.9390869140625,
      "logps/policy_rejected": -396.7261657714844,
      "logps/reference_chosen": -285.54638671875,
      "logps/reference_rejected": -202.1131591796875,
      "loss": 0.2451,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7560728788375854,
      "rewards/margins": 3.7022030353546143,
      "rewards/rejected": -1.9461302757263184,
      "step": 299,
      "tokens/chosen_response_len_mean": 378.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 207.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.24,
      "grad_norm": 2.9338445441961096,
      "learning_rate": 4.704285977497687e-07,
      "logits/policy_chosen": -1.83700692653656,
      "logits/policy_rejected": -2.106710433959961,
      "logits/reference_chosen": -2.9238107204437256,
      "logits/reference_rejected": -2.5823757648468018,
      "logps/policy_chosen": -82.13780975341797,
      "logps/policy_rejected": -493.58087158203125,
      "logps/reference_chosen": -289.5302429199219,
      "logps/reference_rejected": -208.1035919189453,
      "loss": 0.113,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0739243030548096,
      "rewards/margins": 4.928697109222412,
      "rewards/rejected": -2.8547730445861816,
      "step": 300,
      "tokens/chosen_response_len_mean": 445.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 201.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.24,
      "grad_norm": 2.290878044608878,
      "learning_rate": 4.700965963637067e-07,
      "logits/policy_chosen": -1.2083768844604492,
      "logits/policy_rejected": -1.3407108783721924,
      "logits/reference_chosen": -2.4264414310455322,
      "logits/reference_rejected": -2.0838944911956787,
      "logps/policy_chosen": -87.56996154785156,
      "logps/policy_rejected": -337.8468933105469,
      "logps/reference_chosen": -196.71490478515625,
      "logps/reference_rejected": -176.93084716796875,
      "loss": 0.1417,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0914493799209595,
      "rewards/margins": 2.7006096839904785,
      "rewards/rejected": -1.609160304069519,
      "step": 301,
      "tokens/chosen_response_len_mean": 252.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 133.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.24,
      "grad_norm": 3.664882673854998,
      "learning_rate": 4.6976286016486156e-07,
      "logits/policy_chosen": -1.6306910514831543,
      "logits/policy_rejected": -1.7238553762435913,
      "logits/reference_chosen": -3.024641513824463,
      "logits/reference_rejected": -2.9463772773742676,
      "logps/policy_chosen": -103.36473846435547,
      "logps/policy_rejected": -482.4529113769531,
      "logps/reference_chosen": -300.7445068359375,
      "logps/reference_rejected": -272.9915466308594,
      "loss": 0.2465,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9737976789474487,
      "rewards/margins": 4.068410873413086,
      "rewards/rejected": -2.0946133136749268,
      "step": 302,
      "tokens/chosen_response_len_mean": 537.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 378.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.24,
      "grad_norm": 4.07962885346587,
      "learning_rate": 4.694273917837593e-07,
      "logits/policy_chosen": -1.718010663986206,
      "logits/policy_rejected": -1.1793601512908936,
      "logits/reference_chosen": -2.1970267295837402,
      "logits/reference_rejected": -2.242309093475342,
      "logps/policy_chosen": -89.3014144897461,
      "logps/policy_rejected": -925.4991455078125,
      "logps/reference_chosen": -293.8038330078125,
      "logps/reference_rejected": -482.5627136230469,
      "loss": 0.2865,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.0450241565704346,
      "rewards/margins": 6.47438907623291,
      "rewards/rejected": -4.429364204406738,
      "step": 303,
      "tokens/chosen_response_len_mean": 358.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 365.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.24,
      "grad_norm": 2.405932913361,
      "learning_rate": 4.6909019386457927e-07,
      "logits/policy_chosen": -1.6491061449050903,
      "logits/policy_rejected": -1.6597315073013306,
      "logits/reference_chosen": -2.341599464416504,
      "logits/reference_rejected": -2.266160488128662,
      "logps/policy_chosen": -68.28823852539062,
      "logps/policy_rejected": -270.70416259765625,
      "logps/reference_chosen": -157.44268798828125,
      "logps/reference_rejected": -147.8431396484375,
      "loss": 0.2139,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8915444016456604,
      "rewards/margins": 2.120154857635498,
      "rewards/rejected": -1.2286103963851929,
      "step": 304,
      "tokens/chosen_response_len_mean": 237.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 197.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.25,
      "grad_norm": 3.572341442607126,
      "learning_rate": 4.687512690651327e-07,
      "logits/policy_chosen": -2.2066917419433594,
      "logits/policy_rejected": -2.2525765895843506,
      "logits/reference_chosen": -2.9350485801696777,
      "logits/reference_rejected": -2.8024892807006836,
      "logps/policy_chosen": -59.40142822265625,
      "logps/policy_rejected": -203.68032836914062,
      "logps/reference_chosen": -145.8667449951172,
      "logps/reference_rejected": -85.89591979980469,
      "loss": 0.209,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8646531701087952,
      "rewards/margins": 2.042497158050537,
      "rewards/rejected": -1.1778440475463867,
      "step": 305,
      "tokens/chosen_response_len_mean": 184.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 98.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.25,
      "grad_norm": 3.124612250113926,
      "learning_rate": 4.6841062005684255e-07,
      "logits/policy_chosen": -1.8113189935684204,
      "logits/policy_rejected": -1.8169225454330444,
      "logits/reference_chosen": -2.8508493900299072,
      "logits/reference_rejected": -2.9348199367523193,
      "logps/policy_chosen": -35.2087516784668,
      "logps/policy_rejected": -334.8651123046875,
      "logps/reference_chosen": -112.86093139648438,
      "logps/reference_rejected": -148.45632934570312,
      "loss": 0.3251,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7765218019485474,
      "rewards/margins": 2.6406097412109375,
      "rewards/rejected": -1.8640880584716797,
      "step": 306,
      "tokens/chosen_response_len_mean": 320.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 285.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.25,
      "grad_norm": 3.32496586258652,
      "learning_rate": 4.68068249524722e-07,
      "logits/policy_chosen": -2.3824849128723145,
      "logits/policy_rejected": -1.6881012916564941,
      "logits/reference_chosen": -3.0569143295288086,
      "logits/reference_rejected": -2.4748129844665527,
      "logps/policy_chosen": -61.716796875,
      "logps/policy_rejected": -166.7222137451172,
      "logps/reference_chosen": -222.57785034179688,
      "logps/reference_rejected": -93.1553955078125,
      "loss": 0.2575,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6086106300354004,
      "rewards/margins": 2.344278573989868,
      "rewards/rejected": -0.7356680631637573,
      "step": 307,
      "tokens/chosen_response_len_mean": 291.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 310.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.25,
      "grad_norm": 3.3874944482736735,
      "learning_rate": 4.6772416016735323e-07,
      "logits/policy_chosen": -1.7318488359451294,
      "logits/policy_rejected": -2.1800973415374756,
      "logits/reference_chosen": -2.737339973449707,
      "logits/reference_rejected": -2.64113712310791,
      "logps/policy_chosen": -109.86978149414062,
      "logps/policy_rejected": -449.2695007324219,
      "logps/reference_chosen": -305.08203125,
      "logps/reference_rejected": -229.10501098632812,
      "loss": 0.1993,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9521225690841675,
      "rewards/margins": 4.1537675857543945,
      "rewards/rejected": -2.2016448974609375,
      "step": 308,
      "tokens/chosen_response_len_mean": 614.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 206.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.25,
      "grad_norm": 3.3558308202865,
      "learning_rate": 4.673783546968665e-07,
      "logits/policy_chosen": -2.2060866355895996,
      "logits/policy_rejected": -2.37375545501709,
      "logits/reference_chosen": -3.0307087898254395,
      "logits/reference_rejected": -2.645644426345825,
      "logps/policy_chosen": -47.22865676879883,
      "logps/policy_rejected": -480.0763244628906,
      "logps/reference_chosen": -145.3326873779297,
      "logps/reference_rejected": -169.44834899902344,
      "loss": 0.19,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.981040358543396,
      "rewards/margins": 4.087320327758789,
      "rewards/rejected": -3.1062798500061035,
      "step": 309,
      "tokens/chosen_response_len_mean": 243.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 131.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.25,
      "grad_norm": 4.153117211006617,
      "learning_rate": 4.6703083583891835e-07,
      "logits/policy_chosen": -2.103739023208618,
      "logits/policy_rejected": -2.222684383392334,
      "logits/reference_chosen": -2.9488046169281006,
      "logits/reference_rejected": -2.909597158432007,
      "logps/policy_chosen": -72.56431579589844,
      "logps/policy_rejected": -597.1136474609375,
      "logps/reference_chosen": -229.9650115966797,
      "logps/reference_rejected": -291.440673828125,
      "loss": 0.2705,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.5740069150924683,
      "rewards/margins": 4.630736827850342,
      "rewards/rejected": -3.056729793548584,
      "step": 310,
      "tokens/chosen_response_len_mean": 416.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 279.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.25,
      "grad_norm": 6.474334517638787,
      "learning_rate": 4.666816063326705e-07,
      "logits/policy_chosen": -1.8240094184875488,
      "logits/policy_rejected": -1.2601674795150757,
      "logits/reference_chosen": -2.8085074424743652,
      "logits/reference_rejected": -1.7981979846954346,
      "logps/policy_chosen": -75.03584289550781,
      "logps/policy_rejected": -454.2119140625,
      "logps/reference_chosen": -269.4425964355469,
      "logps/reference_rejected": -277.75634765625,
      "loss": 0.2551,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.9440672397613525,
      "rewards/margins": 3.708622932434082,
      "rewards/rejected": -1.7645556926727295,
      "step": 311,
      "tokens/chosen_response_len_mean": 437.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 564.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.25,
      "grad_norm": 5.519032924912619,
      "learning_rate": 4.66330668930768e-07,
      "logits/policy_chosen": -1.546844244003296,
      "logits/policy_rejected": -1.0349215269088745,
      "logits/reference_chosen": -2.6315059661865234,
      "logits/reference_rejected": -2.3254542350769043,
      "logps/policy_chosen": -46.538272857666016,
      "logps/policy_rejected": -340.2102355957031,
      "logps/reference_chosen": -118.1043701171875,
      "logps/reference_rejected": -194.15060424804688,
      "loss": 0.2405,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.7156610488891602,
      "rewards/margins": 2.1762571334838867,
      "rewards/rejected": -1.4605962038040161,
      "step": 312,
      "tokens/chosen_response_len_mean": 286.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 542.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.25,
      "grad_norm": 3.0150528597270467,
      "learning_rate": 4.659780263993177e-07,
      "logits/policy_chosen": -1.8767701387405396,
      "logits/policy_rejected": -1.8039137125015259,
      "logits/reference_chosen": -3.0502028465270996,
      "logits/reference_rejected": -3.0671067237854004,
      "logps/policy_chosen": -54.472354888916016,
      "logps/policy_rejected": -321.30377197265625,
      "logps/reference_chosen": -124.24363708496094,
      "logps/reference_rejected": -180.5016326904297,
      "loss": 0.3478,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.697712779045105,
      "rewards/margins": 2.105734348297119,
      "rewards/rejected": -1.4080214500427246,
      "step": 313,
      "tokens/chosen_response_len_mean": 180.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 186.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.25,
      "grad_norm": 3.338552410487336,
      "learning_rate": 4.656236815178662e-07,
      "logits/policy_chosen": -1.8194024562835693,
      "logits/policy_rejected": -1.8620460033416748,
      "logits/reference_chosen": -2.952608585357666,
      "logits/reference_rejected": -2.880305528640747,
      "logps/policy_chosen": -64.1020736694336,
      "logps/policy_rejected": -668.8197021484375,
      "logps/reference_chosen": -191.9952392578125,
      "logps/reference_rejected": -297.9425964355469,
      "loss": 0.305,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2789316177368164,
      "rewards/margins": 4.987702369689941,
      "rewards/rejected": -3.708770513534546,
      "step": 314,
      "tokens/chosen_response_len_mean": 302.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 252.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.25,
      "grad_norm": 4.216599658137105,
      "learning_rate": 4.652676370793783e-07,
      "logits/policy_chosen": -1.9225913286209106,
      "logits/policy_rejected": -2.21560001373291,
      "logits/reference_chosen": -2.867231845855713,
      "logits/reference_rejected": -2.1903316974639893,
      "logps/policy_chosen": -87.3575439453125,
      "logps/policy_rejected": -352.74395751953125,
      "logps/reference_chosen": -222.33282470703125,
      "logps/reference_rejected": -139.34376525878906,
      "loss": 0.1772,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3497529029846191,
      "rewards/margins": 3.4837546348571777,
      "rewards/rejected": -2.1340014934539795,
      "step": 315,
      "tokens/chosen_response_len_mean": 291.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 90.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.25,
      "grad_norm": 1.9381348585009064,
      "learning_rate": 4.649098958902148e-07,
      "logits/policy_chosen": -1.4626785516738892,
      "logits/policy_rejected": -1.6685293912887573,
      "logits/reference_chosen": -3.0980098247528076,
      "logits/reference_rejected": -2.3443450927734375,
      "logps/policy_chosen": -132.43142700195312,
      "logps/policy_rejected": -1093.05859375,
      "logps/reference_chosen": -439.55596923828125,
      "logps/reference_rejected": -496.19171142578125,
      "loss": 0.1559,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0712451934814453,
      "rewards/margins": 9.039913177490234,
      "rewards/rejected": -5.968668460845947,
      "step": 316,
      "tokens/chosen_response_len_mean": 614.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 437.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.25,
      "grad_norm": 3.3208168275072727,
      "learning_rate": 4.6455046077011017e-07,
      "logits/policy_chosen": -1.6945469379425049,
      "logits/policy_rejected": -1.6586346626281738,
      "logits/reference_chosen": -2.797177314758301,
      "logits/reference_rejected": -2.7550530433654785,
      "logps/policy_chosen": -89.57418823242188,
      "logps/policy_rejected": -376.0241394042969,
      "logps/reference_chosen": -193.11834716796875,
      "logps/reference_rejected": -237.541015625,
      "loss": 0.1793,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.035441517829895,
      "rewards/margins": 2.4202725887298584,
      "rewards/rejected": -1.384831190109253,
      "step": 317,
      "tokens/chosen_response_len_mean": 268.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 252.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.26,
      "grad_norm": 4.834044743941225,
      "learning_rate": 4.6418933455215076e-07,
      "logits/policy_chosen": -1.7103971242904663,
      "logits/policy_rejected": -1.777637004852295,
      "logits/reference_chosen": -2.8807411193847656,
      "logits/reference_rejected": -2.910388946533203,
      "logps/policy_chosen": -43.051544189453125,
      "logps/policy_rejected": -451.1328430175781,
      "logps/reference_chosen": -98.37159729003906,
      "logps/reference_rejected": -200.70199584960938,
      "loss": 0.2363,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5532006025314331,
      "rewards/margins": 3.057508945465088,
      "rewards/rejected": -2.5043087005615234,
      "step": 318,
      "tokens/chosen_response_len_mean": 247.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 208.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.26,
      "grad_norm": 3.0765900633406913,
      "learning_rate": 4.63826520082752e-07,
      "logits/policy_chosen": -1.9023985862731934,
      "logits/policy_rejected": -2.087189197540283,
      "logits/reference_chosen": -2.684725522994995,
      "logits/reference_rejected": -2.7294716835021973,
      "logps/policy_chosen": -29.559532165527344,
      "logps/policy_rejected": -101.22508239746094,
      "logps/reference_chosen": -34.04695129394531,
      "logps/reference_rejected": -49.713741302490234,
      "loss": 0.1553,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.044874198734760284,
      "rewards/margins": 0.5599876046180725,
      "rewards/rejected": -0.5151134133338928,
      "step": 319,
      "tokens/chosen_response_len_mean": 277.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 294.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.26,
      "grad_norm": 3.1173897201990224,
      "learning_rate": 4.6346202022163656e-07,
      "logits/policy_chosen": -1.382790446281433,
      "logits/policy_rejected": -1.732831597328186,
      "logits/reference_chosen": -2.682234764099121,
      "logits/reference_rejected": -2.60119891166687,
      "logps/policy_chosen": -110.54884338378906,
      "logps/policy_rejected": -942.2950439453125,
      "logps/reference_chosen": -358.08880615234375,
      "logps/reference_rejected": -484.7252197265625,
      "loss": 0.2402,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4753994941711426,
      "rewards/margins": 7.051098346710205,
      "rewards/rejected": -4.575698375701904,
      "step": 320,
      "tokens/chosen_response_len_mean": 697.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 564.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.26,
      "grad_norm": 3.703781134184552,
      "learning_rate": 4.6309583784181104e-07,
      "logits/policy_chosen": -1.6343090534210205,
      "logits/policy_rejected": -1.7306853532791138,
      "logits/reference_chosen": -2.6760318279266357,
      "logits/reference_rejected": -2.2841994762420654,
      "logps/policy_chosen": -63.79367446899414,
      "logps/policy_rejected": -231.47938537597656,
      "logps/reference_chosen": -115.32276916503906,
      "logps/reference_rejected": -110.79583740234375,
      "loss": 0.276,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5152909159660339,
      "rewards/margins": 1.7221263647079468,
      "rewards/rejected": -1.2068355083465576,
      "step": 321,
      "tokens/chosen_response_len_mean": 192.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 137.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.26,
      "grad_norm": 3.5327692055358666,
      "learning_rate": 4.62727975829544e-07,
      "logits/policy_chosen": -2.3158786296844482,
      "logits/policy_rejected": -2.3356070518493652,
      "logits/reference_chosen": -2.933302402496338,
      "logits/reference_rejected": -2.826026678085327,
      "logps/policy_chosen": -32.73407745361328,
      "logps/policy_rejected": -107.17605590820312,
      "logps/reference_chosen": -40.871437072753906,
      "logps/reference_rejected": -49.330963134765625,
      "loss": 0.2464,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08137363195419312,
      "rewards/margins": 0.659824550151825,
      "rewards/rejected": -0.5784509181976318,
      "step": 322,
      "tokens/chosen_response_len_mean": 34.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 32.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.26,
      "grad_norm": 4.524743189821704,
      "learning_rate": 4.6235843708434305e-07,
      "logits/policy_chosen": -1.0855112075805664,
      "logits/policy_rejected": -1.2527565956115723,
      "logits/reference_chosen": -2.279548168182373,
      "logits/reference_rejected": -2.0591039657592773,
      "logps/policy_chosen": -65.47724914550781,
      "logps/policy_rejected": -398.4112548828125,
      "logps/reference_chosen": -202.6960906982422,
      "logps/reference_rejected": -213.66448974609375,
      "loss": 0.2768,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3721883296966553,
      "rewards/margins": 3.219656467437744,
      "rewards/rejected": -1.8474678993225098,
      "step": 323,
      "tokens/chosen_response_len_mean": 237.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 194.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.26,
      "grad_norm": 2.9172728498954816,
      "learning_rate": 4.619872245189315e-07,
      "logits/policy_chosen": -1.803147554397583,
      "logits/policy_rejected": -2.121699333190918,
      "logits/reference_chosen": -2.9548394680023193,
      "logits/reference_rejected": -2.4536960124969482,
      "logps/policy_chosen": -96.12008666992188,
      "logps/policy_rejected": -595.5452270507812,
      "logps/reference_chosen": -285.5267333984375,
      "logps/reference_rejected": -236.10336303710938,
      "loss": 0.2238,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8940664529800415,
      "rewards/margins": 5.488485336303711,
      "rewards/rejected": -3.594418525695801,
      "step": 324,
      "tokens/chosen_response_len_mean": 443.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 240.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.26,
      "grad_norm": 3.0831020027878946,
      "learning_rate": 4.616143410592261e-07,
      "logits/policy_chosen": -2.2690231800079346,
      "logits/policy_rejected": -2.37300181388855,
      "logits/reference_chosen": -3.101170539855957,
      "logits/reference_rejected": -2.814063310623169,
      "logps/policy_chosen": -75.43067932128906,
      "logps/policy_rejected": -1028.3170166015625,
      "logps/reference_chosen": -261.9538269042969,
      "logps/reference_rejected": -396.73486328125,
      "loss": 0.2074,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8652312755584717,
      "rewards/margins": 8.181053161621094,
      "rewards/rejected": -6.315821647644043,
      "step": 325,
      "tokens/chosen_response_len_mean": 413.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 280.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.26,
      "grad_norm": 2.9506911486463143,
      "learning_rate": 4.6123978964431376e-07,
      "logits/policy_chosen": -1.961487054824829,
      "logits/policy_rejected": -2.289116144180298,
      "logits/reference_chosen": -3.1111416816711426,
      "logits/reference_rejected": -2.8734333515167236,
      "logps/policy_chosen": -71.76238250732422,
      "logps/policy_rejected": -479.92535400390625,
      "logps/reference_chosen": -267.5050964355469,
      "logps/reference_rejected": -202.55059814453125,
      "loss": 0.2505,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.9574271440505981,
      "rewards/margins": 4.731174468994141,
      "rewards/rejected": -2.773747682571411,
      "step": 326,
      "tokens/chosen_response_len_mean": 445.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 148.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.26,
      "grad_norm": 2.819044425119029,
      "learning_rate": 4.60863573226428e-07,
      "logits/policy_chosen": -1.2217872142791748,
      "logits/policy_rejected": -1.4427027702331543,
      "logits/reference_chosen": -2.5383174419403076,
      "logits/reference_rejected": -2.6582465171813965,
      "logps/policy_chosen": -78.91648864746094,
      "logps/policy_rejected": -792.6351318359375,
      "logps/reference_chosen": -325.96051025390625,
      "logps/reference_rejected": -492.7793884277344,
      "loss": 0.1434,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.470440149307251,
      "rewards/margins": 5.468997955322266,
      "rewards/rejected": -2.9985575675964355,
      "step": 327,
      "tokens/chosen_response_len_mean": 698.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 698.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.26,
      "grad_norm": 2.051994037783105,
      "learning_rate": 4.604856947709262e-07,
      "logits/policy_chosen": -1.699342966079712,
      "logits/policy_rejected": -1.9920241832733154,
      "logits/reference_chosen": -3.1137824058532715,
      "logits/reference_rejected": -2.245004892349243,
      "logps/policy_chosen": -108.17164611816406,
      "logps/policy_rejected": -584.7960205078125,
      "logps/reference_chosen": -348.4723205566406,
      "logps/reference_rejected": -226.91238403320312,
      "loss": 0.154,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4030065536499023,
      "rewards/margins": 5.981842517852783,
      "rewards/rejected": -3.578835964202881,
      "step": 328,
      "tokens/chosen_response_len_mean": 482.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 178.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.26,
      "grad_norm": 2.853440760910482,
      "learning_rate": 4.601061572562659e-07,
      "logits/policy_chosen": -1.1898789405822754,
      "logits/policy_rejected": -1.331478476524353,
      "logits/reference_chosen": -2.611237049102783,
      "logits/reference_rejected": -2.666792869567871,
      "logps/policy_chosen": -48.299530029296875,
      "logps/policy_rejected": -434.9912109375,
      "logps/reference_chosen": -154.5133056640625,
      "logps/reference_rejected": -240.702392578125,
      "loss": 0.2731,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0621377229690552,
      "rewards/margins": 3.00502610206604,
      "rewards/rejected": -1.9428882598876953,
      "step": 329,
      "tokens/chosen_response_len_mean": 265.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 216.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.27,
      "grad_norm": 3.1024957208965094,
      "learning_rate": 4.5972496367398146e-07,
      "logits/policy_chosen": -1.352826714515686,
      "logits/policy_rejected": -1.392114520072937,
      "logits/reference_chosen": -2.7036824226379395,
      "logits/reference_rejected": -2.694230794906616,
      "logps/policy_chosen": -61.405372619628906,
      "logps/policy_rejected": -453.8320007324219,
      "logps/reference_chosen": -191.30259704589844,
      "logps/reference_rejected": -260.45123291015625,
      "loss": 0.2595,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.298972249031067,
      "rewards/margins": 3.2327802181243896,
      "rewards/rejected": -1.933807611465454,
      "step": 330,
      "tokens/chosen_response_len_mean": 460.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 469.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.27,
      "grad_norm": 2.523243805150809,
      "learning_rate": 4.5934211702866046e-07,
      "logits/policy_chosen": -1.6151251792907715,
      "logits/policy_rejected": -1.6389597654342651,
      "logits/reference_chosen": -2.9203569889068604,
      "logits/reference_rejected": -2.905449390411377,
      "logps/policy_chosen": -63.08237075805664,
      "logps/policy_rejected": -630.6449584960938,
      "logps/reference_chosen": -203.67691040039062,
      "logps/reference_rejected": -291.34234619140625,
      "loss": 0.1624,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4059453010559082,
      "rewards/margins": 4.798971176147461,
      "rewards/rejected": -3.3930258750915527,
      "step": 331,
      "tokens/chosen_response_len_mean": 318.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 279.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.27,
      "grad_norm": 3.2573268171576544,
      "learning_rate": 4.5895762033791995e-07,
      "logits/policy_chosen": -2.4760982990264893,
      "logits/policy_rejected": -2.6151270866394043,
      "logits/reference_chosen": -3.021172523498535,
      "logits/reference_rejected": -2.504538059234619,
      "logps/policy_chosen": -40.59739685058594,
      "logps/policy_rejected": -89.36610412597656,
      "logps/reference_chosen": -70.21237182617188,
      "logps/reference_rejected": -43.146018981933594,
      "loss": 0.3215,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.29614976048469543,
      "rewards/margins": 0.7583505511283875,
      "rewards/rejected": -0.4622008204460144,
      "step": 332,
      "tokens/chosen_response_len_mean": 64.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 32.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.27,
      "grad_norm": 3.8626650481777554,
      "learning_rate": 4.58571476632383e-07,
      "logits/policy_chosen": -1.6957329511642456,
      "logits/policy_rejected": -2.019444227218628,
      "logits/reference_chosen": -2.8664603233337402,
      "logits/reference_rejected": -2.8443779945373535,
      "logps/policy_chosen": -86.0948257446289,
      "logps/policy_rejected": -600.858154296875,
      "logps/reference_chosen": -277.8157958984375,
      "logps/reference_rejected": -353.20318603515625,
      "loss": 0.2874,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9172096252441406,
      "rewards/margins": 4.393758773803711,
      "rewards/rejected": -2.4765493869781494,
      "step": 333,
      "tokens/chosen_response_len_mean": 540.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 265.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.27,
      "grad_norm": 4.40990215478422,
      "learning_rate": 4.5818368895565405e-07,
      "logits/policy_chosen": -1.576200008392334,
      "logits/policy_rejected": -1.6361205577850342,
      "logits/reference_chosen": -2.847994089126587,
      "logits/reference_rejected": -2.3715221881866455,
      "logps/policy_chosen": -62.154117584228516,
      "logps/policy_rejected": -340.4329833984375,
      "logps/reference_chosen": -182.93911743164062,
      "logps/reference_rejected": -150.31068420410156,
      "loss": 0.2179,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2078500986099243,
      "rewards/margins": 3.109072685241699,
      "rewards/rejected": -1.9012229442596436,
      "step": 334,
      "tokens/chosen_response_len_mean": 285.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 197.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.27,
      "grad_norm": 2.8983851836796353,
      "learning_rate": 4.5779426036429587e-07,
      "logits/policy_chosen": -2.055795907974243,
      "logits/policy_rejected": -1.6440610885620117,
      "logits/reference_chosen": -2.7132530212402344,
      "logits/reference_rejected": -2.2304413318634033,
      "logps/policy_chosen": -93.63249206542969,
      "logps/policy_rejected": -186.17974853515625,
      "logps/reference_chosen": -258.497314453125,
      "logps/reference_rejected": -183.7268524169922,
      "loss": 0.2936,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6486481428146362,
      "rewards/margins": 1.6731772422790527,
      "rewards/rejected": -0.024528950452804565,
      "step": 335,
      "tokens/chosen_response_len_mean": 367.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 459.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.27,
      "grad_norm": 2.3274853742981976,
      "learning_rate": 4.574031939278047e-07,
      "logits/policy_chosen": -1.8988397121429443,
      "logits/policy_rejected": -1.9968366622924805,
      "logits/reference_chosen": -3.2483341693878174,
      "logits/reference_rejected": -2.948215961456299,
      "logps/policy_chosen": -96.612548828125,
      "logps/policy_rejected": -861.7078247070312,
      "logps/reference_chosen": -347.1319885253906,
      "logps/reference_rejected": -461.7752990722656,
      "loss": 0.206,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.505194664001465,
      "rewards/margins": 6.504519939422607,
      "rewards/rejected": -3.9993255138397217,
      "step": 336,
      "tokens/chosen_response_len_mean": 526.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 455.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.27,
      "grad_norm": 4.109692231324148,
      "learning_rate": 4.570104927285864e-07,
      "logits/policy_chosen": -1.5520960092544556,
      "logits/policy_rejected": -1.7077003717422485,
      "logits/reference_chosen": -2.795018196105957,
      "logits/reference_rejected": -2.788074016571045,
      "logps/policy_chosen": -82.21895599365234,
      "logps/policy_rejected": -586.7863159179688,
      "logps/reference_chosen": -284.3222351074219,
      "logps/reference_rejected": -400.7400817871094,
      "loss": 0.1851,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.0210328102111816,
      "rewards/margins": 3.8814949989318848,
      "rewards/rejected": -1.8604620695114136,
      "step": 337,
      "tokens/chosen_response_len_mean": 517.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 480.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.27,
      "grad_norm": 3.2891497747662277,
      "learning_rate": 4.5661615986193243e-07,
      "logits/policy_chosen": -2.0379586219787598,
      "logits/policy_rejected": -2.2403860092163086,
      "logits/reference_chosen": -2.9786267280578613,
      "logits/reference_rejected": -2.831549882888794,
      "logps/policy_chosen": -86.32127380371094,
      "logps/policy_rejected": -617.3648071289062,
      "logps/reference_chosen": -241.71507263183594,
      "logps/reference_rejected": -268.22021484375,
      "loss": 0.15,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5539380311965942,
      "rewards/margins": 5.045384407043457,
      "rewards/rejected": -3.4914462566375732,
      "step": 338,
      "tokens/chosen_response_len_mean": 496.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 221.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.27,
      "grad_norm": 2.5176895862702615,
      "learning_rate": 4.562201984359946e-07,
      "logits/policy_chosen": -1.5903446674346924,
      "logits/policy_rejected": -1.8955085277557373,
      "logits/reference_chosen": -2.9453320503234863,
      "logits/reference_rejected": -2.7459821701049805,
      "logps/policy_chosen": -67.94086456298828,
      "logps/policy_rejected": -488.8743896484375,
      "logps/reference_chosen": -216.46241760253906,
      "logps/reference_rejected": -206.1934356689453,
      "loss": 0.1801,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.485215425491333,
      "rewards/margins": 4.31202507019043,
      "rewards/rejected": -2.826809883117676,
      "step": 339,
      "tokens/chosen_response_len_mean": 474.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 204.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.27,
      "grad_norm": 2.394686969217896,
      "learning_rate": 4.558226115717616e-07,
      "logits/policy_chosen": -1.1812870502471924,
      "logits/policy_rejected": -1.297174334526062,
      "logits/reference_chosen": -2.829561948776245,
      "logits/reference_rejected": -2.5182604789733887,
      "logps/policy_chosen": -114.76174926757812,
      "logps/policy_rejected": -792.0388793945312,
      "logps/reference_chosen": -383.6563415527344,
      "logps/reference_rejected": -463.0151672363281,
      "loss": 0.1879,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.688946008682251,
      "rewards/margins": 5.979182720184326,
      "rewards/rejected": -3.290236473083496,
      "step": 340,
      "tokens/chosen_response_len_mean": 546.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 485.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.27,
      "grad_norm": 3.0673469023984423,
      "learning_rate": 4.5542340240303367e-07,
      "logits/policy_chosen": -1.7937376499176025,
      "logits/policy_rejected": -1.815042495727539,
      "logits/reference_chosen": -3.0137624740600586,
      "logits/reference_rejected": -2.86102294921875,
      "logps/policy_chosen": -52.81929016113281,
      "logps/policy_rejected": -513.2135620117188,
      "logps/reference_chosen": -155.88839721679688,
      "logps/reference_rejected": -219.87347412109375,
      "loss": 0.271,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0306909084320068,
      "rewards/margins": 3.9640917778015137,
      "rewards/rejected": -2.933401107788086,
      "step": 341,
      "tokens/chosen_response_len_mean": 236.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 219.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.27,
      "grad_norm": 1.9367186293576946,
      "learning_rate": 4.5502257407639806e-07,
      "logits/policy_chosen": -1.552323341369629,
      "logits/policy_rejected": -1.8226666450500488,
      "logits/reference_chosen": -3.2792892456054688,
      "logits/reference_rejected": -2.0340218544006348,
      "logps/policy_chosen": -97.57713317871094,
      "logps/policy_rejected": -627.7135009765625,
      "logps/reference_chosen": -344.95343017578125,
      "logps/reference_rejected": -284.43548583984375,
      "loss": 0.134,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4737627506256104,
      "rewards/margins": 5.906542778015137,
      "rewards/rejected": -3.4327802658081055,
      "step": 342,
      "tokens/chosen_response_len_mean": 452.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 255.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.28,
      "grad_norm": 5.549352474070261,
      "learning_rate": 4.5462012975120453e-07,
      "logits/policy_chosen": -1.2813889980316162,
      "logits/policy_rejected": -1.7645294666290283,
      "logits/reference_chosen": -2.8398776054382324,
      "logits/reference_rejected": -2.596129894256592,
      "logps/policy_chosen": -90.58872985839844,
      "logps/policy_rejected": -583.3743896484375,
      "logps/reference_chosen": -270.8271789550781,
      "logps/reference_rejected": -242.07595825195312,
      "loss": 0.1804,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.802384376525879,
      "rewards/margins": 5.21536922454834,
      "rewards/rejected": -3.412984848022461,
      "step": 343,
      "tokens/chosen_response_len_mean": 581.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 286.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.28,
      "grad_norm": 1.9674810528357884,
      "learning_rate": 4.542160725995401e-07,
      "logits/policy_chosen": -2.343667984008789,
      "logits/policy_rejected": -2.326040029525757,
      "logits/reference_chosen": -2.737659454345703,
      "logits/reference_rejected": -2.802224636077881,
      "logps/policy_chosen": -28.079269409179688,
      "logps/policy_rejected": -175.65020751953125,
      "logps/reference_chosen": -27.471698760986328,
      "logps/reference_rejected": -72.05661010742188,
      "loss": 0.2642,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.006075695157051086,
      "rewards/margins": 1.0298603773117065,
      "rewards/rejected": -1.0359361171722412,
      "step": 344,
      "tokens/chosen_response_len_mean": 86.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 111.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.28,
      "grad_norm": 5.451886317836004,
      "learning_rate": 4.538104058062041e-07,
      "logits/policy_chosen": -1.6862752437591553,
      "logits/policy_rejected": -1.886472225189209,
      "logits/reference_chosen": -2.9743881225585938,
      "logits/reference_rejected": -2.8067820072174072,
      "logps/policy_chosen": -62.13983917236328,
      "logps/policy_rejected": -612.0286254882812,
      "logps/reference_chosen": -192.025634765625,
      "logps/reference_rejected": -268.3218688964844,
      "loss": 0.3178,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2988580465316772,
      "rewards/margins": 4.735925674438477,
      "rewards/rejected": -3.437067747116089,
      "step": 345,
      "tokens/chosen_response_len_mean": 359.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 269.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.28,
      "grad_norm": 4.435509463597182,
      "learning_rate": 4.5340313256868337e-07,
      "logits/policy_chosen": -1.5953540802001953,
      "logits/policy_rejected": -1.0541170835494995,
      "logits/reference_chosen": -2.2541754245758057,
      "logits/reference_rejected": -1.9815850257873535,
      "logps/policy_chosen": -91.02217864990234,
      "logps/policy_rejected": -648.81787109375,
      "logps/reference_chosen": -300.9630432128906,
      "logps/reference_rejected": -305.43585205078125,
      "loss": 0.288,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0994083881378174,
      "rewards/margins": 5.533228874206543,
      "rewards/rejected": -3.4338202476501465,
      "step": 346,
      "tokens/chosen_response_len_mean": 344.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 248.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.28,
      "grad_norm": 4.398404543060366,
      "learning_rate": 4.529942560971266e-07,
      "logits/policy_chosen": -1.4266515970230103,
      "logits/policy_rejected": -1.49228835105896,
      "logits/reference_chosen": -2.7921342849731445,
      "logits/reference_rejected": -2.7268803119659424,
      "logps/policy_chosen": -120.46929931640625,
      "logps/policy_rejected": -746.52880859375,
      "logps/reference_chosen": -357.797607421875,
      "logps/reference_rejected": -444.6593322753906,
      "loss": 0.2106,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3732829093933105,
      "rewards/margins": 5.3919782638549805,
      "rewards/rejected": -3.01869535446167,
      "step": 347,
      "tokens/chosen_response_len_mean": 441.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 367.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.28,
      "grad_norm": 2.688781964809469,
      "learning_rate": 4.5258377961431937e-07,
      "logits/policy_chosen": -1.0297237634658813,
      "logits/policy_rejected": -0.6758372187614441,
      "logits/reference_chosen": -2.3045077323913574,
      "logits/reference_rejected": -1.6970428228378296,
      "logps/policy_chosen": -84.93526458740234,
      "logps/policy_rejected": -675.788330078125,
      "logps/reference_chosen": -266.78472900390625,
      "logps/reference_rejected": -359.9700927734375,
      "loss": 0.1233,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.818494439125061,
      "rewards/margins": 4.976676940917969,
      "rewards/rejected": -3.158182382583618,
      "step": 348,
      "tokens/chosen_response_len_mean": 318.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 289.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.28,
      "grad_norm": 3.214059022448137,
      "learning_rate": 4.5217170635565855e-07,
      "logits/policy_chosen": -2.1661670207977295,
      "logits/policy_rejected": -2.1412088871002197,
      "logits/reference_chosen": -2.990678310394287,
      "logits/reference_rejected": -2.691145181655884,
      "logps/policy_chosen": -27.263290405273438,
      "logps/policy_rejected": -180.78501892089844,
      "logps/reference_chosen": -51.59008026123047,
      "logps/reference_rejected": -96.38774108886719,
      "loss": 0.2374,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.24326787889003754,
      "rewards/margins": 1.0872406959533691,
      "rewards/rejected": -0.8439728617668152,
      "step": 349,
      "tokens/chosen_response_len_mean": 106.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 89.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.28,
      "grad_norm": 3.518670555830533,
      "learning_rate": 4.5175803956912694e-07,
      "logits/policy_chosen": -1.9499872922897339,
      "logits/policy_rejected": -2.1933279037475586,
      "logits/reference_chosen": -3.137690305709839,
      "logits/reference_rejected": -2.8975977897644043,
      "logps/policy_chosen": -66.71675109863281,
      "logps/policy_rejected": -577.4403076171875,
      "logps/reference_chosen": -220.90902709960938,
      "logps/reference_rejected": -255.65286254882812,
      "loss": 0.1443,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5419225692749023,
      "rewards/margins": 4.759797096252441,
      "rewards/rejected": -3.217874526977539,
      "step": 350,
      "tokens/chosen_response_len_mean": 485.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 242.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.28,
      "grad_norm": 3.699314909735531,
      "learning_rate": 4.5134278251526766e-07,
      "logits/policy_chosen": -1.4733799695968628,
      "logits/policy_rejected": -1.4901491403579712,
      "logits/reference_chosen": -2.594074249267578,
      "logits/reference_rejected": -2.557436466217041,
      "logps/policy_chosen": -65.0804443359375,
      "logps/policy_rejected": -769.9380493164062,
      "logps/reference_chosen": -200.39883422851562,
      "logps/reference_rejected": -388.98785400390625,
      "loss": 0.2727,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3531838655471802,
      "rewards/margins": 5.162685394287109,
      "rewards/rejected": -3.8095014095306396,
      "step": 351,
      "tokens/chosen_response_len_mean": 433.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 425.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.28,
      "grad_norm": 2.955343714828839,
      "learning_rate": 4.509259384671582e-07,
      "logits/policy_chosen": -1.5196211338043213,
      "logits/policy_rejected": -1.566109299659729,
      "logits/reference_chosen": -3.052445888519287,
      "logits/reference_rejected": -2.340404510498047,
      "logps/policy_chosen": -115.84571838378906,
      "logps/policy_rejected": -830.0971069335938,
      "logps/reference_chosen": -340.6599426269531,
      "logps/reference_rejected": -353.662109375,
      "loss": 0.175,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2481420040130615,
      "rewards/margins": 7.012492656707764,
      "rewards/rejected": -4.764350414276123,
      "step": 352,
      "tokens/chosen_response_len_mean": 470.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 305.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.28,
      "grad_norm": 3.500797998192056,
      "learning_rate": 4.5050751071038514e-07,
      "logits/policy_chosen": -2.069139003753662,
      "logits/policy_rejected": -2.066579580307007,
      "logits/reference_chosen": -3.0677695274353027,
      "logits/reference_rejected": -3.032003402709961,
      "logps/policy_chosen": -59.17474365234375,
      "logps/policy_rejected": -444.245361328125,
      "logps/reference_chosen": -161.52598571777344,
      "logps/reference_rejected": -251.91793823242188,
      "loss": 0.2833,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0235124826431274,
      "rewards/margins": 2.946786642074585,
      "rewards/rejected": -1.9232741594314575,
      "step": 353,
      "tokens/chosen_response_len_mean": 381.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 321.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.28,
      "grad_norm": 2.1150271947388437,
      "learning_rate": 4.500875025430176e-07,
      "logits/policy_chosen": -1.5964065790176392,
      "logits/policy_rejected": -1.8535802364349365,
      "logits/reference_chosen": -3.0790903568267822,
      "logits/reference_rejected": -2.013641595840454,
      "logps/policy_chosen": -91.34295654296875,
      "logps/policy_rejected": -1013.7203369140625,
      "logps/reference_chosen": -323.52752685546875,
      "logps/reference_rejected": -508.953369140625,
      "loss": 0.082,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.321845769882202,
      "rewards/margins": 7.3695149421691895,
      "rewards/rejected": -5.047668933868408,
      "step": 354,
      "tokens/chosen_response_len_mean": 414.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 346.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.29,
      "grad_norm": 4.458337777869365,
      "learning_rate": 4.496659172755818e-07,
      "logits/policy_chosen": -1.3642287254333496,
      "logits/policy_rejected": -1.410393476486206,
      "logits/reference_chosen": -2.549731492996216,
      "logits/reference_rejected": -2.4937071800231934,
      "logps/policy_chosen": -111.19947814941406,
      "logps/policy_rejected": -509.53564453125,
      "logps/reference_chosen": -215.30552673339844,
      "logps/reference_rejected": -308.2844543457031,
      "loss": 0.3226,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0410603284835815,
      "rewards/margins": 3.053572416305542,
      "rewards/rejected": -2.01251220703125,
      "step": 355,
      "tokens/chosen_response_len_mean": 330.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 318.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.29,
      "grad_norm": 4.024586930018457,
      "learning_rate": 4.4924275823103466e-07,
      "logits/policy_chosen": -1.8358407020568848,
      "logits/policy_rejected": -1.9370864629745483,
      "logits/reference_chosen": -3.0807526111602783,
      "logits/reference_rejected": -2.92172908782959,
      "logps/policy_chosen": -74.08454132080078,
      "logps/policy_rejected": -655.4462890625,
      "logps/reference_chosen": -287.81585693359375,
      "logps/reference_rejected": -331.1325378417969,
      "loss": 0.1674,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.137312889099121,
      "rewards/margins": 5.380450248718262,
      "rewards/rejected": -3.2431373596191406,
      "step": 356,
      "tokens/chosen_response_len_mean": 392.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 230.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.29,
      "grad_norm": 2.5198509782294924,
      "learning_rate": 4.4881802874473774e-07,
      "logits/policy_chosen": -1.2551157474517822,
      "logits/policy_rejected": -1.7098793983459473,
      "logits/reference_chosen": -2.8631482124328613,
      "logits/reference_rejected": -1.495772123336792,
      "logps/policy_chosen": -56.709354400634766,
      "logps/policy_rejected": -329.5363464355469,
      "logps/reference_chosen": -180.67190551757812,
      "logps/reference_rejected": -128.47109985351562,
      "loss": 0.1685,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2396254539489746,
      "rewards/margins": 3.2502779960632324,
      "rewards/rejected": -2.0106523036956787,
      "step": 357,
      "tokens/chosen_response_len_mean": 321.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 157.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.29,
      "grad_norm": 2.7881215108282253,
      "learning_rate": 4.4839173216443084e-07,
      "logits/policy_chosen": -2.051104784011841,
      "logits/policy_rejected": -2.1336636543273926,
      "logits/reference_chosen": -2.95450758934021,
      "logits/reference_rejected": -2.7968645095825195,
      "logps/policy_chosen": -36.93220520019531,
      "logps/policy_rejected": -377.7947082519531,
      "logps/reference_chosen": -118.31900787353516,
      "logps/reference_rejected": -182.75210571289062,
      "loss": 0.1298,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8138680458068848,
      "rewards/margins": 2.764294147491455,
      "rewards/rejected": -1.9504261016845703,
      "step": 358,
      "tokens/chosen_response_len_mean": 222.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 194.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.29,
      "grad_norm": 2.9202638151581564,
      "learning_rate": 4.479638718502056e-07,
      "logits/policy_chosen": -2.800821304321289,
      "logits/policy_rejected": -2.74906063079834,
      "logits/reference_chosen": -2.9278526306152344,
      "logits/reference_rejected": -3.074326515197754,
      "logps/policy_chosen": -26.134109497070312,
      "logps/policy_rejected": -101.59957122802734,
      "logps/reference_chosen": -21.818992614746094,
      "logps/reference_rejected": -37.89555358886719,
      "loss": 0.245,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.04315117001533508,
      "rewards/margins": 0.5938889980316162,
      "rewards/rejected": -0.6370401382446289,
      "step": 359,
      "tokens/chosen_response_len_mean": 14.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 28.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.29,
      "grad_norm": 2.2402578949629,
      "learning_rate": 4.475344511744794e-07,
      "logits/policy_chosen": -1.5277884006500244,
      "logits/policy_rejected": -1.1120015382766724,
      "logits/reference_chosen": -3.0673985481262207,
      "logits/reference_rejected": -1.979531168937683,
      "logps/policy_chosen": -129.73883056640625,
      "logps/policy_rejected": -1009.8583984375,
      "logps/reference_chosen": -419.92962646484375,
      "logps/reference_rejected": -410.1859130859375,
      "loss": 0.131,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9019076824188232,
      "rewards/margins": 8.898632049560547,
      "rewards/rejected": -5.996725082397461,
      "step": 360,
      "tokens/chosen_response_len_mean": 535.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 503.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.29,
      "grad_norm": 2.2362340915683405,
      "learning_rate": 4.471034735219682e-07,
      "logits/policy_chosen": -1.8571796417236328,
      "logits/policy_rejected": -2.0071451663970947,
      "logits/reference_chosen": -3.0822551250457764,
      "logits/reference_rejected": -2.8071236610412598,
      "logps/policy_chosen": -113.39871215820312,
      "logps/policy_rejected": -765.5164184570312,
      "logps/reference_chosen": -314.39288330078125,
      "logps/reference_rejected": -295.49212646484375,
      "loss": 0.286,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.009941577911377,
      "rewards/margins": 6.7101850509643555,
      "rewards/rejected": -4.7002434730529785,
      "step": 361,
      "tokens/chosen_response_len_mean": 370.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 250.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.29,
      "grad_norm": 2.9958059876475645,
      "learning_rate": 4.4667094228966006e-07,
      "logits/policy_chosen": -1.3339217901229858,
      "logits/policy_rejected": -1.4387969970703125,
      "logits/reference_chosen": -2.5392210483551025,
      "logits/reference_rejected": -2.5316193103790283,
      "logps/policy_chosen": -65.20072174072266,
      "logps/policy_rejected": -141.90574645996094,
      "logps/reference_chosen": -153.89601135253906,
      "logps/reference_rejected": -161.5721893310547,
      "loss": 0.3208,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.8869529962539673,
      "rewards/margins": 0.6902884840965271,
      "rewards/rejected": 0.19666451215744019,
      "step": 362,
      "tokens/chosen_response_len_mean": 182.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 179.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.29,
      "grad_norm": 2.714261614183403,
      "learning_rate": 4.462368608867888e-07,
      "logits/policy_chosen": -2.0045056343078613,
      "logits/policy_rejected": -2.136458158493042,
      "logits/reference_chosen": -2.8513717651367188,
      "logits/reference_rejected": -2.6678905487060547,
      "logps/policy_chosen": -57.24700164794922,
      "logps/policy_rejected": -487.1899719238281,
      "logps/reference_chosen": -193.75621032714844,
      "logps/reference_rejected": -158.42803955078125,
      "loss": 0.2195,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3650922775268555,
      "rewards/margins": 4.652711868286133,
      "rewards/rejected": -3.287619113922119,
      "step": 363,
      "tokens/chosen_response_len_mean": 301.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 134.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.29,
      "grad_norm": 3.0348087146187934,
      "learning_rate": 4.4580123273480643e-07,
      "logits/policy_chosen": -1.2777620553970337,
      "logits/policy_rejected": -1.6785014867782593,
      "logits/reference_chosen": -2.842172622680664,
      "logits/reference_rejected": -2.3908252716064453,
      "logps/policy_chosen": -62.06177520751953,
      "logps/policy_rejected": -431.913330078125,
      "logps/reference_chosen": -187.41107177734375,
      "logps/reference_rejected": -209.33456420898438,
      "loss": 0.1306,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2534929513931274,
      "rewards/margins": 3.4792802333831787,
      "rewards/rejected": -2.225787401199341,
      "step": 364,
      "tokens/chosen_response_len_mean": 385.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 249.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.29,
      "grad_norm": 4.751862010403708,
      "learning_rate": 4.453640612673566e-07,
      "logits/policy_chosen": -1.219412088394165,
      "logits/policy_rejected": -1.36615788936615,
      "logits/reference_chosen": -2.3538596630096436,
      "logits/reference_rejected": -2.0649256706237793,
      "logps/policy_chosen": -101.12548828125,
      "logps/policy_rejected": -373.8370361328125,
      "logps/reference_chosen": -326.5352783203125,
      "logps/reference_rejected": -239.3107147216797,
      "loss": 0.1992,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2540977001190186,
      "rewards/margins": 3.599360942840576,
      "rewards/rejected": -1.3452633619308472,
      "step": 365,
      "tokens/chosen_response_len_mean": 465.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 235.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.29,
      "grad_norm": 3.4845170475354874,
      "learning_rate": 4.4492534993024744e-07,
      "logits/policy_chosen": -1.7572671175003052,
      "logits/policy_rejected": -1.7631720304489136,
      "logits/reference_chosen": -2.83882737159729,
      "logits/reference_rejected": -2.7936973571777344,
      "logps/policy_chosen": -79.20316314697266,
      "logps/policy_rejected": -479.2203063964844,
      "logps/reference_chosen": -240.270263671875,
      "logps/reference_rejected": -315.88555908203125,
      "loss": 0.2738,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.610671043395996,
      "rewards/margins": 3.2440185546875,
      "rewards/rejected": -1.633347511291504,
      "step": 366,
      "tokens/chosen_response_len_mean": 446.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 421.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.3,
      "grad_norm": 1.7048313796935655,
      "learning_rate": 4.4448510218142466e-07,
      "logits/policy_chosen": -1.9213192462921143,
      "logits/policy_rejected": -1.650076150894165,
      "logits/reference_chosen": -3.0253703594207764,
      "logits/reference_rejected": -2.3654181957244873,
      "logps/policy_chosen": -61.119239807128906,
      "logps/policy_rejected": -502.1123962402344,
      "logps/reference_chosen": -204.41744995117188,
      "logps/reference_rejected": -221.98794555664062,
      "loss": 0.1403,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4329822063446045,
      "rewards/margins": 4.234225749969482,
      "rewards/rejected": -2.801244020462036,
      "step": 367,
      "tokens/chosen_response_len_mean": 211.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 156.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.3,
      "grad_norm": 4.368110245343919,
      "learning_rate": 4.440433214909436e-07,
      "logits/policy_chosen": -1.6689025163650513,
      "logits/policy_rejected": -0.940861701965332,
      "logits/reference_chosen": -2.9102039337158203,
      "logits/reference_rejected": -2.0719833374023438,
      "logps/policy_chosen": -48.4640007019043,
      "logps/policy_rejected": -306.5902099609375,
      "logps/reference_chosen": -142.2967529296875,
      "logps/reference_rejected": -147.8551025390625,
      "loss": 0.2671,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.938327431678772,
      "rewards/margins": 2.5256781578063965,
      "rewards/rejected": -1.587350845336914,
      "step": 368,
      "tokens/chosen_response_len_mean": 255.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 382.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.3,
      "grad_norm": 4.421616617478599,
      "learning_rate": 4.4360001134094274e-07,
      "logits/policy_chosen": -1.7381703853607178,
      "logits/policy_rejected": -1.873031497001648,
      "logits/reference_chosen": -2.868490695953369,
      "logits/reference_rejected": -2.679164171218872,
      "logps/policy_chosen": -37.81551742553711,
      "logps/policy_rejected": -327.39471435546875,
      "logps/reference_chosen": -100.18566131591797,
      "logps/reference_rejected": -154.6254425048828,
      "loss": 0.2761,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6237015128135681,
      "rewards/margins": 2.3513944149017334,
      "rewards/rejected": -1.7276926040649414,
      "step": 369,
      "tokens/chosen_response_len_mean": 260.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 207.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.3,
      "grad_norm": 3.55457857405225,
      "learning_rate": 4.431551752256155e-07,
      "logits/policy_chosen": -1.4202126264572144,
      "logits/policy_rejected": -1.435547113418579,
      "logits/reference_chosen": -2.81121563911438,
      "logits/reference_rejected": -2.563999652862549,
      "logps/policy_chosen": -81.54356384277344,
      "logps/policy_rejected": -511.54022216796875,
      "logps/reference_chosen": -295.8115539550781,
      "logps/reference_rejected": -280.714111328125,
      "loss": 0.0897,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1426796913146973,
      "rewards/margins": 4.4509406089782715,
      "rewards/rejected": -2.308260917663574,
      "step": 370,
      "tokens/chosen_response_len_mean": 406.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 281.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.3,
      "grad_norm": 3.6157257883700984,
      "learning_rate": 4.4270881665118344e-07,
      "logits/policy_chosen": -1.7662705183029175,
      "logits/policy_rejected": -1.9252934455871582,
      "logits/reference_chosen": -2.995553493499756,
      "logits/reference_rejected": -2.469679594039917,
      "logps/policy_chosen": -77.08085632324219,
      "logps/policy_rejected": -437.26995849609375,
      "logps/reference_chosen": -275.45654296875,
      "logps/reference_rejected": -245.6436767578125,
      "loss": 0.2254,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9837571382522583,
      "rewards/margins": 3.900019884109497,
      "rewards/rejected": -1.9162627458572388,
      "step": 371,
      "tokens/chosen_response_len_mean": 352.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 245.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.3,
      "grad_norm": 2.739985089644356,
      "learning_rate": 4.422609391358677e-07,
      "logits/policy_chosen": -1.491788387298584,
      "logits/policy_rejected": -1.8629511594772339,
      "logits/reference_chosen": -2.8367395401000977,
      "logits/reference_rejected": -2.385693073272705,
      "logps/policy_chosen": -93.3801498413086,
      "logps/policy_rejected": -564.1484985351562,
      "logps/reference_chosen": -333.2955322265625,
      "logps/reference_rejected": -284.53338623046875,
      "loss": 0.154,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.399153709411621,
      "rewards/margins": 5.195304870605469,
      "rewards/rejected": -2.7961511611938477,
      "step": 372,
      "tokens/chosen_response_len_mean": 537.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 220.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.3,
      "grad_norm": 2.8326056798898325,
      "learning_rate": 4.418115462098623e-07,
      "logits/policy_chosen": -1.6880661249160767,
      "logits/policy_rejected": -1.8501157760620117,
      "logits/reference_chosen": -2.9830873012542725,
      "logits/reference_rejected": -2.656494140625,
      "logps/policy_chosen": -87.76195526123047,
      "logps/policy_rejected": -603.6038208007812,
      "logps/reference_chosen": -330.90252685546875,
      "logps/reference_rejected": -258.64581298828125,
      "loss": 0.1816,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4314053058624268,
      "rewards/margins": 5.880985260009766,
      "rewards/rejected": -3.449579954147339,
      "step": 373,
      "tokens/chosen_response_len_mean": 536.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 377.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.3,
      "grad_norm": 4.790162201367013,
      "learning_rate": 4.4136064141530547e-07,
      "logits/policy_chosen": -1.7553645372390747,
      "logits/policy_rejected": -1.6190441846847534,
      "logits/reference_chosen": -2.7890069484710693,
      "logits/reference_rejected": -2.1159238815307617,
      "logps/policy_chosen": -67.688232421875,
      "logps/policy_rejected": -195.291748046875,
      "logps/reference_chosen": -179.9830322265625,
      "logps/reference_rejected": -114.24205780029297,
      "loss": 0.1921,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1229479312896729,
      "rewards/margins": 1.9334447383880615,
      "rewards/rejected": -0.8104968070983887,
      "step": 374,
      "tokens/chosen_response_len_mean": 431.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 405.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.3,
      "grad_norm": 1.2307538383465877,
      "learning_rate": 4.409082283062523e-07,
      "logits/policy_chosen": -1.5772703886032104,
      "logits/policy_rejected": -1.6785141229629517,
      "logits/reference_chosen": -3.0875000953674316,
      "logits/reference_rejected": -2.7498016357421875,
      "logps/policy_chosen": -74.79031372070312,
      "logps/policy_rejected": -750.72998046875,
      "logps/reference_chosen": -317.4873352050781,
      "logps/reference_rejected": -347.156982421875,
      "loss": 0.0851,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4269702434539795,
      "rewards/margins": 6.462699890136719,
      "rewards/rejected": -4.03572940826416,
      "step": 375,
      "tokens/chosen_response_len_mean": 458.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 365.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.3,
      "grad_norm": 3.5542378235491863,
      "learning_rate": 4.4045431044864616e-07,
      "logits/policy_chosen": -1.6069151163101196,
      "logits/policy_rejected": -1.7103867530822754,
      "logits/reference_chosen": -2.6566100120544434,
      "logits/reference_rejected": -2.6409106254577637,
      "logps/policy_chosen": -87.49906921386719,
      "logps/policy_rejected": -665.1824951171875,
      "logps/reference_chosen": -305.02557373046875,
      "logps/reference_rejected": -320.46221923828125,
      "loss": 0.2368,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.175264835357666,
      "rewards/margins": 5.622467517852783,
      "rewards/rejected": -3.447202682495117,
      "step": 376,
      "tokens/chosen_response_len_mean": 411.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 312.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.3,
      "grad_norm": 5.281625587606173,
      "learning_rate": 4.3999889142029126e-07,
      "logits/policy_chosen": -2.1449053287506104,
      "logits/policy_rejected": -2.194610118865967,
      "logits/reference_chosen": -2.8323793411254883,
      "logits/reference_rejected": -2.804175853729248,
      "logps/policy_chosen": -30.32268524169922,
      "logps/policy_rejected": -170.59368896484375,
      "logps/reference_chosen": -67.28926086425781,
      "logps/reference_rejected": -86.84202575683594,
      "loss": 0.3036,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3696657121181488,
      "rewards/margins": 1.20718252658844,
      "rewards/rejected": -0.8375166654586792,
      "step": 377,
      "tokens/chosen_response_len_mean": 183.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 105.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.3,
      "grad_norm": 1.9215327012324088,
      "learning_rate": 4.395419748108241e-07,
      "logits/policy_chosen": -1.883148431777954,
      "logits/policy_rejected": -1.9562857151031494,
      "logits/reference_chosen": -3.1288890838623047,
      "logits/reference_rejected": -2.9048759937286377,
      "logps/policy_chosen": -83.87744903564453,
      "logps/policy_rejected": -814.286865234375,
      "logps/reference_chosen": -300.5325622558594,
      "logps/reference_rejected": -364.4478759765625,
      "loss": 0.1434,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.166551113128662,
      "rewards/margins": 6.66494083404541,
      "rewards/rejected": -4.498389720916748,
      "step": 378,
      "tokens/chosen_response_len_mean": 353.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 284.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.3,
      "grad_norm": 3.0494184783882075,
      "learning_rate": 4.390835642216848e-07,
      "logits/policy_chosen": -1.1829478740692139,
      "logits/policy_rejected": -1.6148120164871216,
      "logits/reference_chosen": -2.8968608379364014,
      "logits/reference_rejected": -2.4937872886657715,
      "logps/policy_chosen": -101.84024047851562,
      "logps/policy_rejected": -1095.6416015625,
      "logps/reference_chosen": -356.5980224609375,
      "logps/reference_rejected": -464.4998474121094,
      "loss": 0.0851,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5475780963897705,
      "rewards/margins": 8.85899543762207,
      "rewards/rejected": -6.311417579650879,
      "step": 379,
      "tokens/chosen_response_len_mean": 750.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 420.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.31,
      "grad_norm": 1.9839846610083902,
      "learning_rate": 4.386236632660897e-07,
      "logits/policy_chosen": -1.9111047983169556,
      "logits/policy_rejected": -1.440907597541809,
      "logits/reference_chosen": -2.8704569339752197,
      "logits/reference_rejected": -2.470503091812134,
      "logps/policy_chosen": -63.218116760253906,
      "logps/policy_rejected": -509.4794616699219,
      "logps/reference_chosen": -221.18771362304688,
      "logps/reference_rejected": -222.36109924316406,
      "loss": 0.0839,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5796959400177002,
      "rewards/margins": 4.4508795738220215,
      "rewards/rejected": -2.871183395385742,
      "step": 380,
      "tokens/chosen_response_len_mean": 366.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 483.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.31,
      "grad_norm": 3.547953034306201,
      "learning_rate": 4.3816227556900174e-07,
      "logits/policy_chosen": -2.1110708713531494,
      "logits/policy_rejected": -1.5525974035263062,
      "logits/reference_chosen": -3.074847459793091,
      "logits/reference_rejected": -2.299776315689087,
      "logps/policy_chosen": -53.559783935546875,
      "logps/policy_rejected": -340.3197937011719,
      "logps/reference_chosen": -210.47630310058594,
      "logps/reference_rejected": -170.03268432617188,
      "loss": 0.2292,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5691652297973633,
      "rewards/margins": 3.27203631401062,
      "rewards/rejected": -1.7028712034225464,
      "step": 381,
      "tokens/chosen_response_len_mean": 434.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 429.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.31,
      "grad_norm": 4.4598115746078175,
      "learning_rate": 4.376994047671028e-07,
      "logits/policy_chosen": -1.8473024368286133,
      "logits/policy_rejected": -1.9738845825195312,
      "logits/reference_chosen": -2.9645516872406006,
      "logits/reference_rejected": -2.810880184173584,
      "logps/policy_chosen": -37.34379196166992,
      "logps/policy_rejected": -328.9185485839844,
      "logps/reference_chosen": -105.24017333984375,
      "logps/reference_rejected": -159.31472778320312,
      "loss": 0.1772,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6789637804031372,
      "rewards/margins": 2.375002145767212,
      "rewards/rejected": -1.6960381269454956,
      "step": 382,
      "tokens/chosen_response_len_mean": 204.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 216.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.31,
      "grad_norm": 2.895844402337817,
      "learning_rate": 4.3723505450876454e-07,
      "logits/policy_chosen": -1.5201809406280518,
      "logits/policy_rejected": -1.8310809135437012,
      "logits/reference_chosen": -2.652768135070801,
      "logits/reference_rejected": -2.6164894104003906,
      "logps/policy_chosen": -89.18895721435547,
      "logps/policy_rejected": -672.33349609375,
      "logps/reference_chosen": -299.8074951171875,
      "logps/reference_rejected": -316.7373962402344,
      "loss": 0.1478,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1061851978302,
      "rewards/margins": 5.66214656829834,
      "rewards/rejected": -3.5559608936309814,
      "step": 383,
      "tokens/chosen_response_len_mean": 622.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 480.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.31,
      "grad_norm": 3.2525327057187328,
      "learning_rate": 4.3676922845401963e-07,
      "logits/policy_chosen": -1.4135372638702393,
      "logits/policy_rejected": -1.6460597515106201,
      "logits/reference_chosen": -2.6158249378204346,
      "logits/reference_rejected": -2.269045352935791,
      "logps/policy_chosen": -36.46479797363281,
      "logps/policy_rejected": -372.2850341796875,
      "logps/reference_chosen": -101.00570678710938,
      "logps/reference_rejected": -160.78067016601562,
      "loss": 0.1283,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.645409107208252,
      "rewards/margins": 2.76045298576355,
      "rewards/rejected": -2.115043878555298,
      "step": 384,
      "tokens/chosen_response_len_mean": 362.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 215.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.31,
      "grad_norm": 2.683005455867468,
      "learning_rate": 4.363019302745334e-07,
      "logits/policy_chosen": -2.2419838905334473,
      "logits/policy_rejected": -2.421705961227417,
      "logits/reference_chosen": -3.0781402587890625,
      "logits/reference_rejected": -2.9199631214141846,
      "logps/policy_chosen": -49.374847412109375,
      "logps/policy_rejected": -359.7245178222656,
      "logps/reference_chosen": -158.96978759765625,
      "logps/reference_rejected": -127.78620910644531,
      "loss": 0.2265,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0959495306015015,
      "rewards/margins": 3.415332317352295,
      "rewards/rejected": -2.319382667541504,
      "step": 385,
      "tokens/chosen_response_len_mean": 254.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 102.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.31,
      "grad_norm": 4.3953584290948156,
      "learning_rate": 4.3583316365357416e-07,
      "logits/policy_chosen": -1.6831576824188232,
      "logits/policy_rejected": -1.8844032287597656,
      "logits/reference_chosen": -2.9197726249694824,
      "logits/reference_rejected": -2.744558811187744,
      "logps/policy_chosen": -82.87364959716797,
      "logps/policy_rejected": -661.0844116210938,
      "logps/reference_chosen": -322.45697021484375,
      "logps/reference_rejected": -285.1172180175781,
      "loss": 0.1246,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3958332538604736,
      "rewards/margins": 6.155505657196045,
      "rewards/rejected": -3.7596724033355713,
      "step": 386,
      "tokens/chosen_response_len_mean": 471.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 260.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.31,
      "grad_norm": 4.3894384552460215,
      "learning_rate": 4.3536293228598474e-07,
      "logits/policy_chosen": -1.8510702848434448,
      "logits/policy_rejected": -1.8795132637023926,
      "logits/reference_chosen": -2.6799912452697754,
      "logits/reference_rejected": -2.4659016132354736,
      "logps/policy_chosen": -32.05808639526367,
      "logps/policy_rejected": -213.7239990234375,
      "logps/reference_chosen": -59.347633361816406,
      "logps/reference_rejected": -68.51786804199219,
      "loss": 0.2542,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2728954553604126,
      "rewards/margins": 1.7249568700790405,
      "rewards/rejected": -1.4520612955093384,
      "step": 387,
      "tokens/chosen_response_len_mean": 63.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 50.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.31,
      "grad_norm": 9.267866458567886,
      "learning_rate": 4.3489123987815325e-07,
      "logits/policy_chosen": -1.4151192903518677,
      "logits/policy_rejected": -1.5208919048309326,
      "logits/reference_chosen": -2.8274762630462646,
      "logits/reference_rejected": -2.905880928039551,
      "logps/policy_chosen": -118.6706771850586,
      "logps/policy_rejected": -557.0550537109375,
      "logps/reference_chosen": -374.5296936035156,
      "logps/reference_rejected": -424.9022521972656,
      "loss": 0.1827,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5585901737213135,
      "rewards/margins": 3.8801183700561523,
      "rewards/rejected": -1.321528434753418,
      "step": 388,
      "tokens/chosen_response_len_mean": 562.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 549.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.31,
      "grad_norm": 4.103907207414104,
      "learning_rate": 4.344180901479836e-07,
      "logits/policy_chosen": -1.689210295677185,
      "logits/policy_rejected": -1.1707533597946167,
      "logits/reference_chosen": -2.7496488094329834,
      "logits/reference_rejected": -2.5293631553649902,
      "logps/policy_chosen": -95.53358459472656,
      "logps/policy_rejected": -641.15380859375,
      "logps/reference_chosen": -288.0838623046875,
      "logps/reference_rejected": -270.31884765625,
      "loss": 0.1837,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.925502896308899,
      "rewards/margins": 5.633852958679199,
      "rewards/rejected": -3.7083497047424316,
      "step": 389,
      "tokens/chosen_response_len_mean": 366.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 445.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.31,
      "grad_norm": 4.588056708293884,
      "learning_rate": 4.339434868248665e-07,
      "logits/policy_chosen": -1.6808419227600098,
      "logits/policy_rejected": -1.7554115056991577,
      "logits/reference_chosen": -2.79362154006958,
      "logits/reference_rejected": -2.6228065490722656,
      "logps/policy_chosen": -59.64699172973633,
      "logps/policy_rejected": -678.405029296875,
      "logps/reference_chosen": -213.72731018066406,
      "logps/reference_rejected": -335.5184326171875,
      "loss": 0.2293,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.54080331325531,
      "rewards/margins": 4.969668865203857,
      "rewards/rejected": -3.428865909576416,
      "step": 390,
      "tokens/chosen_response_len_mean": 348.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 301.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.31,
      "grad_norm": 3.0510324137868525,
      "learning_rate": 4.3346743364965e-07,
      "logits/policy_chosen": -1.049216628074646,
      "logits/policy_rejected": -1.8208162784576416,
      "logits/reference_chosen": -2.591167449951172,
      "logits/reference_rejected": -2.324847459793091,
      "logps/policy_chosen": -121.5611572265625,
      "logps/policy_rejected": -607.802001953125,
      "logps/reference_chosen": -316.2970275878906,
      "logps/reference_rejected": -231.97964477539062,
      "loss": 0.1775,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9473586082458496,
      "rewards/margins": 5.705582141876221,
      "rewards/rejected": -3.758223056793213,
      "step": 391,
      "tokens/chosen_response_len_mean": 472.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 194.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.32,
      "grad_norm": 4.799092230789488,
      "learning_rate": 4.3298993437460984e-07,
      "logits/policy_chosen": -1.3143819570541382,
      "logits/policy_rejected": -1.4681634902954102,
      "logits/reference_chosen": -2.5000357627868652,
      "logits/reference_rejected": -2.514500379562378,
      "logps/policy_chosen": -81.29645538330078,
      "logps/policy_rejected": -878.3502197265625,
      "logps/reference_chosen": -289.78057861328125,
      "logps/reference_rejected": -434.30841064453125,
      "loss": 0.2684,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.084841012954712,
      "rewards/margins": 6.525258541107178,
      "rewards/rejected": -4.440417766571045,
      "step": 392,
      "tokens/chosen_response_len_mean": 493.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 346.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.32,
      "grad_norm": 5.0096860526087985,
      "learning_rate": 4.325109927634202e-07,
      "logits/policy_chosen": -1.745060682296753,
      "logits/policy_rejected": -1.8800654411315918,
      "logits/reference_chosen": -2.9817285537719727,
      "logits/reference_rejected": -2.4398183822631836,
      "logps/policy_chosen": -67.2919921875,
      "logps/policy_rejected": -429.408203125,
      "logps/reference_chosen": -218.21548461914062,
      "logps/reference_rejected": -260.8848571777344,
      "loss": 0.239,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.50923490524292,
      "rewards/margins": 3.1944684982299805,
      "rewards/rejected": -1.6852335929870605,
      "step": 393,
      "tokens/chosen_response_len_mean": 263.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 170.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.32,
      "grad_norm": 3.990192788421323,
      "learning_rate": 4.320306125911236e-07,
      "logits/policy_chosen": -2.1053833961486816,
      "logits/policy_rejected": -2.0984866619110107,
      "logits/reference_chosen": -2.8531177043914795,
      "logits/reference_rejected": -2.862253189086914,
      "logps/policy_chosen": -26.376094818115234,
      "logps/policy_rejected": -305.2973327636719,
      "logps/reference_chosen": -47.91350555419922,
      "logps/reference_rejected": -131.7420196533203,
      "loss": 0.2484,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21537409722805023,
      "rewards/margins": 1.9509271383285522,
      "rewards/rejected": -1.7355530261993408,
      "step": 394,
      "tokens/chosen_response_len_mean": 129.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 149.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.32,
      "grad_norm": 6.374622408586315,
      "learning_rate": 4.315487976441013e-07,
      "logits/policy_chosen": -1.2467350959777832,
      "logits/policy_rejected": -1.5044665336608887,
      "logits/reference_chosen": -2.920576572418213,
      "logits/reference_rejected": -2.6558072566986084,
      "logps/policy_chosen": -138.9432373046875,
      "logps/policy_rejected": -726.0556030273438,
      "logps/reference_chosen": -493.89862060546875,
      "logps/reference_rejected": -350.41497802734375,
      "loss": 0.0826,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.549553394317627,
      "rewards/margins": 7.305959224700928,
      "rewards/rejected": -3.756405830383301,
      "step": 395,
      "tokens/chosen_response_len_mean": 683.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 443.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.32,
      "grad_norm": 2.7801583537276717,
      "learning_rate": 4.31065551720044e-07,
      "logits/policy_chosen": -1.164729118347168,
      "logits/policy_rejected": -1.1291731595993042,
      "logits/reference_chosen": -2.5515594482421875,
      "logits/reference_rejected": -2.443310022354126,
      "logps/policy_chosen": -66.39266967773438,
      "logps/policy_rejected": -558.2879638671875,
      "logps/reference_chosen": -186.88536071777344,
      "logps/reference_rejected": -314.25531005859375,
      "loss": 0.19,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2049269676208496,
      "rewards/margins": 3.6452534198760986,
      "rewards/rejected": -2.440326452255249,
      "step": 396,
      "tokens/chosen_response_len_mean": 448.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 443.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.32,
      "grad_norm": 2.9706769612280617,
      "learning_rate": 4.3058087862792097e-07,
      "logits/policy_chosen": -1.182519555091858,
      "logits/policy_rejected": -1.190351128578186,
      "logits/reference_chosen": -2.546760082244873,
      "logits/reference_rejected": -2.415588617324829,
      "logps/policy_chosen": -64.53900146484375,
      "logps/policy_rejected": -359.20123291015625,
      "logps/reference_chosen": -171.31234741210938,
      "logps/reference_rejected": -217.0635986328125,
      "loss": 0.1348,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0677335262298584,
      "rewards/margins": 2.489109992980957,
      "rewards/rejected": -1.4213764667510986,
      "step": 397,
      "tokens/chosen_response_len_mean": 421.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 385.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.32,
      "grad_norm": 4.30537555824225,
      "learning_rate": 4.300947821879506e-07,
      "logits/policy_chosen": -1.8363888263702393,
      "logits/policy_rejected": -2.059960126876831,
      "logits/reference_chosen": -2.9730725288391113,
      "logits/reference_rejected": -2.7597203254699707,
      "logps/policy_chosen": -53.318687438964844,
      "logps/policy_rejected": -473.90850830078125,
      "logps/reference_chosen": -151.44744873046875,
      "logps/reference_rejected": -189.46267700195312,
      "loss": 0.261,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9812875390052795,
      "rewards/margins": 3.8257455825805664,
      "rewards/rejected": -2.8444581031799316,
      "step": 398,
      "tokens/chosen_response_len_mean": 290.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 171.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.32,
      "grad_norm": 1.3903903548910281,
      "learning_rate": 4.296072662315704e-07,
      "logits/policy_chosen": -1.257875919342041,
      "logits/policy_rejected": -1.1928701400756836,
      "logits/reference_chosen": -2.869974374771118,
      "logits/reference_rejected": -2.889599323272705,
      "logps/policy_chosen": -72.68022155761719,
      "logps/policy_rejected": -879.3326416015625,
      "logps/reference_chosen": -297.6036071777344,
      "logps/reference_rejected": -490.29583740234375,
      "loss": 0.1009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2492337226867676,
      "rewards/margins": 6.139601707458496,
      "rewards/rejected": -3.8903677463531494,
      "step": 399,
      "tokens/chosen_response_len_mean": 611.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 622.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.32,
      "grad_norm": 2.979855295546658,
      "learning_rate": 4.291183346014062e-07,
      "logits/policy_chosen": -2.134671926498413,
      "logits/policy_rejected": -2.2338736057281494,
      "logits/reference_chosen": -3.2482404708862305,
      "logits/reference_rejected": -3.0586061477661133,
      "logps/policy_chosen": -53.64082336425781,
      "logps/policy_rejected": -510.7403869628906,
      "logps/reference_chosen": -155.8409423828125,
      "logps/reference_rejected": -154.61331176757812,
      "loss": 0.2301,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0220012664794922,
      "rewards/margins": 4.5832719802856445,
      "rewards/rejected": -3.5612707138061523,
      "step": 400,
      "tokens/chosen_response_len_mean": 202.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 134.25,
      "tokens/rejected_response_len_std": 0.0
    }
  ],
  "logging_steps": 1,
  "max_steps": 1244,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}