{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.48231511254019294,
  "eval_steps": 100,
  "global_step": 600,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "grad_norm": 2.5493941690669297,
      "learning_rate": 4e-09,
      "logits/policy_chosen": -2.279445171356201,
      "logits/policy_rejected": -2.070641279220581,
      "logits/reference_chosen": -2.262451410293579,
      "logits/reference_rejected": -1.700181484222412,
      "logps/policy_chosen": -259.83294677734375,
      "logps/policy_rejected": -282.3813171386719,
      "logps/reference_chosen": -280.28363037109375,
      "logps/reference_rejected": -290.83868408203125,
      "loss": 0.6746,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.20450688898563385,
      "rewards/margins": 0.11993316560983658,
      "rewards/rejected": 0.08457372337579727,
      "step": 1,
      "tokens/chosen_response_len_mean": 528.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 444.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.0,
      "grad_norm": 2.2490732615654547,
      "learning_rate": 8e-09,
      "logits/policy_chosen": -2.745072364807129,
      "logits/policy_rejected": -2.607792854309082,
      "logits/reference_chosen": -2.701054096221924,
      "logits/reference_rejected": -2.529007911682129,
      "logps/policy_chosen": -147.3030242919922,
      "logps/policy_rejected": -197.99253845214844,
      "logps/reference_chosen": -156.72518920898438,
      "logps/reference_rejected": -206.69471740722656,
      "loss": 0.6863,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.09422144293785095,
      "rewards/margins": 0.0071996524930000305,
      "rewards/rejected": 0.08702179044485092,
      "step": 2,
      "tokens/chosen_response_len_mean": 298.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 277.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.0,
      "grad_norm": 2.2889093862107543,
      "learning_rate": 1.2e-08,
      "logits/policy_chosen": -2.882375478744507,
      "logits/policy_rejected": -2.992877960205078,
      "logits/reference_chosen": -2.748257637023926,
      "logits/reference_rejected": -2.9701337814331055,
      "logps/policy_chosen": -105.87107849121094,
      "logps/policy_rejected": -148.20022583007812,
      "logps/reference_chosen": -112.85089111328125,
      "logps/reference_rejected": -155.7706298828125,
      "loss": 0.6832,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.06979808956384659,
      "rewards/margins": -0.005905959755182266,
      "rewards/rejected": 0.07570404559373856,
      "step": 3,
      "tokens/chosen_response_len_mean": 239.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 254.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.0,
      "grad_norm": 2.2808153966501084,
      "learning_rate": 1.6e-08,
      "logits/policy_chosen": -2.9396884441375732,
      "logits/policy_rejected": -2.9384472370147705,
      "logits/reference_chosen": -2.9226865768432617,
      "logits/reference_rejected": -2.9309639930725098,
      "logps/policy_chosen": -123.63784790039062,
      "logps/policy_rejected": -180.25296020507812,
      "logps/reference_chosen": -129.93817138671875,
      "logps/reference_rejected": -184.21334838867188,
      "loss": 0.674,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.06300327926874161,
      "rewards/margins": 0.023399492725729942,
      "rewards/rejected": 0.039603784680366516,
      "step": 4,
      "tokens/chosen_response_len_mean": 298.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 297.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.0,
      "grad_norm": 2.2660235876486974,
      "learning_rate": 2e-08,
      "logits/policy_chosen": -3.221891164779663,
      "logits/policy_rejected": -2.8190152645111084,
      "logits/reference_chosen": -3.207385778427124,
      "logits/reference_rejected": -2.6983580589294434,
      "logps/policy_chosen": -179.60354614257812,
      "logps/policy_rejected": -206.77960205078125,
      "logps/reference_chosen": -189.15216064453125,
      "logps/reference_rejected": -209.81790161132812,
      "loss": 0.6778,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09548593312501907,
      "rewards/margins": 0.06510297954082489,
      "rewards/rejected": 0.03038296103477478,
      "step": 5,
      "tokens/chosen_response_len_mean": 440.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 232.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.0,
      "grad_norm": 2.0558578581082307,
      "learning_rate": 2.4e-08,
      "logits/policy_chosen": -2.4811384677886963,
      "logits/policy_rejected": -2.4067883491516113,
      "logits/reference_chosen": -2.330796003341675,
      "logits/reference_rejected": -2.4005446434020996,
      "logps/policy_chosen": -148.8319091796875,
      "logps/policy_rejected": -179.03482055664062,
      "logps/reference_chosen": -160.83175659179688,
      "logps/reference_rejected": -189.98275756835938,
      "loss": 0.6989,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.11999829858541489,
      "rewards/margins": 0.010519090108573437,
      "rewards/rejected": 0.10947921127080917,
      "step": 6,
      "tokens/chosen_response_len_mean": 289.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 309.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 2.5447256930451734,
      "learning_rate": 2.8e-08,
      "logits/policy_chosen": -2.2599525451660156,
      "logits/policy_rejected": -2.190964698791504,
      "logits/reference_chosen": -2.1052117347717285,
      "logits/reference_rejected": -2.134941577911377,
      "logps/policy_chosen": -19.338191986083984,
      "logps/policy_rejected": -38.73615646362305,
      "logps/reference_chosen": -20.665983200073242,
      "logps/reference_rejected": -40.615394592285156,
      "loss": 0.684,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01327789481729269,
      "rewards/margins": -0.005514495074748993,
      "rewards/rejected": 0.018792390823364258,
      "step": 7,
      "tokens/chosen_response_len_mean": 34.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 47.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 2.967658233666017,
      "learning_rate": 3.2e-08,
      "logits/policy_chosen": -2.4715514183044434,
      "logits/policy_rejected": -2.4913477897644043,
      "logits/reference_chosen": -2.421693801879883,
      "logits/reference_rejected": -2.4608089923858643,
      "logps/policy_chosen": -29.782058715820312,
      "logps/policy_rejected": -64.6925048828125,
      "logps/reference_chosen": -32.50696563720703,
      "logps/reference_rejected": -66.7137222290039,
      "loss": 0.6712,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.027249068021774292,
      "rewards/margins": 0.00703694112598896,
      "rewards/rejected": 0.020212125033140182,
      "step": 8,
      "tokens/chosen_response_len_mean": 40.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 48.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 3.4470069507233316,
      "learning_rate": 3.6e-08,
      "logits/policy_chosen": -2.7727508544921875,
      "logits/policy_rejected": -2.718796730041504,
      "logits/reference_chosen": -2.676337957382202,
      "logits/reference_rejected": -2.616457223892212,
      "logps/policy_chosen": -108.26776123046875,
      "logps/policy_rejected": -218.6373748779297,
      "logps/reference_chosen": -118.67279815673828,
      "logps/reference_rejected": -223.27325439453125,
      "loss": 0.6796,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.1040503978729248,
      "rewards/margins": 0.05769149586558342,
      "rewards/rejected": 0.046358898282051086,
      "step": 9,
      "tokens/chosen_response_len_mean": 252.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 222.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 2.304016636338041,
      "learning_rate": 4e-08,
      "logits/policy_chosen": -2.5566985607147217,
      "logits/policy_rejected": -2.5395240783691406,
      "logits/reference_chosen": -2.4825587272644043,
      "logits/reference_rejected": -2.498563051223755,
      "logps/policy_chosen": -168.69143676757812,
      "logps/policy_rejected": -287.2906494140625,
      "logps/reference_chosen": -181.2764129638672,
      "logps/reference_rejected": -299.7208557128906,
      "loss": 0.6885,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.1258496791124344,
      "rewards/margins": 0.001547420397400856,
      "rewards/rejected": 0.12430226802825928,
      "step": 10,
      "tokens/chosen_response_len_mean": 518.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 503.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 2.084799466815333,
      "learning_rate": 4.4e-08,
      "logits/policy_chosen": -2.7605199813842773,
      "logits/policy_rejected": -2.7186694145202637,
      "logits/reference_chosen": -2.7691569328308105,
      "logits/reference_rejected": -2.7842984199523926,
      "logps/policy_chosen": -145.88803100585938,
      "logps/policy_rejected": -219.43760681152344,
      "logps/reference_chosen": -155.90982055664062,
      "logps/reference_rejected": -227.8662109375,
      "loss": 0.6893,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.1002180427312851,
      "rewards/margins": 0.015932004898786545,
      "rewards/rejected": 0.08428603410720825,
      "step": 11,
      "tokens/chosen_response_len_mean": 604.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 448.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 2.7628214791467736,
      "learning_rate": 4.8e-08,
      "logits/policy_chosen": -2.9070513248443604,
      "logits/policy_rejected": -2.7778186798095703,
      "logits/reference_chosen": -2.8774163722991943,
      "logits/reference_rejected": -2.6446313858032227,
      "logps/policy_chosen": -101.27095794677734,
      "logps/policy_rejected": -136.12225341796875,
      "logps/reference_chosen": -108.19136047363281,
      "logps/reference_rejected": -140.3483123779297,
      "loss": 0.6673,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.06920397281646729,
      "rewards/margins": 0.026943188160657883,
      "rewards/rejected": 0.0422607846558094,
      "step": 12,
      "tokens/chosen_response_len_mean": 339.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 269.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 2.1795487992104348,
      "learning_rate": 5.1999999999999996e-08,
      "logits/policy_chosen": -2.4187445640563965,
      "logits/policy_rejected": -2.845353364944458,
      "logits/reference_chosen": -2.3343820571899414,
      "logits/reference_rejected": -2.5970327854156494,
      "logps/policy_chosen": -130.312255859375,
      "logps/policy_rejected": -73.26902770996094,
      "logps/reference_chosen": -134.85186767578125,
      "logps/reference_rejected": -73.45825958251953,
      "loss": 0.6772,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.045396123081445694,
      "rewards/margins": 0.04350382089614868,
      "rewards/rejected": 0.0018922993913292885,
      "step": 13,
      "tokens/chosen_response_len_mean": 177.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 101.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 2.7157502561420026,
      "learning_rate": 5.6e-08,
      "logits/policy_chosen": -3.211674928665161,
      "logits/policy_rejected": -3.0040595531463623,
      "logits/reference_chosen": -3.2025675773620605,
      "logits/reference_rejected": -3.0620803833007812,
      "logps/policy_chosen": -195.99533081054688,
      "logps/policy_rejected": -242.1226806640625,
      "logps/reference_chosen": -208.40536499023438,
      "logps/reference_rejected": -252.9475860595703,
      "loss": 0.6781,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.12410032004117966,
      "rewards/margins": 0.015851235017180443,
      "rewards/rejected": 0.10824908316135406,
      "step": 14,
      "tokens/chosen_response_len_mean": 543.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 408.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 3.50208846860952,
      "learning_rate": 6e-08,
      "logits/policy_chosen": -2.6727657318115234,
      "logits/policy_rejected": -2.5919432640075684,
      "logits/reference_chosen": -2.6382195949554443,
      "logits/reference_rejected": -2.417813777923584,
      "logps/policy_chosen": -170.3718719482422,
      "logps/policy_rejected": -223.43052673339844,
      "logps/reference_chosen": -191.32583618164062,
      "logps/reference_rejected": -242.33273315429688,
      "loss": 0.6686,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.209539532661438,
      "rewards/margins": 0.020517464727163315,
      "rewards/rejected": 0.18902206420898438,
      "step": 15,
      "tokens/chosen_response_len_mean": 431.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 403.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 1.6207728056732267,
      "learning_rate": 6.4e-08,
      "logits/policy_chosen": -2.954475164413452,
      "logits/policy_rejected": -2.8040027618408203,
      "logits/reference_chosen": -2.9069364070892334,
      "logits/reference_rejected": -2.8010973930358887,
      "logps/policy_chosen": -54.05670166015625,
      "logps/policy_rejected": -109.65800476074219,
      "logps/reference_chosen": -55.35580825805664,
      "logps/reference_rejected": -108.90874481201172,
      "loss": 0.6824,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.012991041876375675,
      "rewards/margins": 0.02048366516828537,
      "rewards/rejected": -0.007492623291909695,
      "step": 16,
      "tokens/chosen_response_len_mean": 89.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 96.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 2.409299627131989,
      "learning_rate": 6.8e-08,
      "logits/policy_chosen": -2.909968137741089,
      "logits/policy_rejected": -2.84816837310791,
      "logits/reference_chosen": -2.8810513019561768,
      "logits/reference_rejected": -2.6653146743774414,
      "logps/policy_chosen": -119.80767822265625,
      "logps/policy_rejected": -267.8531799316406,
      "logps/reference_chosen": -126.68125915527344,
      "logps/reference_rejected": -275.761474609375,
      "loss": 0.6995,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.06873579323291779,
      "rewards/margins": -0.010347230359911919,
      "rewards/rejected": 0.07908301800489426,
      "step": 17,
      "tokens/chosen_response_len_mean": 314.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 251.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.01,
      "grad_norm": 2.703021449933624,
      "learning_rate": 7.2e-08,
      "logits/policy_chosen": -2.8886208534240723,
      "logits/policy_rejected": -2.603667974472046,
      "logits/reference_chosen": -2.8919739723205566,
      "logits/reference_rejected": -2.336148262023926,
      "logps/policy_chosen": -144.8170166015625,
      "logps/policy_rejected": -144.6788330078125,
      "logps/reference_chosen": -164.3677215576172,
      "logps/reference_rejected": -156.78842163085938,
      "loss": 0.679,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1955069750547409,
      "rewards/margins": 0.07441116869449615,
      "rewards/rejected": 0.12109580636024475,
      "step": 18,
      "tokens/chosen_response_len_mean": 421.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 314.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 1.9691382131351773,
      "learning_rate": 7.599999999999999e-08,
      "logits/policy_chosen": -2.766261339187622,
      "logits/policy_rejected": -2.697242259979248,
      "logits/reference_chosen": -2.710810899734497,
      "logits/reference_rejected": -2.557218551635742,
      "logps/policy_chosen": -150.92379760742188,
      "logps/policy_rejected": -174.591064453125,
      "logps/reference_chosen": -160.51756286621094,
      "logps/reference_rejected": -185.50238037109375,
      "loss": 0.6851,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0959378108382225,
      "rewards/margins": -0.013175351545214653,
      "rewards/rejected": 0.1091131642460823,
      "step": 19,
      "tokens/chosen_response_len_mean": 263.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 253.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 2.1003754961271675,
      "learning_rate": 8e-08,
      "logits/policy_chosen": -2.8774542808532715,
      "logits/policy_rejected": -2.6342382431030273,
      "logits/reference_chosen": -2.742877244949341,
      "logits/reference_rejected": -2.4385361671447754,
      "logps/policy_chosen": -104.96978759765625,
      "logps/policy_rejected": -157.80657958984375,
      "logps/reference_chosen": -107.06401062011719,
      "logps/reference_rejected": -159.94366455078125,
      "loss": 0.6774,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.02094227820634842,
      "rewards/margins": -0.00042874296195805073,
      "rewards/rejected": 0.02137102000415325,
      "step": 20,
      "tokens/chosen_response_len_mean": 111.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 124.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 2.113555044323313,
      "learning_rate": 8.4e-08,
      "logits/policy_chosen": -3.1314477920532227,
      "logits/policy_rejected": -2.8576135635375977,
      "logits/reference_chosen": -3.1075565814971924,
      "logits/reference_rejected": -2.6684985160827637,
      "logps/policy_chosen": -121.34175109863281,
      "logps/policy_rejected": -152.86807250976562,
      "logps/reference_chosen": -127.80372619628906,
      "logps/reference_rejected": -158.0865478515625,
      "loss": 0.6994,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.06461974233388901,
      "rewards/margins": 0.012434981763362885,
      "rewards/rejected": 0.05218476057052612,
      "step": 21,
      "tokens/chosen_response_len_mean": 317.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 203.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 2.8572997186380196,
      "learning_rate": 8.8e-08,
      "logits/policy_chosen": -3.023026466369629,
      "logits/policy_rejected": -2.970913887023926,
      "logits/reference_chosen": -2.9523911476135254,
      "logits/reference_rejected": -2.8476057052612305,
      "logps/policy_chosen": -123.84272003173828,
      "logps/policy_rejected": -210.03341674804688,
      "logps/reference_chosen": -129.73898315429688,
      "logps/reference_rejected": -216.26409912109375,
      "loss": 0.672,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.05896264314651489,
      "rewards/margins": -0.0033443691208958626,
      "rewards/rejected": 0.06230701133608818,
      "step": 22,
      "tokens/chosen_response_len_mean": 241.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 224.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 2.4344792354730034,
      "learning_rate": 9.199999999999999e-08,
      "logits/policy_chosen": -2.45721435546875,
      "logits/policy_rejected": -2.4645514488220215,
      "logits/reference_chosen": -2.429760694503784,
      "logits/reference_rejected": -2.448065757751465,
      "logps/policy_chosen": -107.37543487548828,
      "logps/policy_rejected": -120.32754516601562,
      "logps/reference_chosen": -113.06903839111328,
      "logps/reference_rejected": -126.56575012207031,
      "loss": 0.6748,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.056935906410217285,
      "rewards/margins": -0.005446167662739754,
      "rewards/rejected": 0.06238207593560219,
      "step": 23,
      "tokens/chosen_response_len_mean": 314.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 315.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 3.5341818462448504,
      "learning_rate": 9.6e-08,
      "logits/policy_chosen": -2.5186548233032227,
      "logits/policy_rejected": -2.3946735858917236,
      "logits/reference_chosen": -2.5037264823913574,
      "logits/reference_rejected": -2.4204611778259277,
      "logps/policy_chosen": -71.1528549194336,
      "logps/policy_rejected": -96.15353393554688,
      "logps/reference_chosen": -74.45181274414062,
      "logps/reference_rejected": -93.63554382324219,
      "loss": 0.6593,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03298955038189888,
      "rewards/margins": 0.05816952511668205,
      "rewards/rejected": -0.025179967284202576,
      "step": 24,
      "tokens/chosen_response_len_mean": 247.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 155.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 2.6672688539385514,
      "learning_rate": 1e-07,
      "logits/policy_chosen": -3.052779197692871,
      "logits/policy_rejected": -2.9980740547180176,
      "logits/reference_chosen": -3.0356476306915283,
      "logits/reference_rejected": -2.9704558849334717,
      "logps/policy_chosen": -158.24560546875,
      "logps/policy_rejected": -246.05154418945312,
      "logps/reference_chosen": -166.8022003173828,
      "logps/reference_rejected": -252.77267456054688,
      "loss": 0.6801,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.08556582033634186,
      "rewards/margins": 0.01835446059703827,
      "rewards/rejected": 0.06721135973930359,
      "step": 25,
      "tokens/chosen_response_len_mean": 302.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 309.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 2.157863179356832,
      "learning_rate": 1.0399999999999999e-07,
      "logits/policy_chosen": -2.8244199752807617,
      "logits/policy_rejected": -2.8272688388824463,
      "logits/reference_chosen": -2.8120579719543457,
      "logits/reference_rejected": -2.81868839263916,
      "logps/policy_chosen": -188.75546264648438,
      "logps/policy_rejected": -245.15794372558594,
      "logps/reference_chosen": -201.83450317382812,
      "logps/reference_rejected": -256.3631591796875,
      "loss": 0.6824,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.13079024851322174,
      "rewards/margins": 0.018737995997071266,
      "rewards/rejected": 0.11205224692821503,
      "step": 26,
      "tokens/chosen_response_len_mean": 499.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 500.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 2.6785946054818295,
      "learning_rate": 1.0799999999999999e-07,
      "logits/policy_chosen": -3.041701078414917,
      "logits/policy_rejected": -3.0683014392852783,
      "logits/reference_chosen": -3.021995782852173,
      "logits/reference_rejected": -3.0257372856140137,
      "logps/policy_chosen": -102.09321594238281,
      "logps/policy_rejected": -163.72860717773438,
      "logps/reference_chosen": -109.2179946899414,
      "logps/reference_rejected": -167.15447998046875,
      "loss": 0.6755,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07124768942594528,
      "rewards/margins": 0.036988936364650726,
      "rewards/rejected": 0.03425874561071396,
      "step": 27,
      "tokens/chosen_response_len_mean": 243.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 237.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 3.0106278359730614,
      "learning_rate": 1.12e-07,
      "logits/policy_chosen": -3.058267116546631,
      "logits/policy_rejected": -3.0097880363464355,
      "logits/reference_chosen": -3.000871181488037,
      "logits/reference_rejected": -2.860621690750122,
      "logps/policy_chosen": -177.05154418945312,
      "logps/policy_rejected": -241.18173217773438,
      "logps/reference_chosen": -187.62625122070312,
      "logps/reference_rejected": -242.83314514160156,
      "loss": 0.6642,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.10574702173471451,
      "rewards/margins": 0.08923279494047165,
      "rewards/rejected": 0.01651422120630741,
      "step": 28,
      "tokens/chosen_response_len_mean": 391.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 363.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 1.9714081999119137,
      "learning_rate": 1.16e-07,
      "logits/policy_chosen": -2.7036781311035156,
      "logits/policy_rejected": -2.706456422805786,
      "logits/reference_chosen": -2.6821694374084473,
      "logits/reference_rejected": -2.70880126953125,
      "logps/policy_chosen": -50.51161193847656,
      "logps/policy_rejected": -97.51376342773438,
      "logps/reference_chosen": -53.98440170288086,
      "logps/reference_rejected": -100.45523834228516,
      "loss": 0.6839,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.03472791612148285,
      "rewards/margins": 0.005313164554536343,
      "rewards/rejected": 0.029414750635623932,
      "step": 29,
      "tokens/chosen_response_len_mean": 246.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 215.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 3.3945073696477333,
      "learning_rate": 1.2e-07,
      "logits/policy_chosen": -2.679271936416626,
      "logits/policy_rejected": -2.747288227081299,
      "logits/reference_chosen": -2.648017406463623,
      "logits/reference_rejected": -2.542226552963257,
      "logps/policy_chosen": -181.09521484375,
      "logps/policy_rejected": -349.05694580078125,
      "logps/reference_chosen": -202.54962158203125,
      "logps/reference_rejected": -355.8513488769531,
      "loss": 0.6558,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21454395353794098,
      "rewards/margins": 0.14660006761550903,
      "rewards/rejected": 0.06794389337301254,
      "step": 30,
      "tokens/chosen_response_len_mean": 309.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 268.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.02,
      "grad_norm": 2.729481181496147,
      "learning_rate": 1.24e-07,
      "logits/policy_chosen": -3.1615757942199707,
      "logits/policy_rejected": -2.618534564971924,
      "logits/reference_chosen": -3.148972511291504,
      "logits/reference_rejected": -2.28782320022583,
      "logps/policy_chosen": -244.06692504882812,
      "logps/policy_rejected": -395.2162780761719,
      "logps/reference_chosen": -254.54090881347656,
      "logps/reference_rejected": -397.2469177246094,
      "loss": 0.6873,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10473988205194473,
      "rewards/margins": 0.08443356305360794,
      "rewards/rejected": 0.020306318998336792,
      "step": 31,
      "tokens/chosen_response_len_mean": 410.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 269.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 2.1927971055522133,
      "learning_rate": 1.28e-07,
      "logits/policy_chosen": -2.935199499130249,
      "logits/policy_rejected": -2.862617254257202,
      "logits/reference_chosen": -2.9165947437286377,
      "logits/reference_rejected": -2.809278964996338,
      "logps/policy_chosen": -156.64752197265625,
      "logps/policy_rejected": -224.47006225585938,
      "logps/reference_chosen": -176.1259765625,
      "logps/reference_rejected": -239.154296875,
      "loss": 0.6749,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.19478438794612885,
      "rewards/margins": 0.04794217646121979,
      "rewards/rejected": 0.14684221148490906,
      "step": 32,
      "tokens/chosen_response_len_mean": 545.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 527.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 2.6358653856327257,
      "learning_rate": 1.32e-07,
      "logits/policy_chosen": -2.5262393951416016,
      "logits/policy_rejected": -2.364823341369629,
      "logits/reference_chosen": -2.506678342819214,
      "logits/reference_rejected": -1.6685891151428223,
      "logps/policy_chosen": -104.21517181396484,
      "logps/policy_rejected": -126.63694763183594,
      "logps/reference_chosen": -114.83543395996094,
      "logps/reference_rejected": -136.51809692382812,
      "loss": 0.6837,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.10620267689228058,
      "rewards/margins": 0.007391175255179405,
      "rewards/rejected": 0.09881150722503662,
      "step": 33,
      "tokens/chosen_response_len_mean": 448.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 339.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 2.626247472893481,
      "learning_rate": 1.36e-07,
      "logits/policy_chosen": -3.0164244174957275,
      "logits/policy_rejected": -2.670468330383301,
      "logits/reference_chosen": -2.9480087757110596,
      "logits/reference_rejected": -2.077363967895508,
      "logps/policy_chosen": -144.32318115234375,
      "logps/policy_rejected": -178.09408569335938,
      "logps/reference_chosen": -154.3887939453125,
      "logps/reference_rejected": -182.9420623779297,
      "loss": 0.6748,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1006559506058693,
      "rewards/margins": 0.052176106721162796,
      "rewards/rejected": 0.0484798438847065,
      "step": 34,
      "tokens/chosen_response_len_mean": 407.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 217.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 2.296064376513374,
      "learning_rate": 1.4e-07,
      "logits/policy_chosen": -2.598402738571167,
      "logits/policy_rejected": -2.6598832607269287,
      "logits/reference_chosen": -2.553514003753662,
      "logits/reference_rejected": -2.6580069065093994,
      "logps/policy_chosen": -281.2328186035156,
      "logps/policy_rejected": -179.6129608154297,
      "logps/reference_chosen": -298.13836669921875,
      "logps/reference_rejected": -186.5122528076172,
      "loss": 0.6684,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.16905567049980164,
      "rewards/margins": 0.10006258636713028,
      "rewards/rejected": 0.06899307668209076,
      "step": 35,
      "tokens/chosen_response_len_mean": 377.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 248.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 2.898376793238501,
      "learning_rate": 1.44e-07,
      "logits/policy_chosen": -2.7883753776550293,
      "logits/policy_rejected": -2.6082558631896973,
      "logits/reference_chosen": -2.780714273452759,
      "logits/reference_rejected": -2.583864212036133,
      "logps/policy_chosen": -105.63471984863281,
      "logps/policy_rejected": -245.55908203125,
      "logps/reference_chosen": -110.00314331054688,
      "logps/reference_rejected": -244.04986572265625,
      "loss": 0.6715,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04368434101343155,
      "rewards/margins": 0.05877665430307388,
      "rewards/rejected": -0.015092315152287483,
      "step": 36,
      "tokens/chosen_response_len_mean": 183.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 284.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 2.6935110955386,
      "learning_rate": 1.4799999999999998e-07,
      "logits/policy_chosen": -3.0653228759765625,
      "logits/policy_rejected": -2.882253646850586,
      "logits/reference_chosen": -3.0485243797302246,
      "logits/reference_rejected": -2.6053519248962402,
      "logps/policy_chosen": -186.7164764404297,
      "logps/policy_rejected": -267.58123779296875,
      "logps/reference_chosen": -204.80267333984375,
      "logps/reference_rejected": -272.190185546875,
      "loss": 0.6697,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1808619499206543,
      "rewards/margins": 0.13477247953414917,
      "rewards/rejected": 0.046089474111795425,
      "step": 37,
      "tokens/chosen_response_len_mean": 433.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 321.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 2.597922955352179,
      "learning_rate": 1.5199999999999998e-07,
      "logits/policy_chosen": -2.699632167816162,
      "logits/policy_rejected": -2.7144150733947754,
      "logits/reference_chosen": -2.658318519592285,
      "logits/reference_rejected": -2.691765546798706,
      "logps/policy_chosen": -135.453369140625,
      "logps/policy_rejected": -242.96441650390625,
      "logps/reference_chosen": -146.15704345703125,
      "logps/reference_rejected": -248.74380493164062,
      "loss": 0.6585,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.10703684389591217,
      "rewards/margins": 0.049242909997701645,
      "rewards/rejected": 0.057793937623500824,
      "step": 38,
      "tokens/chosen_response_len_mean": 375.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 387.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 2.079049499920605,
      "learning_rate": 1.56e-07,
      "logits/policy_chosen": -3.077770233154297,
      "logits/policy_rejected": -3.098449945449829,
      "logits/reference_chosen": -3.0287766456604004,
      "logits/reference_rejected": -3.0804648399353027,
      "logps/policy_chosen": -12.474931716918945,
      "logps/policy_rejected": -25.898088455200195,
      "logps/reference_chosen": -12.915287971496582,
      "logps/reference_rejected": -27.526098251342773,
      "loss": 0.6793,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.00440356507897377,
      "rewards/margins": -0.011876537464559078,
      "rewards/rejected": 0.016280101612210274,
      "step": 39,
      "tokens/chosen_response_len_mean": 23.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 25.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 2.9270192927802197,
      "learning_rate": 1.6e-07,
      "logits/policy_chosen": -2.821016550064087,
      "logits/policy_rejected": -2.7184388637542725,
      "logits/reference_chosen": -2.5926756858825684,
      "logits/reference_rejected": -2.7159690856933594,
      "logps/policy_chosen": -115.9990005493164,
      "logps/policy_rejected": -150.73153686523438,
      "logps/reference_chosen": -120.78485107421875,
      "logps/reference_rejected": -157.8815460205078,
      "loss": 0.668,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.04785853251814842,
      "rewards/margins": -0.02364155277609825,
      "rewards/rejected": 0.07150008529424667,
      "step": 40,
      "tokens/chosen_response_len_mean": 179.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 168.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 2.807071363383845,
      "learning_rate": 1.64e-07,
      "logits/policy_chosen": -2.6167349815368652,
      "logits/policy_rejected": -2.5763139724731445,
      "logits/reference_chosen": -2.513126850128174,
      "logits/reference_rejected": -2.5481503009796143,
      "logps/policy_chosen": -213.66064453125,
      "logps/policy_rejected": -274.35003662109375,
      "logps/reference_chosen": -240.58090209960938,
      "logps/reference_rejected": -297.9310607910156,
      "loss": 0.6847,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.26920264959335327,
      "rewards/margins": 0.03339245170354843,
      "rewards/rejected": 0.23581019043922424,
      "step": 41,
      "tokens/chosen_response_len_mean": 468.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 442.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 3.1342954098759788,
      "learning_rate": 1.68e-07,
      "logits/policy_chosen": -2.98238205909729,
      "logits/policy_rejected": -2.6742758750915527,
      "logits/reference_chosen": -2.9786105155944824,
      "logits/reference_rejected": -2.5011706352233887,
      "logps/policy_chosen": -400.793701171875,
      "logps/policy_rejected": -471.21826171875,
      "logps/reference_chosen": -424.1178283691406,
      "logps/reference_rejected": -484.6050720214844,
      "loss": 0.6671,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.23324130475521088,
      "rewards/margins": 0.09937307238578796,
      "rewards/rejected": 0.1338682323694229,
      "step": 42,
      "tokens/chosen_response_len_mean": 593.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 479.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.03,
      "grad_norm": 1.7422493551589444,
      "learning_rate": 1.7199999999999998e-07,
      "logits/policy_chosen": -2.763662338256836,
      "logits/policy_rejected": -2.434535503387451,
      "logits/reference_chosen": -2.710179567337036,
      "logits/reference_rejected": -1.868179440498352,
      "logps/policy_chosen": -77.93124389648438,
      "logps/policy_rejected": -110.21360778808594,
      "logps/reference_chosen": -86.2520751953125,
      "logps/reference_rejected": -110.11550903320312,
      "loss": 0.679,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08320826292037964,
      "rewards/margins": 0.08418922126293182,
      "rewards/rejected": -0.0009809592738747597,
      "step": 43,
      "tokens/chosen_response_len_mean": 152.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 85.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 3.138281711904061,
      "learning_rate": 1.76e-07,
      "logits/policy_chosen": -2.8771750926971436,
      "logits/policy_rejected": -2.9175660610198975,
      "logits/reference_chosen": -2.8088254928588867,
      "logits/reference_rejected": -2.877462863922119,
      "logps/policy_chosen": -37.14703369140625,
      "logps/policy_rejected": -55.703121185302734,
      "logps/reference_chosen": -38.25192642211914,
      "logps/reference_rejected": -56.50023651123047,
      "loss": 0.686,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.011048920452594757,
      "rewards/margins": 0.0030777817592024803,
      "rewards/rejected": 0.007971138693392277,
      "step": 44,
      "tokens/chosen_response_len_mean": 66.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 70.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 3.284990775967774,
      "learning_rate": 1.8e-07,
      "logits/policy_chosen": -2.8581864833831787,
      "logits/policy_rejected": -2.766713857650757,
      "logits/reference_chosen": -2.8522939682006836,
      "logits/reference_rejected": -2.7666430473327637,
      "logps/policy_chosen": -165.7818603515625,
      "logps/policy_rejected": -241.84002685546875,
      "logps/reference_chosen": -181.072998046875,
      "logps/reference_rejected": -250.3277587890625,
      "loss": 0.6537,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.15291133522987366,
      "rewards/margins": 0.06803382933139801,
      "rewards/rejected": 0.08487750589847565,
      "step": 45,
      "tokens/chosen_response_len_mean": 560.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 473.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 2.463717582717747,
      "learning_rate": 1.8399999999999998e-07,
      "logits/policy_chosen": -2.900191068649292,
      "logits/policy_rejected": -2.9184107780456543,
      "logits/reference_chosen": -2.8673532009124756,
      "logits/reference_rejected": -2.892207145690918,
      "logps/policy_chosen": -51.21653747558594,
      "logps/policy_rejected": -107.85240936279297,
      "logps/reference_chosen": -54.76094436645508,
      "logps/reference_rejected": -117.80521392822266,
      "loss": 0.695,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.035444095730781555,
      "rewards/margins": -0.06408397108316422,
      "rewards/rejected": 0.09952806681394577,
      "step": 46,
      "tokens/chosen_response_len_mean": 109.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 108.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 1.9738986795582658,
      "learning_rate": 1.88e-07,
      "logits/policy_chosen": -2.2502527236938477,
      "logits/policy_rejected": -2.245668649673462,
      "logits/reference_chosen": -2.177823781967163,
      "logits/reference_rejected": -2.223409652709961,
      "logps/policy_chosen": -37.820743560791016,
      "logps/policy_rejected": -99.87039184570312,
      "logps/reference_chosen": -44.7762451171875,
      "logps/reference_rejected": -104.45420837402344,
      "loss": 0.6825,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.06955502927303314,
      "rewards/margins": 0.023716777563095093,
      "rewards/rejected": 0.04583824798464775,
      "step": 47,
      "tokens/chosen_response_len_mean": 93.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 110.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 2.8813658188320335,
      "learning_rate": 1.92e-07,
      "logits/policy_chosen": -2.6205151081085205,
      "logits/policy_rejected": -2.5673413276672363,
      "logits/reference_chosen": -2.5005288124084473,
      "logits/reference_rejected": -2.561748504638672,
      "logps/policy_chosen": -65.36790466308594,
      "logps/policy_rejected": -167.23562622070312,
      "logps/reference_chosen": -72.64520263671875,
      "logps/reference_rejected": -172.6849365234375,
      "loss": 0.6706,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.07277299463748932,
      "rewards/margins": 0.018280120566487312,
      "rewards/rejected": 0.054492875933647156,
      "step": 48,
      "tokens/chosen_response_len_mean": 199.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 209.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 2.862318181422901,
      "learning_rate": 1.96e-07,
      "logits/policy_chosen": -2.701411247253418,
      "logits/policy_rejected": -2.8611505031585693,
      "logits/reference_chosen": -2.2373385429382324,
      "logits/reference_rejected": -2.845200777053833,
      "logps/policy_chosen": -126.9096908569336,
      "logps/policy_rejected": -269.68243408203125,
      "logps/reference_chosen": -136.80958557128906,
      "logps/reference_rejected": -277.586181640625,
      "loss": 0.6796,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.09899893403053284,
      "rewards/margins": 0.01996154710650444,
      "rewards/rejected": 0.0790373906493187,
      "step": 49,
      "tokens/chosen_response_len_mean": 262.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 337.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 2.324007938345189,
      "learning_rate": 2e-07,
      "logits/policy_chosen": -2.2418417930603027,
      "logits/policy_rejected": -2.1697628498077393,
      "logits/reference_chosen": -2.142324924468994,
      "logits/reference_rejected": -2.1179537773132324,
      "logps/policy_chosen": -136.041259765625,
      "logps/policy_rejected": -266.0906677246094,
      "logps/reference_chosen": -151.6305694580078,
      "logps/reference_rejected": -273.1885986328125,
      "loss": 0.6677,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.15589313209056854,
      "rewards/margins": 0.08491363376379013,
      "rewards/rejected": 0.07097949832677841,
      "step": 50,
      "tokens/chosen_response_len_mean": 526.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 469.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 2.3504496124001744,
      "learning_rate": 2.0399999999999997e-07,
      "logits/policy_chosen": -2.899348497390747,
      "logits/policy_rejected": -2.8138720989227295,
      "logits/reference_chosen": -2.8722875118255615,
      "logits/reference_rejected": -2.8029227256774902,
      "logps/policy_chosen": -63.48197555541992,
      "logps/policy_rejected": -101.47474670410156,
      "logps/reference_chosen": -68.96565246582031,
      "logps/reference_rejected": -108.34456634521484,
      "loss": 0.6831,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.05483681336045265,
      "rewards/margins": -0.013861382380127907,
      "rewards/rejected": 0.06869819760322571,
      "step": 51,
      "tokens/chosen_response_len_mean": 212.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 181.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 2.4664067603984665,
      "learning_rate": 2.0799999999999998e-07,
      "logits/policy_chosen": -2.710914134979248,
      "logits/policy_rejected": -2.6489503383636475,
      "logits/reference_chosen": -2.6887543201446533,
      "logits/reference_rejected": -2.565074920654297,
      "logps/policy_chosen": -120.6403579711914,
      "logps/policy_rejected": -180.31842041015625,
      "logps/reference_chosen": -129.82321166992188,
      "logps/reference_rejected": -187.31112670898438,
      "loss": 0.6824,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0918285921216011,
      "rewards/margins": 0.021901624277234077,
      "rewards/rejected": 0.06992696225643158,
      "step": 52,
      "tokens/chosen_response_len_mean": 340.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 269.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 3.7734473565962108,
      "learning_rate": 2.12e-07,
      "logits/policy_chosen": -2.9076874256134033,
      "logits/policy_rejected": -2.8471598625183105,
      "logits/reference_chosen": -2.837496280670166,
      "logits/reference_rejected": -2.783947467803955,
      "logps/policy_chosen": -88.39242553710938,
      "logps/policy_rejected": -232.25318908691406,
      "logps/reference_chosen": -96.45068359375,
      "logps/reference_rejected": -237.35009765625,
      "loss": 0.6784,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.08058254420757294,
      "rewards/margins": 0.02961355820298195,
      "rewards/rejected": 0.05096898972988129,
      "step": 53,
      "tokens/chosen_response_len_mean": 195.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 194.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 2.8319761915445367,
      "learning_rate": 2.1599999999999998e-07,
      "logits/policy_chosen": -3.144252300262451,
      "logits/policy_rejected": -3.0141947269439697,
      "logits/reference_chosen": -3.116751194000244,
      "logits/reference_rejected": -2.8330681324005127,
      "logps/policy_chosen": -132.089111328125,
      "logps/policy_rejected": -225.1402587890625,
      "logps/reference_chosen": -136.70138549804688,
      "logps/reference_rejected": -224.64724731445312,
      "loss": 0.6737,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.046122774481773376,
      "rewards/margins": 0.05105280876159668,
      "rewards/rejected": -0.004930033814162016,
      "step": 54,
      "tokens/chosen_response_len_mean": 299.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 257.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.04,
      "grad_norm": 2.947847201514034,
      "learning_rate": 2.1999999999999998e-07,
      "logits/policy_chosen": -2.8887228965759277,
      "logits/policy_rejected": -2.68259596824646,
      "logits/reference_chosen": -2.864752769470215,
      "logits/reference_rejected": -2.628319263458252,
      "logps/policy_chosen": -135.53025817871094,
      "logps/policy_rejected": -133.9132843017578,
      "logps/reference_chosen": -145.9614715576172,
      "logps/reference_rejected": -137.02508544921875,
      "loss": 0.666,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10431211441755295,
      "rewards/margins": 0.07319390773773193,
      "rewards/rejected": 0.031118206679821014,
      "step": 55,
      "tokens/chosen_response_len_mean": 349.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 196.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 2.771906040366205,
      "learning_rate": 2.24e-07,
      "logits/policy_chosen": -2.231527328491211,
      "logits/policy_rejected": -2.0439558029174805,
      "logits/reference_chosen": -2.230081796646118,
      "logits/reference_rejected": -2.080432176589966,
      "logps/policy_chosen": -399.80303955078125,
      "logps/policy_rejected": -446.4791259765625,
      "logps/reference_chosen": -431.472412109375,
      "logps/reference_rejected": -470.2456970214844,
      "loss": 0.6709,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.31669357419013977,
      "rewards/margins": 0.07902783155441284,
      "rewards/rejected": 0.23766574263572693,
      "step": 56,
      "tokens/chosen_response_len_mean": 805.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 711.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 1.7428598517814506,
      "learning_rate": 2.28e-07,
      "logits/policy_chosen": -2.7684319019317627,
      "logits/policy_rejected": -2.71665096282959,
      "logits/reference_chosen": -2.747512102127075,
      "logits/reference_rejected": -2.690124034881592,
      "logps/policy_chosen": -85.4046630859375,
      "logps/policy_rejected": -111.96269226074219,
      "logps/reference_chosen": -96.96040344238281,
      "logps/reference_rejected": -121.33839416503906,
      "loss": 0.6888,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.11555740237236023,
      "rewards/margins": 0.021800296381115913,
      "rewards/rejected": 0.09375710785388947,
      "step": 57,
      "tokens/chosen_response_len_mean": 261.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 248.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 2.6040542171359817,
      "learning_rate": 2.32e-07,
      "logits/policy_chosen": -2.989000082015991,
      "logits/policy_rejected": -2.8631539344787598,
      "logits/reference_chosen": -2.9670627117156982,
      "logits/reference_rejected": -2.5129096508026123,
      "logps/policy_chosen": -87.29337310791016,
      "logps/policy_rejected": -119.82539367675781,
      "logps/reference_chosen": -95.52102661132812,
      "logps/reference_rejected": -122.27404022216797,
      "loss": 0.6657,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.08227652311325073,
      "rewards/margins": 0.05779005587100983,
      "rewards/rejected": 0.024486469104886055,
      "step": 58,
      "tokens/chosen_response_len_mean": 162.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 138.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 3.249420407772776,
      "learning_rate": 2.3599999999999997e-07,
      "logits/policy_chosen": -2.9996423721313477,
      "logits/policy_rejected": -2.974095582962036,
      "logits/reference_chosen": -2.964367151260376,
      "logits/reference_rejected": -2.953282356262207,
      "logps/policy_chosen": -56.97121810913086,
      "logps/policy_rejected": -141.20883178710938,
      "logps/reference_chosen": -58.637413024902344,
      "logps/reference_rejected": -143.08058166503906,
      "loss": 0.679,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0166619960218668,
      "rewards/margins": -0.0020555637311190367,
      "rewards/rejected": 0.018717560917139053,
      "step": 59,
      "tokens/chosen_response_len_mean": 121.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 127.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 3.503891824715986,
      "learning_rate": 2.4e-07,
      "logits/policy_chosen": -3.207751512527466,
      "logits/policy_rejected": -2.8613648414611816,
      "logits/reference_chosen": -3.1973516941070557,
      "logits/reference_rejected": -2.244464874267578,
      "logps/policy_chosen": -115.4853286743164,
      "logps/policy_rejected": -205.64161682128906,
      "logps/reference_chosen": -123.15602111816406,
      "logps/reference_rejected": -206.07244873046875,
      "loss": 0.6713,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0767069011926651,
      "rewards/margins": 0.07239863276481628,
      "rewards/rejected": 0.004308272153139114,
      "step": 60,
      "tokens/chosen_response_len_mean": 262.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 185.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 2.987682806191956,
      "learning_rate": 2.4399999999999996e-07,
      "logits/policy_chosen": -2.65006947517395,
      "logits/policy_rejected": -2.60097599029541,
      "logits/reference_chosen": -2.6217522621154785,
      "logits/reference_rejected": -2.507542610168457,
      "logps/policy_chosen": -184.2285614013672,
      "logps/policy_rejected": -184.65708923339844,
      "logps/reference_chosen": -215.3209228515625,
      "logps/reference_rejected": -187.43243408203125,
      "loss": 0.6592,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.31092363595962524,
      "rewards/margins": 0.2831703722476959,
      "rewards/rejected": 0.027753248810768127,
      "step": 61,
      "tokens/chosen_response_len_mean": 344.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 292.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 3.02187794129979,
      "learning_rate": 2.48e-07,
      "logits/policy_chosen": -3.102184534072876,
      "logits/policy_rejected": -3.0169076919555664,
      "logits/reference_chosen": -3.0980794429779053,
      "logits/reference_rejected": -3.06693434715271,
      "logps/policy_chosen": -180.20120239257812,
      "logps/policy_rejected": -290.76556396484375,
      "logps/reference_chosen": -186.42239379882812,
      "logps/reference_rejected": -299.5263977050781,
      "loss": 0.6663,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06221204251050949,
      "rewards/margins": -0.025396330282092094,
      "rewards/rejected": 0.08760837465524673,
      "step": 62,
      "tokens/chosen_response_len_mean": 628.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 573.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 2.2026851566699364,
      "learning_rate": 2.52e-07,
      "logits/policy_chosen": -2.9184327125549316,
      "logits/policy_rejected": -2.7862141132354736,
      "logits/reference_chosen": -2.9029111862182617,
      "logits/reference_rejected": -2.562068462371826,
      "logps/policy_chosen": -176.180908203125,
      "logps/policy_rejected": -224.06822204589844,
      "logps/reference_chosen": -191.051025390625,
      "logps/reference_rejected": -231.65249633789062,
      "loss": 0.682,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.14870098233222961,
      "rewards/margins": 0.07285812497138977,
      "rewards/rejected": 0.07584285736083984,
      "step": 63,
      "tokens/chosen_response_len_mean": 383.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 315.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 2.6923985748881183,
      "learning_rate": 2.56e-07,
      "logits/policy_chosen": -3.1094112396240234,
      "logits/policy_rejected": -3.059873104095459,
      "logits/reference_chosen": -3.0696709156036377,
      "logits/reference_rejected": -3.0277552604675293,
      "logps/policy_chosen": -109.50044250488281,
      "logps/policy_rejected": -227.6053009033203,
      "logps/reference_chosen": -114.45265197753906,
      "logps/reference_rejected": -232.44473266601562,
      "loss": 0.6689,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.04952218383550644,
      "rewards/margins": 0.0011278600431978703,
      "rewards/rejected": 0.04839432239532471,
      "step": 64,
      "tokens/chosen_response_len_mean": 172.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 173.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 2.3372174586928685,
      "learning_rate": 2.6e-07,
      "logits/policy_chosen": -2.9087581634521484,
      "logits/policy_rejected": -2.973167657852173,
      "logits/reference_chosen": -2.754256248474121,
      "logits/reference_rejected": -2.943002223968506,
      "logps/policy_chosen": -16.885406494140625,
      "logps/policy_rejected": -58.0799560546875,
      "logps/reference_chosen": -16.55327033996582,
      "logps/reference_rejected": -58.852867126464844,
      "loss": 0.6709,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0033213470596820116,
      "rewards/margins": -0.011050481349229813,
      "rewards/rejected": 0.007729134988039732,
      "step": 65,
      "tokens/chosen_response_len_mean": 25.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 37.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 1.8439881768995727,
      "learning_rate": 2.64e-07,
      "logits/policy_chosen": -3.2132272720336914,
      "logits/policy_rejected": -3.156252861022949,
      "logits/reference_chosen": -3.1969757080078125,
      "logits/reference_rejected": -3.008265733718872,
      "logps/policy_chosen": -89.50849914550781,
      "logps/policy_rejected": -105.58460998535156,
      "logps/reference_chosen": -96.482421875,
      "logps/reference_rejected": -111.33866119384766,
      "loss": 0.6855,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.06973925232887268,
      "rewards/margins": 0.0121987946331501,
      "rewards/rejected": 0.05754045769572258,
      "step": 66,
      "tokens/chosen_response_len_mean": 163.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 142.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 1.9679864280666897,
      "learning_rate": 2.68e-07,
      "logits/policy_chosen": -2.443567991256714,
      "logits/policy_rejected": -2.2462878227233887,
      "logits/reference_chosen": -2.3940353393554688,
      "logits/reference_rejected": -1.4334111213684082,
      "logps/policy_chosen": -133.6657257080078,
      "logps/policy_rejected": -126.65130615234375,
      "logps/reference_chosen": -143.77218627929688,
      "logps/reference_rejected": -130.96572875976562,
      "loss": 0.6801,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.10106448829174042,
      "rewards/margins": 0.057920388877391815,
      "rewards/rejected": 0.0431441031396389,
      "step": 67,
      "tokens/chosen_response_len_mean": 360.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 238.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.05,
      "grad_norm": 2.3444172227352817,
      "learning_rate": 2.72e-07,
      "logits/policy_chosen": -2.447854518890381,
      "logits/policy_rejected": -2.4554026126861572,
      "logits/reference_chosen": -2.4224636554718018,
      "logits/reference_rejected": -2.44649600982666,
      "logps/policy_chosen": -116.48789978027344,
      "logps/policy_rejected": -196.64881896972656,
      "logps/reference_chosen": -129.35415649414062,
      "logps/reference_rejected": -209.97300720214844,
      "loss": 0.6714,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.12866270542144775,
      "rewards/margins": -0.004579143598675728,
      "rewards/rejected": 0.13324186205863953,
      "step": 68,
      "tokens/chosen_response_len_mean": 627.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 597.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 1.9355500776493433,
      "learning_rate": 2.7600000000000004e-07,
      "logits/policy_chosen": -3.001335620880127,
      "logits/policy_rejected": -2.8170900344848633,
      "logits/reference_chosen": -2.9873673915863037,
      "logits/reference_rejected": -2.8246562480926514,
      "logps/policy_chosen": -122.1274642944336,
      "logps/policy_rejected": -174.14083862304688,
      "logps/reference_chosen": -129.572509765625,
      "logps/reference_rejected": -175.02163696289062,
      "loss": 0.6814,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07445055991411209,
      "rewards/margins": 0.06564265489578247,
      "rewards/rejected": 0.008807908743619919,
      "step": 69,
      "tokens/chosen_response_len_mean": 316.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 212.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 3.1880887553688613,
      "learning_rate": 2.8e-07,
      "logits/policy_chosen": -2.666834831237793,
      "logits/policy_rejected": -2.602752208709717,
      "logits/reference_chosen": -2.5575764179229736,
      "logits/reference_rejected": -2.594496965408325,
      "logps/policy_chosen": -144.6117401123047,
      "logps/policy_rejected": -323.0421447753906,
      "logps/reference_chosen": -156.17013549804688,
      "logps/reference_rejected": -333.77972412109375,
      "loss": 0.6725,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.11558418720960617,
      "rewards/margins": 0.00820855051279068,
      "rewards/rejected": 0.10737563669681549,
      "step": 70,
      "tokens/chosen_response_len_mean": 319.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 349.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 2.476324701775898,
      "learning_rate": 2.8399999999999995e-07,
      "logits/policy_chosen": -2.8258767127990723,
      "logits/policy_rejected": -2.808089017868042,
      "logits/reference_chosen": -2.8227195739746094,
      "logits/reference_rejected": -2.8310012817382812,
      "logps/policy_chosen": -141.63214111328125,
      "logps/policy_rejected": -178.0096893310547,
      "logps/reference_chosen": -151.22640991210938,
      "logps/reference_rejected": -182.14822387695312,
      "loss": 0.6621,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09594269841909409,
      "rewards/margins": 0.054557494819164276,
      "rewards/rejected": 0.04138519987463951,
      "step": 71,
      "tokens/chosen_response_len_mean": 486.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 302.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 2.699352211419919,
      "learning_rate": 2.88e-07,
      "logits/policy_chosen": -2.597538709640503,
      "logits/policy_rejected": -2.41672945022583,
      "logits/reference_chosen": -2.5760765075683594,
      "logits/reference_rejected": -1.9195555448532104,
      "logps/policy_chosen": -247.6851043701172,
      "logps/policy_rejected": -382.4797058105469,
      "logps/reference_chosen": -263.0463562011719,
      "logps/reference_rejected": -387.8023376464844,
      "loss": 0.6657,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.15361259877681732,
      "rewards/margins": 0.10038647800683975,
      "rewards/rejected": 0.05322612449526787,
      "step": 72,
      "tokens/chosen_response_len_mean": 411.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 344.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 2.5969468603738255,
      "learning_rate": 2.9199999999999997e-07,
      "logits/policy_chosen": -2.7702274322509766,
      "logits/policy_rejected": -2.5614566802978516,
      "logits/reference_chosen": -2.7665748596191406,
      "logits/reference_rejected": -2.3978798389434814,
      "logps/policy_chosen": -72.09918212890625,
      "logps/policy_rejected": -68.87893676757812,
      "logps/reference_chosen": -80.68479919433594,
      "logps/reference_rejected": -67.986328125,
      "loss": 0.6635,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.08585613965988159,
      "rewards/margins": 0.0947822630405426,
      "rewards/rejected": -0.008926120586693287,
      "step": 73,
      "tokens/chosen_response_len_mean": 205.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 90.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 2.8994112666814575,
      "learning_rate": 2.9599999999999995e-07,
      "logits/policy_chosen": -2.9491050243377686,
      "logits/policy_rejected": -2.9374337196350098,
      "logits/reference_chosen": -2.924247980117798,
      "logits/reference_rejected": -2.8732385635375977,
      "logps/policy_chosen": -163.5004425048828,
      "logps/policy_rejected": -186.22500610351562,
      "logps/reference_chosen": -180.18331909179688,
      "logps/reference_rejected": -202.66912841796875,
      "loss": 0.6673,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.16682888567447662,
      "rewards/margins": 0.00238775834441185,
      "rewards/rejected": 0.16444112360477448,
      "step": 74,
      "tokens/chosen_response_len_mean": 365.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 331.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 2.1648523975476306,
      "learning_rate": 3e-07,
      "logits/policy_chosen": -2.1604349613189697,
      "logits/policy_rejected": -2.168158769607544,
      "logits/reference_chosen": -2.1622910499572754,
      "logits/reference_rejected": -2.1353628635406494,
      "logps/policy_chosen": -88.18841552734375,
      "logps/policy_rejected": -119.30603790283203,
      "logps/reference_chosen": -93.11495971679688,
      "logps/reference_rejected": -123.05696868896484,
      "loss": 0.6881,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.04926542192697525,
      "rewards/margins": 0.011756086722016335,
      "rewards/rejected": 0.037509337067604065,
      "step": 75,
      "tokens/chosen_response_len_mean": 258.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 199.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 2.403809139260171,
      "learning_rate": 3.0399999999999997e-07,
      "logits/policy_chosen": -2.830843925476074,
      "logits/policy_rejected": -2.8032288551330566,
      "logits/reference_chosen": -2.8209171295166016,
      "logits/reference_rejected": -2.797828435897827,
      "logps/policy_chosen": -69.43246459960938,
      "logps/policy_rejected": -162.20603942871094,
      "logps/reference_chosen": -74.48234558105469,
      "logps/reference_rejected": -164.12376403808594,
      "loss": 0.6658,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05049885809421539,
      "rewards/margins": 0.03132178634405136,
      "rewards/rejected": 0.019177071750164032,
      "step": 76,
      "tokens/chosen_response_len_mean": 142.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 126.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 2.854272678012172,
      "learning_rate": 3.08e-07,
      "logits/policy_chosen": -3.0018627643585205,
      "logits/policy_rejected": -2.6590089797973633,
      "logits/reference_chosen": -2.995105028152466,
      "logits/reference_rejected": -2.385610580444336,
      "logps/policy_chosen": -85.27000427246094,
      "logps/policy_rejected": -132.92564392089844,
      "logps/reference_chosen": -90.3302001953125,
      "logps/reference_rejected": -134.35781860351562,
      "loss": 0.6608,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.05060192570090294,
      "rewards/margins": 0.03628016635775566,
      "rewards/rejected": 0.014321764931082726,
      "step": 77,
      "tokens/chosen_response_len_mean": 363.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 320.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 2.7505136356983133,
      "learning_rate": 3.12e-07,
      "logits/policy_chosen": -2.0338382720947266,
      "logits/policy_rejected": -2.276106357574463,
      "logits/reference_chosen": -2.0206451416015625,
      "logits/reference_rejected": -2.060913562774658,
      "logps/policy_chosen": -232.1044464111328,
      "logps/policy_rejected": -146.2607879638672,
      "logps/reference_chosen": -250.27952575683594,
      "logps/reference_rejected": -149.79270935058594,
      "loss": 0.6679,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.18175065517425537,
      "rewards/margins": 0.14643150568008423,
      "rewards/rejected": 0.03531915321946144,
      "step": 78,
      "tokens/chosen_response_len_mean": 521.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 307.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 2.4952987336105665,
      "learning_rate": 3.1599999999999997e-07,
      "logits/policy_chosen": -2.8599331378936768,
      "logits/policy_rejected": -2.8679845333099365,
      "logits/reference_chosen": -2.7079713344573975,
      "logits/reference_rejected": -2.4389915466308594,
      "logps/policy_chosen": -148.4790496826172,
      "logps/policy_rejected": -277.09405517578125,
      "logps/reference_chosen": -156.32412719726562,
      "logps/reference_rejected": -281.2509765625,
      "loss": 0.6709,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0784510001540184,
      "rewards/margins": 0.03688175603747368,
      "rewards/rejected": 0.041569244116544724,
      "step": 79,
      "tokens/chosen_response_len_mean": 276.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 336.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.06,
      "grad_norm": 2.3479129695327297,
      "learning_rate": 3.2e-07,
      "logits/policy_chosen": -1.8025144338607788,
      "logits/policy_rejected": -1.9184333086013794,
      "logits/reference_chosen": -1.7402411699295044,
      "logits/reference_rejected": -1.906538724899292,
      "logps/policy_chosen": -79.56036376953125,
      "logps/policy_rejected": -185.89906311035156,
      "logps/reference_chosen": -88.7896499633789,
      "logps/reference_rejected": -198.23004150390625,
      "loss": 0.6921,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.09229286015033722,
      "rewards/margins": -0.03101687878370285,
      "rewards/rejected": 0.12330973893404007,
      "step": 80,
      "tokens/chosen_response_len_mean": 432.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 410.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 1.4788277018262954,
      "learning_rate": 3.24e-07,
      "logits/policy_chosen": -2.7751684188842773,
      "logits/policy_rejected": -2.7188775539398193,
      "logits/reference_chosen": -2.774470567703247,
      "logits/reference_rejected": -2.3951942920684814,
      "logps/policy_chosen": -64.14806365966797,
      "logps/policy_rejected": -91.78968048095703,
      "logps/reference_chosen": -70.32818603515625,
      "logps/reference_rejected": -92.31802368164062,
      "loss": 0.6813,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06180126219987869,
      "rewards/margins": 0.05651785433292389,
      "rewards/rejected": 0.005283408332616091,
      "step": 81,
      "tokens/chosen_response_len_mean": 345.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 316.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.8935594664890503,
      "learning_rate": 3.28e-07,
      "logits/policy_chosen": -2.936018943786621,
      "logits/policy_rejected": -2.9313597679138184,
      "logits/reference_chosen": -2.9295578002929688,
      "logits/reference_rejected": -2.9169669151306152,
      "logps/policy_chosen": -141.27691650390625,
      "logps/policy_rejected": -264.863037109375,
      "logps/reference_chosen": -152.80865478515625,
      "logps/reference_rejected": -274.9104919433594,
      "loss": 0.6644,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11531716585159302,
      "rewards/margins": 0.014842674136161804,
      "rewards/rejected": 0.10047448426485062,
      "step": 82,
      "tokens/chosen_response_len_mean": 394.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 390.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.4649587565099678,
      "learning_rate": 3.32e-07,
      "logits/policy_chosen": -2.9249117374420166,
      "logits/policy_rejected": -2.9090662002563477,
      "logits/reference_chosen": -2.868709087371826,
      "logits/reference_rejected": -2.898679256439209,
      "logps/policy_chosen": -154.240234375,
      "logps/policy_rejected": -297.08160400390625,
      "logps/reference_chosen": -161.24609375,
      "logps/reference_rejected": -297.5753479003906,
      "loss": 0.6687,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07005855441093445,
      "rewards/margins": 0.06512109935283661,
      "rewards/rejected": 0.004937455989420414,
      "step": 83,
      "tokens/chosen_response_len_mean": 334.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 309.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.5355636198848175,
      "learning_rate": 3.36e-07,
      "logits/policy_chosen": -2.9488754272460938,
      "logits/policy_rejected": -2.9289844036102295,
      "logits/reference_chosen": -2.890922784805298,
      "logits/reference_rejected": -2.870727062225342,
      "logps/policy_chosen": -18.58037757873535,
      "logps/policy_rejected": -47.89413833618164,
      "logps/reference_chosen": -19.83108901977539,
      "logps/reference_rejected": -48.526878356933594,
      "loss": 0.6755,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.012507104314863682,
      "rewards/margins": 0.006179675459861755,
      "rewards/rejected": 0.006327427923679352,
      "step": 84,
      "tokens/chosen_response_len_mean": 27.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 32.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.5319776077967746,
      "learning_rate": 3.4000000000000003e-07,
      "logits/policy_chosen": -2.7499167919158936,
      "logits/policy_rejected": -2.690596103668213,
      "logits/reference_chosen": -2.73750638961792,
      "logits/reference_rejected": -2.6311354637145996,
      "logps/policy_chosen": -123.83195495605469,
      "logps/policy_rejected": -194.00137329101562,
      "logps/reference_chosen": -139.81263732910156,
      "logps/reference_rejected": -196.44784545898438,
      "loss": 0.6523,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.15980683267116547,
      "rewards/margins": 0.13534212112426758,
      "rewards/rejected": 0.024464718997478485,
      "step": 85,
      "tokens/chosen_response_len_mean": 606.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 519.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.007472503017741,
      "learning_rate": 3.4399999999999996e-07,
      "logits/policy_chosen": -2.9675540924072266,
      "logits/policy_rejected": -2.9613142013549805,
      "logits/reference_chosen": -2.9432828426361084,
      "logits/reference_rejected": -2.9374613761901855,
      "logps/policy_chosen": -12.885274887084961,
      "logps/policy_rejected": -34.69200134277344,
      "logps/reference_chosen": -13.591547012329102,
      "logps/reference_rejected": -35.15093994140625,
      "loss": 0.6775,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.007062721066176891,
      "rewards/margins": 0.002473349217325449,
      "rewards/rejected": 0.004589371383190155,
      "step": 86,
      "tokens/chosen_response_len_mean": 11.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 12.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.3366851120609398,
      "learning_rate": 3.4799999999999994e-07,
      "logits/policy_chosen": -2.510186195373535,
      "logits/policy_rejected": -2.3673720359802246,
      "logits/reference_chosen": -2.5208425521850586,
      "logits/reference_rejected": -2.307051658630371,
      "logps/policy_chosen": -103.87987518310547,
      "logps/policy_rejected": -118.88311767578125,
      "logps/reference_chosen": -114.24214172363281,
      "logps/reference_rejected": -118.43659973144531,
      "loss": 0.664,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.10362273454666138,
      "rewards/margins": 0.10808798670768738,
      "rewards/rejected": -0.004465251229703426,
      "step": 87,
      "tokens/chosen_response_len_mean": 216.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 106.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.4579695955138874,
      "learning_rate": 3.52e-07,
      "logits/policy_chosen": -2.9871153831481934,
      "logits/policy_rejected": -2.8764257431030273,
      "logits/reference_chosen": -2.9515204429626465,
      "logits/reference_rejected": -2.883835792541504,
      "logps/policy_chosen": -151.91807556152344,
      "logps/policy_rejected": -141.33587646484375,
      "logps/reference_chosen": -162.7379608154297,
      "logps/reference_rejected": -147.37442016601562,
      "loss": 0.6695,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10819891095161438,
      "rewards/margins": 0.04781348258256912,
      "rewards/rejected": 0.06038542836904526,
      "step": 88,
      "tokens/chosen_response_len_mean": 299.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 223.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.2124030442195703,
      "learning_rate": 3.5599999999999996e-07,
      "logits/policy_chosen": -3.097362995147705,
      "logits/policy_rejected": -2.821688652038574,
      "logits/reference_chosen": -3.0792651176452637,
      "logits/reference_rejected": -2.6497914791107178,
      "logps/policy_chosen": -106.85523986816406,
      "logps/policy_rejected": -142.70628356933594,
      "logps/reference_chosen": -114.98326873779297,
      "logps/reference_rejected": -145.11463928222656,
      "loss": 0.6769,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08128029853105545,
      "rewards/margins": 0.05719655007123947,
      "rewards/rejected": 0.02408374845981598,
      "step": 89,
      "tokens/chosen_response_len_mean": 288.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 241.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.726548509097342,
      "learning_rate": 3.6e-07,
      "logits/policy_chosen": -2.911972761154175,
      "logits/policy_rejected": -2.7545974254608154,
      "logits/reference_chosen": -2.8914873600006104,
      "logits/reference_rejected": -2.4686713218688965,
      "logps/policy_chosen": -90.4947509765625,
      "logps/policy_rejected": -144.89512634277344,
      "logps/reference_chosen": -100.72172546386719,
      "logps/reference_rejected": -144.84951782226562,
      "loss": 0.6578,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1022697389125824,
      "rewards/margins": 0.10272593796253204,
      "rewards/rejected": -0.0004562041722238064,
      "step": 90,
      "tokens/chosen_response_len_mean": 254.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 112.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.2900988215150644,
      "learning_rate": 3.64e-07,
      "logits/policy_chosen": -2.8383259773254395,
      "logits/policy_rejected": -2.824490547180176,
      "logits/reference_chosen": -2.8273422718048096,
      "logits/reference_rejected": -2.8102381229400635,
      "logps/policy_chosen": -132.3096466064453,
      "logps/policy_rejected": -264.598388671875,
      "logps/reference_chosen": -139.56736755371094,
      "logps/reference_rejected": -264.87725830078125,
      "loss": 0.6825,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.07257718592882156,
      "rewards/margins": 0.0697886273264885,
      "rewards/rejected": 0.002788563258945942,
      "step": 91,
      "tokens/chosen_response_len_mean": 395.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 397.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.8545865680492692,
      "learning_rate": 3.6799999999999996e-07,
      "logits/policy_chosen": -2.7706384658813477,
      "logits/policy_rejected": -2.5063834190368652,
      "logits/reference_chosen": -2.7504565715789795,
      "logits/reference_rejected": -1.929320216178894,
      "logps/policy_chosen": -126.52993774414062,
      "logps/policy_rejected": -134.23519897460938,
      "logps/reference_chosen": -138.69195556640625,
      "logps/reference_rejected": -138.32545471191406,
      "loss": 0.6536,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.12162025272846222,
      "rewards/margins": 0.08071767538785934,
      "rewards/rejected": 0.040902577340602875,
      "step": 92,
      "tokens/chosen_response_len_mean": 315.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 201.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.07,
      "grad_norm": 2.284769139553738,
      "learning_rate": 3.72e-07,
      "logits/policy_chosen": -2.872082233428955,
      "logits/policy_rejected": -2.6889710426330566,
      "logits/reference_chosen": -2.8072874546051025,
      "logits/reference_rejected": -2.4844841957092285,
      "logps/policy_chosen": -74.13471984863281,
      "logps/policy_rejected": -116.40203857421875,
      "logps/reference_chosen": -80.3674545288086,
      "logps/reference_rejected": -114.1785888671875,
      "loss": 0.6602,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06232735142111778,
      "rewards/margins": 0.08456189930438995,
      "rewards/rejected": -0.022234544157981873,
      "step": 93,
      "tokens/chosen_response_len_mean": 162.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 95.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 1.898688154588326,
      "learning_rate": 3.76e-07,
      "logits/policy_chosen": -2.469783306121826,
      "logits/policy_rejected": -2.3272128105163574,
      "logits/reference_chosen": -2.425753593444824,
      "logits/reference_rejected": -2.3546535968780518,
      "logps/policy_chosen": -158.62905883789062,
      "logps/policy_rejected": -291.726318359375,
      "logps/reference_chosen": -168.30911254882812,
      "logps/reference_rejected": -291.2076110839844,
      "loss": 0.6805,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09680052101612091,
      "rewards/margins": 0.10198744386434555,
      "rewards/rejected": -0.005186920985579491,
      "step": 94,
      "tokens/chosen_response_len_mean": 359.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 302.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 2.9949420570538363,
      "learning_rate": 3.7999999999999996e-07,
      "logits/policy_chosen": -2.8909621238708496,
      "logits/policy_rejected": -2.906135082244873,
      "logits/reference_chosen": -2.792299747467041,
      "logits/reference_rejected": -2.848114252090454,
      "logps/policy_chosen": -50.312339782714844,
      "logps/policy_rejected": -105.15270233154297,
      "logps/reference_chosen": -52.856605529785156,
      "logps/reference_rejected": -108.09819030761719,
      "loss": 0.6483,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02544267475605011,
      "rewards/margins": -0.004012217745184898,
      "rewards/rejected": 0.029454896226525307,
      "step": 95,
      "tokens/chosen_response_len_mean": 116.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 120.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 2.4241955868213405,
      "learning_rate": 3.84e-07,
      "logits/policy_chosen": -2.789196014404297,
      "logits/policy_rejected": -2.543649911880493,
      "logits/reference_chosen": -2.7888166904449463,
      "logits/reference_rejected": -2.5362277030944824,
      "logps/policy_chosen": -127.26165771484375,
      "logps/policy_rejected": -104.05661010742188,
      "logps/reference_chosen": -139.64187622070312,
      "logps/reference_rejected": -108.60116577148438,
      "loss": 0.6656,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.12380214780569077,
      "rewards/margins": 0.07835659384727478,
      "rewards/rejected": 0.04544556140899658,
      "step": 96,
      "tokens/chosen_response_len_mean": 381.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 272.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 3.4704291601036683,
      "learning_rate": 3.88e-07,
      "logits/policy_chosen": -2.9072158336639404,
      "logits/policy_rejected": -2.845478057861328,
      "logits/reference_chosen": -2.882009983062744,
      "logits/reference_rejected": -2.943215847015381,
      "logps/policy_chosen": -199.15065002441406,
      "logps/policy_rejected": -345.61346435546875,
      "logps/reference_chosen": -208.939697265625,
      "logps/reference_rejected": -351.9114990234375,
      "loss": 0.6532,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09789040684700012,
      "rewards/margins": 0.0349101684987545,
      "rewards/rejected": 0.06298023462295532,
      "step": 97,
      "tokens/chosen_response_len_mean": 443.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 334.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 2.413585232959013,
      "learning_rate": 3.92e-07,
      "logits/policy_chosen": -2.581110954284668,
      "logits/policy_rejected": -2.5821545124053955,
      "logits/reference_chosen": -2.6214118003845215,
      "logits/reference_rejected": -2.6232800483703613,
      "logps/policy_chosen": -66.4002914428711,
      "logps/policy_rejected": -92.362060546875,
      "logps/reference_chosen": -73.42770385742188,
      "logps/reference_rejected": -96.90080261230469,
      "loss": 0.6682,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07027411460876465,
      "rewards/margins": 0.02488674223423004,
      "rewards/rejected": 0.0453873835504055,
      "step": 98,
      "tokens/chosen_response_len_mean": 369.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 353.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 2.566883896226423,
      "learning_rate": 3.96e-07,
      "logits/policy_chosen": -3.13887357711792,
      "logits/policy_rejected": -3.1172709465026855,
      "logits/reference_chosen": -3.127457618713379,
      "logits/reference_rejected": -3.1038241386413574,
      "logps/policy_chosen": -131.76913452148438,
      "logps/policy_rejected": -274.8529052734375,
      "logps/reference_chosen": -142.9169158935547,
      "logps/reference_rejected": -277.9893798828125,
      "loss": 0.68,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.11147774755954742,
      "rewards/margins": 0.08011295646429062,
      "rewards/rejected": 0.031364794820547104,
      "step": 99,
      "tokens/chosen_response_len_mean": 291.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 296.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 2.7837569692907294,
      "learning_rate": 4e-07,
      "logits/policy_chosen": -3.046888589859009,
      "logits/policy_rejected": -2.9924631118774414,
      "logits/reference_chosen": -3.0203585624694824,
      "logits/reference_rejected": -2.9802844524383545,
      "logps/policy_chosen": -96.66795349121094,
      "logps/policy_rejected": -151.02667236328125,
      "logps/reference_chosen": -106.40546417236328,
      "logps/reference_rejected": -156.30604553222656,
      "loss": 0.6494,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.09737517684698105,
      "rewards/margins": 0.04458136856555939,
      "rewards/rejected": 0.05279380828142166,
      "step": 100,
      "tokens/chosen_response_len_mean": 246.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 213.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 2.5533090774871385,
      "learning_rate": 4.04e-07,
      "logits/policy_chosen": -3.0745935440063477,
      "logits/policy_rejected": -2.9924702644348145,
      "logits/reference_chosen": -3.0464026927948,
      "logits/reference_rejected": -2.9301576614379883,
      "logps/policy_chosen": -130.01454162597656,
      "logps/policy_rejected": -273.28997802734375,
      "logps/reference_chosen": -138.4945831298828,
      "logps/reference_rejected": -277.6722106933594,
      "loss": 0.6659,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08480045944452286,
      "rewards/margins": 0.04097801819443703,
      "rewards/rejected": 0.04382243752479553,
      "step": 101,
      "tokens/chosen_response_len_mean": 351.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 294.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 2.658053156359227,
      "learning_rate": 4.0799999999999995e-07,
      "logits/policy_chosen": -2.9965174198150635,
      "logits/policy_rejected": -2.756721019744873,
      "logits/reference_chosen": -3.0305070877075195,
      "logits/reference_rejected": -2.5150346755981445,
      "logps/policy_chosen": -150.61041259765625,
      "logps/policy_rejected": -145.7916717529297,
      "logps/reference_chosen": -169.15542602539062,
      "logps/reference_rejected": -147.83485412597656,
      "loss": 0.6388,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18545001745224,
      "rewards/margins": 0.16501817107200623,
      "rewards/rejected": 0.02043186128139496,
      "step": 102,
      "tokens/chosen_response_len_mean": 276.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 211.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 1.8527975992911931,
      "learning_rate": 4.12e-07,
      "logits/policy_chosen": -2.4530792236328125,
      "logits/policy_rejected": -2.4156646728515625,
      "logits/reference_chosen": -2.4479713439941406,
      "logits/reference_rejected": -2.3904917240142822,
      "logps/policy_chosen": -173.00030517578125,
      "logps/policy_rejected": -199.19415283203125,
      "logps/reference_chosen": -189.02093505859375,
      "logps/reference_rejected": -209.84201049804688,
      "loss": 0.6666,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16020633280277252,
      "rewards/margins": 0.05372770130634308,
      "rewards/rejected": 0.10647863149642944,
      "step": 103,
      "tokens/chosen_response_len_mean": 309.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 310.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 2.5555937206717187,
      "learning_rate": 4.1599999999999997e-07,
      "logits/policy_chosen": -3.0911600589752197,
      "logits/policy_rejected": -3.0214200019836426,
      "logits/reference_chosen": -3.089872360229492,
      "logits/reference_rejected": -2.882385730743408,
      "logps/policy_chosen": -153.61936950683594,
      "logps/policy_rejected": -227.18930053710938,
      "logps/reference_chosen": -176.565673828125,
      "logps/reference_rejected": -239.63970947265625,
      "loss": 0.665,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22946298122406006,
      "rewards/margins": 0.10495913028717041,
      "rewards/rejected": 0.12450385838747025,
      "step": 104,
      "tokens/chosen_response_len_mean": 368.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 301.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.08,
      "grad_norm": 2.0081558824498096,
      "learning_rate": 4.1999999999999995e-07,
      "logits/policy_chosen": -2.958939790725708,
      "logits/policy_rejected": -2.80196213722229,
      "logits/reference_chosen": -2.944668769836426,
      "logits/reference_rejected": -2.6392948627471924,
      "logps/policy_chosen": -37.521568298339844,
      "logps/policy_rejected": -74.32329559326172,
      "logps/reference_chosen": -41.80109786987305,
      "logps/reference_rejected": -72.32585906982422,
      "loss": 0.664,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0427953377366066,
      "rewards/margins": 0.06276971846818924,
      "rewards/rejected": -0.01997438445687294,
      "step": 105,
      "tokens/chosen_response_len_mean": 105.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 85.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 2.7526269849819696,
      "learning_rate": 4.24e-07,
      "logits/policy_chosen": -3.100095272064209,
      "logits/policy_rejected": -2.8839964866638184,
      "logits/reference_chosen": -3.0812511444091797,
      "logits/reference_rejected": -2.936655044555664,
      "logps/policy_chosen": -207.67698669433594,
      "logps/policy_rejected": -240.35739135742188,
      "logps/reference_chosen": -223.25807189941406,
      "logps/reference_rejected": -239.6046142578125,
      "loss": 0.6494,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.15581092238426208,
      "rewards/margins": 0.16333873569965363,
      "rewards/rejected": -0.007527794688940048,
      "step": 106,
      "tokens/chosen_response_len_mean": 392.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 203.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 1.9552389682760483,
      "learning_rate": 4.2799999999999997e-07,
      "logits/policy_chosen": -2.922211170196533,
      "logits/policy_rejected": -2.94032621383667,
      "logits/reference_chosen": -2.9038805961608887,
      "logits/reference_rejected": -2.9326558113098145,
      "logps/policy_chosen": -142.2540283203125,
      "logps/policy_rejected": -238.67227172851562,
      "logps/reference_chosen": -151.64273071289062,
      "logps/reference_rejected": -239.12753295898438,
      "loss": 0.6753,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09388697892427444,
      "rewards/margins": 0.08933433890342712,
      "rewards/rejected": 0.004552632570266724,
      "step": 107,
      "tokens/chosen_response_len_mean": 347.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 352.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 3.014699368900832,
      "learning_rate": 4.3199999999999995e-07,
      "logits/policy_chosen": -3.1775548458099365,
      "logits/policy_rejected": -3.03210186958313,
      "logits/reference_chosen": -3.1847496032714844,
      "logits/reference_rejected": -2.531522750854492,
      "logps/policy_chosen": -143.90025329589844,
      "logps/policy_rejected": -291.92388916015625,
      "logps/reference_chosen": -160.6278839111328,
      "logps/reference_rejected": -297.986572265625,
      "loss": 0.6498,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.16727623343467712,
      "rewards/margins": 0.10664979368448257,
      "rewards/rejected": 0.060626450926065445,
      "step": 108,
      "tokens/chosen_response_len_mean": 411.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 381.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 3.3247511691725236,
      "learning_rate": 4.36e-07,
      "logits/policy_chosen": -3.0070884227752686,
      "logits/policy_rejected": -3.006807565689087,
      "logits/reference_chosen": -2.9760210514068604,
      "logits/reference_rejected": -2.982029438018799,
      "logps/policy_chosen": -117.77191925048828,
      "logps/policy_rejected": -288.87689208984375,
      "logps/reference_chosen": -130.60389709472656,
      "logps/reference_rejected": -292.3191833496094,
      "loss": 0.6517,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.12831971049308777,
      "rewards/margins": 0.09389686584472656,
      "rewards/rejected": 0.034422844648361206,
      "step": 109,
      "tokens/chosen_response_len_mean": 318.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 302.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 3.158975765008515,
      "learning_rate": 4.3999999999999997e-07,
      "logits/policy_chosen": -2.50785756111145,
      "logits/policy_rejected": -2.504612684249878,
      "logits/reference_chosen": -2.5301709175109863,
      "logits/reference_rejected": -2.5272464752197266,
      "logps/policy_chosen": -136.28662109375,
      "logps/policy_rejected": -200.89352416992188,
      "logps/reference_chosen": -158.42433166503906,
      "logps/reference_rejected": -212.5179443359375,
      "loss": 0.638,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.22137700021266937,
      "rewards/margins": 0.10513283312320709,
      "rewards/rejected": 0.11624416708946228,
      "step": 110,
      "tokens/chosen_response_len_mean": 608.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 525.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 2.8278528577842095,
      "learning_rate": 4.44e-07,
      "logits/policy_chosen": -2.725069999694824,
      "logits/policy_rejected": -2.732088565826416,
      "logits/reference_chosen": -2.6485838890075684,
      "logits/reference_rejected": -2.76961612701416,
      "logps/policy_chosen": -178.09320068359375,
      "logps/policy_rejected": -237.05970764160156,
      "logps/reference_chosen": -192.22482299804688,
      "logps/reference_rejected": -248.2169647216797,
      "loss": 0.6253,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.14131632447242737,
      "rewards/margins": 0.029743948951363564,
      "rewards/rejected": 0.11157237738370895,
      "step": 111,
      "tokens/chosen_response_len_mean": 589.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 556.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 3.4191485129114714,
      "learning_rate": 4.48e-07,
      "logits/policy_chosen": -2.7914977073669434,
      "logits/policy_rejected": -2.804826259613037,
      "logits/reference_chosen": -2.7042627334594727,
      "logits/reference_rejected": -2.640076160430908,
      "logps/policy_chosen": -146.0531768798828,
      "logps/policy_rejected": -232.5332794189453,
      "logps/reference_chosen": -168.23060607910156,
      "logps/reference_rejected": -237.49801635742188,
      "loss": 0.6295,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22177428007125854,
      "rewards/margins": 0.172126904129982,
      "rewards/rejected": 0.049647387117147446,
      "step": 112,
      "tokens/chosen_response_len_mean": 438.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 396.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 2.331597318165628,
      "learning_rate": 4.5199999999999997e-07,
      "logits/policy_chosen": -3.1060757637023926,
      "logits/policy_rejected": -2.961545467376709,
      "logits/reference_chosen": -3.1052682399749756,
      "logits/reference_rejected": -2.833801746368408,
      "logps/policy_chosen": -127.66785430908203,
      "logps/policy_rejected": -196.163330078125,
      "logps/reference_chosen": -137.27505493164062,
      "logps/reference_rejected": -198.14578247070312,
      "loss": 0.6605,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09607198089361191,
      "rewards/margins": 0.07624746859073639,
      "rewards/rejected": 0.01982451230287552,
      "step": 113,
      "tokens/chosen_response_len_mean": 314.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 291.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 2.750538166446364,
      "learning_rate": 4.56e-07,
      "logits/policy_chosen": -2.7500100135803223,
      "logits/policy_rejected": -2.681910514831543,
      "logits/reference_chosen": -2.789179563522339,
      "logits/reference_rejected": -2.3359477519989014,
      "logps/policy_chosen": -85.98888397216797,
      "logps/policy_rejected": -100.74612426757812,
      "logps/reference_chosen": -98.14977264404297,
      "logps/reference_rejected": -102.4841537475586,
      "loss": 0.6408,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.12160897254943848,
      "rewards/margins": 0.10422859340906143,
      "rewards/rejected": 0.017380371689796448,
      "step": 114,
      "tokens/chosen_response_len_mean": 256.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 223.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 2.5440555248438996,
      "learning_rate": 4.6e-07,
      "logits/policy_chosen": -2.971923351287842,
      "logits/policy_rejected": -2.989212989807129,
      "logits/reference_chosen": -2.9268057346343994,
      "logits/reference_rejected": -2.919334650039673,
      "logps/policy_chosen": -98.5408706665039,
      "logps/policy_rejected": -166.144775390625,
      "logps/reference_chosen": -109.1729965209961,
      "logps/reference_rejected": -167.0085906982422,
      "loss": 0.6505,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10632127523422241,
      "rewards/margins": 0.09768319875001907,
      "rewards/rejected": 0.008638077415525913,
      "step": 115,
      "tokens/chosen_response_len_mean": 229.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 254.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 2.4244019154261474,
      "learning_rate": 4.64e-07,
      "logits/policy_chosen": -2.8886287212371826,
      "logits/policy_rejected": -2.922623872756958,
      "logits/reference_chosen": -2.8570451736450195,
      "logits/reference_rejected": -2.888643741607666,
      "logps/policy_chosen": -113.93293762207031,
      "logps/policy_rejected": -225.3817596435547,
      "logps/reference_chosen": -120.10133361816406,
      "logps/reference_rejected": -223.7104034423828,
      "loss": 0.6579,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06168397143483162,
      "rewards/margins": 0.0783977136015892,
      "rewards/rejected": -0.016713742166757584,
      "step": 116,
      "tokens/chosen_response_len_mean": 330.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 318.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 2.400248188855275,
      "learning_rate": 4.68e-07,
      "logits/policy_chosen": -2.8977608680725098,
      "logits/policy_rejected": -2.9003448486328125,
      "logits/reference_chosen": -2.8988900184631348,
      "logits/reference_rejected": -2.9036717414855957,
      "logps/policy_chosen": -74.31845092773438,
      "logps/policy_rejected": -92.14234924316406,
      "logps/reference_chosen": -78.81231689453125,
      "logps/reference_rejected": -93.07196807861328,
      "loss": 0.6588,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04493863508105278,
      "rewards/margins": 0.035642486065626144,
      "rewards/rejected": 0.00929614994674921,
      "step": 117,
      "tokens/chosen_response_len_mean": 275.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 274.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.09,
      "grad_norm": 2.4323431386681595,
      "learning_rate": 4.7199999999999994e-07,
      "logits/policy_chosen": -2.6670236587524414,
      "logits/policy_rejected": -2.518751621246338,
      "logits/reference_chosen": -2.666407585144043,
      "logits/reference_rejected": -2.5256845951080322,
      "logps/policy_chosen": -92.44061279296875,
      "logps/policy_rejected": -151.3938446044922,
      "logps/reference_chosen": -100.54401397705078,
      "logps/reference_rejected": -153.23507690429688,
      "loss": 0.6474,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08103407174348831,
      "rewards/margins": 0.06262162327766418,
      "rewards/rejected": 0.01841244101524353,
      "step": 118,
      "tokens/chosen_response_len_mean": 302.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 332.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 2.9673553403256707,
      "learning_rate": 4.76e-07,
      "logits/policy_chosen": -2.9274580478668213,
      "logits/policy_rejected": -2.8594419956207275,
      "logits/reference_chosen": -2.9505772590637207,
      "logits/reference_rejected": -2.850255012512207,
      "logps/policy_chosen": -169.86106872558594,
      "logps/policy_rejected": -316.6098327636719,
      "logps/reference_chosen": -191.77130126953125,
      "logps/reference_rejected": -317.0496520996094,
      "loss": 0.6273,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21910235285758972,
      "rewards/margins": 0.21470384299755096,
      "rewards/rejected": 0.004398513585329056,
      "step": 119,
      "tokens/chosen_response_len_mean": 431.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 374.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 2.3505092556042304,
      "learning_rate": 4.8e-07,
      "logits/policy_chosen": -2.814655303955078,
      "logits/policy_rejected": -2.5319275856018066,
      "logits/reference_chosen": -2.8454208374023438,
      "logits/reference_rejected": -2.315487861633301,
      "logps/policy_chosen": -50.07554626464844,
      "logps/policy_rejected": -110.84510040283203,
      "logps/reference_chosen": -60.05486297607422,
      "logps/reference_rejected": -106.57113647460938,
      "loss": 0.6368,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09979310631752014,
      "rewards/margins": 0.14253273606300354,
      "rewards/rejected": -0.0427396334707737,
      "step": 120,
      "tokens/chosen_response_len_mean": 267.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 187.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 2.8274965378312586,
      "learning_rate": 4.839999999999999e-07,
      "logits/policy_chosen": -2.942972421646118,
      "logits/policy_rejected": -2.8031089305877686,
      "logits/reference_chosen": -2.961139678955078,
      "logits/reference_rejected": -2.5472524166107178,
      "logps/policy_chosen": -130.38365173339844,
      "logps/policy_rejected": -191.11968994140625,
      "logps/reference_chosen": -159.17897033691406,
      "logps/reference_rejected": -199.572021484375,
      "loss": 0.6282,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2879532277584076,
      "rewards/margins": 0.2034299373626709,
      "rewards/rejected": 0.0845232680439949,
      "step": 121,
      "tokens/chosen_response_len_mean": 420.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 307.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 2.886588252399639,
      "learning_rate": 4.879999999999999e-07,
      "logits/policy_chosen": -2.853999376296997,
      "logits/policy_rejected": -2.480708122253418,
      "logits/reference_chosen": -2.710318088531494,
      "logits/reference_rejected": -1.9466197490692139,
      "logps/policy_chosen": -167.15240478515625,
      "logps/policy_rejected": -201.3469696044922,
      "logps/reference_chosen": -181.32119750976562,
      "logps/reference_rejected": -195.66384887695312,
      "loss": 0.6126,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.14168795943260193,
      "rewards/margins": 0.1985190212726593,
      "rewards/rejected": -0.056831058114767075,
      "step": 122,
      "tokens/chosen_response_len_mean": 326.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 201.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 2.826829476556088,
      "learning_rate": 4.92e-07,
      "logits/policy_chosen": -2.465135335922241,
      "logits/policy_rejected": -2.4350569248199463,
      "logits/reference_chosen": -2.4818878173828125,
      "logits/reference_rejected": -2.3954761028289795,
      "logps/policy_chosen": -100.21925354003906,
      "logps/policy_rejected": -125.87073516845703,
      "logps/reference_chosen": -126.35533905029297,
      "logps/reference_rejected": -144.03236389160156,
      "loss": 0.6328,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.26136085391044617,
      "rewards/margins": 0.07974465936422348,
      "rewards/rejected": 0.1816161870956421,
      "step": 123,
      "tokens/chosen_response_len_mean": 426.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 408.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 2.5289041457286774,
      "learning_rate": 4.96e-07,
      "logits/policy_chosen": -2.6869404315948486,
      "logits/policy_rejected": -2.773239850997925,
      "logits/reference_chosen": -2.653362512588501,
      "logits/reference_rejected": -2.5821008682250977,
      "logps/policy_chosen": -71.6850357055664,
      "logps/policy_rejected": -108.95183563232422,
      "logps/reference_chosen": -77.52404022216797,
      "logps/reference_rejected": -107.73311614990234,
      "loss": 0.6376,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05839002877473831,
      "rewards/margins": 0.07057718932628632,
      "rewards/rejected": -0.012187155894935131,
      "step": 124,
      "tokens/chosen_response_len_mean": 229.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 211.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 3.7452560936185804,
      "learning_rate": 5e-07,
      "logits/policy_chosen": -2.8474879264831543,
      "logits/policy_rejected": -2.77992844581604,
      "logits/reference_chosen": -2.8415565490722656,
      "logits/reference_rejected": -2.658388376235962,
      "logps/policy_chosen": -85.31348419189453,
      "logps/policy_rejected": -100.18048095703125,
      "logps/reference_chosen": -91.71619415283203,
      "logps/reference_rejected": -100.39627838134766,
      "loss": 0.6215,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0640270784497261,
      "rewards/margins": 0.06186922267079353,
      "rewards/rejected": 0.00215785950422287,
      "step": 125,
      "tokens/chosen_response_len_mean": 91.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 71.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 2.791877501388191,
      "learning_rate": 4.999990147435196e-07,
      "logits/policy_chosen": -2.772634506225586,
      "logits/policy_rejected": -2.7663345336914062,
      "logits/reference_chosen": -2.8182811737060547,
      "logits/reference_rejected": -2.821171760559082,
      "logps/policy_chosen": -114.60906982421875,
      "logps/policy_rejected": -122.4944839477539,
      "logps/reference_chosen": -141.17056274414062,
      "logps/reference_rejected": -135.64431762695312,
      "loss": 0.63,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2656148672103882,
      "rewards/margins": 0.13411659002304077,
      "rewards/rejected": 0.1314982771873474,
      "step": 126,
      "tokens/chosen_response_len_mean": 458.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 332.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 2.692910890812567,
      "learning_rate": 4.99996058981844e-07,
      "logits/policy_chosen": -2.5161116123199463,
      "logits/policy_rejected": -2.3703317642211914,
      "logits/reference_chosen": -2.4154701232910156,
      "logits/reference_rejected": -2.4023566246032715,
      "logps/policy_chosen": -72.83512115478516,
      "logps/policy_rejected": -136.28607177734375,
      "logps/reference_chosen": -82.63104248046875,
      "logps/reference_rejected": -140.04986572265625,
      "loss": 0.6369,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09795919060707092,
      "rewards/margins": 0.06032119691371918,
      "rewards/rejected": 0.037637993693351746,
      "step": 127,
      "tokens/chosen_response_len_mean": 416.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 410.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 2.859419025019686,
      "learning_rate": 4.999911327382711e-07,
      "logits/policy_chosen": -2.0673351287841797,
      "logits/policy_rejected": -1.9885450601577759,
      "logits/reference_chosen": -2.1499083042144775,
      "logits/reference_rejected": -1.6471858024597168,
      "logps/policy_chosen": -57.06989669799805,
      "logps/policy_rejected": -87.03865814208984,
      "logps/reference_chosen": -69.13873291015625,
      "logps/reference_rejected": -83.46507263183594,
      "loss": 0.5998,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.12068840861320496,
      "rewards/margins": 0.15642426908016205,
      "rewards/rejected": -0.035735856741666794,
      "step": 128,
      "tokens/chosen_response_len_mean": 343.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 194.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 2.8759886658807288,
      "learning_rate": 4.999842360516294e-07,
      "logits/policy_chosen": -3.1045093536376953,
      "logits/policy_rejected": -3.1223738193511963,
      "logits/reference_chosen": -3.09148907661438,
      "logits/reference_rejected": -3.091776132583618,
      "logps/policy_chosen": -40.253719329833984,
      "logps/policy_rejected": -93.64241027832031,
      "logps/reference_chosen": -43.66665267944336,
      "logps/reference_rejected": -92.32077026367188,
      "loss": 0.5883,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03412933275103569,
      "rewards/margins": 0.047345712780952454,
      "rewards/rejected": -0.01321637723594904,
      "step": 129,
      "tokens/chosen_response_len_mean": 134.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 142.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.1,
      "grad_norm": 2.396608033911008,
      "learning_rate": 4.999753689762792e-07,
      "logits/policy_chosen": -2.7669222354888916,
      "logits/policy_rejected": -3.0244264602661133,
      "logits/reference_chosen": -2.7182681560516357,
      "logits/reference_rejected": -3.016852855682373,
      "logps/policy_chosen": -49.25522994995117,
      "logps/policy_rejected": -209.12026977539062,
      "logps/reference_chosen": -58.14155197143555,
      "logps/reference_rejected": -213.75338745117188,
      "loss": 0.6532,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08886321634054184,
      "rewards/margins": 0.04253204166889191,
      "rewards/rejected": 0.04633117839694023,
      "step": 130,
      "tokens/chosen_response_len_mean": 140.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 397.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 2.8338085411833127,
      "learning_rate": 4.999645315821111e-07,
      "logits/policy_chosen": -2.177436590194702,
      "logits/policy_rejected": -2.1923160552978516,
      "logits/reference_chosen": -2.23976993560791,
      "logits/reference_rejected": -2.227308511734009,
      "logps/policy_chosen": -92.42753601074219,
      "logps/policy_rejected": -104.93522644042969,
      "logps/reference_chosen": -111.59609985351562,
      "logps/reference_rejected": -120.66436004638672,
      "loss": 0.6227,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19168560206890106,
      "rewards/margins": 0.03439423814415932,
      "rewards/rejected": 0.15729136765003204,
      "step": 131,
      "tokens/chosen_response_len_mean": 428.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 393.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 2.537615272647266,
      "learning_rate": 4.99951723954546e-07,
      "logits/policy_chosen": -2.6528267860412598,
      "logits/policy_rejected": -2.7896065711975098,
      "logits/reference_chosen": -2.3186991214752197,
      "logits/reference_rejected": -2.7976412773132324,
      "logps/policy_chosen": -35.68031311035156,
      "logps/policy_rejected": -72.86119079589844,
      "logps/reference_chosen": -42.128787994384766,
      "logps/reference_rejected": -76.47801208496094,
      "loss": 0.6365,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06448473036289215,
      "rewards/margins": 0.028316427022218704,
      "rewards/rejected": 0.03616830334067345,
      "step": 132,
      "tokens/chosen_response_len_mean": 226.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 273.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 2.952022770646515,
      "learning_rate": 4.999369461945343e-07,
      "logits/policy_chosen": -3.1463866233825684,
      "logits/policy_rejected": -2.8932509422302246,
      "logits/reference_chosen": -3.141604423522949,
      "logits/reference_rejected": -2.444639205932617,
      "logps/policy_chosen": -83.71105194091797,
      "logps/policy_rejected": -113.13372802734375,
      "logps/reference_chosen": -94.0230941772461,
      "logps/reference_rejected": -108.04383850097656,
      "loss": 0.5892,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10312040150165558,
      "rewards/margins": 0.15401925146579742,
      "rewards/rejected": -0.05089886486530304,
      "step": 133,
      "tokens/chosen_response_len_mean": 249.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 178.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 3.540843119893174,
      "learning_rate": 4.999201984185552e-07,
      "logits/policy_chosen": -3.0958125591278076,
      "logits/policy_rejected": -3.042353868484497,
      "logits/reference_chosen": -3.133866310119629,
      "logits/reference_rejected": -3.0950450897216797,
      "logps/policy_chosen": -236.39605712890625,
      "logps/policy_rejected": -330.2846374511719,
      "logps/reference_chosen": -259.0321350097656,
      "logps/reference_rejected": -336.391357421875,
      "loss": 0.5663,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22636055946350098,
      "rewards/margins": 0.16529357433319092,
      "rewards/rejected": 0.06106698513031006,
      "step": 134,
      "tokens/chosen_response_len_mean": 717.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 639.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 2.981444512825392,
      "learning_rate": 4.999014807586154e-07,
      "logits/policy_chosen": -2.3856396675109863,
      "logits/policy_rejected": -2.1970813274383545,
      "logits/reference_chosen": -2.4693095684051514,
      "logits/reference_rejected": -1.5482537746429443,
      "logps/policy_chosen": -142.9355926513672,
      "logps/policy_rejected": -189.60543823242188,
      "logps/reference_chosen": -164.72271728515625,
      "logps/reference_rejected": -189.248046875,
      "loss": 0.6065,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2178712785243988,
      "rewards/margins": 0.22144514322280884,
      "rewards/rejected": -0.003573855385184288,
      "step": 135,
      "tokens/chosen_response_len_mean": 339.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 276.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 2.4024504350993654,
      "learning_rate": 4.998807933622484e-07,
      "logits/policy_chosen": -3.1550679206848145,
      "logits/policy_rejected": -3.1557650566101074,
      "logits/reference_chosen": -3.1761221885681152,
      "logits/reference_rejected": -3.135535955429077,
      "logps/policy_chosen": -118.20024871826172,
      "logps/policy_rejected": -264.43560791015625,
      "logps/reference_chosen": -130.25527954101562,
      "logps/reference_rejected": -264.16741943359375,
      "loss": 0.6336,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.12055033445358276,
      "rewards/margins": 0.12323223054409027,
      "rewards/rejected": -0.002681899815797806,
      "step": 136,
      "tokens/chosen_response_len_mean": 354.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 386.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 2.8382305196380684,
      "learning_rate": 4.998581363925135e-07,
      "logits/policy_chosen": -2.2047266960144043,
      "logits/policy_rejected": -2.433465003967285,
      "logits/reference_chosen": -2.275531530380249,
      "logits/reference_rejected": -2.245540142059326,
      "logps/policy_chosen": -378.1270751953125,
      "logps/policy_rejected": -344.7900085449219,
      "logps/reference_chosen": -416.6529541015625,
      "logps/reference_rejected": -347.785888671875,
      "loss": 0.5804,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3852587938308716,
      "rewards/margins": 0.3552997410297394,
      "rewards/rejected": 0.02995903044939041,
      "step": 137,
      "tokens/chosen_response_len_mean": 584.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 427.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 2.1385240477591636,
      "learning_rate": 4.998335100279941e-07,
      "logits/policy_chosen": -2.6462714672088623,
      "logits/policy_rejected": -2.658045768737793,
      "logits/reference_chosen": -2.729498863220215,
      "logits/reference_rejected": -2.694502830505371,
      "logps/policy_chosen": -93.70889282226562,
      "logps/policy_rejected": -182.60643005371094,
      "logps/reference_chosen": -114.53910827636719,
      "logps/reference_rejected": -185.69065856933594,
      "loss": 0.6218,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2083020955324173,
      "rewards/margins": 0.17745977640151978,
      "rewards/rejected": 0.030842319130897522,
      "step": 138,
      "tokens/chosen_response_len_mean": 282.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 247.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 2.116685308038921,
      "learning_rate": 4.998069144627964e-07,
      "logits/policy_chosen": -2.870461940765381,
      "logits/policy_rejected": -2.6627469062805176,
      "logits/reference_chosen": -2.924255132675171,
      "logits/reference_rejected": -2.6669623851776123,
      "logps/policy_chosen": -149.49957275390625,
      "logps/policy_rejected": -133.25009155273438,
      "logps/reference_chosen": -191.32994079589844,
      "logps/reference_rejected": -157.3734588623047,
      "loss": 0.6358,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4183034896850586,
      "rewards/margins": 0.1770700067281723,
      "rewards/rejected": 0.2412334829568863,
      "step": 139,
      "tokens/chosen_response_len_mean": 457.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 308.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 2.5870338720368387,
      "learning_rate": 4.99778349906548e-07,
      "logits/policy_chosen": -2.9523699283599854,
      "logits/policy_rejected": -2.9597270488739014,
      "logits/reference_chosen": -2.9815268516540527,
      "logits/reference_rejected": -2.9721755981445312,
      "logps/policy_chosen": -45.849361419677734,
      "logps/policy_rejected": -106.37723541259766,
      "logps/reference_chosen": -56.053375244140625,
      "logps/reference_rejected": -102.900390625,
      "loss": 0.6334,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10204015672206879,
      "rewards/margins": 0.13680851459503174,
      "rewards/rejected": -0.03476836904883385,
      "step": 140,
      "tokens/chosen_response_len_mean": 174.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 184.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 3.1171856643556626,
      "learning_rate": 4.997478165843964e-07,
      "logits/policy_chosen": -2.8171520233154297,
      "logits/policy_rejected": -2.8751494884490967,
      "logits/reference_chosen": -2.9145286083221436,
      "logits/reference_rejected": -2.9242196083068848,
      "logps/policy_chosen": -170.06459045410156,
      "logps/policy_rejected": -296.7114562988281,
      "logps/reference_chosen": -197.37327575683594,
      "logps/reference_rejected": -302.43438720703125,
      "loss": 0.5862,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.27308669686317444,
      "rewards/margins": 0.21585765480995178,
      "rewards/rejected": 0.057229042053222656,
      "step": 141,
      "tokens/chosen_response_len_mean": 416.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 422.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 3.0084655845363835,
      "learning_rate": 4.997153147370065e-07,
      "logits/policy_chosen": -2.85317325592041,
      "logits/policy_rejected": -2.8581879138946533,
      "logits/reference_chosen": -2.7449471950531006,
      "logits/reference_rejected": -2.9342892169952393,
      "logps/policy_chosen": -95.86734008789062,
      "logps/policy_rejected": -255.03668212890625,
      "logps/reference_chosen": -114.49011993408203,
      "logps/reference_rejected": -234.34010314941406,
      "loss": 0.5691,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18622779846191406,
      "rewards/margins": 0.39319345355033875,
      "rewards/rejected": -0.20696566998958588,
      "step": 142,
      "tokens/chosen_response_len_mean": 348.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 233.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.11,
      "grad_norm": 2.8051570403665753,
      "learning_rate": 4.996808446205598e-07,
      "logits/policy_chosen": -2.9738688468933105,
      "logits/policy_rejected": -2.8889176845550537,
      "logits/reference_chosen": -3.072221517562866,
      "logits/reference_rejected": -2.744046449661255,
      "logps/policy_chosen": -106.87481689453125,
      "logps/policy_rejected": -214.15478515625,
      "logps/reference_chosen": -126.41868591308594,
      "logps/reference_rejected": -216.4077606201172,
      "loss": 0.6244,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19543880224227905,
      "rewards/margins": 0.17290906608104706,
      "rewards/rejected": 0.022529736161231995,
      "step": 143,
      "tokens/chosen_response_len_mean": 340.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 310.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 3.1099351541314864,
      "learning_rate": 4.996444065067515e-07,
      "logits/policy_chosen": -2.849621534347534,
      "logits/policy_rejected": -2.8288486003875732,
      "logits/reference_chosen": -2.9862475395202637,
      "logits/reference_rejected": -2.9479100704193115,
      "logps/policy_chosen": -331.7215576171875,
      "logps/policy_rejected": -571.941650390625,
      "logps/reference_chosen": -382.40533447265625,
      "logps/reference_rejected": -549.6744995117188,
      "loss": 0.5803,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5068381428718567,
      "rewards/margins": 0.7295098304748535,
      "rewards/rejected": -0.2226717472076416,
      "step": 144,
      "tokens/chosen_response_len_mean": 687.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 550.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 2.712783580670268,
      "learning_rate": 4.996060006827887e-07,
      "logits/policy_chosen": -2.882990837097168,
      "logits/policy_rejected": -2.8665995597839355,
      "logits/reference_chosen": -2.9348299503326416,
      "logits/reference_rejected": -2.9280238151550293,
      "logps/policy_chosen": -75.93386840820312,
      "logps/policy_rejected": -159.058837890625,
      "logps/reference_chosen": -91.52436828613281,
      "logps/reference_rejected": -146.97247314453125,
      "loss": 0.5965,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.15590497851371765,
      "rewards/margins": 0.2767685055732727,
      "rewards/rejected": -0.12086354941129684,
      "step": 145,
      "tokens/chosen_response_len_mean": 248.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 191.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 2.364187725655967,
      "learning_rate": 4.995656274513881e-07,
      "logits/policy_chosen": -3.0416619777679443,
      "logits/policy_rejected": -2.995178699493408,
      "logits/reference_chosen": -3.053309440612793,
      "logits/reference_rejected": -3.0008325576782227,
      "logps/policy_chosen": -90.75337982177734,
      "logps/policy_rejected": -127.47672271728516,
      "logps/reference_chosen": -102.95506286621094,
      "logps/reference_rejected": -132.29486083984375,
      "loss": 0.6275,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1220167949795723,
      "rewards/margins": 0.07383548468351364,
      "rewards/rejected": 0.04818130284547806,
      "step": 146,
      "tokens/chosen_response_len_mean": 403.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 352.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 3.163944044368287,
      "learning_rate": 4.995232871307736e-07,
      "logits/policy_chosen": -3.0712456703186035,
      "logits/policy_rejected": -2.916374921798706,
      "logits/reference_chosen": -3.115262746810913,
      "logits/reference_rejected": -2.6205530166625977,
      "logps/policy_chosen": -104.03935241699219,
      "logps/policy_rejected": -156.13577270507812,
      "logps/reference_chosen": -121.57315063476562,
      "logps/reference_rejected": -153.22605895996094,
      "loss": 0.583,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17533791065216064,
      "rewards/margins": 0.2044350802898407,
      "rewards/rejected": -0.029097169637680054,
      "step": 147,
      "tokens/chosen_response_len_mean": 273.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 197.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 2.8981364214624588,
      "learning_rate": 4.994789800546737e-07,
      "logits/policy_chosen": -2.519289970397949,
      "logits/policy_rejected": -2.3818559646606445,
      "logits/reference_chosen": -2.5594940185546875,
      "logits/reference_rejected": -2.2251200675964355,
      "logps/policy_chosen": -64.7489242553711,
      "logps/policy_rejected": -90.21540832519531,
      "logps/reference_chosen": -80.6169204711914,
      "logps/reference_rejected": -92.92266845703125,
      "loss": 0.5887,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.15867996215820312,
      "rewards/margins": 0.13160738348960876,
      "rewards/rejected": 0.027072593569755554,
      "step": 148,
      "tokens/chosen_response_len_mean": 252.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 195.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 2.7118945452462073,
      "learning_rate": 4.994327065723192e-07,
      "logits/policy_chosen": -2.6516730785369873,
      "logits/policy_rejected": -2.5235743522644043,
      "logits/reference_chosen": -2.629504442214966,
      "logits/reference_rejected": -2.4736945629119873,
      "logps/policy_chosen": -80.69113159179688,
      "logps/policy_rejected": -134.16526794433594,
      "logps/reference_chosen": -103.33537292480469,
      "logps/reference_rejected": -144.43687438964844,
      "loss": 0.5924,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2264423370361328,
      "rewards/margins": 0.12372627854347229,
      "rewards/rejected": 0.10271605104207993,
      "step": 149,
      "tokens/chosen_response_len_mean": 244.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 244.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 2.5381060398035205,
      "learning_rate": 4.9938446704844e-07,
      "logits/policy_chosen": -2.8819587230682373,
      "logits/policy_rejected": -2.9721384048461914,
      "logits/reference_chosen": -2.7422800064086914,
      "logits/reference_rejected": -2.9950976371765137,
      "logps/policy_chosen": -57.52130126953125,
      "logps/policy_rejected": -120.23619079589844,
      "logps/reference_chosen": -59.788352966308594,
      "logps/reference_rejected": -115.03448486328125,
      "loss": 0.619,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.022670526057481766,
      "rewards/margins": 0.07468754798173904,
      "rewards/rejected": -0.05201702564954758,
      "step": 150,
      "tokens/chosen_response_len_mean": 145.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 157.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 2.8034120172472052,
      "learning_rate": 4.993342618632626e-07,
      "logits/policy_chosen": -3.0852291584014893,
      "logits/policy_rejected": -2.92305850982666,
      "logits/reference_chosen": -3.162454128265381,
      "logits/reference_rejected": -2.4117138385772705,
      "logps/policy_chosen": -101.7428207397461,
      "logps/policy_rejected": -242.9844207763672,
      "logps/reference_chosen": -119.85108184814453,
      "logps/reference_rejected": -215.4344940185547,
      "loss": 0.5702,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18108254671096802,
      "rewards/margins": 0.45658186078071594,
      "rewards/rejected": -0.2754993140697479,
      "step": 151,
      "tokens/chosen_response_len_mean": 318.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 234.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 2.9459543515337776,
      "learning_rate": 4.992820914125069e-07,
      "logits/policy_chosen": -2.8646183013916016,
      "logits/policy_rejected": -2.754851818084717,
      "logits/reference_chosen": -3.0292770862579346,
      "logits/reference_rejected": -2.5228896141052246,
      "logps/policy_chosen": -150.56875610351562,
      "logps/policy_rejected": -277.60125732421875,
      "logps/reference_chosen": -179.77772521972656,
      "logps/reference_rejected": -275.3694152832031,
      "loss": 0.586,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2920897901058197,
      "rewards/margins": 0.31440818309783936,
      "rewards/rejected": -0.022318381816148758,
      "step": 152,
      "tokens/chosen_response_len_mean": 543.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 491.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 2.9972427731212954,
      "learning_rate": 4.99227956107383e-07,
      "logits/policy_chosen": -2.133497714996338,
      "logits/policy_rejected": -2.275707721710205,
      "logits/reference_chosen": -2.4100184440612793,
      "logits/reference_rejected": -2.27435040473938,
      "logps/policy_chosen": -149.89268493652344,
      "logps/policy_rejected": -225.88043212890625,
      "logps/reference_chosen": -195.48631286621094,
      "logps/reference_rejected": -232.33531188964844,
      "loss": 0.5712,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4559362828731537,
      "rewards/margins": 0.3913875222206116,
      "rewards/rejected": 0.06454875320196152,
      "step": 153,
      "tokens/chosen_response_len_mean": 551.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 501.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 2.6455429598648053,
      "learning_rate": 4.991718563745882e-07,
      "logits/policy_chosen": -3.022681951522827,
      "logits/policy_rejected": -3.033928155899048,
      "logits/reference_chosen": -3.0507426261901855,
      "logits/reference_rejected": -3.0070152282714844,
      "logps/policy_chosen": -17.34947395324707,
      "logps/policy_rejected": -55.760353088378906,
      "logps/reference_chosen": -20.67547607421875,
      "logps/reference_rejected": -50.72275924682617,
      "loss": 0.5857,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.03326002135872841,
      "rewards/margins": 0.08363597095012665,
      "rewards/rejected": -0.05037594214081764,
      "step": 154,
      "tokens/chosen_response_len_mean": 28.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 28.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.12,
      "grad_norm": 2.188503580126624,
      "learning_rate": 4.991137926563035e-07,
      "logits/policy_chosen": -2.8026483058929443,
      "logits/policy_rejected": -2.840829372406006,
      "logits/reference_chosen": -2.866769790649414,
      "logits/reference_rejected": -2.868910789489746,
      "logps/policy_chosen": -45.941871643066406,
      "logps/policy_rejected": -90.47376251220703,
      "logps/reference_chosen": -53.47924041748047,
      "logps/reference_rejected": -84.99751281738281,
      "loss": 0.6102,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.07537368685007095,
      "rewards/margins": 0.1301361620426178,
      "rewards/rejected": -0.054762475192546844,
      "step": 155,
      "tokens/chosen_response_len_mean": 173.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 133.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.5667127080808974,
      "learning_rate": 4.990537654101903e-07,
      "logits/policy_chosen": -2.980593681335449,
      "logits/policy_rejected": -3.0731048583984375,
      "logits/reference_chosen": -3.066528797149658,
      "logits/reference_rejected": -3.1092565059661865,
      "logps/policy_chosen": -92.3929672241211,
      "logps/policy_rejected": -223.2591552734375,
      "logps/reference_chosen": -117.49671936035156,
      "logps/reference_rejected": -224.93527221679688,
      "loss": 0.5677,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.25103750824928284,
      "rewards/margins": 0.2342762053012848,
      "rewards/rejected": 0.016761288046836853,
      "step": 156,
      "tokens/chosen_response_len_mean": 291.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 321.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.742556033679143,
      "learning_rate": 4.989917751093862e-07,
      "logits/policy_chosen": -2.9195590019226074,
      "logits/policy_rejected": -2.7180957794189453,
      "logits/reference_chosen": -3.0530753135681152,
      "logits/reference_rejected": -2.1471331119537354,
      "logps/policy_chosen": -86.05413818359375,
      "logps/policy_rejected": -171.83856201171875,
      "logps/reference_chosen": -108.33873748779297,
      "logps/reference_rejected": -158.24143981933594,
      "loss": 0.5624,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22284595668315887,
      "rewards/margins": 0.35881713032722473,
      "rewards/rejected": -0.13597117364406586,
      "step": 157,
      "tokens/chosen_response_len_mean": 300.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 193.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.320383809339318,
      "learning_rate": 4.989278222425021e-07,
      "logits/policy_chosen": -2.9507193565368652,
      "logits/policy_rejected": -3.0587496757507324,
      "logits/reference_chosen": -3.0194993019104004,
      "logits/reference_rejected": -3.0736992359161377,
      "logps/policy_chosen": -103.41101837158203,
      "logps/policy_rejected": -312.6053466796875,
      "logps/reference_chosen": -126.41258239746094,
      "logps/reference_rejected": -292.84515380859375,
      "loss": 0.5961,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2300155907869339,
      "rewards/margins": 0.4276174306869507,
      "rewards/rejected": -0.19760185480117798,
      "step": 158,
      "tokens/chosen_response_len_mean": 338.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 361.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.516259723460725,
      "learning_rate": 4.988619073136179e-07,
      "logits/policy_chosen": -2.578339099884033,
      "logits/policy_rejected": -2.6243929862976074,
      "logits/reference_chosen": -2.651563882827759,
      "logits/reference_rejected": -2.7075228691101074,
      "logps/policy_chosen": -60.65949249267578,
      "logps/policy_rejected": -124.75245666503906,
      "logps/reference_chosen": -77.07334899902344,
      "logps/reference_rejected": -121.59002685546875,
      "loss": 0.5969,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16413861513137817,
      "rewards/margins": 0.19576290249824524,
      "rewards/rejected": -0.031624287366867065,
      "step": 159,
      "tokens/chosen_response_len_mean": 344.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 347.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.715123355655463,
      "learning_rate": 4.987940308422783e-07,
      "logits/policy_chosen": -2.0114169120788574,
      "logits/policy_rejected": -2.0045886039733887,
      "logits/reference_chosen": -2.092721700668335,
      "logits/reference_rejected": -1.7612866163253784,
      "logps/policy_chosen": -84.94512176513672,
      "logps/policy_rejected": -133.75323486328125,
      "logps/reference_chosen": -107.3111801147461,
      "logps/reference_rejected": -128.45538330078125,
      "loss": 0.5549,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22366058826446533,
      "rewards/margins": 0.27663907408714294,
      "rewards/rejected": -0.05297844856977463,
      "step": 160,
      "tokens/chosen_response_len_mean": 425.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 374.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.6648898364230735,
      "learning_rate": 4.987241933634892e-07,
      "logits/policy_chosen": -2.6987576484680176,
      "logits/policy_rejected": -2.7210419178009033,
      "logits/reference_chosen": -2.804107189178467,
      "logits/reference_rejected": -2.7485485076904297,
      "logps/policy_chosen": -101.68658447265625,
      "logps/policy_rejected": -234.5952606201172,
      "logps/reference_chosen": -123.14704132080078,
      "logps/reference_rejected": -228.36183166503906,
      "loss": 0.5769,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21460454165935516,
      "rewards/margins": 0.276938796043396,
      "rewards/rejected": -0.06233424320816994,
      "step": 161,
      "tokens/chosen_response_len_mean": 409.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 407.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.82572590776649,
      "learning_rate": 4.986523954277134e-07,
      "logits/policy_chosen": -2.5539088249206543,
      "logits/policy_rejected": -2.5690407752990723,
      "logits/reference_chosen": -2.634115695953369,
      "logits/reference_rejected": -2.615056037902832,
      "logps/policy_chosen": -37.4378662109375,
      "logps/policy_rejected": -93.33854675292969,
      "logps/reference_chosen": -45.8214225769043,
      "logps/reference_rejected": -89.81768798828125,
      "loss": 0.5852,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.08383554965257645,
      "rewards/margins": 0.11904413998126984,
      "rewards/rejected": -0.03520859405398369,
      "step": 162,
      "tokens/chosen_response_len_mean": 228.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 240.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.7170854054640863,
      "learning_rate": 4.985786376008658e-07,
      "logits/policy_chosen": -2.820416212081909,
      "logits/policy_rejected": -2.79477858543396,
      "logits/reference_chosen": -2.9494552612304688,
      "logits/reference_rejected": -2.837876081466675,
      "logps/policy_chosen": -71.54603576660156,
      "logps/policy_rejected": -156.01953125,
      "logps/reference_chosen": -90.15393829345703,
      "logps/reference_rejected": -154.65646362304688,
      "loss": 0.5657,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1860790103673935,
      "rewards/margins": 0.1997096687555313,
      "rewards/rejected": -0.01363067515194416,
      "step": 163,
      "tokens/chosen_response_len_mean": 302.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 296.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.3740765633517396,
      "learning_rate": 4.985029204643094e-07,
      "logits/policy_chosen": -2.747786045074463,
      "logits/policy_rejected": -2.7663800716400146,
      "logits/reference_chosen": -2.8246467113494873,
      "logits/reference_rejected": -2.6436727046966553,
      "logps/policy_chosen": -58.324737548828125,
      "logps/policy_rejected": -223.62765502929688,
      "logps/reference_chosen": -85.9714126586914,
      "logps/reference_rejected": -203.6089630126953,
      "loss": 0.6018,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.27646681666374207,
      "rewards/margins": 0.4766535758972168,
      "rewards/rejected": -0.20018675923347473,
      "step": 164,
      "tokens/chosen_response_len_mean": 223.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 216.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.188806416183682,
      "learning_rate": 4.984252446148508e-07,
      "logits/policy_chosen": -2.701019287109375,
      "logits/policy_rejected": -2.729661226272583,
      "logits/reference_chosen": -2.8306493759155273,
      "logits/reference_rejected": -2.8457367420196533,
      "logps/policy_chosen": -154.78518676757812,
      "logps/policy_rejected": -214.3829345703125,
      "logps/reference_chosen": -194.75637817382812,
      "logps/reference_rejected": -246.5928497314453,
      "loss": 0.6206,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.39971184730529785,
      "rewards/margins": 0.07761266827583313,
      "rewards/rejected": 0.3220991790294647,
      "step": 165,
      "tokens/chosen_response_len_mean": 429.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 445.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.8889287147059104,
      "learning_rate": 4.983456106647348e-07,
      "logits/policy_chosen": -2.861962080001831,
      "logits/policy_rejected": -2.8529515266418457,
      "logits/reference_chosen": -3.064021587371826,
      "logits/reference_rejected": -2.927248001098633,
      "logps/policy_chosen": -146.7970428466797,
      "logps/policy_rejected": -283.18389892578125,
      "logps/reference_chosen": -181.90396118164062,
      "logps/reference_rejected": -277.1955871582031,
      "loss": 0.5642,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.35106930136680603,
      "rewards/margins": 0.4109525978565216,
      "rewards/rejected": -0.05988328531384468,
      "step": 166,
      "tokens/chosen_response_len_mean": 468.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 421.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.13,
      "grad_norm": 2.9841019270826292,
      "learning_rate": 4.982640192416404e-07,
      "logits/policy_chosen": -2.6893157958984375,
      "logits/policy_rejected": -2.681230068206787,
      "logits/reference_chosen": -2.948234796524048,
      "logits/reference_rejected": -2.248135566711426,
      "logps/policy_chosen": -122.96820831298828,
      "logps/policy_rejected": -197.66404724121094,
      "logps/reference_chosen": -155.029541015625,
      "logps/reference_rejected": -194.3126678466797,
      "loss": 0.5642,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.32061344385147095,
      "rewards/margins": 0.35412728786468506,
      "rewards/rejected": -0.0335138775408268,
      "step": 167,
      "tokens/chosen_response_len_mean": 452.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 366.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 2.766687221527344,
      "learning_rate": 4.981804709886755e-07,
      "logits/policy_chosen": -2.4212305545806885,
      "logits/policy_rejected": -2.4506514072418213,
      "logits/reference_chosen": -2.6806509494781494,
      "logits/reference_rejected": -2.608292579650879,
      "logps/policy_chosen": -60.471797943115234,
      "logps/policy_rejected": -120.82171630859375,
      "logps/reference_chosen": -92.31079864501953,
      "logps/reference_rejected": -130.82656860351562,
      "loss": 0.54,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.31838998198509216,
      "rewards/margins": 0.21834133565425873,
      "rewards/rejected": 0.10004864633083344,
      "step": 168,
      "tokens/chosen_response_len_mean": 324.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 310.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 2.1828651372724637,
      "learning_rate": 4.980949665643719e-07,
      "logits/policy_chosen": -2.7441906929016113,
      "logits/policy_rejected": -2.7218408584594727,
      "logits/reference_chosen": -2.8971827030181885,
      "logits/reference_rejected": -2.860405445098877,
      "logps/policy_chosen": -92.7531509399414,
      "logps/policy_rejected": -210.39797973632812,
      "logps/reference_chosen": -123.75934600830078,
      "logps/reference_rejected": -200.2626190185547,
      "loss": 0.5755,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3100619614124298,
      "rewards/margins": 0.41141554713249207,
      "rewards/rejected": -0.10135360062122345,
      "step": 169,
      "tokens/chosen_response_len_mean": 517.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 465.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 2.9989447327164425,
      "learning_rate": 4.980075066426795e-07,
      "logits/policy_chosen": -1.6698166131973267,
      "logits/policy_rejected": -1.7434771060943604,
      "logits/reference_chosen": -1.803000569343567,
      "logits/reference_rejected": -1.8658090829849243,
      "logps/policy_chosen": -61.05796813964844,
      "logps/policy_rejected": -89.74996948242188,
      "logps/reference_chosen": -77.35304260253906,
      "logps/reference_rejected": -98.12295532226562,
      "loss": 0.5561,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1629507839679718,
      "rewards/margins": 0.07922087609767914,
      "rewards/rejected": 0.08372990787029266,
      "step": 170,
      "tokens/chosen_response_len_mean": 243.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 245.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 2.5554431416025243,
      "learning_rate": 4.979180919129624e-07,
      "logits/policy_chosen": -2.85784912109375,
      "logits/policy_rejected": -2.838977098464966,
      "logits/reference_chosen": -2.8757684230804443,
      "logits/reference_rejected": -2.8608899116516113,
      "logps/policy_chosen": -53.759185791015625,
      "logps/policy_rejected": -90.618408203125,
      "logps/reference_chosen": -62.647789001464844,
      "logps/reference_rejected": -86.81078338623047,
      "loss": 0.5702,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.08888600766658783,
      "rewards/margins": 0.1269623190164566,
      "rewards/rejected": -0.03807631880044937,
      "step": 171,
      "tokens/chosen_response_len_mean": 207.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 209.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 2.7869612955855776,
      "learning_rate": 4.978267230799918e-07,
      "logits/policy_chosen": -2.0658278465270996,
      "logits/policy_rejected": -2.2496557235717773,
      "logits/reference_chosen": -2.1524839401245117,
      "logits/reference_rejected": -2.3013737201690674,
      "logps/policy_chosen": -86.49990844726562,
      "logps/policy_rejected": -197.741943359375,
      "logps/reference_chosen": -109.14804077148438,
      "logps/reference_rejected": -182.33616638183594,
      "loss": 0.5549,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22648128867149353,
      "rewards/margins": 0.380539208650589,
      "rewards/rejected": -0.15405790507793427,
      "step": 172,
      "tokens/chosen_response_len_mean": 443.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 439.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 2.9395952260087506,
      "learning_rate": 4.977334008639417e-07,
      "logits/policy_chosen": -2.605189800262451,
      "logits/policy_rejected": -2.5856776237487793,
      "logits/reference_chosen": -2.778407335281372,
      "logits/reference_rejected": -2.793548583984375,
      "logps/policy_chosen": -112.27513122558594,
      "logps/policy_rejected": -266.2890625,
      "logps/reference_chosen": -140.7950439453125,
      "logps/reference_rejected": -241.72708129882812,
      "loss": 0.5051,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.28519904613494873,
      "rewards/margins": 0.5308188796043396,
      "rewards/rejected": -0.24561986327171326,
      "step": 173,
      "tokens/chosen_response_len_mean": 462.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 293.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 2.7423154795634153,
      "learning_rate": 4.976381260003826e-07,
      "logits/policy_chosen": -2.7179572582244873,
      "logits/policy_rejected": -2.756556510925293,
      "logits/reference_chosen": -2.8246583938598633,
      "logits/reference_rejected": -2.793865203857422,
      "logps/policy_chosen": -80.221923828125,
      "logps/policy_rejected": -204.06829833984375,
      "logps/reference_chosen": -96.3402328491211,
      "logps/reference_rejected": -181.1507568359375,
      "loss": 0.5916,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16118302941322327,
      "rewards/margins": 0.3903582692146301,
      "rewards/rejected": -0.22917523980140686,
      "step": 174,
      "tokens/chosen_response_len_mean": 307.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 307.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 2.8390846969584946,
      "learning_rate": 4.97540899240276e-07,
      "logits/policy_chosen": -2.6149544715881348,
      "logits/policy_rejected": -2.7186009883880615,
      "logits/reference_chosen": -2.755988597869873,
      "logits/reference_rejected": -2.8059897422790527,
      "logps/policy_chosen": -83.78113555908203,
      "logps/policy_rejected": -158.60336303710938,
      "logps/reference_chosen": -105.53045654296875,
      "logps/reference_rejected": -161.6735076904297,
      "loss": 0.5707,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2174932360649109,
      "rewards/margins": 0.18679174780845642,
      "rewards/rejected": 0.030701473355293274,
      "step": 175,
      "tokens/chosen_response_len_mean": 227.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 221.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 1.928131402745428,
      "learning_rate": 4.974417213499681e-07,
      "logits/policy_chosen": -2.97554874420166,
      "logits/policy_rejected": -3.0169177055358887,
      "logits/reference_chosen": -3.1351237297058105,
      "logits/reference_rejected": -3.054442882537842,
      "logps/policy_chosen": -155.18731689453125,
      "logps/policy_rejected": -306.79449462890625,
      "logps/reference_chosen": -198.6287841796875,
      "logps/reference_rejected": -296.349609375,
      "loss": 0.5884,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4344147741794586,
      "rewards/margins": 0.5388633608818054,
      "rewards/rejected": -0.1044485792517662,
      "step": 176,
      "tokens/chosen_response_len_mean": 399.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 380.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 2.5891349823773604,
      "learning_rate": 4.973405931111844e-07,
      "logits/policy_chosen": -2.475595474243164,
      "logits/policy_rejected": -2.5474050045013428,
      "logits/reference_chosen": -2.574971914291382,
      "logits/reference_rejected": -2.7303850650787354,
      "logps/policy_chosen": -60.937068939208984,
      "logps/policy_rejected": -196.1728515625,
      "logps/reference_chosen": -93.98703002929688,
      "logps/reference_rejected": -193.26068115234375,
      "loss": 0.5721,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.330499529838562,
      "rewards/margins": 0.3596211075782776,
      "rewards/rejected": -0.029121577739715576,
      "step": 177,
      "tokens/chosen_response_len_mean": 277.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 290.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 3.186008328730442,
      "learning_rate": 4.972375153210227e-07,
      "logits/policy_chosen": -2.73168683052063,
      "logits/policy_rejected": -2.80218505859375,
      "logits/reference_chosen": -2.8586461544036865,
      "logits/reference_rejected": -2.856407642364502,
      "logps/policy_chosen": -41.751312255859375,
      "logps/policy_rejected": -121.0931396484375,
      "logps/reference_chosen": -52.108253479003906,
      "logps/reference_rejected": -104.97093200683594,
      "loss": 0.5198,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1035693883895874,
      "rewards/margins": 0.26479148864746094,
      "rewards/rejected": -0.16122210025787354,
      "step": 178,
      "tokens/chosen_response_len_mean": 131.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 131.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 2.4038166772062186,
      "learning_rate": 4.971324887919477e-07,
      "logits/policy_chosen": -2.768160820007324,
      "logits/policy_rejected": -2.2585337162017822,
      "logits/reference_chosen": -2.8858625888824463,
      "logits/reference_rejected": -2.4065427780151367,
      "logps/policy_chosen": -67.46320343017578,
      "logps/policy_rejected": -281.3670654296875,
      "logps/reference_chosen": -94.11170196533203,
      "logps/reference_rejected": -263.22698974609375,
      "loss": 0.5596,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.26648497581481934,
      "rewards/margins": 0.4478859305381775,
      "rewards/rejected": -0.18140093982219696,
      "step": 179,
      "tokens/chosen_response_len_mean": 239.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 333.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.14,
      "grad_norm": 2.530913530757402,
      "learning_rate": 4.970255143517838e-07,
      "logits/policy_chosen": -2.6580893993377686,
      "logits/policy_rejected": -2.717331886291504,
      "logits/reference_chosen": -2.90189790725708,
      "logits/reference_rejected": -2.8899142742156982,
      "logps/policy_chosen": -129.58033752441406,
      "logps/policy_rejected": -165.34884643554688,
      "logps/reference_chosen": -179.34756469726562,
      "logps/reference_rejected": -170.3519287109375,
      "loss": 0.5522,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4976721405982971,
      "rewards/margins": 0.4476412534713745,
      "rewards/rejected": 0.050030872225761414,
      "step": 180,
      "tokens/chosen_response_len_mean": 500.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 318.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 2.633909183828016,
      "learning_rate": 4.969165928437091e-07,
      "logits/policy_chosen": -2.570159912109375,
      "logits/policy_rejected": -2.5334911346435547,
      "logits/reference_chosen": -2.760227680206299,
      "logits/reference_rejected": -2.3865246772766113,
      "logps/policy_chosen": -55.27775955200195,
      "logps/policy_rejected": -119.49053955078125,
      "logps/reference_chosen": -72.7205581665039,
      "logps/reference_rejected": -111.53352355957031,
      "loss": 0.5199,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17442797124385834,
      "rewards/margins": 0.2539982497692108,
      "rewards/rejected": -0.07957027852535248,
      "step": 181,
      "tokens/chosen_response_len_mean": 307.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 267.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 2.3888599137710167,
      "learning_rate": 4.968057251262486e-07,
      "logits/policy_chosen": -2.6059675216674805,
      "logits/policy_rejected": -2.395353078842163,
      "logits/reference_chosen": -2.9607644081115723,
      "logits/reference_rejected": -1.8757669925689697,
      "logps/policy_chosen": -74.94976043701172,
      "logps/policy_rejected": -134.52859497070312,
      "logps/reference_chosen": -111.65763092041016,
      "logps/reference_rejected": -96.27473449707031,
      "loss": 0.5536,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.36707866191864014,
      "rewards/margins": 0.7496173977851868,
      "rewards/rejected": -0.38253867626190186,
      "step": 182,
      "tokens/chosen_response_len_mean": 351.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 148.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 2.826021624132104,
      "learning_rate": 4.966929120732675e-07,
      "logits/policy_chosen": -2.3233590126037598,
      "logits/policy_rejected": -2.4008846282958984,
      "logits/reference_chosen": -2.5622141361236572,
      "logits/reference_rejected": -2.381047248840332,
      "logps/policy_chosen": -185.09976196289062,
      "logps/policy_rejected": -336.95941162109375,
      "logps/reference_chosen": -236.7534637451172,
      "logps/reference_rejected": -320.8805847167969,
      "loss": 0.5087,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.516537070274353,
      "rewards/margins": 0.6773253679275513,
      "rewards/rejected": -0.16078826785087585,
      "step": 183,
      "tokens/chosen_response_len_mean": 441.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 384.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 2.26864738127188,
      "learning_rate": 4.965781545739638e-07,
      "logits/policy_chosen": -2.7422099113464355,
      "logits/policy_rejected": -2.7613723278045654,
      "logits/reference_chosen": -2.9637043476104736,
      "logits/reference_rejected": -2.921086549758911,
      "logps/policy_chosen": -45.85551452636719,
      "logps/policy_rejected": -118.8583984375,
      "logps/reference_chosen": -65.04806518554688,
      "logps/reference_rejected": -119.62251281738281,
      "loss": 0.5676,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.191925510764122,
      "rewards/margins": 0.1842842549085617,
      "rewards/rejected": 0.007641255855560303,
      "step": 184,
      "tokens/chosen_response_len_mean": 266.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 242.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 2.8182732561665023,
      "learning_rate": 4.964614535328625e-07,
      "logits/policy_chosen": -2.523998975753784,
      "logits/policy_rejected": -2.4280190467834473,
      "logits/reference_chosen": -2.717611789703369,
      "logits/reference_rejected": -2.5818982124328613,
      "logps/policy_chosen": -250.7300262451172,
      "logps/policy_rejected": -131.51202392578125,
      "logps/reference_chosen": -304.0659484863281,
      "logps/reference_rejected": -136.93946838378906,
      "loss": 0.5284,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.533359169960022,
      "rewards/margins": 0.47908464074134827,
      "rewards/rejected": 0.0542745515704155,
      "step": 185,
      "tokens/chosen_response_len_mean": 653.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 300.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 2.658487252908494,
      "learning_rate": 4.963428098698069e-07,
      "logits/policy_chosen": -2.8848061561584473,
      "logits/policy_rejected": -2.887396812438965,
      "logits/reference_chosen": -3.0770082473754883,
      "logits/reference_rejected": -3.012545347213745,
      "logps/policy_chosen": -26.844032287597656,
      "logps/policy_rejected": -64.93582916259766,
      "logps/reference_chosen": -37.33853530883789,
      "logps/reference_rejected": -57.781166076660156,
      "loss": 0.5617,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10494504123926163,
      "rewards/margins": 0.17649173736572266,
      "rewards/rejected": -0.07154669612646103,
      "step": 186,
      "tokens/chosen_response_len_mean": 101.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 99.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 3.192921534395214,
      "learning_rate": 4.962222245199527e-07,
      "logits/policy_chosen": -2.5711615085601807,
      "logits/policy_rejected": -2.556881904602051,
      "logits/reference_chosen": -2.9153711795806885,
      "logits/reference_rejected": -2.4689133167266846,
      "logps/policy_chosen": -79.51646423339844,
      "logps/policy_rejected": -212.90431213378906,
      "logps/reference_chosen": -108.06434631347656,
      "logps/reference_rejected": -189.78848266601562,
      "loss": 0.5117,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.285478800535202,
      "rewards/margins": 0.5166370868682861,
      "rewards/rejected": -0.2311583161354065,
      "step": 187,
      "tokens/chosen_response_len_mean": 368.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 330.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 3.185300778929063,
      "learning_rate": 4.960996984337598e-07,
      "logits/policy_chosen": -2.322800397872925,
      "logits/policy_rejected": -2.180086851119995,
      "logits/reference_chosen": -2.647416830062866,
      "logits/reference_rejected": -2.404383420944214,
      "logps/policy_chosen": -88.6377944946289,
      "logps/policy_rejected": -154.1780242919922,
      "logps/reference_chosen": -146.73565673828125,
      "logps/reference_rejected": -158.4337921142578,
      "loss": 0.4879,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5809786319732666,
      "rewards/margins": 0.5384209156036377,
      "rewards/rejected": 0.042557742446660995,
      "step": 188,
      "tokens/chosen_response_len_mean": 496.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 375.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 3.018235573859185,
      "learning_rate": 4.959752325769852e-07,
      "logits/policy_chosen": -2.469785213470459,
      "logits/policy_rejected": -2.580235481262207,
      "logits/reference_chosen": -2.680108070373535,
      "logits/reference_rejected": -2.8124053478240967,
      "logps/policy_chosen": -57.80915069580078,
      "logps/policy_rejected": -210.04135131835938,
      "logps/reference_chosen": -90.7098617553711,
      "logps/reference_rejected": -191.3103485107422,
      "loss": 0.5062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.329007089138031,
      "rewards/margins": 0.5163170099258423,
      "rewards/rejected": -0.18730990588665009,
      "step": 189,
      "tokens/chosen_response_len_mean": 332.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 348.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 3.1505949518170437,
      "learning_rate": 4.958488279306753e-07,
      "logits/policy_chosen": -2.3762245178222656,
      "logits/policy_rejected": -2.466752767562866,
      "logits/reference_chosen": -2.847898483276367,
      "logits/reference_rejected": -2.2204389572143555,
      "logps/policy_chosen": -104.57975006103516,
      "logps/policy_rejected": -250.35414123535156,
      "logps/reference_chosen": -159.56100463867188,
      "logps/reference_rejected": -225.86146545410156,
      "loss": 0.522,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5498126149177551,
      "rewards/margins": 0.7947396039962769,
      "rewards/rejected": -0.24492698907852173,
      "step": 190,
      "tokens/chosen_response_len_mean": 512.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 435.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 3.056135297935867,
      "learning_rate": 4.957204854911578e-07,
      "logits/policy_chosen": -2.485416889190674,
      "logits/policy_rejected": -2.3778135776519775,
      "logits/reference_chosen": -2.856623411178589,
      "logits/reference_rejected": -2.3928959369659424,
      "logps/policy_chosen": -83.4710693359375,
      "logps/policy_rejected": -267.17791748046875,
      "logps/reference_chosen": -124.20071411132812,
      "logps/reference_rejected": -209.21670532226562,
      "loss": 0.5036,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.40729638934135437,
      "rewards/margins": 0.9869081377983093,
      "rewards/rejected": -0.5796117782592773,
      "step": 191,
      "tokens/chosen_response_len_mean": 528.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 410.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.15,
      "grad_norm": 2.503765131980355,
      "learning_rate": 4.955902062700349e-07,
      "logits/policy_chosen": -2.3622069358825684,
      "logits/policy_rejected": -2.4439258575439453,
      "logits/reference_chosen": -2.7395248413085938,
      "logits/reference_rejected": -2.6338257789611816,
      "logps/policy_chosen": -76.7771224975586,
      "logps/policy_rejected": -262.30841064453125,
      "logps/reference_chosen": -117.23094940185547,
      "logps/reference_rejected": -224.7442626953125,
      "loss": 0.5445,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.40453827381134033,
      "rewards/margins": 0.7801797986030579,
      "rewards/rejected": -0.37564149498939514,
      "step": 192,
      "tokens/chosen_response_len_mean": 359.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 367.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.16,
      "grad_norm": 2.9703660280881734,
      "learning_rate": 4.954579912941737e-07,
      "logits/policy_chosen": -2.622279644012451,
      "logits/policy_rejected": -2.6470251083374023,
      "logits/reference_chosen": -2.853696823120117,
      "logits/reference_rejected": -2.840925931930542,
      "logps/policy_chosen": -38.201744079589844,
      "logps/policy_rejected": -138.85470581054688,
      "logps/reference_chosen": -63.35798645019531,
      "logps/reference_rejected": -113.46359252929688,
      "loss": 0.4864,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2515623867511749,
      "rewards/margins": 0.5054734349250793,
      "rewards/rejected": -0.25391101837158203,
      "step": 193,
      "tokens/chosen_response_len_mean": 157.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 169.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.16,
      "grad_norm": 2.9246186619159347,
      "learning_rate": 4.953238416057e-07,
      "logits/policy_chosen": -2.605665683746338,
      "logits/policy_rejected": -2.5566539764404297,
      "logits/reference_chosen": -2.9360122680664062,
      "logits/reference_rejected": -2.068471908569336,
      "logps/policy_chosen": -84.89468383789062,
      "logps/policy_rejected": -218.09048461914062,
      "logps/reference_chosen": -133.76991271972656,
      "logps/reference_rejected": -183.8456573486328,
      "loss": 0.4881,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.48875218629837036,
      "rewards/margins": 0.8312005996704102,
      "rewards/rejected": -0.3424483835697174,
      "step": 194,
      "tokens/chosen_response_len_mean": 360.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 238.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.16,
      "grad_norm": 2.80153859644754,
      "learning_rate": 4.951877582619881e-07,
      "logits/policy_chosen": -2.8731517791748047,
      "logits/policy_rejected": -2.975919246673584,
      "logits/reference_chosen": -3.0668981075286865,
      "logits/reference_rejected": -3.1014294624328613,
      "logps/policy_chosen": -24.165613174438477,
      "logps/policy_rejected": -61.2584342956543,
      "logps/reference_chosen": -23.939205169677734,
      "logps/reference_rejected": -43.215232849121094,
      "loss": 0.5741,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.002264055423438549,
      "rewards/margins": 0.1781679093837738,
      "rewards/rejected": -0.18043196201324463,
      "step": 195,
      "tokens/chosen_response_len_mean": 62.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 63.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.16,
      "grad_norm": 2.7527879974178955,
      "learning_rate": 4.950497423356542e-07,
      "logits/policy_chosen": -2.9940450191497803,
      "logits/policy_rejected": -2.710885763168335,
      "logits/reference_chosen": -3.169459581375122,
      "logits/reference_rejected": -2.7128515243530273,
      "logps/policy_chosen": -120.05692291259766,
      "logps/policy_rejected": -105.24632263183594,
      "logps/reference_chosen": -160.91629028320312,
      "logps/reference_rejected": -78.08905029296875,
      "loss": 0.5115,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4085935354232788,
      "rewards/margins": 0.6801662445068359,
      "rewards/rejected": -0.27157267928123474,
      "step": 196,
      "tokens/chosen_response_len_mean": 332.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 53.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.16,
      "grad_norm": 2.473954328567208,
      "learning_rate": 4.949097949145471e-07,
      "logits/policy_chosen": -2.994095802307129,
      "logits/policy_rejected": -2.9911117553710938,
      "logits/reference_chosen": -3.0934770107269287,
      "logits/reference_rejected": -3.05942440032959,
      "logps/policy_chosen": -54.1805419921875,
      "logps/policy_rejected": -158.96043395996094,
      "logps/reference_chosen": -74.52861785888672,
      "logps/reference_rejected": -128.6375732421875,
      "loss": 0.4544,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2034807801246643,
      "rewards/margins": 0.5067094564437866,
      "rewards/rejected": -0.3032286465167999,
      "step": 197,
      "tokens/chosen_response_len_mean": 182.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 165.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.16,
      "grad_norm": 2.6028329307586944,
      "learning_rate": 4.947679171017394e-07,
      "logits/policy_chosen": -2.7241291999816895,
      "logits/policy_rejected": -2.8773388862609863,
      "logits/reference_chosen": -2.9862842559814453,
      "logits/reference_rejected": -2.9953157901763916,
      "logps/policy_chosen": -56.624752044677734,
      "logps/policy_rejected": -255.8826141357422,
      "logps/reference_chosen": -101.90193176269531,
      "logps/reference_rejected": -203.60231018066406,
      "loss": 0.491,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4527718722820282,
      "rewards/margins": 0.9755749106407166,
      "rewards/rejected": -0.5228029489517212,
      "step": 198,
      "tokens/chosen_response_len_mean": 225.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 217.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.16,
      "grad_norm": 2.8264702439738474,
      "learning_rate": 4.946241100155197e-07,
      "logits/policy_chosen": -2.6785213947296143,
      "logits/policy_rejected": -2.800018310546875,
      "logits/reference_chosen": -2.9413294792175293,
      "logits/reference_rejected": -2.913431406021118,
      "logps/policy_chosen": -38.42972183227539,
      "logps/policy_rejected": -164.37850952148438,
      "logps/reference_chosen": -65.64104461669922,
      "logps/reference_rejected": -128.12234497070312,
      "loss": 0.4973,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.27211323380470276,
      "rewards/margins": 0.6346749067306519,
      "rewards/rejected": -0.3625616431236267,
      "step": 199,
      "tokens/chosen_response_len_mean": 312.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 278.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.16,
      "grad_norm": 3.0030443321611644,
      "learning_rate": 4.944783747893825e-07,
      "logits/policy_chosen": -2.727503538131714,
      "logits/policy_rejected": -2.6386661529541016,
      "logits/reference_chosen": -2.9862632751464844,
      "logits/reference_rejected": -2.1630351543426514,
      "logps/policy_chosen": -84.61045837402344,
      "logps/policy_rejected": -191.14212036132812,
      "logps/reference_chosen": -125.34469604492188,
      "logps/reference_rejected": -150.1243896484375,
      "loss": 0.4714,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4073423147201538,
      "rewards/margins": 0.8175195455551147,
      "rewards/rejected": -0.41017720103263855,
      "step": 200,
      "tokens/chosen_response_len_mean": 264.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 164.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.16,
      "grad_norm": 2.7775892532799955,
      "learning_rate": 4.943307125720206e-07,
      "logits/policy_chosen": -2.4299516677856445,
      "logits/policy_rejected": -2.5677711963653564,
      "logits/reference_chosen": -2.907986640930176,
      "logits/reference_rejected": -2.6046524047851562,
      "logps/policy_chosen": -107.82955169677734,
      "logps/policy_rejected": -325.7413635253906,
      "logps/reference_chosen": -176.93951416015625,
      "logps/reference_rejected": -240.94747924804688,
      "loss": 0.425,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6910995841026306,
      "rewards/margins": 1.5390385389328003,
      "rewards/rejected": -0.8479388952255249,
      "step": 201,
      "tokens/chosen_response_len_mean": 608.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 298.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.16,
      "grad_norm": 2.8271865552266267,
      "learning_rate": 4.941811245273153e-07,
      "logits/policy_chosen": -2.6319420337677,
      "logits/policy_rejected": -2.5893003940582275,
      "logits/reference_chosen": -3.000640392303467,
      "logits/reference_rejected": -2.3574352264404297,
      "logps/policy_chosen": -64.42079162597656,
      "logps/policy_rejected": -202.20693969726562,
      "logps/reference_chosen": -102.7632064819336,
      "logps/reference_rejected": -181.3625030517578,
      "loss": 0.3885,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3834240138530731,
      "rewards/margins": 0.5918683409690857,
      "rewards/rejected": -0.20844432711601257,
      "step": 202,
      "tokens/chosen_response_len_mean": 263.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 237.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.16,
      "grad_norm": 2.804030062412289,
      "learning_rate": 4.940296118343273e-07,
      "logits/policy_chosen": -2.741957187652588,
      "logits/policy_rejected": -2.7859504222869873,
      "logits/reference_chosen": -2.956801414489746,
      "logits/reference_rejected": -2.9367804527282715,
      "logps/policy_chosen": -72.28400421142578,
      "logps/policy_rejected": -123.73040771484375,
      "logps/reference_chosen": -100.23980712890625,
      "logps/reference_rejected": -129.1441650390625,
      "loss": 0.5117,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2795579731464386,
      "rewards/margins": 0.22542047500610352,
      "rewards/rejected": 0.05413751304149628,
      "step": 203,
      "tokens/chosen_response_len_mean": 226.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 221.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.16,
      "grad_norm": 3.1378106559973666,
      "learning_rate": 4.938761756872874e-07,
      "logits/policy_chosen": -2.6754446029663086,
      "logits/policy_rejected": -2.5880160331726074,
      "logits/reference_chosen": -3.173189163208008,
      "logits/reference_rejected": -2.022594451904297,
      "logps/policy_chosen": -59.6561393737793,
      "logps/policy_rejected": -180.80789184570312,
      "logps/reference_chosen": -116.20298767089844,
      "logps/reference_rejected": -130.52818298339844,
      "loss": 0.4056,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5654685497283936,
      "rewards/margins": 1.0682657957077026,
      "rewards/rejected": -0.5027973055839539,
      "step": 204,
      "tokens/chosen_response_len_mean": 284.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 165.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.16,
      "grad_norm": 3.1932802336332298,
      "learning_rate": 4.937208172955876e-07,
      "logits/policy_chosen": -2.549724578857422,
      "logits/policy_rejected": -2.6621100902557373,
      "logits/reference_chosen": -2.8741047382354736,
      "logits/reference_rejected": -2.9152588844299316,
      "logps/policy_chosen": -46.50154113769531,
      "logps/policy_rejected": -164.41668701171875,
      "logps/reference_chosen": -80.38411712646484,
      "logps/reference_rejected": -138.61923217773438,
      "loss": 0.47,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3388257622718811,
      "rewards/margins": 0.5968001484870911,
      "rewards/rejected": -0.25797438621520996,
      "step": 205,
      "tokens/chosen_response_len_mean": 209.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 216.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.17,
      "grad_norm": 3.3325761570853016,
      "learning_rate": 4.935635378837702e-07,
      "logits/policy_chosen": -2.4038610458374023,
      "logits/policy_rejected": -2.402017831802368,
      "logits/reference_chosen": -2.797759771347046,
      "logits/reference_rejected": -2.5829825401306152,
      "logps/policy_chosen": -44.91532516479492,
      "logps/policy_rejected": -160.6805419921875,
      "logps/reference_chosen": -74.13213348388672,
      "logps/reference_rejected": -132.3360595703125,
      "loss": 0.4302,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2921680212020874,
      "rewards/margins": 0.5756129026412964,
      "rewards/rejected": -0.2834448516368866,
      "step": 206,
      "tokens/chosen_response_len_mean": 232.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 251.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.17,
      "grad_norm": 2.7368863153844414,
      "learning_rate": 4.934043386915202e-07,
      "logits/policy_chosen": -2.5967376232147217,
      "logits/policy_rejected": -2.661456823348999,
      "logits/reference_chosen": -2.750797748565674,
      "logits/reference_rejected": -2.847586154937744,
      "logps/policy_chosen": -31.504817962646484,
      "logps/policy_rejected": -81.93124389648438,
      "logps/reference_chosen": -47.813560485839844,
      "logps/reference_rejected": -76.41636657714844,
      "loss": 0.4935,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16308742761611938,
      "rewards/margins": 0.21823616325855255,
      "rewards/rejected": -0.05514874309301376,
      "step": 207,
      "tokens/chosen_response_len_mean": 135.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 144.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.17,
      "grad_norm": 3.2289580903320956,
      "learning_rate": 4.932432209736538e-07,
      "logits/policy_chosen": -2.779569149017334,
      "logits/policy_rejected": -2.719834089279175,
      "logits/reference_chosen": -3.0510177612304688,
      "logits/reference_rejected": -2.049421787261963,
      "logps/policy_chosen": -39.49280548095703,
      "logps/policy_rejected": -97.85432434082031,
      "logps/reference_chosen": -61.321197509765625,
      "logps/reference_rejected": -68.41065979003906,
      "loss": 0.4074,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21828393638134003,
      "rewards/margins": 0.5127205848693848,
      "rewards/rejected": -0.29443657398223877,
      "step": 208,
      "tokens/chosen_response_len_mean": 127.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 47.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.17,
      "grad_norm": 3.6234390581200833,
      "learning_rate": 4.93080186000109e-07,
      "logits/policy_chosen": -2.0586435794830322,
      "logits/policy_rejected": -2.2156870365142822,
      "logits/reference_chosen": -2.455575466156006,
      "logits/reference_rejected": -2.4480810165405273,
      "logps/policy_chosen": -179.0718994140625,
      "logps/policy_rejected": -132.7694854736328,
      "logps/reference_chosen": -197.98556518554688,
      "logps/reference_rejected": -119.65494537353516,
      "loss": 0.4632,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18913674354553223,
      "rewards/margins": 0.32028234004974365,
      "rewards/rejected": -0.13114558160305023,
      "step": 209,
      "tokens/chosen_response_len_mean": 428.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 281.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.17,
      "grad_norm": 3.033273237981518,
      "learning_rate": 4.92915235055936e-07,
      "logits/policy_chosen": -2.756727933883667,
      "logits/policy_rejected": -2.7047367095947266,
      "logits/reference_chosen": -3.014847755432129,
      "logits/reference_rejected": -2.7174530029296875,
      "logps/policy_chosen": -55.59083938598633,
      "logps/policy_rejected": -117.91790008544922,
      "logps/reference_chosen": -91.90667724609375,
      "logps/reference_rejected": -112.02029418945312,
      "loss": 0.4147,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.36315834522247314,
      "rewards/margins": 0.4221343994140625,
      "rewards/rejected": -0.05897604674100876,
      "step": 210,
      "tokens/chosen_response_len_mean": 255.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 197.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.17,
      "grad_norm": 2.632232149269225,
      "learning_rate": 4.927483694412868e-07,
      "logits/policy_chosen": -2.6325759887695312,
      "logits/policy_rejected": -2.4173927307128906,
      "logits/reference_chosen": -2.4960875511169434,
      "logits/reference_rejected": -2.534782648086548,
      "logps/policy_chosen": -24.59671974182129,
      "logps/policy_rejected": -98.01979064941406,
      "logps/reference_chosen": -36.76423645019531,
      "logps/reference_rejected": -77.22918701171875,
      "loss": 0.4171,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.12167516350746155,
      "rewards/margins": 0.32958126068115234,
      "rewards/rejected": -0.2079060971736908,
      "step": 211,
      "tokens/chosen_response_len_mean": 114.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 95.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.17,
      "grad_norm": 2.6677296030035,
      "learning_rate": 4.925795904714048e-07,
      "logits/policy_chosen": -2.6325550079345703,
      "logits/policy_rejected": -2.7918920516967773,
      "logits/reference_chosen": -3.0179286003112793,
      "logits/reference_rejected": -2.9879252910614014,
      "logps/policy_chosen": -68.28926849365234,
      "logps/policy_rejected": -233.2633514404297,
      "logps/reference_chosen": -110.06537628173828,
      "logps/reference_rejected": -193.7027587890625,
      "loss": 0.3524,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.41776108741760254,
      "rewards/margins": 0.8133671283721924,
      "rewards/rejected": -0.39560604095458984,
      "step": 212,
      "tokens/chosen_response_len_mean": 270.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 280.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.17,
      "grad_norm": 3.4599652097860303,
      "learning_rate": 4.924088994766144e-07,
      "logits/policy_chosen": -2.2819485664367676,
      "logits/policy_rejected": -2.3868868350982666,
      "logits/reference_chosen": -2.75758695602417,
      "logits/reference_rejected": -2.2176506519317627,
      "logps/policy_chosen": -94.18050384521484,
      "logps/policy_rejected": -299.33758544921875,
      "logps/reference_chosen": -170.72991943359375,
      "logps/reference_rejected": -266.3306579589844,
      "loss": 0.4024,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.765494167804718,
      "rewards/margins": 1.0955636501312256,
      "rewards/rejected": -0.3300694227218628,
      "step": 213,
      "tokens/chosen_response_len_mean": 548.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 511.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.17,
      "grad_norm": 2.821041192679951,
      "learning_rate": 4.922362978023111e-07,
      "logits/policy_chosen": -2.624894618988037,
      "logits/policy_rejected": -2.621846914291382,
      "logits/reference_chosen": -3.108778476715088,
      "logits/reference_rejected": -3.036085367202759,
      "logps/policy_chosen": -128.86549377441406,
      "logps/policy_rejected": -284.0617370605469,
      "logps/reference_chosen": -219.0366973876953,
      "logps/reference_rejected": -296.7759094238281,
      "loss": 0.4239,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9017120003700256,
      "rewards/margins": 0.7745702266693115,
      "rewards/rejected": 0.12714174389839172,
      "step": 214,
      "tokens/chosen_response_len_mean": 571.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 543.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.17,
      "grad_norm": 3.0002891083329857,
      "learning_rate": 4.9206178680895e-07,
      "logits/policy_chosen": -1.7852914333343506,
      "logits/policy_rejected": -1.8987267017364502,
      "logits/reference_chosen": -1.9375059604644775,
      "logits/reference_rejected": -2.050326108932495,
      "logps/policy_chosen": -52.338626861572266,
      "logps/policy_rejected": -93.70428466796875,
      "logps/reference_chosen": -71.64360046386719,
      "logps/reference_rejected": -91.83578491210938,
      "loss": 0.4021,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19304978847503662,
      "rewards/margins": 0.2117348462343216,
      "rewards/rejected": -0.01868506893515587,
      "step": 215,
      "tokens/chosen_response_len_mean": 359.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 364.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.17,
      "grad_norm": 3.157069335949432,
      "learning_rate": 4.918853678720361e-07,
      "logits/policy_chosen": -2.6262574195861816,
      "logits/policy_rejected": -2.659903049468994,
      "logits/reference_chosen": -2.9374420642852783,
      "logits/reference_rejected": -2.76753568649292,
      "logps/policy_chosen": -38.7560920715332,
      "logps/policy_rejected": -142.21218872070312,
      "logps/reference_chosen": -69.5650634765625,
      "logps/reference_rejected": -97.97706604003906,
      "loss": 0.5087,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3080897629261017,
      "rewards/margins": 0.7504409551620483,
      "rewards/rejected": -0.44235122203826904,
      "step": 216,
      "tokens/chosen_response_len_mean": 248.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 202.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.17,
      "grad_norm": 2.8169264822166338,
      "learning_rate": 4.917070423821125e-07,
      "logits/policy_chosen": -2.4803824424743652,
      "logits/policy_rejected": -2.511946439743042,
      "logits/reference_chosen": -2.749699592590332,
      "logits/reference_rejected": -2.704967975616455,
      "logps/policy_chosen": -83.62620544433594,
      "logps/policy_rejected": -296.3905029296875,
      "logps/reference_chosen": -138.813720703125,
      "logps/reference_rejected": -214.48797607421875,
      "loss": 0.3658,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5518751740455627,
      "rewards/margins": 1.3709005117416382,
      "rewards/rejected": -0.8190253376960754,
      "step": 217,
      "tokens/chosen_response_len_mean": 493.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 406.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.18,
      "grad_norm": 2.7162895030534195,
      "learning_rate": 4.915268117447497e-07,
      "logits/policy_chosen": -2.778090476989746,
      "logits/policy_rejected": -2.7868547439575195,
      "logits/reference_chosen": -3.147437572479248,
      "logits/reference_rejected": -2.9738471508026123,
      "logps/policy_chosen": -58.98277282714844,
      "logps/policy_rejected": -193.54931640625,
      "logps/reference_chosen": -103.57978057861328,
      "logps/reference_rejected": -130.34754943847656,
      "loss": 0.4043,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4459700584411621,
      "rewards/margins": 1.077987790107727,
      "rewards/rejected": -0.6320176720619202,
      "step": 218,
      "tokens/chosen_response_len_mean": 233.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 185.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.18,
      "grad_norm": 3.6458168828683926,
      "learning_rate": 4.913446773805354e-07,
      "logits/policy_chosen": -2.333178997039795,
      "logits/policy_rejected": -2.3620800971984863,
      "logits/reference_chosen": -3.10478138923645,
      "logits/reference_rejected": -2.8959479331970215,
      "logps/policy_chosen": -90.79602813720703,
      "logps/policy_rejected": -286.6520690917969,
      "logps/reference_chosen": -163.10195922851562,
      "logps/reference_rejected": -245.46189880371094,
      "loss": 0.4028,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.723059356212616,
      "rewards/margins": 1.1349611282348633,
      "rewards/rejected": -0.4119018018245697,
      "step": 219,
      "tokens/chosen_response_len_mean": 469.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 442.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.18,
      "grad_norm": 3.233765475571934,
      "learning_rate": 4.911606407250617e-07,
      "logits/policy_chosen": -2.2644901275634766,
      "logits/policy_rejected": -2.4729533195495605,
      "logits/reference_chosen": -2.8000075817108154,
      "logits/reference_rejected": -2.8166446685791016,
      "logps/policy_chosen": -30.913768768310547,
      "logps/policy_rejected": -142.3131561279297,
      "logps/reference_chosen": -51.407657623291016,
      "logps/reference_rejected": -95.70589447021484,
      "loss": 0.4707,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2049388736486435,
      "rewards/margins": 0.6710115671157837,
      "rewards/rejected": -0.4660727083683014,
      "step": 220,
      "tokens/chosen_response_len_mean": 272.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 233.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.18,
      "grad_norm": 2.816944510019336,
      "learning_rate": 4.909747032289152e-07,
      "logits/policy_chosen": -1.9074110984802246,
      "logits/policy_rejected": -2.0913877487182617,
      "logits/reference_chosen": -2.375129461288452,
      "logits/reference_rejected": -2.261173725128174,
      "logps/policy_chosen": -131.19876098632812,
      "logps/policy_rejected": -256.4100646972656,
      "logps/reference_chosen": -193.19882202148438,
      "logps/reference_rejected": -205.95828247070312,
      "loss": 0.4362,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6200005412101746,
      "rewards/margins": 1.1245182752609253,
      "rewards/rejected": -0.5045177936553955,
      "step": 221,
      "tokens/chosen_response_len_mean": 388.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 280.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.18,
      "grad_norm": 3.102088864340547,
      "learning_rate": 4.90786866357665e-07,
      "logits/policy_chosen": -2.299764633178711,
      "logits/policy_rejected": -2.4341142177581787,
      "logits/reference_chosen": -3.09466290473938,
      "logits/reference_rejected": -2.8344757556915283,
      "logps/policy_chosen": -104.12731170654297,
      "logps/policy_rejected": -403.4098205566406,
      "logps/reference_chosen": -205.33453369140625,
      "logps/reference_rejected": -297.8888854980469,
      "loss": 0.4303,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0120720863342285,
      "rewards/margins": 2.067281484603882,
      "rewards/rejected": -1.0552091598510742,
      "step": 222,
      "tokens/chosen_response_len_mean": 487.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 350.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.18,
      "grad_norm": 2.6980157978791506,
      "learning_rate": 4.905971315918508e-07,
      "logits/policy_chosen": -2.2736315727233887,
      "logits/policy_rejected": -2.342026472091675,
      "logits/reference_chosen": -2.8474254608154297,
      "logits/reference_rejected": -2.5709283351898193,
      "logps/policy_chosen": -85.25898742675781,
      "logps/policy_rejected": -367.0793151855469,
      "logps/reference_chosen": -168.6109161376953,
      "logps/reference_rejected": -258.70306396484375,
      "loss": 0.3805,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8335192203521729,
      "rewards/margins": 1.9172817468643188,
      "rewards/rejected": -1.0837624073028564,
      "step": 223,
      "tokens/chosen_response_len_mean": 459.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 381.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.18,
      "grad_norm": 3.023484025148734,
      "learning_rate": 4.904055004269721e-07,
      "logits/policy_chosen": -2.454890251159668,
      "logits/policy_rejected": -2.5622761249542236,
      "logits/reference_chosen": -2.9128506183624268,
      "logits/reference_rejected": -2.8558335304260254,
      "logps/policy_chosen": -36.11943435668945,
      "logps/policy_rejected": -159.4558563232422,
      "logps/reference_chosen": -75.32926177978516,
      "logps/reference_rejected": -124.79375457763672,
      "loss": 0.3956,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.39209815859794617,
      "rewards/margins": 0.7387192845344543,
      "rewards/rejected": -0.3466210961341858,
      "step": 224,
      "tokens/chosen_response_len_mean": 181.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 169.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.18,
      "grad_norm": 2.879025667733998,
      "learning_rate": 4.902119743734755e-07,
      "logits/policy_chosen": -2.664403200149536,
      "logits/policy_rejected": -2.587775230407715,
      "logits/reference_chosen": -3.1453912258148193,
      "logits/reference_rejected": -2.816845417022705,
      "logps/policy_chosen": -49.29490661621094,
      "logps/policy_rejected": -94.93446350097656,
      "logps/reference_chosen": -71.66400146484375,
      "logps/reference_rejected": -87.74375915527344,
      "loss": 0.3954,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.22369101643562317,
      "rewards/margins": 0.2955980896949768,
      "rewards/rejected": -0.07190708816051483,
      "step": 225,
      "tokens/chosen_response_len_mean": 198.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 179.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.18,
      "grad_norm": 3.246164993624403,
      "learning_rate": 4.900165549567435e-07,
      "logits/policy_chosen": -2.3957905769348145,
      "logits/policy_rejected": -2.423283100128174,
      "logits/reference_chosen": -2.991285800933838,
      "logits/reference_rejected": -2.787295341491699,
      "logps/policy_chosen": -88.65385437011719,
      "logps/policy_rejected": -120.2602767944336,
      "logps/reference_chosen": -112.840087890625,
      "logps/reference_rejected": -75.72503662109375,
      "loss": 0.3168,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.24186235666275024,
      "rewards/margins": 0.6872147917747498,
      "rewards/rejected": -0.44535237550735474,
      "step": 226,
      "tokens/chosen_response_len_mean": 422.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 400.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.18,
      "grad_norm": 3.0799697537609085,
      "learning_rate": 4.898192437170821e-07,
      "logits/policy_chosen": -2.530318021774292,
      "logits/policy_rejected": -2.645883083343506,
      "logits/reference_chosen": -3.1499884128570557,
      "logits/reference_rejected": -2.5675787925720215,
      "logps/policy_chosen": -54.94190216064453,
      "logps/policy_rejected": -261.4326171875,
      "logps/reference_chosen": -134.1338348388672,
      "logps/reference_rejected": -190.62583923339844,
      "loss": 0.3325,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7919193506240845,
      "rewards/margins": 1.4999868869781494,
      "rewards/rejected": -0.7080676555633545,
      "step": 227,
      "tokens/chosen_response_len_mean": 255.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 204.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.18,
      "grad_norm": 2.659841913817482,
      "learning_rate": 4.896200422097086e-07,
      "logits/policy_chosen": -1.9145750999450684,
      "logits/policy_rejected": -2.0923149585723877,
      "logits/reference_chosen": -2.8488292694091797,
      "logits/reference_rejected": -2.7205710411071777,
      "logps/policy_chosen": -107.40567016601562,
      "logps/policy_rejected": -427.5948486328125,
      "logps/reference_chosen": -223.86032104492188,
      "logps/reference_rejected": -329.6167297363281,
      "loss": 0.1916,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1645466089248657,
      "rewards/margins": 2.1443278789520264,
      "rewards/rejected": -0.9797812700271606,
      "step": 228,
      "tokens/chosen_response_len_mean": 517.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 439.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.18,
      "grad_norm": 3.234230812286525,
      "learning_rate": 4.894189520047397e-07,
      "logits/policy_chosen": -2.2693915367126465,
      "logits/policy_rejected": -2.3958592414855957,
      "logits/reference_chosen": -3.0886495113372803,
      "logits/reference_rejected": -2.8636574745178223,
      "logps/policy_chosen": -51.258445739746094,
      "logps/policy_rejected": -378.75201416015625,
      "logps/reference_chosen": -118.88182830810547,
      "logps/reference_rejected": -246.41397094726562,
      "loss": 0.3282,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6762338280677795,
      "rewards/margins": 1.9996143579483032,
      "rewards/rejected": -1.3233805894851685,
      "step": 229,
      "tokens/chosen_response_len_mean": 271.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 266.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.18,
      "grad_norm": 3.457038560417765,
      "learning_rate": 4.892159746871788e-07,
      "logits/policy_chosen": -2.3888025283813477,
      "logits/policy_rejected": -2.411865234375,
      "logits/reference_chosen": -3.154548168182373,
      "logits/reference_rejected": -2.1464414596557617,
      "logps/policy_chosen": -57.5999870300293,
      "logps/policy_rejected": -173.8509063720703,
      "logps/reference_chosen": -131.91171264648438,
      "logps/reference_rejected": -113.2002182006836,
      "loss": 0.3683,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7431172728538513,
      "rewards/margins": 1.3496241569519043,
      "rewards/rejected": -0.6065068244934082,
      "step": 230,
      "tokens/chosen_response_len_mean": 285.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 94.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.19,
      "grad_norm": 3.453784172007788,
      "learning_rate": 4.890111118569036e-07,
      "logits/policy_chosen": -2.0798964500427246,
      "logits/policy_rejected": -2.2078676223754883,
      "logits/reference_chosen": -2.8997390270233154,
      "logits/reference_rejected": -2.4869751930236816,
      "logps/policy_chosen": -75.02999877929688,
      "logps/policy_rejected": -229.92823791503906,
      "logps/reference_chosen": -143.95648193359375,
      "logps/reference_rejected": -181.41172790527344,
      "loss": 0.327,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6892647743225098,
      "rewards/margins": 1.1744297742843628,
      "rewards/rejected": -0.485164999961853,
      "step": 231,
      "tokens/chosen_response_len_mean": 280.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 238.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.19,
      "grad_norm": 3.239627424882494,
      "learning_rate": 4.888043651286537e-07,
      "logits/policy_chosen": -2.5193958282470703,
      "logits/policy_rejected": -2.452767848968506,
      "logits/reference_chosen": -3.070887565612793,
      "logits/reference_rejected": -2.4067721366882324,
      "logps/policy_chosen": -44.861671447753906,
      "logps/policy_rejected": -245.7529296875,
      "logps/reference_chosen": -78.50507354736328,
      "logps/reference_rejected": -147.85055541992188,
      "loss": 0.3734,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.336434006690979,
      "rewards/margins": 1.315457820892334,
      "rewards/rejected": -0.979023814201355,
      "step": 232,
      "tokens/chosen_response_len_mean": 251.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 159.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.19,
      "grad_norm": 3.085937654519379,
      "learning_rate": 4.885957361320172e-07,
      "logits/policy_chosen": -1.9149789810180664,
      "logits/policy_rejected": -2.088989734649658,
      "logits/reference_chosen": -2.9695346355438232,
      "logits/reference_rejected": -2.6794815063476562,
      "logps/policy_chosen": -73.91265869140625,
      "logps/policy_rejected": -403.6729431152344,
      "logps/reference_chosen": -209.8995819091797,
      "logps/reference_rejected": -327.79486083984375,
      "loss": 0.2662,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.359869360923767,
      "rewards/margins": 2.118650197982788,
      "rewards/rejected": -0.7587809562683105,
      "step": 233,
      "tokens/chosen_response_len_mean": 403.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 360.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.19,
      "grad_norm": 2.7768750875070762,
      "learning_rate": 4.88385226511419e-07,
      "logits/policy_chosen": -1.917398452758789,
      "logits/policy_rejected": -2.089282989501953,
      "logits/reference_chosen": -3.025069236755371,
      "logits/reference_rejected": -2.1790575981140137,
      "logps/policy_chosen": -98.6364974975586,
      "logps/policy_rejected": -710.46142578125,
      "logps/reference_chosen": -247.2069854736328,
      "logps/reference_rejected": -409.91046142578125,
      "loss": 0.2487,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.485705018043518,
      "rewards/margins": 4.491214752197266,
      "rewards/rejected": -3.005509853363037,
      "step": 234,
      "tokens/chosen_response_len_mean": 516.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 393.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.19,
      "grad_norm": 2.6357031102373694,
      "learning_rate": 4.881728379261067e-07,
      "logits/policy_chosen": -2.5067009925842285,
      "logits/policy_rejected": -2.6151890754699707,
      "logits/reference_chosen": -2.879830837249756,
      "logits/reference_rejected": -2.289306879043579,
      "logps/policy_chosen": -28.403209686279297,
      "logps/policy_rejected": -94.16014099121094,
      "logps/reference_chosen": -50.78627014160156,
      "logps/reference_rejected": -61.18941116333008,
      "loss": 0.3383,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.22383059561252594,
      "rewards/margins": 0.553537905216217,
      "rewards/rejected": -0.3297072649002075,
      "step": 235,
      "tokens/chosen_response_len_mean": 130.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 70.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.19,
      "grad_norm": 2.5416162412883967,
      "learning_rate": 4.879585720501382e-07,
      "logits/policy_chosen": -2.079270362854004,
      "logits/policy_rejected": -2.1965620517730713,
      "logits/reference_chosen": -2.69492769241333,
      "logits/reference_rejected": -2.7009551525115967,
      "logps/policy_chosen": -47.36531448364258,
      "logps/policy_rejected": -210.86045837402344,
      "logps/reference_chosen": -109.4851303100586,
      "logps/reference_rejected": -157.570556640625,
      "loss": 0.4301,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6211981177330017,
      "rewards/margins": 1.1540971994400024,
      "rewards/rejected": -0.5328990817070007,
      "step": 236,
      "tokens/chosen_response_len_mean": 382.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 342.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.19,
      "grad_norm": 3.3640626360746606,
      "learning_rate": 4.877424305723682e-07,
      "logits/policy_chosen": -2.3050661087036133,
      "logits/policy_rejected": -2.362180709838867,
      "logits/reference_chosen": -2.8899943828582764,
      "logits/reference_rejected": -2.970350503921509,
      "logps/policy_chosen": -58.130821228027344,
      "logps/policy_rejected": -187.88409423828125,
      "logps/reference_chosen": -105.74044036865234,
      "logps/reference_rejected": -162.71900939941406,
      "loss": 0.3679,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4760962128639221,
      "rewards/margins": 0.7277471423149109,
      "rewards/rejected": -0.2516508996486664,
      "step": 237,
      "tokens/chosen_response_len_mean": 321.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 332.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.19,
      "grad_norm": 3.092634828507096,
      "learning_rate": 4.87524415196435e-07,
      "logits/policy_chosen": -1.7353723049163818,
      "logits/policy_rejected": -1.8133890628814697,
      "logits/reference_chosen": -2.299503803253174,
      "logits/reference_rejected": -1.7435356378555298,
      "logps/policy_chosen": -25.773319244384766,
      "logps/policy_rejected": -84.91059875488281,
      "logps/reference_chosen": -49.502906799316406,
      "logps/reference_rejected": -59.85688781738281,
      "loss": 0.4402,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.23729589581489563,
      "rewards/margins": 0.4878329634666443,
      "rewards/rejected": -0.25053703784942627,
      "step": 238,
      "tokens/chosen_response_len_mean": 180.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 131.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.19,
      "grad_norm": 2.30111282627728,
      "learning_rate": 4.873045276407472e-07,
      "logits/policy_chosen": -2.3006367683410645,
      "logits/policy_rejected": -2.373878240585327,
      "logits/reference_chosen": -3.0563242435455322,
      "logits/reference_rejected": -2.862274408340454,
      "logps/policy_chosen": -60.457550048828125,
      "logps/policy_rejected": -362.2609558105469,
      "logps/reference_chosen": -138.99404907226562,
      "logps/reference_rejected": -211.34681701660156,
      "loss": 0.263,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7853649258613586,
      "rewards/margins": 2.294506072998047,
      "rewards/rejected": -1.5091413259506226,
      "step": 239,
      "tokens/chosen_response_len_mean": 323.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 253.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.19,
      "grad_norm": 3.3647443577329184,
      "learning_rate": 4.870827696384697e-07,
      "logits/policy_chosen": -1.7883644104003906,
      "logits/policy_rejected": -1.9111793041229248,
      "logits/reference_chosen": -2.7533793449401855,
      "logits/reference_rejected": -2.8058459758758545,
      "logps/policy_chosen": -63.6192512512207,
      "logps/policy_rejected": -472.13812255859375,
      "logps/reference_chosen": -182.55494689941406,
      "logps/reference_rejected": -324.2348327636719,
      "loss": 0.4491,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.189356803894043,
      "rewards/margins": 2.6683895587921143,
      "rewards/rejected": -1.4790325164794922,
      "step": 240,
      "tokens/chosen_response_len_mean": 425.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 412.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.19,
      "grad_norm": 2.8850200026957507,
      "learning_rate": 4.868591429375109e-07,
      "logits/policy_chosen": -2.5384299755096436,
      "logits/policy_rejected": -2.5679569244384766,
      "logits/reference_chosen": -3.137737274169922,
      "logits/reference_rejected": -2.9568300247192383,
      "logps/policy_chosen": -52.663124084472656,
      "logps/policy_rejected": -150.413818359375,
      "logps/reference_chosen": -119.31211853027344,
      "logps/reference_rejected": -144.43154907226562,
      "loss": 0.3981,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6664900183677673,
      "rewards/margins": 0.7263126373291016,
      "rewards/rejected": -0.059822604060173035,
      "step": 241,
      "tokens/chosen_response_len_mean": 247.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 232.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.19,
      "grad_norm": 2.740311113779816,
      "learning_rate": 4.866336493005077e-07,
      "logits/policy_chosen": -2.3799917697906494,
      "logits/policy_rejected": -2.3980116844177246,
      "logits/reference_chosen": -2.8105556964874268,
      "logits/reference_rejected": -2.789691925048828,
      "logps/policy_chosen": -86.99842834472656,
      "logps/policy_rejected": -245.59523010253906,
      "logps/reference_chosen": -164.35971069335938,
      "logps/reference_rejected": -204.79449462890625,
      "loss": 0.3237,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7736127376556396,
      "rewards/margins": 1.1816203594207764,
      "rewards/rejected": -0.4080074727535248,
      "step": 242,
      "tokens/chosen_response_len_mean": 234.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 228.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.2,
      "grad_norm": 3.5286650522231846,
      "learning_rate": 4.86406290504813e-07,
      "logits/policy_chosen": -2.302743434906006,
      "logits/policy_rejected": -2.3800129890441895,
      "logits/reference_chosen": -3.100501537322998,
      "logits/reference_rejected": -2.502018690109253,
      "logps/policy_chosen": -56.89078140258789,
      "logps/policy_rejected": -293.7001953125,
      "logps/reference_chosen": -124.03472900390625,
      "logps/reference_rejected": -186.7720489501953,
      "loss": 0.262,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.671439528465271,
      "rewards/margins": 1.7407207489013672,
      "rewards/rejected": -1.0692812204360962,
      "step": 243,
      "tokens/chosen_response_len_mean": 347.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 276.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.2,
      "grad_norm": 3.590713146645003,
      "learning_rate": 4.861770683424803e-07,
      "logits/policy_chosen": -2.072890043258667,
      "logits/policy_rejected": -1.9427573680877686,
      "logits/reference_chosen": -2.6935362815856934,
      "logits/reference_rejected": -2.673877000808716,
      "logps/policy_chosen": -45.185890197753906,
      "logps/policy_rejected": -246.34381103515625,
      "logps/reference_chosen": -113.1274642944336,
      "logps/reference_rejected": -189.79290771484375,
      "loss": 0.365,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6794158220291138,
      "rewards/margins": 1.2449246644973755,
      "rewards/rejected": -0.5655089616775513,
      "step": 244,
      "tokens/chosen_response_len_mean": 282.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 369.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.2,
      "grad_norm": 2.610774413059978,
      "learning_rate": 4.859459846202507e-07,
      "logits/policy_chosen": -2.30108380317688,
      "logits/policy_rejected": -2.4250240325927734,
      "logits/reference_chosen": -3.1003952026367188,
      "logits/reference_rejected": -2.8910207748413086,
      "logps/policy_chosen": -113.18220520019531,
      "logps/policy_rejected": -571.716064453125,
      "logps/reference_chosen": -273.4241943359375,
      "logps/reference_rejected": -353.7633056640625,
      "loss": 0.3468,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.6024198532104492,
      "rewards/margins": 3.781947135925293,
      "rewards/rejected": -2.1795272827148438,
      "step": 245,
      "tokens/chosen_response_len_mean": 495.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 352.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.2,
      "grad_norm": 3.1327030609878674,
      "learning_rate": 4.857130411595381e-07,
      "logits/policy_chosen": -1.9849697351455688,
      "logits/policy_rejected": -2.2542386054992676,
      "logits/reference_chosen": -2.8174877166748047,
      "logits/reference_rejected": -2.6481382846832275,
      "logps/policy_chosen": -65.47968292236328,
      "logps/policy_rejected": -309.6473083496094,
      "logps/reference_chosen": -125.56417846679688,
      "logps/reference_rejected": -215.66690063476562,
      "loss": 0.2941,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6008449792861938,
      "rewards/margins": 1.5406490564346313,
      "rewards/rejected": -0.9398040771484375,
      "step": 246,
      "tokens/chosen_response_len_mean": 314.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 219.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.2,
      "grad_norm": 2.8949654648587106,
      "learning_rate": 4.854782397964148e-07,
      "logits/policy_chosen": -2.4038028717041016,
      "logits/policy_rejected": -2.535879373550415,
      "logits/reference_chosen": -3.110623836517334,
      "logits/reference_rejected": -2.892681360244751,
      "logps/policy_chosen": -42.45046615600586,
      "logps/policy_rejected": -223.6653594970703,
      "logps/reference_chosen": -108.52606201171875,
      "logps/reference_rejected": -122.96241760253906,
      "loss": 0.1793,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6607558727264404,
      "rewards/margins": 1.667785406112671,
      "rewards/rejected": -1.0070295333862305,
      "step": 247,
      "tokens/chosen_response_len_mean": 224.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 148.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.2,
      "grad_norm": 3.2467096251740397,
      "learning_rate": 4.852415823815975e-07,
      "logits/policy_chosen": -2.0340566635131836,
      "logits/policy_rejected": -2.007965087890625,
      "logits/reference_chosen": -2.2954137325286865,
      "logits/reference_rejected": -2.1136422157287598,
      "logps/policy_chosen": -48.099082946777344,
      "logps/policy_rejected": -218.53590393066406,
      "logps/reference_chosen": -83.52349090576172,
      "logps/reference_rejected": -118.22250366210938,
      "loss": 0.3545,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3542441725730896,
      "rewards/margins": 1.3573782444000244,
      "rewards/rejected": -1.0031341314315796,
      "step": 248,
      "tokens/chosen_response_len_mean": 159.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 128.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.2,
      "grad_norm": 2.4511654553576294,
      "learning_rate": 4.850030707804322e-07,
      "logits/policy_chosen": -2.421999454498291,
      "logits/policy_rejected": -2.4670658111572266,
      "logits/reference_chosen": -3.0133814811706543,
      "logits/reference_rejected": -2.3799855709075928,
      "logps/policy_chosen": -53.002376556396484,
      "logps/policy_rejected": -286.0945739746094,
      "logps/reference_chosen": -161.1949462890625,
      "logps/reference_rejected": -157.0173797607422,
      "loss": 0.3471,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.081925630569458,
      "rewards/margins": 2.372697353363037,
      "rewards/rejected": -1.2907719612121582,
      "step": 249,
      "tokens/chosen_response_len_mean": 254.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 189.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.2,
      "grad_norm": 3.2467420827995683,
      "learning_rate": 4.847627068728794e-07,
      "logits/policy_chosen": -2.1038780212402344,
      "logits/policy_rejected": -2.1949315071105957,
      "logits/reference_chosen": -3.1234359741210938,
      "logits/reference_rejected": -2.4281110763549805,
      "logps/policy_chosen": -33.008514404296875,
      "logps/policy_rejected": -167.2099151611328,
      "logps/reference_chosen": -75.09925842285156,
      "logps/reference_rejected": -86.3980484008789,
      "loss": 0.3442,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4209074079990387,
      "rewards/margins": 1.2290260791778564,
      "rewards/rejected": -0.8081187009811401,
      "step": 250,
      "tokens/chosen_response_len_mean": 105.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 71.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.2,
      "grad_norm": 2.2929712454735145,
      "learning_rate": 4.845204925535003e-07,
      "logits/policy_chosen": -1.7899479866027832,
      "logits/policy_rejected": -1.9353580474853516,
      "logits/reference_chosen": -2.701301336288452,
      "logits/reference_rejected": -2.7325639724731445,
      "logps/policy_chosen": -50.73257827758789,
      "logps/policy_rejected": -341.6130676269531,
      "logps/reference_chosen": -127.89730834960938,
      "logps/reference_rejected": -177.73052978515625,
      "loss": 0.1825,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7716473340988159,
      "rewards/margins": 2.4104726314544678,
      "rewards/rejected": -1.638825535774231,
      "step": 251,
      "tokens/chosen_response_len_mean": 542.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 416.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.2,
      "grad_norm": 2.718487080105349,
      "learning_rate": 4.842764297314406e-07,
      "logits/policy_chosen": -1.9718434810638428,
      "logits/policy_rejected": -2.002476692199707,
      "logits/reference_chosen": -2.571845293045044,
      "logits/reference_rejected": -2.6133267879486084,
      "logps/policy_chosen": -41.810699462890625,
      "logps/policy_rejected": -177.89076232910156,
      "logps/reference_chosen": -95.01820373535156,
      "logps/reference_rejected": -145.2048797607422,
      "loss": 0.2304,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5320751070976257,
      "rewards/margins": 0.8589338064193726,
      "rewards/rejected": -0.3268587589263916,
      "step": 252,
      "tokens/chosen_response_len_mean": 206.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 208.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.2,
      "grad_norm": 3.7180834260951325,
      "learning_rate": 4.840305203304159e-07,
      "logits/policy_chosen": -2.030735969543457,
      "logits/policy_rejected": -2.1171207427978516,
      "logits/reference_chosen": -2.9690611362457275,
      "logits/reference_rejected": -2.7168192863464355,
      "logps/policy_chosen": -41.81312561035156,
      "logps/policy_rejected": -201.77508544921875,
      "logps/reference_chosen": -100.09971618652344,
      "logps/reference_rejected": -119.85955047607422,
      "loss": 0.3607,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5828658938407898,
      "rewards/margins": 1.4020211696624756,
      "rewards/rejected": -0.8191554546356201,
      "step": 253,
      "tokens/chosen_response_len_mean": 273.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 156.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.2,
      "grad_norm": 2.6023063627206815,
      "learning_rate": 4.837827662886971e-07,
      "logits/policy_chosen": -2.1987662315368652,
      "logits/policy_rejected": -2.3438453674316406,
      "logits/reference_chosen": -3.212616443634033,
      "logits/reference_rejected": -3.013723134994507,
      "logps/policy_chosen": -59.346351623535156,
      "logps/policy_rejected": -576.636962890625,
      "logps/reference_chosen": -173.44329833984375,
      "logps/reference_rejected": -289.7512512207031,
      "loss": 0.2857,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1409695148468018,
      "rewards/margins": 4.009826183319092,
      "rewards/rejected": -2.86885666847229,
      "step": 254,
      "tokens/chosen_response_len_mean": 327.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 266.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.2,
      "grad_norm": 2.968765579985845,
      "learning_rate": 4.835331695590943e-07,
      "logits/policy_chosen": -2.362076759338379,
      "logits/policy_rejected": -2.3499341011047363,
      "logits/reference_chosen": -3.076436996459961,
      "logits/reference_rejected": -2.8915157318115234,
      "logps/policy_chosen": -24.552675247192383,
      "logps/policy_rejected": -53.35011291503906,
      "logps/reference_chosen": -56.392234802246094,
      "logps/reference_rejected": -45.72980499267578,
      "loss": 0.3399,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.31839558482170105,
      "rewards/margins": 0.39459866285324097,
      "rewards/rejected": -0.07620306313037872,
      "step": 255,
      "tokens/chosen_response_len_mean": 143.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 110.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.21,
      "grad_norm": 2.8370570975666873,
      "learning_rate": 4.832817321089418e-07,
      "logits/policy_chosen": -2.4410159587860107,
      "logits/policy_rejected": -2.4411826133728027,
      "logits/reference_chosen": -2.7271342277526855,
      "logits/reference_rejected": -2.732917308807373,
      "logps/policy_chosen": -31.23749351501465,
      "logps/policy_rejected": -98.78245544433594,
      "logps/reference_chosen": -43.81713104248047,
      "logps/reference_rejected": -72.64688873291016,
      "loss": 0.4067,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1257963329553604,
      "rewards/margins": 0.3871520459651947,
      "rewards/rejected": -0.2613556981086731,
      "step": 256,
      "tokens/chosen_response_len_mean": 45.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 46.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.21,
      "grad_norm": 2.7797560376214,
      "learning_rate": 4.830284559200827e-07,
      "logits/policy_chosen": -1.952653408050537,
      "logits/policy_rejected": -2.1609013080596924,
      "logits/reference_chosen": -3.012723922729492,
      "logits/reference_rejected": -2.739423990249634,
      "logps/policy_chosen": -57.349365234375,
      "logps/policy_rejected": -257.4586486816406,
      "logps/reference_chosen": -145.12765502929688,
      "logps/reference_rejected": -142.13612365722656,
      "loss": 0.2769,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8777830004692078,
      "rewards/margins": 2.03100848197937,
      "rewards/rejected": -1.153225302696228,
      "step": 257,
      "tokens/chosen_response_len_mean": 332.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 215.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.21,
      "grad_norm": 2.6056834165338865,
      "learning_rate": 4.827733429888531e-07,
      "logits/policy_chosen": -2.5333569049835205,
      "logits/policy_rejected": -2.632535934448242,
      "logits/reference_chosen": -3.0343687534332275,
      "logits/reference_rejected": -2.7431583404541016,
      "logps/policy_chosen": -51.503787994384766,
      "logps/policy_rejected": -205.5330810546875,
      "logps/reference_chosen": -112.60284423828125,
      "logps/reference_rejected": -109.25039672851562,
      "loss": 0.2703,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6109905242919922,
      "rewards/margins": 1.573817253112793,
      "rewards/rejected": -0.962826669216156,
      "step": 258,
      "tokens/chosen_response_len_mean": 209.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 112.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.21,
      "grad_norm": 2.7017642438408416,
      "learning_rate": 4.825163953260663e-07,
      "logits/policy_chosen": -2.209869146347046,
      "logits/policy_rejected": -2.233635902404785,
      "logits/reference_chosen": -3.0763533115386963,
      "logits/reference_rejected": -2.9993386268615723,
      "logps/policy_chosen": -72.18954467773438,
      "logps/policy_rejected": -188.30348205566406,
      "logps/reference_chosen": -163.45166015625,
      "logps/reference_rejected": -172.4595947265625,
      "loss": 0.2949,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9126211404800415,
      "rewards/margins": 1.0710599422454834,
      "rewards/rejected": -0.1584388017654419,
      "step": 259,
      "tokens/chosen_response_len_mean": 309.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 269.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.21,
      "grad_norm": 2.7221262346156205,
      "learning_rate": 4.82257614956997e-07,
      "logits/policy_chosen": -1.0971046686172485,
      "logits/policy_rejected": -1.240858793258667,
      "logits/reference_chosen": -2.244776964187622,
      "logits/reference_rejected": -1.5502854585647583,
      "logps/policy_chosen": -91.92208862304688,
      "logps/policy_rejected": -343.6435241699219,
      "logps/reference_chosen": -204.17926025390625,
      "logps/reference_rejected": -210.630615234375,
      "loss": 0.2794,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.122571587562561,
      "rewards/margins": 2.452700614929199,
      "rewards/rejected": -1.3301290273666382,
      "step": 260,
      "tokens/chosen_response_len_mean": 590.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 508.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.21,
      "grad_norm": 3.8028582614963775,
      "learning_rate": 4.819970039213655e-07,
      "logits/policy_chosen": -2.1693973541259766,
      "logits/policy_rejected": -2.2979447841644287,
      "logits/reference_chosen": -3.102466106414795,
      "logits/reference_rejected": -2.547724723815918,
      "logps/policy_chosen": -59.068519592285156,
      "logps/policy_rejected": -445.1409606933594,
      "logps/reference_chosen": -156.0995635986328,
      "logps/reference_rejected": -305.90802001953125,
      "loss": 0.3756,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9703104496002197,
      "rewards/margins": 2.362639904022217,
      "rewards/rejected": -1.3923293352127075,
      "step": 261,
      "tokens/chosen_response_len_mean": 229.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 209.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.21,
      "grad_norm": 3.239958730128674,
      "learning_rate": 4.817345642733218e-07,
      "logits/policy_chosen": -2.0956246852874756,
      "logits/policy_rejected": -2.182451009750366,
      "logits/reference_chosen": -3.003354072570801,
      "logits/reference_rejected": -2.506141424179077,
      "logps/policy_chosen": -42.89665603637695,
      "logps/policy_rejected": -128.68064880371094,
      "logps/reference_chosen": -79.88959503173828,
      "logps/reference_rejected": -72.20051574707031,
      "loss": 0.2773,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.36992937326431274,
      "rewards/margins": 0.9347306489944458,
      "rewards/rejected": -0.5648012757301331,
      "step": 262,
      "tokens/chosen_response_len_mean": 154.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 111.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.21,
      "grad_norm": 2.7801336555461136,
      "learning_rate": 4.814702980814284e-07,
      "logits/policy_chosen": -2.0152645111083984,
      "logits/policy_rejected": -2.148892879486084,
      "logits/reference_chosen": -3.0190117359161377,
      "logits/reference_rejected": -2.243680238723755,
      "logps/policy_chosen": -57.687110900878906,
      "logps/policy_rejected": -464.7051696777344,
      "logps/reference_chosen": -180.52468872070312,
      "logps/reference_rejected": -230.62245178222656,
      "loss": 0.1516,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2283756732940674,
      "rewards/margins": 3.5692026615142822,
      "rewards/rejected": -2.340827226638794,
      "step": 263,
      "tokens/chosen_response_len_mean": 365.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 254.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.21,
      "grad_norm": 2.66395215116379,
      "learning_rate": 4.812042074286453e-07,
      "logits/policy_chosen": -2.0517807006835938,
      "logits/policy_rejected": -2.1628949642181396,
      "logits/reference_chosen": -3.143606662750244,
      "logits/reference_rejected": -2.7492570877075195,
      "logps/policy_chosen": -114.81196594238281,
      "logps/policy_rejected": -593.2320556640625,
      "logps/reference_chosen": -350.7387390136719,
      "logps/reference_rejected": -264.07171630859375,
      "loss": 0.3429,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3592677116394043,
      "rewards/margins": 5.650871276855469,
      "rewards/rejected": -3.2916030883789062,
      "step": 264,
      "tokens/chosen_response_len_mean": 589.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 265.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.21,
      "grad_norm": 2.579532955538129,
      "learning_rate": 4.809362944123128e-07,
      "logits/policy_chosen": -2.0034396648406982,
      "logits/policy_rejected": -2.1076416969299316,
      "logits/reference_chosen": -2.9764785766601562,
      "logits/reference_rejected": -2.718217611312866,
      "logps/policy_chosen": -56.3231086730957,
      "logps/policy_rejected": -294.1711730957031,
      "logps/reference_chosen": -122.71052551269531,
      "logps/reference_rejected": -174.21401977539062,
      "loss": 0.3059,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6638741493225098,
      "rewards/margins": 1.8634452819824219,
      "rewards/rejected": -1.1995712518692017,
      "step": 265,
      "tokens/chosen_response_len_mean": 196.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 169.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.21,
      "grad_norm": 2.6877602739497,
      "learning_rate": 4.806665611441354e-07,
      "logits/policy_chosen": -2.2234950065612793,
      "logits/policy_rejected": -2.2985267639160156,
      "logits/reference_chosen": -3.1664934158325195,
      "logits/reference_rejected": -2.4548802375793457,
      "logps/policy_chosen": -53.821807861328125,
      "logps/policy_rejected": -135.544921875,
      "logps/reference_chosen": -155.88006591796875,
      "logps/reference_rejected": -98.04533386230469,
      "loss": 0.3466,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.020582675933838,
      "rewards/margins": 1.3955786228179932,
      "rewards/rejected": -0.3749959468841553,
      "step": 266,
      "tokens/chosen_response_len_mean": 369.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 267.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.21,
      "grad_norm": 2.691839396443011,
      "learning_rate": 4.803950097501643e-07,
      "logits/policy_chosen": -2.3689355850219727,
      "logits/policy_rejected": -2.406802177429199,
      "logits/reference_chosen": -3.110931158065796,
      "logits/reference_rejected": -2.629725694656372,
      "logps/policy_chosen": -61.78421401977539,
      "logps/policy_rejected": -293.3619079589844,
      "logps/reference_chosen": -205.92092895507812,
      "logps/reference_rejected": -133.5921630859375,
      "loss": 0.3107,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4413671493530273,
      "rewards/margins": 3.039064407348633,
      "rewards/rejected": -1.597697377204895,
      "step": 267,
      "tokens/chosen_response_len_mean": 334.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 111.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.22,
      "grad_norm": 4.332732574683176,
      "learning_rate": 4.80121642370782e-07,
      "logits/policy_chosen": -1.435086965560913,
      "logits/policy_rejected": -1.555901288986206,
      "logits/reference_chosen": -2.4242351055145264,
      "logits/reference_rejected": -2.2076950073242188,
      "logps/policy_chosen": -122.86267852783203,
      "logps/policy_rejected": -370.72705078125,
      "logps/reference_chosen": -256.2296142578125,
      "logps/reference_rejected": -221.7090301513672,
      "loss": 0.2893,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3336691856384277,
      "rewards/margins": 2.8238492012023926,
      "rewards/rejected": -1.4901798963546753,
      "step": 268,
      "tokens/chosen_response_len_mean": 474.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 229.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.22,
      "grad_norm": 3.679173265554199,
      "learning_rate": 4.798464611606842e-07,
      "logits/policy_chosen": -1.9900972843170166,
      "logits/policy_rejected": -0.7595923542976379,
      "logits/reference_chosen": -2.307075023651123,
      "logits/reference_rejected": -2.1424033641815186,
      "logps/policy_chosen": -39.70943832397461,
      "logps/policy_rejected": -101.0633544921875,
      "logps/reference_chosen": -61.08944320678711,
      "logps/reference_rejected": -77.70144653320312,
      "loss": 0.3654,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.21380005776882172,
      "rewards/margins": 0.44741907715797424,
      "rewards/rejected": -0.23361904919147491,
      "step": 269,
      "tokens/chosen_response_len_mean": 73.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 361.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.22,
      "grad_norm": 2.860165012641077,
      "learning_rate": 4.795694682888635e-07,
      "logits/policy_chosen": -1.8247418403625488,
      "logits/policy_rejected": -2.0296616554260254,
      "logits/reference_chosen": -2.9217374324798584,
      "logits/reference_rejected": -2.3353545665740967,
      "logps/policy_chosen": -60.13949966430664,
      "logps/policy_rejected": -374.595947265625,
      "logps/reference_chosen": -168.12413024902344,
      "logps/reference_rejected": -216.00479125976562,
      "loss": 0.1644,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0798462629318237,
      "rewards/margins": 2.665757656097412,
      "rewards/rejected": -1.5859116315841675,
      "step": 270,
      "tokens/chosen_response_len_mean": 366.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 259.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.22,
      "grad_norm": 2.8914101024190764,
      "learning_rate": 4.79290665938592e-07,
      "logits/policy_chosen": -2.393129587173462,
      "logits/policy_rejected": -2.532593250274658,
      "logits/reference_chosen": -3.0000436305999756,
      "logits/reference_rejected": -2.605870246887207,
      "logps/policy_chosen": -32.41455841064453,
      "logps/policy_rejected": -120.82582092285156,
      "logps/reference_chosen": -80.18354034423828,
      "logps/reference_rejected": -52.557701110839844,
      "loss": 0.2656,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4776897728443146,
      "rewards/margins": 1.1603708267211914,
      "rewards/rejected": -0.6826812028884888,
      "step": 271,
      "tokens/chosen_response_len_mean": 264.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 82.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.22,
      "grad_norm": 1.9537358053696072,
      "learning_rate": 4.790100563074044e-07,
      "logits/policy_chosen": -1.9999793767929077,
      "logits/policy_rejected": -2.0931291580200195,
      "logits/reference_chosen": -2.969376564025879,
      "logits/reference_rejected": -1.7754515409469604,
      "logps/policy_chosen": -88.11581420898438,
      "logps/policy_rejected": -398.7203369140625,
      "logps/reference_chosen": -281.721435546875,
      "logps/reference_rejected": -166.79583740234375,
      "loss": 0.3137,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.936056137084961,
      "rewards/margins": 4.255301475524902,
      "rewards/rejected": -2.3192451000213623,
      "step": 272,
      "tokens/chosen_response_len_mean": 415.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 262.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.22,
      "grad_norm": 2.7544872780237926,
      "learning_rate": 4.787276416070804e-07,
      "logits/policy_chosen": -2.0469934940338135,
      "logits/policy_rejected": -1.6853868961334229,
      "logits/reference_chosen": -2.1769094467163086,
      "logits/reference_rejected": -2.2352547645568848,
      "logps/policy_chosen": -33.57738494873047,
      "logps/policy_rejected": -220.69619750976562,
      "logps/reference_chosen": -67.25520324707031,
      "logps/reference_rejected": -119.21714782714844,
      "loss": 0.298,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.3367781937122345,
      "rewards/margins": 1.351568579673767,
      "rewards/rejected": -1.014790415763855,
      "step": 273,
      "tokens/chosen_response_len_mean": 89.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 110.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.22,
      "grad_norm": 2.311066363535201,
      "learning_rate": 4.784434240636271e-07,
      "logits/policy_chosen": -1.7351186275482178,
      "logits/policy_rejected": -1.8542585372924805,
      "logits/reference_chosen": -3.0118846893310547,
      "logits/reference_rejected": -2.1115365028381348,
      "logps/policy_chosen": -82.1336898803711,
      "logps/policy_rejected": -439.58245849609375,
      "logps/reference_chosen": -232.67691040039062,
      "logps/reference_rejected": -256.78057861328125,
      "loss": 0.1728,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.50543212890625,
      "rewards/margins": 3.3334507942199707,
      "rewards/rejected": -1.8280186653137207,
      "step": 274,
      "tokens/chosen_response_len_mean": 609.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 465.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.22,
      "grad_norm": 2.9956815155001992,
      "learning_rate": 4.78157405917262e-07,
      "logits/policy_chosen": -1.8693939447402954,
      "logits/policy_rejected": -1.9391303062438965,
      "logits/reference_chosen": -2.969177484512329,
      "logits/reference_rejected": -2.4460105895996094,
      "logps/policy_chosen": -86.36649322509766,
      "logps/policy_rejected": -566.4467163085938,
      "logps/reference_chosen": -261.6522216796875,
      "logps/reference_rejected": -263.3449401855469,
      "loss": 0.2451,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7528573274612427,
      "rewards/margins": 4.783874988555908,
      "rewards/rejected": -3.031017780303955,
      "step": 275,
      "tokens/chosen_response_len_mean": 339.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 251.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.22,
      "grad_norm": 3.2803570213469144,
      "learning_rate": 4.778695894223952e-07,
      "logits/policy_chosen": -2.3628909587860107,
      "logits/policy_rejected": -2.3530797958374023,
      "logits/reference_chosen": -2.9668712615966797,
      "logits/reference_rejected": -2.966824531555176,
      "logps/policy_chosen": -54.248233795166016,
      "logps/policy_rejected": -327.43170166015625,
      "logps/reference_chosen": -146.84291076660156,
      "logps/reference_rejected": -196.168212890625,
      "loss": 0.3313,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9259467124938965,
      "rewards/margins": 2.238581418991089,
      "rewards/rejected": -1.3126345872879028,
      "step": 276,
      "tokens/chosen_response_len_mean": 289.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 287.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.22,
      "grad_norm": 3.3737296079999517,
      "learning_rate": 4.775799768476108e-07,
      "logits/policy_chosen": -1.623067855834961,
      "logits/policy_rejected": -1.66438627243042,
      "logits/reference_chosen": -2.50834321975708,
      "logits/reference_rejected": -2.3877673149108887,
      "logps/policy_chosen": -32.76015853881836,
      "logps/policy_rejected": -140.15492248535156,
      "logps/reference_chosen": -57.18016052246094,
      "logps/reference_rejected": -86.19461822509766,
      "loss": 0.2575,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.24420003592967987,
      "rewards/margins": 0.7838031053543091,
      "rewards/rejected": -0.5396031141281128,
      "step": 277,
      "tokens/chosen_response_len_mean": 176.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 170.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.22,
      "grad_norm": 9.199685663575346,
      "learning_rate": 4.772885704756505e-07,
      "logits/policy_chosen": -1.915112018585205,
      "logits/policy_rejected": -2.189150333404541,
      "logits/reference_chosen": -3.0652871131896973,
      "logits/reference_rejected": -2.8179969787597656,
      "logps/policy_chosen": -88.47130584716797,
      "logps/policy_rejected": -441.1356506347656,
      "logps/reference_chosen": -312.25732421875,
      "logps/reference_rejected": -313.57989501953125,
      "loss": 0.3432,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2378599643707275,
      "rewards/margins": 3.5134174823760986,
      "rewards/rejected": -1.2755577564239502,
      "step": 278,
      "tokens/chosen_response_len_mean": 416.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 258.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.22,
      "grad_norm": 2.568407460041794,
      "learning_rate": 4.769953726033942e-07,
      "logits/policy_chosen": -2.126152515411377,
      "logits/policy_rejected": -2.1976869106292725,
      "logits/reference_chosen": -2.918933391571045,
      "logits/reference_rejected": -2.9181439876556396,
      "logps/policy_chosen": -55.46541213989258,
      "logps/policy_rejected": -393.147705078125,
      "logps/reference_chosen": -175.48330688476562,
      "logps/reference_rejected": -261.3435363769531,
      "loss": 0.1778,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.200179100036621,
      "rewards/margins": 2.518220901489258,
      "rewards/rejected": -1.3180416822433472,
      "step": 279,
      "tokens/chosen_response_len_mean": 215.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 210.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.23,
      "grad_norm": 3.1302891031635984,
      "learning_rate": 4.767003855418429e-07,
      "logits/policy_chosen": -2.0603606700897217,
      "logits/policy_rejected": -2.1878933906555176,
      "logits/reference_chosen": -3.049166679382324,
      "logits/reference_rejected": -2.4211201667785645,
      "logps/policy_chosen": -70.95120239257812,
      "logps/policy_rejected": -435.4041442871094,
      "logps/reference_chosen": -203.48745727539062,
      "logps/reference_rejected": -209.47271728515625,
      "loss": 0.1805,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3253624439239502,
      "rewards/margins": 3.5846762657165527,
      "rewards/rejected": -2.2593138217926025,
      "step": 280,
      "tokens/chosen_response_len_mean": 273.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 158.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.23,
      "grad_norm": 3.0759421417192003,
      "learning_rate": 4.764036116160997e-07,
      "logits/policy_chosen": -1.9647924900054932,
      "logits/policy_rejected": -2.0003933906555176,
      "logits/reference_chosen": -3.1076905727386475,
      "logits/reference_rejected": -2.8670225143432617,
      "logps/policy_chosen": -52.48387145996094,
      "logps/policy_rejected": -527.5159912109375,
      "logps/reference_chosen": -158.17994689941406,
      "logps/reference_rejected": -246.14735412597656,
      "loss": 0.1958,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0569607019424438,
      "rewards/margins": 3.8706471920013428,
      "rewards/rejected": -2.8136863708496094,
      "step": 281,
      "tokens/chosen_response_len_mean": 214.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 191.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.23,
      "grad_norm": 2.7361755546386277,
      "learning_rate": 4.761050531653523e-07,
      "logits/policy_chosen": -1.5984692573547363,
      "logits/policy_rejected": -1.8361363410949707,
      "logits/reference_chosen": -2.7592179775238037,
      "logits/reference_rejected": -2.4084267616271973,
      "logps/policy_chosen": -80.7918930053711,
      "logps/policy_rejected": -345.93499755859375,
      "logps/reference_chosen": -228.01821899414062,
      "logps/reference_rejected": -187.31529235839844,
      "loss": 0.1742,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4722633361816406,
      "rewards/margins": 3.058460235595703,
      "rewards/rejected": -1.586196780204773,
      "step": 282,
      "tokens/chosen_response_len_mean": 634.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 348.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.23,
      "grad_norm": 3.522563546814339,
      "learning_rate": 4.758047125428537e-07,
      "logits/policy_chosen": -1.7348957061767578,
      "logits/policy_rejected": -2.081566333770752,
      "logits/reference_chosen": -2.989297389984131,
      "logits/reference_rejected": -2.2824132442474365,
      "logps/policy_chosen": -58.18089294433594,
      "logps/policy_rejected": -389.5596923828125,
      "logps/reference_chosen": -171.86676025390625,
      "logps/reference_rejected": -148.6519775390625,
      "loss": 0.2789,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1368587017059326,
      "rewards/margins": 3.54593563079834,
      "rewards/rejected": -2.409076690673828,
      "step": 283,
      "tokens/chosen_response_len_mean": 379.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 131.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.23,
      "grad_norm": 3.4857525347681495,
      "learning_rate": 4.7550259211590436e-07,
      "logits/policy_chosen": -2.155935764312744,
      "logits/policy_rejected": -2.212632894515991,
      "logits/reference_chosen": -3.005995750427246,
      "logits/reference_rejected": -2.9473025798797607,
      "logps/policy_chosen": -46.34202575683594,
      "logps/policy_rejected": -374.5079345703125,
      "logps/reference_chosen": -132.93014526367188,
      "logps/reference_rejected": -190.07769775390625,
      "loss": 0.2694,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8658812642097473,
      "rewards/margins": 2.710183620452881,
      "rewards/rejected": -1.8443021774291992,
      "step": 284,
      "tokens/chosen_response_len_mean": 127.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 132.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.23,
      "grad_norm": 2.146664751577214,
      "learning_rate": 4.751986942658332e-07,
      "logits/policy_chosen": -1.8320860862731934,
      "logits/policy_rejected": -1.9627416133880615,
      "logits/reference_chosen": -3.0087828636169434,
      "logits/reference_rejected": -2.798800230026245,
      "logps/policy_chosen": -99.31060028076172,
      "logps/policy_rejected": -590.6198120117188,
      "logps/reference_chosen": -292.62762451171875,
      "logps/reference_rejected": -301.8166198730469,
      "loss": 0.1743,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9331700801849365,
      "rewards/margins": 4.821201801300049,
      "rewards/rejected": -2.8880317211151123,
      "step": 285,
      "tokens/chosen_response_len_mean": 672.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 380.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.23,
      "grad_norm": 4.0068979390995665,
      "learning_rate": 4.748930213879787e-07,
      "logits/policy_chosen": -1.7841126918792725,
      "logits/policy_rejected": -1.616112232208252,
      "logits/reference_chosen": -2.8974661827087402,
      "logits/reference_rejected": -2.3146543502807617,
      "logps/policy_chosen": -83.1224365234375,
      "logps/policy_rejected": -509.5625305175781,
      "logps/reference_chosen": -248.04489135742188,
      "logps/reference_rejected": -299.60675048828125,
      "loss": 0.2229,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6492245197296143,
      "rewards/margins": 3.7487823963165283,
      "rewards/rejected": -2.099557638168335,
      "step": 286,
      "tokens/chosen_response_len_mean": 312.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 268.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.23,
      "grad_norm": 3.6884808286561905,
      "learning_rate": 4.745855758916704e-07,
      "logits/policy_chosen": -1.9492144584655762,
      "logits/policy_rejected": -2.23884916305542,
      "logits/reference_chosen": -2.644348382949829,
      "logits/reference_rejected": -2.677292823791504,
      "logps/policy_chosen": -167.10963439941406,
      "logps/policy_rejected": -408.90594482421875,
      "logps/reference_chosen": -245.54014587402344,
      "logps/reference_rejected": -228.08700561523438,
      "loss": 0.3101,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.784305214881897,
      "rewards/margins": 2.592494487762451,
      "rewards/rejected": -1.8081893920898438,
      "step": 287,
      "tokens/chosen_response_len_mean": 660.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 374.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.23,
      "grad_norm": 2.9811598150339806,
      "learning_rate": 4.7427636020020964e-07,
      "logits/policy_chosen": -1.6351268291473389,
      "logits/policy_rejected": -1.8411903381347656,
      "logits/reference_chosen": -3.084428310394287,
      "logits/reference_rejected": -2.840968132019043,
      "logps/policy_chosen": -109.2936019897461,
      "logps/policy_rejected": -708.4165649414062,
      "logps/reference_chosen": -367.9963073730469,
      "logps/reference_rejected": -379.46942138671875,
      "loss": 0.1152,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.587027072906494,
      "rewards/margins": 5.876498222351074,
      "rewards/rejected": -3.289471387863159,
      "step": 288,
      "tokens/chosen_response_len_mean": 711.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 495.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.23,
      "grad_norm": 4.220920032892386,
      "learning_rate": 4.7396537675085046e-07,
      "logits/policy_chosen": -1.8225438594818115,
      "logits/policy_rejected": -1.9494044780731201,
      "logits/reference_chosen": -2.518173933029175,
      "logits/reference_rejected": -2.711627721786499,
      "logps/policy_chosen": -47.87618637084961,
      "logps/policy_rejected": -247.03065490722656,
      "logps/reference_chosen": -69.38465881347656,
      "logps/reference_rejected": -130.92787170410156,
      "loss": 0.2582,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21508470177650452,
      "rewards/margins": 1.3761123418807983,
      "rewards/rejected": -1.1610276699066162,
      "step": 289,
      "tokens/chosen_response_len_mean": 285.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 109.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.23,
      "grad_norm": 3.2644411706471614,
      "learning_rate": 4.7365262799478065e-07,
      "logits/policy_chosen": -1.631312370300293,
      "logits/policy_rejected": -1.8853410482406616,
      "logits/reference_chosen": -2.863694906234741,
      "logits/reference_rejected": -2.2361578941345215,
      "logps/policy_chosen": -87.65226745605469,
      "logps/policy_rejected": -368.45916748046875,
      "logps/reference_chosen": -175.37078857421875,
      "logps/reference_rejected": -180.3101043701172,
      "loss": 0.3482,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8771853446960449,
      "rewards/margins": 2.758676052093506,
      "rewards/rejected": -1.881490707397461,
      "step": 290,
      "tokens/chosen_response_len_mean": 238.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 142.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.23,
      "grad_norm": 4.164740816732374,
      "learning_rate": 4.733381163971021e-07,
      "logits/policy_chosen": -2.0723226070404053,
      "logits/policy_rejected": -2.1896250247955322,
      "logits/reference_chosen": -2.9462733268737793,
      "logits/reference_rejected": -2.904141902923584,
      "logps/policy_chosen": -82.70597839355469,
      "logps/policy_rejected": -706.9203491210938,
      "logps/reference_chosen": -254.06658935546875,
      "logps/reference_rejected": -371.26043701171875,
      "loss": 0.2392,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7136059999465942,
      "rewards/margins": 5.070204734802246,
      "rewards/rejected": -3.3565988540649414,
      "step": 291,
      "tokens/chosen_response_len_mean": 440.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 360.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.23,
      "grad_norm": 2.591496235271577,
      "learning_rate": 4.7302184443681135e-07,
      "logits/policy_chosen": -1.7894690036773682,
      "logits/policy_rejected": -1.8064680099487305,
      "logits/reference_chosen": -2.837014675140381,
      "logits/reference_rejected": -2.7575623989105225,
      "logps/policy_chosen": -70.64473724365234,
      "logps/policy_rejected": -504.9518737792969,
      "logps/reference_chosen": -204.25482177734375,
      "logps/reference_rejected": -285.4273376464844,
      "loss": 0.1799,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3361008167266846,
      "rewards/margins": 3.531346321105957,
      "rewards/rejected": -2.1952455043792725,
      "step": 292,
      "tokens/chosen_response_len_mean": 480.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 399.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.24,
      "grad_norm": 3.3860324397417396,
      "learning_rate": 4.7270381460678064e-07,
      "logits/policy_chosen": -1.842910885810852,
      "logits/policy_rejected": -1.6502610445022583,
      "logits/reference_chosen": -2.704162120819092,
      "logits/reference_rejected": -2.680164098739624,
      "logps/policy_chosen": -49.48709487915039,
      "logps/policy_rejected": -214.38595581054688,
      "logps/reference_chosen": -82.64054107666016,
      "logps/reference_rejected": -121.0648193359375,
      "loss": 0.2579,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3315344452857971,
      "rewards/margins": 1.264745831489563,
      "rewards/rejected": -0.9332114458084106,
      "step": 293,
      "tokens/chosen_response_len_mean": 103.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 131.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.24,
      "grad_norm": 3.188225158781283,
      "learning_rate": 4.7238402941373743e-07,
      "logits/policy_chosen": -1.6677745580673218,
      "logits/policy_rejected": -1.6923861503601074,
      "logits/reference_chosen": -2.790350914001465,
      "logits/reference_rejected": -2.7842464447021484,
      "logps/policy_chosen": -69.78219604492188,
      "logps/policy_rejected": -374.1966552734375,
      "logps/reference_chosen": -223.26382446289062,
      "logps/reference_rejected": -258.4195861816406,
      "loss": 0.2856,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5348161458969116,
      "rewards/margins": 2.692586898803711,
      "rewards/rejected": -1.1577708721160889,
      "step": 294,
      "tokens/chosen_response_len_mean": 361.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 321.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.24,
      "grad_norm": 2.8300164573379623,
      "learning_rate": 4.7206249137824525e-07,
      "logits/policy_chosen": -1.2584807872772217,
      "logits/policy_rejected": -1.1313340663909912,
      "logits/reference_chosen": -2.185845136642456,
      "logits/reference_rejected": -1.8424296379089355,
      "logps/policy_chosen": -100.97215270996094,
      "logps/policy_rejected": -257.4559631347656,
      "logps/reference_chosen": -234.6266326904297,
      "logps/reference_rejected": -179.3731689453125,
      "loss": 0.3429,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3365448713302612,
      "rewards/margins": 2.117372512817383,
      "rewards/rejected": -0.7808278203010559,
      "step": 295,
      "tokens/chosen_response_len_mean": 382.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 457.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.24,
      "grad_norm": 3.830130538309461,
      "learning_rate": 4.717392030346835e-07,
      "logits/policy_chosen": -2.0780117511749268,
      "logits/policy_rejected": -2.133204698562622,
      "logits/reference_chosen": -2.9950385093688965,
      "logits/reference_rejected": -2.916698932647705,
      "logps/policy_chosen": -83.94184875488281,
      "logps/policy_rejected": -504.6825866699219,
      "logps/reference_chosen": -269.20159912109375,
      "logps/reference_rejected": -335.22650146484375,
      "loss": 0.2967,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.8525973558425903,
      "rewards/margins": 3.5471582412719727,
      "rewards/rejected": -1.6945607662200928,
      "step": 296,
      "tokens/chosen_response_len_mean": 454.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 379.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.24,
      "grad_norm": 6.893378090531771,
      "learning_rate": 4.714141669312277e-07,
      "logits/policy_chosen": -1.9810442924499512,
      "logits/policy_rejected": -2.0455687046051025,
      "logits/reference_chosen": -2.7563416957855225,
      "logits/reference_rejected": -2.617710590362549,
      "logps/policy_chosen": -73.21089935302734,
      "logps/policy_rejected": -340.0194091796875,
      "logps/reference_chosen": -221.56581115722656,
      "logps/reference_rejected": -196.06849670410156,
      "loss": 0.3247,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4835491180419922,
      "rewards/margins": 2.923058032989502,
      "rewards/rejected": -1.4395091533660889,
      "step": 297,
      "tokens/chosen_response_len_mean": 321.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 218.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.24,
      "grad_norm": 2.56574676261905,
      "learning_rate": 4.710873856298293e-07,
      "logits/policy_chosen": -1.895401954650879,
      "logits/policy_rejected": -1.9954261779785156,
      "logits/reference_chosen": -2.9610021114349365,
      "logits/reference_rejected": -2.7275171279907227,
      "logps/policy_chosen": -93.83104705810547,
      "logps/policy_rejected": -574.4371948242188,
      "logps/reference_chosen": -263.5850524902344,
      "logps/reference_rejected": -328.8235168457031,
      "loss": 0.1995,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6975396871566772,
      "rewards/margins": 4.1536760330200195,
      "rewards/rejected": -2.456136465072632,
      "step": 298,
      "tokens/chosen_response_len_mean": 543.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 433.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.24,
      "grad_norm": 2.62583819131534,
      "learning_rate": 4.7075886170619536e-07,
      "logits/policy_chosen": -1.3762860298156738,
      "logits/policy_rejected": -1.4836927652359009,
      "logits/reference_chosen": -2.8365345001220703,
      "logits/reference_rejected": -2.107300281524658,
      "logps/policy_chosen": -109.9390869140625,
      "logps/policy_rejected": -396.7261657714844,
      "logps/reference_chosen": -285.54638671875,
      "logps/reference_rejected": -202.1131591796875,
      "loss": 0.2451,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7560728788375854,
      "rewards/margins": 3.7022030353546143,
      "rewards/rejected": -1.9461302757263184,
      "step": 299,
      "tokens/chosen_response_len_mean": 378.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 207.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.24,
      "grad_norm": 2.9338445441961096,
      "learning_rate": 4.704285977497687e-07,
      "logits/policy_chosen": -1.83700692653656,
      "logits/policy_rejected": -2.106710433959961,
      "logits/reference_chosen": -2.9238107204437256,
      "logits/reference_rejected": -2.5823757648468018,
      "logps/policy_chosen": -82.13780975341797,
      "logps/policy_rejected": -493.58087158203125,
      "logps/reference_chosen": -289.5302429199219,
      "logps/reference_rejected": -208.1035919189453,
      "loss": 0.113,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0739243030548096,
      "rewards/margins": 4.928697109222412,
      "rewards/rejected": -2.8547730445861816,
      "step": 300,
      "tokens/chosen_response_len_mean": 445.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 201.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.24,
      "grad_norm": 2.290878044608878,
      "learning_rate": 4.700965963637067e-07,
      "logits/policy_chosen": -1.2083768844604492,
      "logits/policy_rejected": -1.3407108783721924,
      "logits/reference_chosen": -2.4264414310455322,
      "logits/reference_rejected": -2.0838944911956787,
      "logps/policy_chosen": -87.56996154785156,
      "logps/policy_rejected": -337.8468933105469,
      "logps/reference_chosen": -196.71490478515625,
      "logps/reference_rejected": -176.93084716796875,
      "loss": 0.1417,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0914493799209595,
      "rewards/margins": 2.7006096839904785,
      "rewards/rejected": -1.609160304069519,
      "step": 301,
      "tokens/chosen_response_len_mean": 252.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 133.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.24,
      "grad_norm": 3.664882673854998,
      "learning_rate": 4.6976286016486156e-07,
      "logits/policy_chosen": -1.6306910514831543,
      "logits/policy_rejected": -1.7238553762435913,
      "logits/reference_chosen": -3.024641513824463,
      "logits/reference_rejected": -2.9463772773742676,
      "logps/policy_chosen": -103.36473846435547,
      "logps/policy_rejected": -482.4529113769531,
      "logps/reference_chosen": -300.7445068359375,
      "logps/reference_rejected": -272.9915466308594,
      "loss": 0.2465,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9737976789474487,
      "rewards/margins": 4.068410873413086,
      "rewards/rejected": -2.0946133136749268,
      "step": 302,
      "tokens/chosen_response_len_mean": 537.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 378.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.24,
      "grad_norm": 4.07962885346587,
      "learning_rate": 4.694273917837593e-07,
      "logits/policy_chosen": -1.718010663986206,
      "logits/policy_rejected": -1.1793601512908936,
      "logits/reference_chosen": -2.1970267295837402,
      "logits/reference_rejected": -2.242309093475342,
      "logps/policy_chosen": -89.3014144897461,
      "logps/policy_rejected": -925.4991455078125,
      "logps/reference_chosen": -293.8038330078125,
      "logps/reference_rejected": -482.5627136230469,
      "loss": 0.2865,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.0450241565704346,
      "rewards/margins": 6.47438907623291,
      "rewards/rejected": -4.429364204406738,
      "step": 303,
      "tokens/chosen_response_len_mean": 358.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 365.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.24,
      "grad_norm": 2.405932913361,
      "learning_rate": 4.6909019386457927e-07,
      "logits/policy_chosen": -1.6491061449050903,
      "logits/policy_rejected": -1.6597315073013306,
      "logits/reference_chosen": -2.341599464416504,
      "logits/reference_rejected": -2.266160488128662,
      "logps/policy_chosen": -68.28823852539062,
      "logps/policy_rejected": -270.70416259765625,
      "logps/reference_chosen": -157.44268798828125,
      "logps/reference_rejected": -147.8431396484375,
      "loss": 0.2139,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8915444016456604,
      "rewards/margins": 2.120154857635498,
      "rewards/rejected": -1.2286103963851929,
      "step": 304,
      "tokens/chosen_response_len_mean": 237.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 197.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.25,
      "grad_norm": 3.572341442607126,
      "learning_rate": 4.687512690651327e-07,
      "logits/policy_chosen": -2.2066917419433594,
      "logits/policy_rejected": -2.2525765895843506,
      "logits/reference_chosen": -2.9350485801696777,
      "logits/reference_rejected": -2.8024892807006836,
      "logps/policy_chosen": -59.40142822265625,
      "logps/policy_rejected": -203.68032836914062,
      "logps/reference_chosen": -145.8667449951172,
      "logps/reference_rejected": -85.89591979980469,
      "loss": 0.209,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8646531701087952,
      "rewards/margins": 2.042497158050537,
      "rewards/rejected": -1.1778440475463867,
      "step": 305,
      "tokens/chosen_response_len_mean": 184.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 98.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.25,
      "grad_norm": 3.124612250113926,
      "learning_rate": 4.6841062005684255e-07,
      "logits/policy_chosen": -1.8113189935684204,
      "logits/policy_rejected": -1.8169225454330444,
      "logits/reference_chosen": -2.8508493900299072,
      "logits/reference_rejected": -2.9348199367523193,
      "logps/policy_chosen": -35.2087516784668,
      "logps/policy_rejected": -334.8651123046875,
      "logps/reference_chosen": -112.86093139648438,
      "logps/reference_rejected": -148.45632934570312,
      "loss": 0.3251,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7765218019485474,
      "rewards/margins": 2.6406097412109375,
      "rewards/rejected": -1.8640880584716797,
      "step": 306,
      "tokens/chosen_response_len_mean": 320.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 285.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.25,
      "grad_norm": 3.32496586258652,
      "learning_rate": 4.68068249524722e-07,
      "logits/policy_chosen": -2.3824849128723145,
      "logits/policy_rejected": -1.6881012916564941,
      "logits/reference_chosen": -3.0569143295288086,
      "logits/reference_rejected": -2.4748129844665527,
      "logps/policy_chosen": -61.716796875,
      "logps/policy_rejected": -166.7222137451172,
      "logps/reference_chosen": -222.57785034179688,
      "logps/reference_rejected": -93.1553955078125,
      "loss": 0.2575,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6086106300354004,
      "rewards/margins": 2.344278573989868,
      "rewards/rejected": -0.7356680631637573,
      "step": 307,
      "tokens/chosen_response_len_mean": 291.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 310.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.25,
      "grad_norm": 3.3874944482736735,
      "learning_rate": 4.6772416016735323e-07,
      "logits/policy_chosen": -1.7318488359451294,
      "logits/policy_rejected": -2.1800973415374756,
      "logits/reference_chosen": -2.737339973449707,
      "logits/reference_rejected": -2.64113712310791,
      "logps/policy_chosen": -109.86978149414062,
      "logps/policy_rejected": -449.2695007324219,
      "logps/reference_chosen": -305.08203125,
      "logps/reference_rejected": -229.10501098632812,
      "loss": 0.1993,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9521225690841675,
      "rewards/margins": 4.1537675857543945,
      "rewards/rejected": -2.2016448974609375,
      "step": 308,
      "tokens/chosen_response_len_mean": 614.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 206.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.25,
      "grad_norm": 3.3558308202865,
      "learning_rate": 4.673783546968665e-07,
      "logits/policy_chosen": -2.2060866355895996,
      "logits/policy_rejected": -2.37375545501709,
      "logits/reference_chosen": -3.0307087898254395,
      "logits/reference_rejected": -2.645644426345825,
      "logps/policy_chosen": -47.22865676879883,
      "logps/policy_rejected": -480.0763244628906,
      "logps/reference_chosen": -145.3326873779297,
      "logps/reference_rejected": -169.44834899902344,
      "loss": 0.19,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.981040358543396,
      "rewards/margins": 4.087320327758789,
      "rewards/rejected": -3.1062798500061035,
      "step": 309,
      "tokens/chosen_response_len_mean": 243.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 131.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.25,
      "grad_norm": 4.153117211006617,
      "learning_rate": 4.6703083583891835e-07,
      "logits/policy_chosen": -2.103739023208618,
      "logits/policy_rejected": -2.222684383392334,
      "logits/reference_chosen": -2.9488046169281006,
      "logits/reference_rejected": -2.909597158432007,
      "logps/policy_chosen": -72.56431579589844,
      "logps/policy_rejected": -597.1136474609375,
      "logps/reference_chosen": -229.9650115966797,
      "logps/reference_rejected": -291.440673828125,
      "loss": 0.2705,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.5740069150924683,
      "rewards/margins": 4.630736827850342,
      "rewards/rejected": -3.056729793548584,
      "step": 310,
      "tokens/chosen_response_len_mean": 416.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 279.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.25,
      "grad_norm": 6.474334517638787,
      "learning_rate": 4.666816063326705e-07,
      "logits/policy_chosen": -1.8240094184875488,
      "logits/policy_rejected": -1.2601674795150757,
      "logits/reference_chosen": -2.8085074424743652,
      "logits/reference_rejected": -1.7981979846954346,
      "logps/policy_chosen": -75.03584289550781,
      "logps/policy_rejected": -454.2119140625,
      "logps/reference_chosen": -269.4425964355469,
      "logps/reference_rejected": -277.75634765625,
      "loss": 0.2551,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.9440672397613525,
      "rewards/margins": 3.708622932434082,
      "rewards/rejected": -1.7645556926727295,
      "step": 311,
      "tokens/chosen_response_len_mean": 437.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 564.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.25,
      "grad_norm": 5.519032924912619,
      "learning_rate": 4.66330668930768e-07,
      "logits/policy_chosen": -1.546844244003296,
      "logits/policy_rejected": -1.0349215269088745,
      "logits/reference_chosen": -2.6315059661865234,
      "logits/reference_rejected": -2.3254542350769043,
      "logps/policy_chosen": -46.538272857666016,
      "logps/policy_rejected": -340.2102355957031,
      "logps/reference_chosen": -118.1043701171875,
      "logps/reference_rejected": -194.15060424804688,
      "loss": 0.2405,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.7156610488891602,
      "rewards/margins": 2.1762571334838867,
      "rewards/rejected": -1.4605962038040161,
      "step": 312,
      "tokens/chosen_response_len_mean": 286.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 542.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.25,
      "grad_norm": 3.0150528597270467,
      "learning_rate": 4.659780263993177e-07,
      "logits/policy_chosen": -1.8767701387405396,
      "logits/policy_rejected": -1.8039137125015259,
      "logits/reference_chosen": -3.0502028465270996,
      "logits/reference_rejected": -3.0671067237854004,
      "logps/policy_chosen": -54.472354888916016,
      "logps/policy_rejected": -321.30377197265625,
      "logps/reference_chosen": -124.24363708496094,
      "logps/reference_rejected": -180.5016326904297,
      "loss": 0.3478,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.697712779045105,
      "rewards/margins": 2.105734348297119,
      "rewards/rejected": -1.4080214500427246,
      "step": 313,
      "tokens/chosen_response_len_mean": 180.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 186.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.25,
      "grad_norm": 3.338552410487336,
      "learning_rate": 4.656236815178662e-07,
      "logits/policy_chosen": -1.8194024562835693,
      "logits/policy_rejected": -1.8620460033416748,
      "logits/reference_chosen": -2.952608585357666,
      "logits/reference_rejected": -2.880305528640747,
      "logps/policy_chosen": -64.1020736694336,
      "logps/policy_rejected": -668.8197021484375,
      "logps/reference_chosen": -191.9952392578125,
      "logps/reference_rejected": -297.9425964355469,
      "loss": 0.305,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2789316177368164,
      "rewards/margins": 4.987702369689941,
      "rewards/rejected": -3.708770513534546,
      "step": 314,
      "tokens/chosen_response_len_mean": 302.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 252.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.25,
      "grad_norm": 4.216599658137105,
      "learning_rate": 4.652676370793783e-07,
      "logits/policy_chosen": -1.9225913286209106,
      "logits/policy_rejected": -2.21560001373291,
      "logits/reference_chosen": -2.867231845855713,
      "logits/reference_rejected": -2.1903316974639893,
      "logps/policy_chosen": -87.3575439453125,
      "logps/policy_rejected": -352.74395751953125,
      "logps/reference_chosen": -222.33282470703125,
      "logps/reference_rejected": -139.34376525878906,
      "loss": 0.1772,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3497529029846191,
      "rewards/margins": 3.4837546348571777,
      "rewards/rejected": -2.1340014934539795,
      "step": 315,
      "tokens/chosen_response_len_mean": 291.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 90.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.25,
      "grad_norm": 1.9381348585009064,
      "learning_rate": 4.649098958902148e-07,
      "logits/policy_chosen": -1.4626785516738892,
      "logits/policy_rejected": -1.6685293912887573,
      "logits/reference_chosen": -3.0980098247528076,
      "logits/reference_rejected": -2.3443450927734375,
      "logps/policy_chosen": -132.43142700195312,
      "logps/policy_rejected": -1093.05859375,
      "logps/reference_chosen": -439.55596923828125,
      "logps/reference_rejected": -496.19171142578125,
      "loss": 0.1559,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.0712451934814453,
      "rewards/margins": 9.039913177490234,
      "rewards/rejected": -5.968668460845947,
      "step": 316,
      "tokens/chosen_response_len_mean": 614.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 437.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.25,
      "grad_norm": 3.3208168275072727,
      "learning_rate": 4.6455046077011017e-07,
      "logits/policy_chosen": -1.6945469379425049,
      "logits/policy_rejected": -1.6586346626281738,
      "logits/reference_chosen": -2.797177314758301,
      "logits/reference_rejected": -2.7550530433654785,
      "logps/policy_chosen": -89.57418823242188,
      "logps/policy_rejected": -376.0241394042969,
      "logps/reference_chosen": -193.11834716796875,
      "logps/reference_rejected": -237.541015625,
      "loss": 0.1793,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.035441517829895,
      "rewards/margins": 2.4202725887298584,
      "rewards/rejected": -1.384831190109253,
      "step": 317,
      "tokens/chosen_response_len_mean": 268.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 252.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.26,
      "grad_norm": 4.834044743941225,
      "learning_rate": 4.6418933455215076e-07,
      "logits/policy_chosen": -1.7103971242904663,
      "logits/policy_rejected": -1.777637004852295,
      "logits/reference_chosen": -2.8807411193847656,
      "logits/reference_rejected": -2.910388946533203,
      "logps/policy_chosen": -43.051544189453125,
      "logps/policy_rejected": -451.1328430175781,
      "logps/reference_chosen": -98.37159729003906,
      "logps/reference_rejected": -200.70199584960938,
      "loss": 0.2363,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5532006025314331,
      "rewards/margins": 3.057508945465088,
      "rewards/rejected": -2.5043087005615234,
      "step": 318,
      "tokens/chosen_response_len_mean": 247.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 208.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.26,
      "grad_norm": 3.0765900633406913,
      "learning_rate": 4.63826520082752e-07,
      "logits/policy_chosen": -1.9023985862731934,
      "logits/policy_rejected": -2.087189197540283,
      "logits/reference_chosen": -2.684725522994995,
      "logits/reference_rejected": -2.7294716835021973,
      "logps/policy_chosen": -29.559532165527344,
      "logps/policy_rejected": -101.22508239746094,
      "logps/reference_chosen": -34.04695129394531,
      "logps/reference_rejected": -49.713741302490234,
      "loss": 0.1553,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.044874198734760284,
      "rewards/margins": 0.5599876046180725,
      "rewards/rejected": -0.5151134133338928,
      "step": 319,
      "tokens/chosen_response_len_mean": 277.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 294.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.26,
      "grad_norm": 3.1173897201990224,
      "learning_rate": 4.6346202022163656e-07,
      "logits/policy_chosen": -1.382790446281433,
      "logits/policy_rejected": -1.732831597328186,
      "logits/reference_chosen": -2.682234764099121,
      "logits/reference_rejected": -2.60119891166687,
      "logps/policy_chosen": -110.54884338378906,
      "logps/policy_rejected": -942.2950439453125,
      "logps/reference_chosen": -358.08880615234375,
      "logps/reference_rejected": -484.7252197265625,
      "loss": 0.2402,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4753994941711426,
      "rewards/margins": 7.051098346710205,
      "rewards/rejected": -4.575698375701904,
      "step": 320,
      "tokens/chosen_response_len_mean": 697.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 564.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.26,
      "grad_norm": 3.703781134184552,
      "learning_rate": 4.6309583784181104e-07,
      "logits/policy_chosen": -1.6343090534210205,
      "logits/policy_rejected": -1.7306853532791138,
      "logits/reference_chosen": -2.6760318279266357,
      "logits/reference_rejected": -2.2841994762420654,
      "logps/policy_chosen": -63.79367446899414,
      "logps/policy_rejected": -231.47938537597656,
      "logps/reference_chosen": -115.32276916503906,
      "logps/reference_rejected": -110.79583740234375,
      "loss": 0.276,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5152909159660339,
      "rewards/margins": 1.7221263647079468,
      "rewards/rejected": -1.2068355083465576,
      "step": 321,
      "tokens/chosen_response_len_mean": 192.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 137.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.26,
      "grad_norm": 3.5327692055358666,
      "learning_rate": 4.62727975829544e-07,
      "logits/policy_chosen": -2.3158786296844482,
      "logits/policy_rejected": -2.3356070518493652,
      "logits/reference_chosen": -2.933302402496338,
      "logits/reference_rejected": -2.826026678085327,
      "logps/policy_chosen": -32.73407745361328,
      "logps/policy_rejected": -107.17605590820312,
      "logps/reference_chosen": -40.871437072753906,
      "logps/reference_rejected": -49.330963134765625,
      "loss": 0.2464,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08137363195419312,
      "rewards/margins": 0.659824550151825,
      "rewards/rejected": -0.5784509181976318,
      "step": 322,
      "tokens/chosen_response_len_mean": 34.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 32.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.26,
      "grad_norm": 4.524743189821704,
      "learning_rate": 4.6235843708434305e-07,
      "logits/policy_chosen": -1.0855112075805664,
      "logits/policy_rejected": -1.2527565956115723,
      "logits/reference_chosen": -2.279548168182373,
      "logits/reference_rejected": -2.0591039657592773,
      "logps/policy_chosen": -65.47724914550781,
      "logps/policy_rejected": -398.4112548828125,
      "logps/reference_chosen": -202.6960906982422,
      "logps/reference_rejected": -213.66448974609375,
      "loss": 0.2768,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3721883296966553,
      "rewards/margins": 3.219656467437744,
      "rewards/rejected": -1.8474678993225098,
      "step": 323,
      "tokens/chosen_response_len_mean": 237.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 194.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.26,
      "grad_norm": 2.9172728498954816,
      "learning_rate": 4.619872245189315e-07,
      "logits/policy_chosen": -1.803147554397583,
      "logits/policy_rejected": -2.121699333190918,
      "logits/reference_chosen": -2.9548394680023193,
      "logits/reference_rejected": -2.4536960124969482,
      "logps/policy_chosen": -96.12008666992188,
      "logps/policy_rejected": -595.5452270507812,
      "logps/reference_chosen": -285.5267333984375,
      "logps/reference_rejected": -236.10336303710938,
      "loss": 0.2238,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8940664529800415,
      "rewards/margins": 5.488485336303711,
      "rewards/rejected": -3.594418525695801,
      "step": 324,
      "tokens/chosen_response_len_mean": 443.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 240.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.26,
      "grad_norm": 3.0831020027878946,
      "learning_rate": 4.616143410592261e-07,
      "logits/policy_chosen": -2.2690231800079346,
      "logits/policy_rejected": -2.37300181388855,
      "logits/reference_chosen": -3.101170539855957,
      "logits/reference_rejected": -2.814063310623169,
      "logps/policy_chosen": -75.43067932128906,
      "logps/policy_rejected": -1028.3170166015625,
      "logps/reference_chosen": -261.9538269042969,
      "logps/reference_rejected": -396.73486328125,
      "loss": 0.2074,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8652312755584717,
      "rewards/margins": 8.181053161621094,
      "rewards/rejected": -6.315821647644043,
      "step": 325,
      "tokens/chosen_response_len_mean": 413.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 280.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.26,
      "grad_norm": 2.9506911486463143,
      "learning_rate": 4.6123978964431376e-07,
      "logits/policy_chosen": -1.961487054824829,
      "logits/policy_rejected": -2.289116144180298,
      "logits/reference_chosen": -3.1111416816711426,
      "logits/reference_rejected": -2.8734333515167236,
      "logps/policy_chosen": -71.76238250732422,
      "logps/policy_rejected": -479.92535400390625,
      "logps/reference_chosen": -267.5050964355469,
      "logps/reference_rejected": -202.55059814453125,
      "loss": 0.2505,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.9574271440505981,
      "rewards/margins": 4.731174468994141,
      "rewards/rejected": -2.773747682571411,
      "step": 326,
      "tokens/chosen_response_len_mean": 445.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 148.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.26,
      "grad_norm": 2.819044425119029,
      "learning_rate": 4.60863573226428e-07,
      "logits/policy_chosen": -1.2217872142791748,
      "logits/policy_rejected": -1.4427027702331543,
      "logits/reference_chosen": -2.5383174419403076,
      "logits/reference_rejected": -2.6582465171813965,
      "logps/policy_chosen": -78.91648864746094,
      "logps/policy_rejected": -792.6351318359375,
      "logps/reference_chosen": -325.96051025390625,
      "logps/reference_rejected": -492.7793884277344,
      "loss": 0.1434,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.470440149307251,
      "rewards/margins": 5.468997955322266,
      "rewards/rejected": -2.9985575675964355,
      "step": 327,
      "tokens/chosen_response_len_mean": 698.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 698.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.26,
      "grad_norm": 2.051994037783105,
      "learning_rate": 4.604856947709262e-07,
      "logits/policy_chosen": -1.699342966079712,
      "logits/policy_rejected": -1.9920241832733154,
      "logits/reference_chosen": -3.1137824058532715,
      "logits/reference_rejected": -2.245004892349243,
      "logps/policy_chosen": -108.17164611816406,
      "logps/policy_rejected": -584.7960205078125,
      "logps/reference_chosen": -348.4723205566406,
      "logps/reference_rejected": -226.91238403320312,
      "loss": 0.154,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4030065536499023,
      "rewards/margins": 5.981842517852783,
      "rewards/rejected": -3.578835964202881,
      "step": 328,
      "tokens/chosen_response_len_mean": 482.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 178.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.26,
      "grad_norm": 2.853440760910482,
      "learning_rate": 4.601061572562659e-07,
      "logits/policy_chosen": -1.1898789405822754,
      "logits/policy_rejected": -1.331478476524353,
      "logits/reference_chosen": -2.611237049102783,
      "logits/reference_rejected": -2.666792869567871,
      "logps/policy_chosen": -48.299530029296875,
      "logps/policy_rejected": -434.9912109375,
      "logps/reference_chosen": -154.5133056640625,
      "logps/reference_rejected": -240.702392578125,
      "loss": 0.2731,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0621377229690552,
      "rewards/margins": 3.00502610206604,
      "rewards/rejected": -1.9428882598876953,
      "step": 329,
      "tokens/chosen_response_len_mean": 265.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 216.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.27,
      "grad_norm": 3.1024957208965094,
      "learning_rate": 4.5972496367398146e-07,
      "logits/policy_chosen": -1.352826714515686,
      "logits/policy_rejected": -1.392114520072937,
      "logits/reference_chosen": -2.7036824226379395,
      "logits/reference_rejected": -2.694230794906616,
      "logps/policy_chosen": -61.405372619628906,
      "logps/policy_rejected": -453.8320007324219,
      "logps/reference_chosen": -191.30259704589844,
      "logps/reference_rejected": -260.45123291015625,
      "loss": 0.2595,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.298972249031067,
      "rewards/margins": 3.2327802181243896,
      "rewards/rejected": -1.933807611465454,
      "step": 330,
      "tokens/chosen_response_len_mean": 460.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 469.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.27,
      "grad_norm": 2.523243805150809,
      "learning_rate": 4.5934211702866046e-07,
      "logits/policy_chosen": -1.6151251792907715,
      "logits/policy_rejected": -1.6389597654342651,
      "logits/reference_chosen": -2.9203569889068604,
      "logits/reference_rejected": -2.905449390411377,
      "logps/policy_chosen": -63.08237075805664,
      "logps/policy_rejected": -630.6449584960938,
      "logps/reference_chosen": -203.67691040039062,
      "logps/reference_rejected": -291.34234619140625,
      "loss": 0.1624,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4059453010559082,
      "rewards/margins": 4.798971176147461,
      "rewards/rejected": -3.3930258750915527,
      "step": 331,
      "tokens/chosen_response_len_mean": 318.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 279.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.27,
      "grad_norm": 3.2573268171576544,
      "learning_rate": 4.5895762033791995e-07,
      "logits/policy_chosen": -2.4760982990264893,
      "logits/policy_rejected": -2.6151270866394043,
      "logits/reference_chosen": -3.021172523498535,
      "logits/reference_rejected": -2.504538059234619,
      "logps/policy_chosen": -40.59739685058594,
      "logps/policy_rejected": -89.36610412597656,
      "logps/reference_chosen": -70.21237182617188,
      "logps/reference_rejected": -43.146018981933594,
      "loss": 0.3215,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.29614976048469543,
      "rewards/margins": 0.7583505511283875,
      "rewards/rejected": -0.4622008204460144,
      "step": 332,
      "tokens/chosen_response_len_mean": 64.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 32.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.27,
      "grad_norm": 3.8626650481777554,
      "learning_rate": 4.58571476632383e-07,
      "logits/policy_chosen": -1.6957329511642456,
      "logits/policy_rejected": -2.019444227218628,
      "logits/reference_chosen": -2.8664603233337402,
      "logits/reference_rejected": -2.8443779945373535,
      "logps/policy_chosen": -86.0948257446289,
      "logps/policy_rejected": -600.858154296875,
      "logps/reference_chosen": -277.8157958984375,
      "logps/reference_rejected": -353.20318603515625,
      "loss": 0.2874,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9172096252441406,
      "rewards/margins": 4.393758773803711,
      "rewards/rejected": -2.4765493869781494,
      "step": 333,
      "tokens/chosen_response_len_mean": 540.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 265.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.27,
      "grad_norm": 4.40990215478422,
      "learning_rate": 4.5818368895565405e-07,
      "logits/policy_chosen": -1.576200008392334,
      "logits/policy_rejected": -1.6361205577850342,
      "logits/reference_chosen": -2.847994089126587,
      "logits/reference_rejected": -2.3715221881866455,
      "logps/policy_chosen": -62.154117584228516,
      "logps/policy_rejected": -340.4329833984375,
      "logps/reference_chosen": -182.93911743164062,
      "logps/reference_rejected": -150.31068420410156,
      "loss": 0.2179,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2078500986099243,
      "rewards/margins": 3.109072685241699,
      "rewards/rejected": -1.9012229442596436,
      "step": 334,
      "tokens/chosen_response_len_mean": 285.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 197.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.27,
      "grad_norm": 2.8983851836796353,
      "learning_rate": 4.5779426036429587e-07,
      "logits/policy_chosen": -2.055795907974243,
      "logits/policy_rejected": -1.6440610885620117,
      "logits/reference_chosen": -2.7132530212402344,
      "logits/reference_rejected": -2.2304413318634033,
      "logps/policy_chosen": -93.63249206542969,
      "logps/policy_rejected": -186.17974853515625,
      "logps/reference_chosen": -258.497314453125,
      "logps/reference_rejected": -183.7268524169922,
      "loss": 0.2936,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6486481428146362,
      "rewards/margins": 1.6731772422790527,
      "rewards/rejected": -0.024528950452804565,
      "step": 335,
      "tokens/chosen_response_len_mean": 367.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 459.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.27,
      "grad_norm": 2.3274853742981976,
      "learning_rate": 4.574031939278047e-07,
      "logits/policy_chosen": -1.8988397121429443,
      "logits/policy_rejected": -1.9968366622924805,
      "logits/reference_chosen": -3.2483341693878174,
      "logits/reference_rejected": -2.948215961456299,
      "logps/policy_chosen": -96.612548828125,
      "logps/policy_rejected": -861.7078247070312,
      "logps/reference_chosen": -347.1319885253906,
      "logps/reference_rejected": -461.7752990722656,
      "loss": 0.206,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.505194664001465,
      "rewards/margins": 6.504519939422607,
      "rewards/rejected": -3.9993255138397217,
      "step": 336,
      "tokens/chosen_response_len_mean": 526.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 455.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.27,
      "grad_norm": 4.109692231324148,
      "learning_rate": 4.570104927285864e-07,
      "logits/policy_chosen": -1.5520960092544556,
      "logits/policy_rejected": -1.7077003717422485,
      "logits/reference_chosen": -2.795018196105957,
      "logits/reference_rejected": -2.788074016571045,
      "logps/policy_chosen": -82.21895599365234,
      "logps/policy_rejected": -586.7863159179688,
      "logps/reference_chosen": -284.3222351074219,
      "logps/reference_rejected": -400.7400817871094,
      "loss": 0.1851,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.0210328102111816,
      "rewards/margins": 3.8814949989318848,
      "rewards/rejected": -1.8604620695114136,
      "step": 337,
      "tokens/chosen_response_len_mean": 517.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 480.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.27,
      "grad_norm": 3.2891497747662277,
      "learning_rate": 4.5661615986193243e-07,
      "logits/policy_chosen": -2.0379586219787598,
      "logits/policy_rejected": -2.2403860092163086,
      "logits/reference_chosen": -2.9786267280578613,
      "logits/reference_rejected": -2.831549882888794,
      "logps/policy_chosen": -86.32127380371094,
      "logps/policy_rejected": -617.3648071289062,
      "logps/reference_chosen": -241.71507263183594,
      "logps/reference_rejected": -268.22021484375,
      "loss": 0.15,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5539380311965942,
      "rewards/margins": 5.045384407043457,
      "rewards/rejected": -3.4914462566375732,
      "step": 338,
      "tokens/chosen_response_len_mean": 496.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 221.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.27,
      "grad_norm": 2.5176895862702615,
      "learning_rate": 4.562201984359946e-07,
      "logits/policy_chosen": -1.5903446674346924,
      "logits/policy_rejected": -1.8955085277557373,
      "logits/reference_chosen": -2.9453320503234863,
      "logits/reference_rejected": -2.7459821701049805,
      "logps/policy_chosen": -67.94086456298828,
      "logps/policy_rejected": -488.8743896484375,
      "logps/reference_chosen": -216.46241760253906,
      "logps/reference_rejected": -206.1934356689453,
      "loss": 0.1801,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.485215425491333,
      "rewards/margins": 4.31202507019043,
      "rewards/rejected": -2.826809883117676,
      "step": 339,
      "tokens/chosen_response_len_mean": 474.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 204.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.27,
      "grad_norm": 2.394686969217896,
      "learning_rate": 4.558226115717616e-07,
      "logits/policy_chosen": -1.1812870502471924,
      "logits/policy_rejected": -1.297174334526062,
      "logits/reference_chosen": -2.829561948776245,
      "logits/reference_rejected": -2.5182604789733887,
      "logps/policy_chosen": -114.76174926757812,
      "logps/policy_rejected": -792.0388793945312,
      "logps/reference_chosen": -383.6563415527344,
      "logps/reference_rejected": -463.0151672363281,
      "loss": 0.1879,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.688946008682251,
      "rewards/margins": 5.979182720184326,
      "rewards/rejected": -3.290236473083496,
      "step": 340,
      "tokens/chosen_response_len_mean": 546.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 485.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.27,
      "grad_norm": 3.0673469023984423,
      "learning_rate": 4.5542340240303367e-07,
      "logits/policy_chosen": -1.7937376499176025,
      "logits/policy_rejected": -1.815042495727539,
      "logits/reference_chosen": -3.0137624740600586,
      "logits/reference_rejected": -2.86102294921875,
      "logps/policy_chosen": -52.81929016113281,
      "logps/policy_rejected": -513.2135620117188,
      "logps/reference_chosen": -155.88839721679688,
      "logps/reference_rejected": -219.87347412109375,
      "loss": 0.271,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0306909084320068,
      "rewards/margins": 3.9640917778015137,
      "rewards/rejected": -2.933401107788086,
      "step": 341,
      "tokens/chosen_response_len_mean": 236.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 219.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.27,
      "grad_norm": 1.9367186293576946,
      "learning_rate": 4.5502257407639806e-07,
      "logits/policy_chosen": -1.552323341369629,
      "logits/policy_rejected": -1.8226666450500488,
      "logits/reference_chosen": -3.2792892456054688,
      "logits/reference_rejected": -2.0340218544006348,
      "logps/policy_chosen": -97.57713317871094,
      "logps/policy_rejected": -627.7135009765625,
      "logps/reference_chosen": -344.95343017578125,
      "logps/reference_rejected": -284.43548583984375,
      "loss": 0.134,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4737627506256104,
      "rewards/margins": 5.906542778015137,
      "rewards/rejected": -3.4327802658081055,
      "step": 342,
      "tokens/chosen_response_len_mean": 452.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 255.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.28,
      "grad_norm": 5.549352474070261,
      "learning_rate": 4.5462012975120453e-07,
      "logits/policy_chosen": -1.2813889980316162,
      "logits/policy_rejected": -1.7645294666290283,
      "logits/reference_chosen": -2.8398776054382324,
      "logits/reference_rejected": -2.596129894256592,
      "logps/policy_chosen": -90.58872985839844,
      "logps/policy_rejected": -583.3743896484375,
      "logps/reference_chosen": -270.8271789550781,
      "logps/reference_rejected": -242.07595825195312,
      "loss": 0.1804,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.802384376525879,
      "rewards/margins": 5.21536922454834,
      "rewards/rejected": -3.412984848022461,
      "step": 343,
      "tokens/chosen_response_len_mean": 581.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 286.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.28,
      "grad_norm": 1.9674810528357884,
      "learning_rate": 4.542160725995401e-07,
      "logits/policy_chosen": -2.343667984008789,
      "logits/policy_rejected": -2.326040029525757,
      "logits/reference_chosen": -2.737659454345703,
      "logits/reference_rejected": -2.802224636077881,
      "logps/policy_chosen": -28.079269409179688,
      "logps/policy_rejected": -175.65020751953125,
      "logps/reference_chosen": -27.471698760986328,
      "logps/reference_rejected": -72.05661010742188,
      "loss": 0.2642,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.006075695157051086,
      "rewards/margins": 1.0298603773117065,
      "rewards/rejected": -1.0359361171722412,
      "step": 344,
      "tokens/chosen_response_len_mean": 86.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 111.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.28,
      "grad_norm": 5.451886317836004,
      "learning_rate": 4.538104058062041e-07,
      "logits/policy_chosen": -1.6862752437591553,
      "logits/policy_rejected": -1.886472225189209,
      "logits/reference_chosen": -2.9743881225585938,
      "logits/reference_rejected": -2.8067820072174072,
      "logps/policy_chosen": -62.13983917236328,
      "logps/policy_rejected": -612.0286254882812,
      "logps/reference_chosen": -192.025634765625,
      "logps/reference_rejected": -268.3218688964844,
      "loss": 0.3178,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2988580465316772,
      "rewards/margins": 4.735925674438477,
      "rewards/rejected": -3.437067747116089,
      "step": 345,
      "tokens/chosen_response_len_mean": 359.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 269.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.28,
      "grad_norm": 4.435509463597182,
      "learning_rate": 4.5340313256868337e-07,
      "logits/policy_chosen": -1.5953540802001953,
      "logits/policy_rejected": -1.0541170835494995,
      "logits/reference_chosen": -2.2541754245758057,
      "logits/reference_rejected": -1.9815850257873535,
      "logps/policy_chosen": -91.02217864990234,
      "logps/policy_rejected": -648.81787109375,
      "logps/reference_chosen": -300.9630432128906,
      "logps/reference_rejected": -305.43585205078125,
      "loss": 0.288,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0994083881378174,
      "rewards/margins": 5.533228874206543,
      "rewards/rejected": -3.4338202476501465,
      "step": 346,
      "tokens/chosen_response_len_mean": 344.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 248.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.28,
      "grad_norm": 4.398404543060366,
      "learning_rate": 4.529942560971266e-07,
      "logits/policy_chosen": -1.4266515970230103,
      "logits/policy_rejected": -1.49228835105896,
      "logits/reference_chosen": -2.7921342849731445,
      "logits/reference_rejected": -2.7268803119659424,
      "logps/policy_chosen": -120.46929931640625,
      "logps/policy_rejected": -746.52880859375,
      "logps/reference_chosen": -357.797607421875,
      "logps/reference_rejected": -444.6593322753906,
      "loss": 0.2106,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3732829093933105,
      "rewards/margins": 5.3919782638549805,
      "rewards/rejected": -3.01869535446167,
      "step": 347,
      "tokens/chosen_response_len_mean": 441.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 367.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.28,
      "grad_norm": 2.688781964809469,
      "learning_rate": 4.5258377961431937e-07,
      "logits/policy_chosen": -1.0297237634658813,
      "logits/policy_rejected": -0.6758372187614441,
      "logits/reference_chosen": -2.3045077323913574,
      "logits/reference_rejected": -1.6970428228378296,
      "logps/policy_chosen": -84.93526458740234,
      "logps/policy_rejected": -675.788330078125,
      "logps/reference_chosen": -266.78472900390625,
      "logps/reference_rejected": -359.9700927734375,
      "loss": 0.1233,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.818494439125061,
      "rewards/margins": 4.976676940917969,
      "rewards/rejected": -3.158182382583618,
      "step": 348,
      "tokens/chosen_response_len_mean": 318.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 289.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.28,
      "grad_norm": 3.214059022448137,
      "learning_rate": 4.5217170635565855e-07,
      "logits/policy_chosen": -2.1661670207977295,
      "logits/policy_rejected": -2.1412088871002197,
      "logits/reference_chosen": -2.990678310394287,
      "logits/reference_rejected": -2.691145181655884,
      "logps/policy_chosen": -27.263290405273438,
      "logps/policy_rejected": -180.78501892089844,
      "logps/reference_chosen": -51.59008026123047,
      "logps/reference_rejected": -96.38774108886719,
      "loss": 0.2374,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.24326787889003754,
      "rewards/margins": 1.0872406959533691,
      "rewards/rejected": -0.8439728617668152,
      "step": 349,
      "tokens/chosen_response_len_mean": 106.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 89.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.28,
      "grad_norm": 3.518670555830533,
      "learning_rate": 4.5175803956912694e-07,
      "logits/policy_chosen": -1.9499872922897339,
      "logits/policy_rejected": -2.1933279037475586,
      "logits/reference_chosen": -3.137690305709839,
      "logits/reference_rejected": -2.8975977897644043,
      "logps/policy_chosen": -66.71675109863281,
      "logps/policy_rejected": -577.4403076171875,
      "logps/reference_chosen": -220.90902709960938,
      "logps/reference_rejected": -255.65286254882812,
      "loss": 0.1443,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5419225692749023,
      "rewards/margins": 4.759797096252441,
      "rewards/rejected": -3.217874526977539,
      "step": 350,
      "tokens/chosen_response_len_mean": 485.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 242.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.28,
      "grad_norm": 3.699314909735531,
      "learning_rate": 4.5134278251526766e-07,
      "logits/policy_chosen": -1.4733799695968628,
      "logits/policy_rejected": -1.4901491403579712,
      "logits/reference_chosen": -2.594074249267578,
      "logits/reference_rejected": -2.557436466217041,
      "logps/policy_chosen": -65.0804443359375,
      "logps/policy_rejected": -769.9380493164062,
      "logps/reference_chosen": -200.39883422851562,
      "logps/reference_rejected": -388.98785400390625,
      "loss": 0.2727,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3531838655471802,
      "rewards/margins": 5.162685394287109,
      "rewards/rejected": -3.8095014095306396,
      "step": 351,
      "tokens/chosen_response_len_mean": 433.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 425.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.28,
      "grad_norm": 2.955343714828839,
      "learning_rate": 4.509259384671582e-07,
      "logits/policy_chosen": -1.5196211338043213,
      "logits/policy_rejected": -1.566109299659729,
      "logits/reference_chosen": -3.052445888519287,
      "logits/reference_rejected": -2.340404510498047,
      "logps/policy_chosen": -115.84571838378906,
      "logps/policy_rejected": -830.0971069335938,
      "logps/reference_chosen": -340.6599426269531,
      "logps/reference_rejected": -353.662109375,
      "loss": 0.175,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2481420040130615,
      "rewards/margins": 7.012492656707764,
      "rewards/rejected": -4.764350414276123,
      "step": 352,
      "tokens/chosen_response_len_mean": 470.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 305.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.28,
      "grad_norm": 3.500797998192056,
      "learning_rate": 4.5050751071038514e-07,
      "logits/policy_chosen": -2.069139003753662,
      "logits/policy_rejected": -2.066579580307007,
      "logits/reference_chosen": -3.0677695274353027,
      "logits/reference_rejected": -3.032003402709961,
      "logps/policy_chosen": -59.17474365234375,
      "logps/policy_rejected": -444.245361328125,
      "logps/reference_chosen": -161.52598571777344,
      "logps/reference_rejected": -251.91793823242188,
      "loss": 0.2833,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0235124826431274,
      "rewards/margins": 2.946786642074585,
      "rewards/rejected": -1.9232741594314575,
      "step": 353,
      "tokens/chosen_response_len_mean": 381.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 321.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.28,
      "grad_norm": 2.1150271947388437,
      "learning_rate": 4.500875025430176e-07,
      "logits/policy_chosen": -1.5964065790176392,
      "logits/policy_rejected": -1.8535802364349365,
      "logits/reference_chosen": -3.0790903568267822,
      "logits/reference_rejected": -2.013641595840454,
      "logps/policy_chosen": -91.34295654296875,
      "logps/policy_rejected": -1013.7203369140625,
      "logps/reference_chosen": -323.52752685546875,
      "logps/reference_rejected": -508.953369140625,
      "loss": 0.082,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.321845769882202,
      "rewards/margins": 7.3695149421691895,
      "rewards/rejected": -5.047668933868408,
      "step": 354,
      "tokens/chosen_response_len_mean": 414.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 346.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.29,
      "grad_norm": 4.458337777869365,
      "learning_rate": 4.496659172755818e-07,
      "logits/policy_chosen": -1.3642287254333496,
      "logits/policy_rejected": -1.410393476486206,
      "logits/reference_chosen": -2.549731492996216,
      "logits/reference_rejected": -2.4937071800231934,
      "logps/policy_chosen": -111.19947814941406,
      "logps/policy_rejected": -509.53564453125,
      "logps/reference_chosen": -215.30552673339844,
      "logps/reference_rejected": -308.2844543457031,
      "loss": 0.3226,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0410603284835815,
      "rewards/margins": 3.053572416305542,
      "rewards/rejected": -2.01251220703125,
      "step": 355,
      "tokens/chosen_response_len_mean": 330.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 318.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.29,
      "grad_norm": 4.024586930018457,
      "learning_rate": 4.4924275823103466e-07,
      "logits/policy_chosen": -1.8358407020568848,
      "logits/policy_rejected": -1.9370864629745483,
      "logits/reference_chosen": -3.0807526111602783,
      "logits/reference_rejected": -2.92172908782959,
      "logps/policy_chosen": -74.08454132080078,
      "logps/policy_rejected": -655.4462890625,
      "logps/reference_chosen": -287.81585693359375,
      "logps/reference_rejected": -331.1325378417969,
      "loss": 0.1674,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.137312889099121,
      "rewards/margins": 5.380450248718262,
      "rewards/rejected": -3.2431373596191406,
      "step": 356,
      "tokens/chosen_response_len_mean": 392.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 230.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.29,
      "grad_norm": 2.5198509782294924,
      "learning_rate": 4.4881802874473774e-07,
      "logits/policy_chosen": -1.2551157474517822,
      "logits/policy_rejected": -1.7098793983459473,
      "logits/reference_chosen": -2.8631482124328613,
      "logits/reference_rejected": -1.495772123336792,
      "logps/policy_chosen": -56.709354400634766,
      "logps/policy_rejected": -329.5363464355469,
      "logps/reference_chosen": -180.67190551757812,
      "logps/reference_rejected": -128.47109985351562,
      "loss": 0.1685,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2396254539489746,
      "rewards/margins": 3.2502779960632324,
      "rewards/rejected": -2.0106523036956787,
      "step": 357,
      "tokens/chosen_response_len_mean": 321.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 157.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.29,
      "grad_norm": 2.7881215108282253,
      "learning_rate": 4.4839173216443084e-07,
      "logits/policy_chosen": -2.051104784011841,
      "logits/policy_rejected": -2.1336636543273926,
      "logits/reference_chosen": -2.95450758934021,
      "logits/reference_rejected": -2.7968645095825195,
      "logps/policy_chosen": -36.93220520019531,
      "logps/policy_rejected": -377.7947082519531,
      "logps/reference_chosen": -118.31900787353516,
      "logps/reference_rejected": -182.75210571289062,
      "loss": 0.1298,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8138680458068848,
      "rewards/margins": 2.764294147491455,
      "rewards/rejected": -1.9504261016845703,
      "step": 358,
      "tokens/chosen_response_len_mean": 222.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 194.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.29,
      "grad_norm": 2.9202638151581564,
      "learning_rate": 4.479638718502056e-07,
      "logits/policy_chosen": -2.800821304321289,
      "logits/policy_rejected": -2.74906063079834,
      "logits/reference_chosen": -2.9278526306152344,
      "logits/reference_rejected": -3.074326515197754,
      "logps/policy_chosen": -26.134109497070312,
      "logps/policy_rejected": -101.59957122802734,
      "logps/reference_chosen": -21.818992614746094,
      "logps/reference_rejected": -37.89555358886719,
      "loss": 0.245,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.04315117001533508,
      "rewards/margins": 0.5938889980316162,
      "rewards/rejected": -0.6370401382446289,
      "step": 359,
      "tokens/chosen_response_len_mean": 14.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 28.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.29,
      "grad_norm": 2.2402578949629,
      "learning_rate": 4.475344511744794e-07,
      "logits/policy_chosen": -1.5277884006500244,
      "logits/policy_rejected": -1.1120015382766724,
      "logits/reference_chosen": -3.0673985481262207,
      "logits/reference_rejected": -1.979531168937683,
      "logps/policy_chosen": -129.73883056640625,
      "logps/policy_rejected": -1009.8583984375,
      "logps/reference_chosen": -419.92962646484375,
      "logps/reference_rejected": -410.1859130859375,
      "loss": 0.131,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9019076824188232,
      "rewards/margins": 8.898632049560547,
      "rewards/rejected": -5.996725082397461,
      "step": 360,
      "tokens/chosen_response_len_mean": 535.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 503.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.29,
      "grad_norm": 2.2362340915683405,
      "learning_rate": 4.471034735219682e-07,
      "logits/policy_chosen": -1.8571796417236328,
      "logits/policy_rejected": -2.0071451663970947,
      "logits/reference_chosen": -3.0822551250457764,
      "logits/reference_rejected": -2.8071236610412598,
      "logps/policy_chosen": -113.39871215820312,
      "logps/policy_rejected": -765.5164184570312,
      "logps/reference_chosen": -314.39288330078125,
      "logps/reference_rejected": -295.49212646484375,
      "loss": 0.286,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.009941577911377,
      "rewards/margins": 6.7101850509643555,
      "rewards/rejected": -4.7002434730529785,
      "step": 361,
      "tokens/chosen_response_len_mean": 370.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 250.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.29,
      "grad_norm": 2.9958059876475645,
      "learning_rate": 4.4667094228966006e-07,
      "logits/policy_chosen": -1.3339217901229858,
      "logits/policy_rejected": -1.4387969970703125,
      "logits/reference_chosen": -2.5392210483551025,
      "logits/reference_rejected": -2.5316193103790283,
      "logps/policy_chosen": -65.20072174072266,
      "logps/policy_rejected": -141.90574645996094,
      "logps/reference_chosen": -153.89601135253906,
      "logps/reference_rejected": -161.5721893310547,
      "loss": 0.3208,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.8869529962539673,
      "rewards/margins": 0.6902884840965271,
      "rewards/rejected": 0.19666451215744019,
      "step": 362,
      "tokens/chosen_response_len_mean": 182.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 179.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.29,
      "grad_norm": 2.714261614183403,
      "learning_rate": 4.462368608867888e-07,
      "logits/policy_chosen": -2.0045056343078613,
      "logits/policy_rejected": -2.136458158493042,
      "logits/reference_chosen": -2.8513717651367188,
      "logits/reference_rejected": -2.6678905487060547,
      "logps/policy_chosen": -57.24700164794922,
      "logps/policy_rejected": -487.1899719238281,
      "logps/reference_chosen": -193.75621032714844,
      "logps/reference_rejected": -158.42803955078125,
      "loss": 0.2195,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3650922775268555,
      "rewards/margins": 4.652711868286133,
      "rewards/rejected": -3.287619113922119,
      "step": 363,
      "tokens/chosen_response_len_mean": 301.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 134.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.29,
      "grad_norm": 3.0348087146187934,
      "learning_rate": 4.4580123273480643e-07,
      "logits/policy_chosen": -1.2777620553970337,
      "logits/policy_rejected": -1.6785014867782593,
      "logits/reference_chosen": -2.842172622680664,
      "logits/reference_rejected": -2.3908252716064453,
      "logps/policy_chosen": -62.06177520751953,
      "logps/policy_rejected": -431.913330078125,
      "logps/reference_chosen": -187.41107177734375,
      "logps/reference_rejected": -209.33456420898438,
      "loss": 0.1306,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2534929513931274,
      "rewards/margins": 3.4792802333831787,
      "rewards/rejected": -2.225787401199341,
      "step": 364,
      "tokens/chosen_response_len_mean": 385.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 249.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.29,
      "grad_norm": 4.751862010403708,
      "learning_rate": 4.453640612673566e-07,
      "logits/policy_chosen": -1.219412088394165,
      "logits/policy_rejected": -1.36615788936615,
      "logits/reference_chosen": -2.3538596630096436,
      "logits/reference_rejected": -2.0649256706237793,
      "logps/policy_chosen": -101.12548828125,
      "logps/policy_rejected": -373.8370361328125,
      "logps/reference_chosen": -326.5352783203125,
      "logps/reference_rejected": -239.3107147216797,
      "loss": 0.1992,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2540977001190186,
      "rewards/margins": 3.599360942840576,
      "rewards/rejected": -1.3452633619308472,
      "step": 365,
      "tokens/chosen_response_len_mean": 465.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 235.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.29,
      "grad_norm": 3.4845170475354874,
      "learning_rate": 4.4492534993024744e-07,
      "logits/policy_chosen": -1.7572671175003052,
      "logits/policy_rejected": -1.7631720304489136,
      "logits/reference_chosen": -2.83882737159729,
      "logits/reference_rejected": -2.7936973571777344,
      "logps/policy_chosen": -79.20316314697266,
      "logps/policy_rejected": -479.2203063964844,
      "logps/reference_chosen": -240.270263671875,
      "logps/reference_rejected": -315.88555908203125,
      "loss": 0.2738,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.610671043395996,
      "rewards/margins": 3.2440185546875,
      "rewards/rejected": -1.633347511291504,
      "step": 366,
      "tokens/chosen_response_len_mean": 446.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 421.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.3,
      "grad_norm": 1.7048313796935655,
      "learning_rate": 4.4448510218142466e-07,
      "logits/policy_chosen": -1.9213192462921143,
      "logits/policy_rejected": -1.650076150894165,
      "logits/reference_chosen": -3.0253703594207764,
      "logits/reference_rejected": -2.3654181957244873,
      "logps/policy_chosen": -61.119239807128906,
      "logps/policy_rejected": -502.1123962402344,
      "logps/reference_chosen": -204.41744995117188,
      "logps/reference_rejected": -221.98794555664062,
      "loss": 0.1403,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4329822063446045,
      "rewards/margins": 4.234225749969482,
      "rewards/rejected": -2.801244020462036,
      "step": 367,
      "tokens/chosen_response_len_mean": 211.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 156.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.3,
      "grad_norm": 4.368110245343919,
      "learning_rate": 4.440433214909436e-07,
      "logits/policy_chosen": -1.6689025163650513,
      "logits/policy_rejected": -0.940861701965332,
      "logits/reference_chosen": -2.9102039337158203,
      "logits/reference_rejected": -2.0719833374023438,
      "logps/policy_chosen": -48.4640007019043,
      "logps/policy_rejected": -306.5902099609375,
      "logps/reference_chosen": -142.2967529296875,
      "logps/reference_rejected": -147.8551025390625,
      "loss": 0.2671,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.938327431678772,
      "rewards/margins": 2.5256781578063965,
      "rewards/rejected": -1.587350845336914,
      "step": 368,
      "tokens/chosen_response_len_mean": 255.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 382.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.3,
      "grad_norm": 4.421616617478599,
      "learning_rate": 4.4360001134094274e-07,
      "logits/policy_chosen": -1.7381703853607178,
      "logits/policy_rejected": -1.873031497001648,
      "logits/reference_chosen": -2.868490695953369,
      "logits/reference_rejected": -2.679164171218872,
      "logps/policy_chosen": -37.81551742553711,
      "logps/policy_rejected": -327.39471435546875,
      "logps/reference_chosen": -100.18566131591797,
      "logps/reference_rejected": -154.6254425048828,
      "loss": 0.2761,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6237015128135681,
      "rewards/margins": 2.3513944149017334,
      "rewards/rejected": -1.7276926040649414,
      "step": 369,
      "tokens/chosen_response_len_mean": 260.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 207.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.3,
      "grad_norm": 3.55457857405225,
      "learning_rate": 4.431551752256155e-07,
      "logits/policy_chosen": -1.4202126264572144,
      "logits/policy_rejected": -1.435547113418579,
      "logits/reference_chosen": -2.81121563911438,
      "logits/reference_rejected": -2.563999652862549,
      "logps/policy_chosen": -81.54356384277344,
      "logps/policy_rejected": -511.54022216796875,
      "logps/reference_chosen": -295.8115539550781,
      "logps/reference_rejected": -280.714111328125,
      "loss": 0.0897,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1426796913146973,
      "rewards/margins": 4.4509406089782715,
      "rewards/rejected": -2.308260917663574,
      "step": 370,
      "tokens/chosen_response_len_mean": 406.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 281.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.3,
      "grad_norm": 3.6157257883700984,
      "learning_rate": 4.4270881665118344e-07,
      "logits/policy_chosen": -1.7662705183029175,
      "logits/policy_rejected": -1.9252934455871582,
      "logits/reference_chosen": -2.995553493499756,
      "logits/reference_rejected": -2.469679594039917,
      "logps/policy_chosen": -77.08085632324219,
      "logps/policy_rejected": -437.26995849609375,
      "logps/reference_chosen": -275.45654296875,
      "logps/reference_rejected": -245.6436767578125,
      "loss": 0.2254,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9837571382522583,
      "rewards/margins": 3.900019884109497,
      "rewards/rejected": -1.9162627458572388,
      "step": 371,
      "tokens/chosen_response_len_mean": 352.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 245.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.3,
      "grad_norm": 2.739985089644356,
      "learning_rate": 4.422609391358677e-07,
      "logits/policy_chosen": -1.491788387298584,
      "logits/policy_rejected": -1.8629511594772339,
      "logits/reference_chosen": -2.8367395401000977,
      "logits/reference_rejected": -2.385693073272705,
      "logps/policy_chosen": -93.3801498413086,
      "logps/policy_rejected": -564.1484985351562,
      "logps/reference_chosen": -333.2955322265625,
      "logps/reference_rejected": -284.53338623046875,
      "loss": 0.154,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.399153709411621,
      "rewards/margins": 5.195304870605469,
      "rewards/rejected": -2.7961511611938477,
      "step": 372,
      "tokens/chosen_response_len_mean": 537.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 220.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.3,
      "grad_norm": 2.8326056798898325,
      "learning_rate": 4.418115462098623e-07,
      "logits/policy_chosen": -1.6880661249160767,
      "logits/policy_rejected": -1.8501157760620117,
      "logits/reference_chosen": -2.9830873012542725,
      "logits/reference_rejected": -2.656494140625,
      "logps/policy_chosen": -87.76195526123047,
      "logps/policy_rejected": -603.6038208007812,
      "logps/reference_chosen": -330.90252685546875,
      "logps/reference_rejected": -258.64581298828125,
      "loss": 0.1816,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4314053058624268,
      "rewards/margins": 5.880985260009766,
      "rewards/rejected": -3.449579954147339,
      "step": 373,
      "tokens/chosen_response_len_mean": 536.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 377.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.3,
      "grad_norm": 4.790162201367013,
      "learning_rate": 4.4136064141530547e-07,
      "logits/policy_chosen": -1.7553645372390747,
      "logits/policy_rejected": -1.6190441846847534,
      "logits/reference_chosen": -2.7890069484710693,
      "logits/reference_rejected": -2.1159238815307617,
      "logps/policy_chosen": -67.688232421875,
      "logps/policy_rejected": -195.291748046875,
      "logps/reference_chosen": -179.9830322265625,
      "logps/reference_rejected": -114.24205780029297,
      "loss": 0.1921,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1229479312896729,
      "rewards/margins": 1.9334447383880615,
      "rewards/rejected": -0.8104968070983887,
      "step": 374,
      "tokens/chosen_response_len_mean": 431.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 405.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.3,
      "grad_norm": 1.2307538383465877,
      "learning_rate": 4.409082283062523e-07,
      "logits/policy_chosen": -1.5772703886032104,
      "logits/policy_rejected": -1.6785141229629517,
      "logits/reference_chosen": -3.0875000953674316,
      "logits/reference_rejected": -2.7498016357421875,
      "logps/policy_chosen": -74.79031372070312,
      "logps/policy_rejected": -750.72998046875,
      "logps/reference_chosen": -317.4873352050781,
      "logps/reference_rejected": -347.156982421875,
      "loss": 0.0851,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4269702434539795,
      "rewards/margins": 6.462699890136719,
      "rewards/rejected": -4.03572940826416,
      "step": 375,
      "tokens/chosen_response_len_mean": 458.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 365.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.3,
      "grad_norm": 3.5542378235491863,
      "learning_rate": 4.4045431044864616e-07,
      "logits/policy_chosen": -1.6069151163101196,
      "logits/policy_rejected": -1.7103867530822754,
      "logits/reference_chosen": -2.6566100120544434,
      "logits/reference_rejected": -2.6409106254577637,
      "logps/policy_chosen": -87.49906921386719,
      "logps/policy_rejected": -665.1824951171875,
      "logps/reference_chosen": -305.02557373046875,
      "logps/reference_rejected": -320.46221923828125,
      "loss": 0.2368,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.175264835357666,
      "rewards/margins": 5.622467517852783,
      "rewards/rejected": -3.447202682495117,
      "step": 376,
      "tokens/chosen_response_len_mean": 411.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 312.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.3,
      "grad_norm": 5.281625587606173,
      "learning_rate": 4.3999889142029126e-07,
      "logits/policy_chosen": -2.1449053287506104,
      "logits/policy_rejected": -2.194610118865967,
      "logits/reference_chosen": -2.8323793411254883,
      "logits/reference_rejected": -2.804175853729248,
      "logps/policy_chosen": -30.32268524169922,
      "logps/policy_rejected": -170.59368896484375,
      "logps/reference_chosen": -67.28926086425781,
      "logps/reference_rejected": -86.84202575683594,
      "loss": 0.3036,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3696657121181488,
      "rewards/margins": 1.20718252658844,
      "rewards/rejected": -0.8375166654586792,
      "step": 377,
      "tokens/chosen_response_len_mean": 183.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 105.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.3,
      "grad_norm": 1.9215327012324088,
      "learning_rate": 4.395419748108241e-07,
      "logits/policy_chosen": -1.883148431777954,
      "logits/policy_rejected": -1.9562857151031494,
      "logits/reference_chosen": -3.1288890838623047,
      "logits/reference_rejected": -2.9048759937286377,
      "logps/policy_chosen": -83.87744903564453,
      "logps/policy_rejected": -814.286865234375,
      "logps/reference_chosen": -300.5325622558594,
      "logps/reference_rejected": -364.4478759765625,
      "loss": 0.1434,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.166551113128662,
      "rewards/margins": 6.66494083404541,
      "rewards/rejected": -4.498389720916748,
      "step": 378,
      "tokens/chosen_response_len_mean": 353.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 284.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.3,
      "grad_norm": 3.0494184783882075,
      "learning_rate": 4.390835642216848e-07,
      "logits/policy_chosen": -1.1829478740692139,
      "logits/policy_rejected": -1.6148120164871216,
      "logits/reference_chosen": -2.8968608379364014,
      "logits/reference_rejected": -2.4937872886657715,
      "logps/policy_chosen": -101.84024047851562,
      "logps/policy_rejected": -1095.6416015625,
      "logps/reference_chosen": -356.5980224609375,
      "logps/reference_rejected": -464.4998474121094,
      "loss": 0.0851,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5475780963897705,
      "rewards/margins": 8.85899543762207,
      "rewards/rejected": -6.311417579650879,
      "step": 379,
      "tokens/chosen_response_len_mean": 750.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 420.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.31,
      "grad_norm": 1.9839846610083902,
      "learning_rate": 4.386236632660897e-07,
      "logits/policy_chosen": -1.9111047983169556,
      "logits/policy_rejected": -1.440907597541809,
      "logits/reference_chosen": -2.8704569339752197,
      "logits/reference_rejected": -2.470503091812134,
      "logps/policy_chosen": -63.218116760253906,
      "logps/policy_rejected": -509.4794616699219,
      "logps/reference_chosen": -221.18771362304688,
      "logps/reference_rejected": -222.36109924316406,
      "loss": 0.0839,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5796959400177002,
      "rewards/margins": 4.4508795738220215,
      "rewards/rejected": -2.871183395385742,
      "step": 380,
      "tokens/chosen_response_len_mean": 366.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 483.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.31,
      "grad_norm": 3.547953034306201,
      "learning_rate": 4.3816227556900174e-07,
      "logits/policy_chosen": -2.1110708713531494,
      "logits/policy_rejected": -1.5525974035263062,
      "logits/reference_chosen": -3.074847459793091,
      "logits/reference_rejected": -2.299776315689087,
      "logps/policy_chosen": -53.559783935546875,
      "logps/policy_rejected": -340.3197937011719,
      "logps/reference_chosen": -210.47630310058594,
      "logps/reference_rejected": -170.03268432617188,
      "loss": 0.2292,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5691652297973633,
      "rewards/margins": 3.27203631401062,
      "rewards/rejected": -1.7028712034225464,
      "step": 381,
      "tokens/chosen_response_len_mean": 434.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 429.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.31,
      "grad_norm": 4.4598115746078175,
      "learning_rate": 4.376994047671028e-07,
      "logits/policy_chosen": -1.8473024368286133,
      "logits/policy_rejected": -1.9738845825195312,
      "logits/reference_chosen": -2.9645516872406006,
      "logits/reference_rejected": -2.810880184173584,
      "logps/policy_chosen": -37.34379196166992,
      "logps/policy_rejected": -328.9185485839844,
      "logps/reference_chosen": -105.24017333984375,
      "logps/reference_rejected": -159.31472778320312,
      "loss": 0.1772,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6789637804031372,
      "rewards/margins": 2.375002145767212,
      "rewards/rejected": -1.6960381269454956,
      "step": 382,
      "tokens/chosen_response_len_mean": 204.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 216.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.31,
      "grad_norm": 2.895844402337817,
      "learning_rate": 4.3723505450876454e-07,
      "logits/policy_chosen": -1.5201809406280518,
      "logits/policy_rejected": -1.8310809135437012,
      "logits/reference_chosen": -2.652768135070801,
      "logits/reference_rejected": -2.6164894104003906,
      "logps/policy_chosen": -89.18895721435547,
      "logps/policy_rejected": -672.33349609375,
      "logps/reference_chosen": -299.8074951171875,
      "logps/reference_rejected": -316.7373962402344,
      "loss": 0.1478,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1061851978302,
      "rewards/margins": 5.66214656829834,
      "rewards/rejected": -3.5559608936309814,
      "step": 383,
      "tokens/chosen_response_len_mean": 622.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 480.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.31,
      "grad_norm": 3.2525327057187328,
      "learning_rate": 4.3676922845401963e-07,
      "logits/policy_chosen": -1.4135372638702393,
      "logits/policy_rejected": -1.6460597515106201,
      "logits/reference_chosen": -2.6158249378204346,
      "logits/reference_rejected": -2.269045352935791,
      "logps/policy_chosen": -36.46479797363281,
      "logps/policy_rejected": -372.2850341796875,
      "logps/reference_chosen": -101.00570678710938,
      "logps/reference_rejected": -160.78067016601562,
      "loss": 0.1283,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.645409107208252,
      "rewards/margins": 2.76045298576355,
      "rewards/rejected": -2.115043878555298,
      "step": 384,
      "tokens/chosen_response_len_mean": 362.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 215.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.31,
      "grad_norm": 2.683005455867468,
      "learning_rate": 4.363019302745334e-07,
      "logits/policy_chosen": -2.2419838905334473,
      "logits/policy_rejected": -2.421705961227417,
      "logits/reference_chosen": -3.0781402587890625,
      "logits/reference_rejected": -2.9199631214141846,
      "logps/policy_chosen": -49.374847412109375,
      "logps/policy_rejected": -359.7245178222656,
      "logps/reference_chosen": -158.96978759765625,
      "logps/reference_rejected": -127.78620910644531,
      "loss": 0.2265,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0959495306015015,
      "rewards/margins": 3.415332317352295,
      "rewards/rejected": -2.319382667541504,
      "step": 385,
      "tokens/chosen_response_len_mean": 254.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 102.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.31,
      "grad_norm": 4.3953584290948156,
      "learning_rate": 4.3583316365357416e-07,
      "logits/policy_chosen": -1.6831576824188232,
      "logits/policy_rejected": -1.8844032287597656,
      "logits/reference_chosen": -2.9197726249694824,
      "logits/reference_rejected": -2.744558811187744,
      "logps/policy_chosen": -82.87364959716797,
      "logps/policy_rejected": -661.0844116210938,
      "logps/reference_chosen": -322.45697021484375,
      "logps/reference_rejected": -285.1172180175781,
      "loss": 0.1246,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3958332538604736,
      "rewards/margins": 6.155505657196045,
      "rewards/rejected": -3.7596724033355713,
      "step": 386,
      "tokens/chosen_response_len_mean": 471.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 260.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.31,
      "grad_norm": 4.3894384552460215,
      "learning_rate": 4.3536293228598474e-07,
      "logits/policy_chosen": -1.8510702848434448,
      "logits/policy_rejected": -1.8795132637023926,
      "logits/reference_chosen": -2.6799912452697754,
      "logits/reference_rejected": -2.4659016132354736,
      "logps/policy_chosen": -32.05808639526367,
      "logps/policy_rejected": -213.7239990234375,
      "logps/reference_chosen": -59.347633361816406,
      "logps/reference_rejected": -68.51786804199219,
      "loss": 0.2542,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2728954553604126,
      "rewards/margins": 1.7249568700790405,
      "rewards/rejected": -1.4520612955093384,
      "step": 387,
      "tokens/chosen_response_len_mean": 63.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 50.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.31,
      "grad_norm": 9.267866458567886,
      "learning_rate": 4.3489123987815325e-07,
      "logits/policy_chosen": -1.4151192903518677,
      "logits/policy_rejected": -1.5208919048309326,
      "logits/reference_chosen": -2.8274762630462646,
      "logits/reference_rejected": -2.905880928039551,
      "logps/policy_chosen": -118.6706771850586,
      "logps/policy_rejected": -557.0550537109375,
      "logps/reference_chosen": -374.5296936035156,
      "logps/reference_rejected": -424.9022521972656,
      "loss": 0.1827,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5585901737213135,
      "rewards/margins": 3.8801183700561523,
      "rewards/rejected": -1.321528434753418,
      "step": 388,
      "tokens/chosen_response_len_mean": 562.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 549.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.31,
      "grad_norm": 4.103907207414104,
      "learning_rate": 4.344180901479836e-07,
      "logits/policy_chosen": -1.689210295677185,
      "logits/policy_rejected": -1.1707533597946167,
      "logits/reference_chosen": -2.7496488094329834,
      "logits/reference_rejected": -2.5293631553649902,
      "logps/policy_chosen": -95.53358459472656,
      "logps/policy_rejected": -641.15380859375,
      "logps/reference_chosen": -288.0838623046875,
      "logps/reference_rejected": -270.31884765625,
      "loss": 0.1837,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.925502896308899,
      "rewards/margins": 5.633852958679199,
      "rewards/rejected": -3.7083497047424316,
      "step": 389,
      "tokens/chosen_response_len_mean": 366.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 445.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.31,
      "grad_norm": 4.588056708293884,
      "learning_rate": 4.339434868248665e-07,
      "logits/policy_chosen": -1.6808419227600098,
      "logits/policy_rejected": -1.7554115056991577,
      "logits/reference_chosen": -2.79362154006958,
      "logits/reference_rejected": -2.6228065490722656,
      "logps/policy_chosen": -59.64699172973633,
      "logps/policy_rejected": -678.405029296875,
      "logps/reference_chosen": -213.72731018066406,
      "logps/reference_rejected": -335.5184326171875,
      "loss": 0.2293,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.54080331325531,
      "rewards/margins": 4.969668865203857,
      "rewards/rejected": -3.428865909576416,
      "step": 390,
      "tokens/chosen_response_len_mean": 348.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 301.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.31,
      "grad_norm": 3.0510324137868525,
      "learning_rate": 4.3346743364965e-07,
      "logits/policy_chosen": -1.049216628074646,
      "logits/policy_rejected": -1.8208162784576416,
      "logits/reference_chosen": -2.591167449951172,
      "logits/reference_rejected": -2.324847459793091,
      "logps/policy_chosen": -121.5611572265625,
      "logps/policy_rejected": -607.802001953125,
      "logps/reference_chosen": -316.2970275878906,
      "logps/reference_rejected": -231.97964477539062,
      "loss": 0.1775,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9473586082458496,
      "rewards/margins": 5.705582141876221,
      "rewards/rejected": -3.758223056793213,
      "step": 391,
      "tokens/chosen_response_len_mean": 472.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 194.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.32,
      "grad_norm": 4.799092230789488,
      "learning_rate": 4.3298993437460984e-07,
      "logits/policy_chosen": -1.3143819570541382,
      "logits/policy_rejected": -1.4681634902954102,
      "logits/reference_chosen": -2.5000357627868652,
      "logits/reference_rejected": -2.514500379562378,
      "logps/policy_chosen": -81.29645538330078,
      "logps/policy_rejected": -878.3502197265625,
      "logps/reference_chosen": -289.78057861328125,
      "logps/reference_rejected": -434.30841064453125,
      "loss": 0.2684,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.084841012954712,
      "rewards/margins": 6.525258541107178,
      "rewards/rejected": -4.440417766571045,
      "step": 392,
      "tokens/chosen_response_len_mean": 493.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 346.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.32,
      "grad_norm": 5.0096860526087985,
      "learning_rate": 4.325109927634202e-07,
      "logits/policy_chosen": -1.745060682296753,
      "logits/policy_rejected": -1.8800654411315918,
      "logits/reference_chosen": -2.9817285537719727,
      "logits/reference_rejected": -2.4398183822631836,
      "logps/policy_chosen": -67.2919921875,
      "logps/policy_rejected": -429.408203125,
      "logps/reference_chosen": -218.21548461914062,
      "logps/reference_rejected": -260.8848571777344,
      "loss": 0.239,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.50923490524292,
      "rewards/margins": 3.1944684982299805,
      "rewards/rejected": -1.6852335929870605,
      "step": 393,
      "tokens/chosen_response_len_mean": 263.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 170.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.32,
      "grad_norm": 3.990192788421323,
      "learning_rate": 4.320306125911236e-07,
      "logits/policy_chosen": -2.1053833961486816,
      "logits/policy_rejected": -2.0984866619110107,
      "logits/reference_chosen": -2.8531177043914795,
      "logits/reference_rejected": -2.862253189086914,
      "logps/policy_chosen": -26.376094818115234,
      "logps/policy_rejected": -305.2973327636719,
      "logps/reference_chosen": -47.91350555419922,
      "logps/reference_rejected": -131.7420196533203,
      "loss": 0.2484,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21537409722805023,
      "rewards/margins": 1.9509271383285522,
      "rewards/rejected": -1.7355530261993408,
      "step": 394,
      "tokens/chosen_response_len_mean": 129.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 149.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.32,
      "grad_norm": 6.374622408586315,
      "learning_rate": 4.315487976441013e-07,
      "logits/policy_chosen": -1.2467350959777832,
      "logits/policy_rejected": -1.5044665336608887,
      "logits/reference_chosen": -2.920576572418213,
      "logits/reference_rejected": -2.6558072566986084,
      "logps/policy_chosen": -138.9432373046875,
      "logps/policy_rejected": -726.0556030273438,
      "logps/reference_chosen": -493.89862060546875,
      "logps/reference_rejected": -350.41497802734375,
      "loss": 0.0826,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.549553394317627,
      "rewards/margins": 7.305959224700928,
      "rewards/rejected": -3.756405830383301,
      "step": 395,
      "tokens/chosen_response_len_mean": 683.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 443.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.32,
      "grad_norm": 2.7801583537276717,
      "learning_rate": 4.31065551720044e-07,
      "logits/policy_chosen": -1.164729118347168,
      "logits/policy_rejected": -1.1291731595993042,
      "logits/reference_chosen": -2.5515594482421875,
      "logits/reference_rejected": -2.443310022354126,
      "logps/policy_chosen": -66.39266967773438,
      "logps/policy_rejected": -558.2879638671875,
      "logps/reference_chosen": -186.88536071777344,
      "logps/reference_rejected": -314.25531005859375,
      "loss": 0.19,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2049269676208496,
      "rewards/margins": 3.6452534198760986,
      "rewards/rejected": -2.440326452255249,
      "step": 396,
      "tokens/chosen_response_len_mean": 448.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 443.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.32,
      "grad_norm": 2.9706769612280617,
      "learning_rate": 4.3058087862792097e-07,
      "logits/policy_chosen": -1.182519555091858,
      "logits/policy_rejected": -1.190351128578186,
      "logits/reference_chosen": -2.546760082244873,
      "logits/reference_rejected": -2.415588617324829,
      "logps/policy_chosen": -64.53900146484375,
      "logps/policy_rejected": -359.20123291015625,
      "logps/reference_chosen": -171.31234741210938,
      "logps/reference_rejected": -217.0635986328125,
      "loss": 0.1348,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0677335262298584,
      "rewards/margins": 2.489109992980957,
      "rewards/rejected": -1.4213764667510986,
      "step": 397,
      "tokens/chosen_response_len_mean": 421.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 385.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.32,
      "grad_norm": 4.30537555824225,
      "learning_rate": 4.300947821879506e-07,
      "logits/policy_chosen": -1.8363888263702393,
      "logits/policy_rejected": -2.059960126876831,
      "logits/reference_chosen": -2.9730725288391113,
      "logits/reference_rejected": -2.7597203254699707,
      "logps/policy_chosen": -53.318687438964844,
      "logps/policy_rejected": -473.90850830078125,
      "logps/reference_chosen": -151.44744873046875,
      "logps/reference_rejected": -189.46267700195312,
      "loss": 0.261,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9812875390052795,
      "rewards/margins": 3.8257455825805664,
      "rewards/rejected": -2.8444581031799316,
      "step": 398,
      "tokens/chosen_response_len_mean": 290.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 171.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.32,
      "grad_norm": 1.3903903548910281,
      "learning_rate": 4.296072662315704e-07,
      "logits/policy_chosen": -1.257875919342041,
      "logits/policy_rejected": -1.1928701400756836,
      "logits/reference_chosen": -2.869974374771118,
      "logits/reference_rejected": -2.889599323272705,
      "logps/policy_chosen": -72.68022155761719,
      "logps/policy_rejected": -879.3326416015625,
      "logps/reference_chosen": -297.6036071777344,
      "logps/reference_rejected": -490.29583740234375,
      "loss": 0.1009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2492337226867676,
      "rewards/margins": 6.139601707458496,
      "rewards/rejected": -3.8903677463531494,
      "step": 399,
      "tokens/chosen_response_len_mean": 611.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 622.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.32,
      "grad_norm": 2.979855295546658,
      "learning_rate": 4.291183346014062e-07,
      "logits/policy_chosen": -2.134671926498413,
      "logits/policy_rejected": -2.2338736057281494,
      "logits/reference_chosen": -3.2482404708862305,
      "logits/reference_rejected": -3.0586061477661133,
      "logps/policy_chosen": -53.64082336425781,
      "logps/policy_rejected": -510.7403869628906,
      "logps/reference_chosen": -155.8409423828125,
      "logps/reference_rejected": -154.61331176757812,
      "loss": 0.2301,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0220012664794922,
      "rewards/margins": 4.5832719802856445,
      "rewards/rejected": -3.5612707138061523,
      "step": 400,
      "tokens/chosen_response_len_mean": 202.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 134.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.32,
      "grad_norm": 4.268553682116388,
      "learning_rate": 4.286279911512426e-07,
      "logits/policy_chosen": -1.2192718982696533,
      "logits/policy_rejected": -1.4112502336502075,
      "logits/reference_chosen": -2.929900646209717,
      "logits/reference_rejected": -2.617180824279785,
      "logps/policy_chosen": -107.41123962402344,
      "logps/policy_rejected": -861.2205200195312,
      "logps/reference_chosen": -457.88720703125,
      "logps/reference_rejected": -415.5411071777344,
      "loss": 0.2105,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.5047597885131836,
      "rewards/margins": 7.961553573608398,
      "rewards/rejected": -4.456793785095215,
      "step": 401,
      "tokens/chosen_response_len_mean": 650.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 464.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.32,
      "grad_norm": 3.201126591732796,
      "learning_rate": 4.28136239745992e-07,
      "logits/policy_chosen": -1.5901544094085693,
      "logits/policy_rejected": -1.7110729217529297,
      "logits/reference_chosen": -2.9678263664245605,
      "logits/reference_rejected": -2.9217777252197266,
      "logps/policy_chosen": -55.258575439453125,
      "logps/policy_rejected": -482.46002197265625,
      "logps/reference_chosen": -186.51856994628906,
      "logps/reference_rejected": -218.13978576660156,
      "loss": 0.1656,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.312600016593933,
      "rewards/margins": 3.9558019638061523,
      "rewards/rejected": -2.643202066421509,
      "step": 402,
      "tokens/chosen_response_len_mean": 319.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 279.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.32,
      "grad_norm": 2.755733841295062,
      "learning_rate": 4.2764308426166457e-07,
      "logits/policy_chosen": -1.209920883178711,
      "logits/policy_rejected": -1.3290446996688843,
      "logits/reference_chosen": -2.521901845932007,
      "logits/reference_rejected": -2.5550456047058105,
      "logps/policy_chosen": -92.68165588378906,
      "logps/policy_rejected": -1207.7777099609375,
      "logps/reference_chosen": -366.5865478515625,
      "logps/reference_rejected": -493.7196044921875,
      "loss": 0.1949,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.739048957824707,
      "rewards/margins": 9.879630088806152,
      "rewards/rejected": -7.140581130981445,
      "step": 403,
      "tokens/chosen_response_len_mean": 612.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 444.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.32,
      "grad_norm": 4.388247895143732,
      "learning_rate": 4.271485285853373e-07,
      "logits/policy_chosen": -2.0190563201904297,
      "logits/policy_rejected": -2.0062270164489746,
      "logits/reference_chosen": -2.7125706672668457,
      "logits/reference_rejected": -2.6811699867248535,
      "logps/policy_chosen": -28.796709060668945,
      "logps/policy_rejected": -91.47296142578125,
      "logps/reference_chosen": -29.286884307861328,
      "logps/reference_rejected": -48.171077728271484,
      "loss": 0.2241,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.004901757463812828,
      "rewards/margins": 0.43792060017585754,
      "rewards/rejected": -0.43301886320114136,
      "step": 404,
      "tokens/chosen_response_len_mean": 22.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 25.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.33,
      "grad_norm": 1.8962188440528207,
      "learning_rate": 4.266525766151238e-07,
      "logits/policy_chosen": -1.1334002017974854,
      "logits/policy_rejected": -1.4689135551452637,
      "logits/reference_chosen": -2.716999053955078,
      "logits/reference_rejected": -2.385608196258545,
      "logps/policy_chosen": -105.0840835571289,
      "logps/policy_rejected": -740.72998046875,
      "logps/reference_chosen": -380.8420104980469,
      "logps/reference_rejected": -305.6466979980469,
      "loss": 0.0838,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7575788497924805,
      "rewards/margins": 7.10841178894043,
      "rewards/rejected": -4.350832939147949,
      "step": 405,
      "tokens/chosen_response_len_mean": 690.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 363.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.33,
      "grad_norm": 2.1111760115546776,
      "learning_rate": 4.261552322601431e-07,
      "logits/policy_chosen": -2.5506460666656494,
      "logits/policy_rejected": -2.641453266143799,
      "logits/reference_chosen": -3.1168787479400635,
      "logits/reference_rejected": -2.71101975440979,
      "logps/policy_chosen": -26.55121612548828,
      "logps/policy_rejected": -170.76358032226562,
      "logps/reference_chosen": -59.65875244140625,
      "logps/reference_rejected": -49.777122497558594,
      "loss": 0.1723,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.33107537031173706,
      "rewards/margins": 1.5409399271011353,
      "rewards/rejected": -1.209864616394043,
      "step": 406,
      "tokens/chosen_response_len_mean": 73.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 35.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.33,
      "grad_norm": 3.260613936203855,
      "learning_rate": 4.2565649944048923e-07,
      "logits/policy_chosen": -1.8775959014892578,
      "logits/policy_rejected": -1.8169511556625366,
      "logits/reference_chosen": -3.01236891746521,
      "logits/reference_rejected": -2.9205944538116455,
      "logps/policy_chosen": -99.25498962402344,
      "logps/policy_rejected": -958.16064453125,
      "logps/reference_chosen": -368.29156494140625,
      "logps/reference_rejected": -485.85137939453125,
      "loss": 0.1257,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.69036602973938,
      "rewards/margins": 7.413458347320557,
      "rewards/rejected": -4.723093032836914,
      "step": 407,
      "tokens/chosen_response_len_mean": 447.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 481.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.33,
      "grad_norm": 1.4994292762921666,
      "learning_rate": 4.2515638208720017e-07,
      "logits/policy_chosen": -1.3916863203048706,
      "logits/policy_rejected": -1.5089915990829468,
      "logits/reference_chosen": -2.8834567070007324,
      "logits/reference_rejected": -2.730672597885132,
      "logps/policy_chosen": -87.06869506835938,
      "logps/policy_rejected": -708.5361328125,
      "logps/reference_chosen": -240.7329559326172,
      "logps/reference_rejected": -328.63568115234375,
      "loss": 0.0956,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5366424322128296,
      "rewards/margins": 5.335647106170654,
      "rewards/rejected": -3.799004554748535,
      "step": 408,
      "tokens/chosen_response_len_mean": 335.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 319.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.33,
      "grad_norm": 3.804336061637174,
      "learning_rate": 4.2465488414222675e-07,
      "logits/policy_chosen": -1.3066346645355225,
      "logits/policy_rejected": -1.5538992881774902,
      "logits/reference_chosen": -2.7491512298583984,
      "logits/reference_rejected": -2.5461301803588867,
      "logps/policy_chosen": -73.69843292236328,
      "logps/policy_rejected": -907.4656372070312,
      "logps/reference_chosen": -274.044677734375,
      "logps/reference_rejected": -401.0233154296875,
      "loss": 0.1424,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.003462314605713,
      "rewards/margins": 7.067885398864746,
      "rewards/rejected": -5.064422607421875,
      "step": 409,
      "tokens/chosen_response_len_mean": 530.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 431.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.33,
      "grad_norm": 3.5780045268337726,
      "learning_rate": 4.241520095584018e-07,
      "logits/policy_chosen": -1.7061024904251099,
      "logits/policy_rejected": -1.8617756366729736,
      "logits/reference_chosen": -2.8783411979675293,
      "logits/reference_rejected": -2.676494598388672,
      "logps/policy_chosen": -77.21218872070312,
      "logps/policy_rejected": -727.73583984375,
      "logps/reference_chosen": -247.0732421875,
      "logps/reference_rejected": -277.2325439453125,
      "loss": 0.1838,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6986103057861328,
      "rewards/margins": 6.203642845153809,
      "rewards/rejected": -4.505032539367676,
      "step": 410,
      "tokens/chosen_response_len_mean": 286.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 222.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.33,
      "grad_norm": 4.526177032986618,
      "learning_rate": 4.236477622994089e-07,
      "logits/policy_chosen": -1.5245254039764404,
      "logits/policy_rejected": -1.8174612522125244,
      "logits/reference_chosen": -2.9701709747314453,
      "logits/reference_rejected": -2.106998920440674,
      "logps/policy_chosen": -56.7745361328125,
      "logps/policy_rejected": -579.046142578125,
      "logps/reference_chosen": -207.30511474609375,
      "logps/reference_rejected": -216.8162841796875,
      "loss": 0.2017,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5053056478500366,
      "rewards/margins": 5.127603530883789,
      "rewards/rejected": -3.6222984790802,
      "step": 411,
      "tokens/chosen_response_len_mean": 336.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 168.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.33,
      "grad_norm": 2.0146954189338393,
      "learning_rate": 4.2314214633975097e-07,
      "logits/policy_chosen": -1.5148265361785889,
      "logits/policy_rejected": -1.7217381000518799,
      "logits/reference_chosen": -2.8511645793914795,
      "logits/reference_rejected": -2.744839668273926,
      "logps/policy_chosen": -110.9912109375,
      "logps/policy_rejected": -652.6676025390625,
      "logps/reference_chosen": -460.8502502441406,
      "logps/reference_rejected": -380.97821044921875,
      "loss": 0.1926,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4985907077789307,
      "rewards/margins": 6.215484619140625,
      "rewards/rejected": -2.7168941497802734,
      "step": 412,
      "tokens/chosen_response_len_mean": 686.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 465.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.33,
      "grad_norm": 3.07418927926233,
      "learning_rate": 4.226351656647194e-07,
      "logits/policy_chosen": -2.375103712081909,
      "logits/policy_rejected": -2.5675175189971924,
      "logits/reference_chosen": -3.0197691917419434,
      "logits/reference_rejected": -2.983851432800293,
      "logps/policy_chosen": -43.199851989746094,
      "logps/policy_rejected": -454.6613464355469,
      "logps/reference_chosen": -176.32766723632812,
      "logps/reference_rejected": -172.3994140625,
      "loss": 0.201,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3312782049179077,
      "rewards/margins": 4.153897285461426,
      "rewards/rejected": -2.8226189613342285,
      "step": 413,
      "tokens/chosen_response_len_mean": 238.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 109.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.33,
      "grad_norm": 3.214234675653117,
      "learning_rate": 4.2212682427036207e-07,
      "logits/policy_chosen": -1.683384895324707,
      "logits/policy_rejected": -1.7757070064544678,
      "logits/reference_chosen": -3.0007071495056152,
      "logits/reference_rejected": -2.7569046020507812,
      "logps/policy_chosen": -83.35159301757812,
      "logps/policy_rejected": -1168.320556640625,
      "logps/reference_chosen": -318.6688232421875,
      "logps/reference_rejected": -487.10333251953125,
      "loss": 0.2492,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3531723022460938,
      "rewards/margins": 9.16534423828125,
      "rewards/rejected": -6.812171936035156,
      "step": 414,
      "tokens/chosen_response_len_mean": 382.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 309.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.33,
      "grad_norm": 7.05534013367573,
      "learning_rate": 4.216171261634521e-07,
      "logits/policy_chosen": -1.2008273601531982,
      "logits/policy_rejected": -1.267887830734253,
      "logits/reference_chosen": -2.889969825744629,
      "logits/reference_rejected": -2.5156002044677734,
      "logps/policy_chosen": -110.30436706542969,
      "logps/policy_rejected": -835.0813598632812,
      "logps/reference_chosen": -343.83172607421875,
      "logps/reference_rejected": -419.806884765625,
      "loss": 0.1776,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.335273504257202,
      "rewards/margins": 6.48801851272583,
      "rewards/rejected": -4.152745246887207,
      "step": 415,
      "tokens/chosen_response_len_mean": 520.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 457.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.33,
      "grad_norm": 4.503495871779908,
      "learning_rate": 4.211060753614565e-07,
      "logits/policy_chosen": -1.5078086853027344,
      "logits/policy_rejected": -1.604093074798584,
      "logits/reference_chosen": -3.161609649658203,
      "logits/reference_rejected": -2.910872459411621,
      "logps/policy_chosen": -113.97125244140625,
      "logps/policy_rejected": -1133.2587890625,
      "logps/reference_chosen": -411.87451171875,
      "logps/reference_rejected": -516.1700439453125,
      "loss": 0.0946,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.979032278060913,
      "rewards/margins": 9.149919509887695,
      "rewards/rejected": -6.170886993408203,
      "step": 416,
      "tokens/chosen_response_len_mean": 528.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 459.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.34,
      "grad_norm": 2.097817299341684,
      "learning_rate": 4.205936758925043e-07,
      "logits/policy_chosen": -1.9427061080932617,
      "logits/policy_rejected": -1.928597092628479,
      "logits/reference_chosen": -3.03009033203125,
      "logits/reference_rejected": -2.9061622619628906,
      "logps/policy_chosen": -150.2655792236328,
      "logps/policy_rejected": -1011.0711059570312,
      "logps/reference_chosen": -301.38134765625,
      "logps/reference_rejected": -405.5292663574219,
      "loss": 0.0702,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5111576318740845,
      "rewards/margins": 7.56657600402832,
      "rewards/rejected": -6.055418968200684,
      "step": 417,
      "tokens/chosen_response_len_mean": 499.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 297.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.34,
      "grad_norm": 2.189913916441873,
      "learning_rate": 4.200799317953544e-07,
      "logits/policy_chosen": -2.0002455711364746,
      "logits/policy_rejected": -2.046023368835449,
      "logits/reference_chosen": -3.081580400466919,
      "logits/reference_rejected": -3.055077075958252,
      "logps/policy_chosen": -82.79331970214844,
      "logps/policy_rejected": -431.25262451171875,
      "logps/reference_chosen": -300.2025451660156,
      "logps/reference_rejected": -329.3789367675781,
      "loss": 0.0963,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1740922927856445,
      "rewards/margins": 3.192829132080078,
      "rewards/rejected": -1.018736720085144,
      "step": 418,
      "tokens/chosen_response_len_mean": 402.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 379.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.34,
      "grad_norm": 1.9957462015690448,
      "learning_rate": 4.1956484711936456e-07,
      "logits/policy_chosen": -1.3971436023712158,
      "logits/policy_rejected": -1.6208505630493164,
      "logits/reference_chosen": -2.894362211227417,
      "logits/reference_rejected": -2.2002763748168945,
      "logps/policy_chosen": -104.73182678222656,
      "logps/policy_rejected": -981.6074829101562,
      "logps/reference_chosen": -377.73626708984375,
      "logps/reference_rejected": -490.827880859375,
      "loss": 0.0868,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7300448417663574,
      "rewards/margins": 7.637840270996094,
      "rewards/rejected": -4.9077959060668945,
      "step": 419,
      "tokens/chosen_response_len_mean": 388.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 388.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.34,
      "grad_norm": 3.2963341725856554,
      "learning_rate": 4.19048425924459e-07,
      "logits/policy_chosen": -0.8859174251556396,
      "logits/policy_rejected": -1.1743956804275513,
      "logits/reference_chosen": -2.209782361984253,
      "logits/reference_rejected": -1.7737939357757568,
      "logps/policy_chosen": -87.35176086425781,
      "logps/policy_rejected": -571.3587036132812,
      "logps/reference_chosen": -229.13536071777344,
      "logps/reference_rejected": -191.48001098632812,
      "loss": 0.1983,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.41783607006073,
      "rewards/margins": 5.216622352600098,
      "rewards/rejected": -3.7987866401672363,
      "step": 420,
      "tokens/chosen_response_len_mean": 264.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 151.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.34,
      "grad_norm": 4.536881518708078,
      "learning_rate": 4.185306722810962e-07,
      "logits/policy_chosen": -1.7436951398849487,
      "logits/policy_rejected": -2.112318515777588,
      "logits/reference_chosen": -2.859280824661255,
      "logits/reference_rejected": -1.5029305219650269,
      "logps/policy_chosen": -50.8330078125,
      "logps/policy_rejected": -336.08221435546875,
      "logps/reference_chosen": -99.3558120727539,
      "logps/reference_rejected": -126.44001007080078,
      "loss": 0.1507,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.48522794246673584,
      "rewards/margins": 2.5816502571105957,
      "rewards/rejected": -2.0964221954345703,
      "step": 421,
      "tokens/chosen_response_len_mean": 164.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 69.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.34,
      "grad_norm": 3.7274301986248,
      "learning_rate": 4.180115902702372e-07,
      "logits/policy_chosen": -1.4023182392120361,
      "logits/policy_rejected": -1.7675896883010864,
      "logits/reference_chosen": -3.0383846759796143,
      "logits/reference_rejected": -2.3115711212158203,
      "logps/policy_chosen": -100.04576110839844,
      "logps/policy_rejected": -859.7716064453125,
      "logps/reference_chosen": -420.9822692871094,
      "logps/reference_rejected": -343.9464111328125,
      "loss": 0.1779,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.209364891052246,
      "rewards/margins": 8.367616653442383,
      "rewards/rejected": -5.1582512855529785,
      "step": 422,
      "tokens/chosen_response_len_mean": 535.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 257.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.34,
      "grad_norm": 2.8716382814297066,
      "learning_rate": 4.1749118398331347e-07,
      "logits/policy_chosen": -1.2414610385894775,
      "logits/policy_rejected": -1.2911479473114014,
      "logits/reference_chosen": -2.841339588165283,
      "logits/reference_rejected": -2.7952871322631836,
      "logps/policy_chosen": -103.96023559570312,
      "logps/policy_rejected": -1012.7462158203125,
      "logps/reference_chosen": -400.96234130859375,
      "logps/reference_rejected": -512.890625,
      "loss": 0.07,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9700207710266113,
      "rewards/margins": 7.9685773849487305,
      "rewards/rejected": -4.998556613922119,
      "step": 423,
      "tokens/chosen_response_len_mean": 660.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 620.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.34,
      "grad_norm": 2.992516243108965,
      "learning_rate": 4.169694575221942e-07,
      "logits/policy_chosen": -1.4850367307662964,
      "logits/policy_rejected": -1.6813836097717285,
      "logits/reference_chosen": -2.949424982070923,
      "logits/reference_rejected": -2.575556755065918,
      "logps/policy_chosen": -79.53746032714844,
      "logps/policy_rejected": -865.638916015625,
      "logps/reference_chosen": -315.7203369140625,
      "logps/reference_rejected": -370.00537109375,
      "loss": 0.1411,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3618288040161133,
      "rewards/margins": 7.318163871765137,
      "rewards/rejected": -4.956335067749023,
      "step": 424,
      "tokens/chosen_response_len_mean": 311.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 229.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.34,
      "grad_norm": 2.8328748406832367,
      "learning_rate": 4.164464149991545e-07,
      "logits/policy_chosen": -1.697274088859558,
      "logits/policy_rejected": -1.8267579078674316,
      "logits/reference_chosen": -3.064708709716797,
      "logits/reference_rejected": -2.918539047241211,
      "logps/policy_chosen": -57.181243896484375,
      "logps/policy_rejected": -458.12835693359375,
      "logps/reference_chosen": -193.5140380859375,
      "logps/reference_rejected": -147.54403686523438,
      "loss": 0.0783,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3633277416229248,
      "rewards/margins": 4.469170570373535,
      "rewards/rejected": -3.1058433055877686,
      "step": 425,
      "tokens/chosen_response_len_mean": 255.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 95.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.34,
      "grad_norm": 2.062408676437582,
      "learning_rate": 4.1592206053684264e-07,
      "logits/policy_chosen": -1.7967188358306885,
      "logits/policy_rejected": -1.874650478363037,
      "logits/reference_chosen": -3.0328989028930664,
      "logits/reference_rejected": -2.632453441619873,
      "logps/policy_chosen": -50.73225402832031,
      "logps/policy_rejected": -545.9600830078125,
      "logps/reference_chosen": -169.25364685058594,
      "logps/reference_rejected": -204.98764038085938,
      "loss": 0.1417,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1852138042449951,
      "rewards/margins": 4.5949387550354,
      "rewards/rejected": -3.409724712371826,
      "step": 426,
      "tokens/chosen_response_len_mean": 177.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 155.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.34,
      "grad_norm": 2.236967596720631,
      "learning_rate": 4.1539639826824764e-07,
      "logits/policy_chosen": -1.5609228610992432,
      "logits/policy_rejected": -1.6843552589416504,
      "logits/reference_chosen": -2.964130163192749,
      "logits/reference_rejected": -2.374593734741211,
      "logps/policy_chosen": -83.80799102783203,
      "logps/policy_rejected": -850.1075439453125,
      "logps/reference_chosen": -335.40484619140625,
      "logps/reference_rejected": -406.99139404296875,
      "loss": 0.1003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5159683227539062,
      "rewards/margins": 6.94713020324707,
      "rewards/rejected": -4.431161403656006,
      "step": 427,
      "tokens/chosen_response_len_mean": 432.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 358.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.34,
      "grad_norm": 4.021426985536353,
      "learning_rate": 4.148694323366667e-07,
      "logits/policy_chosen": -2.2846264839172363,
      "logits/policy_rejected": -2.474949359893799,
      "logits/reference_chosen": -3.068881034851074,
      "logits/reference_rejected": -2.916999340057373,
      "logps/policy_chosen": -54.07254409790039,
      "logps/policy_rejected": -278.12530517578125,
      "logps/reference_chosen": -202.6640625,
      "logps/reference_rejected": -123.41377258300781,
      "loss": 0.2022,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.485914945602417,
      "rewards/margins": 3.0330300331115723,
      "rewards/rejected": -1.5471153259277344,
      "step": 428,
      "tokens/chosen_response_len_mean": 227.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 102.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.34,
      "grad_norm": 5.345801600983718,
      "learning_rate": 4.143411668956728e-07,
      "logits/policy_chosen": -1.6848000288009644,
      "logits/policy_rejected": -1.7459708452224731,
      "logits/reference_chosen": -2.5471320152282715,
      "logits/reference_rejected": -2.352344512939453,
      "logps/policy_chosen": -55.883670806884766,
      "logps/policy_rejected": -331.67059326171875,
      "logps/reference_chosen": -164.4464111328125,
      "logps/reference_rejected": -162.13949584960938,
      "loss": 0.2186,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0856271982192993,
      "rewards/margins": 2.780938148498535,
      "rewards/rejected": -1.6953109502792358,
      "step": 429,
      "tokens/chosen_response_len_mean": 207.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 93.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.35,
      "grad_norm": 2.7420590797197613,
      "learning_rate": 4.1381160610908135e-07,
      "logits/policy_chosen": -1.2698513269424438,
      "logits/policy_rejected": -1.3237286806106567,
      "logits/reference_chosen": -2.986090660095215,
      "logits/reference_rejected": -2.4016549587249756,
      "logps/policy_chosen": -59.47589111328125,
      "logps/policy_rejected": -624.0321655273438,
      "logps/reference_chosen": -226.86578369140625,
      "logps/reference_rejected": -271.3048400878906,
      "loss": 0.1937,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6738988161087036,
      "rewards/margins": 5.201171875,
      "rewards/rejected": -3.527272939682007,
      "step": 430,
      "tokens/chosen_response_len_mean": 318.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 237.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.35,
      "grad_norm": 17.51425723640511,
      "learning_rate": 4.1328075415091797e-07,
      "logits/policy_chosen": -1.3769862651824951,
      "logits/policy_rejected": -1.4253036975860596,
      "logits/reference_chosen": -2.627458095550537,
      "logits/reference_rejected": -2.8932135105133057,
      "logps/policy_chosen": -118.02972412109375,
      "logps/policy_rejected": -1191.05712890625,
      "logps/reference_chosen": -446.54656982421875,
      "logps/reference_rejected": -551.8095703125,
      "loss": 0.3468,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.2851686477661133,
      "rewards/margins": 9.677642822265625,
      "rewards/rejected": -6.392475128173828,
      "step": 431,
      "tokens/chosen_response_len_mean": 365.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 322.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.35,
      "grad_norm": 3.0063842553120814,
      "learning_rate": 4.127486152053854e-07,
      "logits/policy_chosen": -1.8279298543930054,
      "logits/policy_rejected": -1.8977150917053223,
      "logits/reference_chosen": -2.812507152557373,
      "logits/reference_rejected": -2.5587706565856934,
      "logps/policy_chosen": -50.766231536865234,
      "logps/policy_rejected": -422.5538635253906,
      "logps/reference_chosen": -181.3826141357422,
      "logps/reference_rejected": -220.7954559326172,
      "loss": 0.2338,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3061637878417969,
      "rewards/margins": 3.3237476348876953,
      "rewards/rejected": -2.0175840854644775,
      "step": 432,
      "tokens/chosen_response_len_mean": 257.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 212.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.35,
      "grad_norm": 3.6070845301119636,
      "learning_rate": 4.1221519346683025e-07,
      "logits/policy_chosen": -1.7052745819091797,
      "logits/policy_rejected": -1.8579838275909424,
      "logits/reference_chosen": -2.9674618244171143,
      "logits/reference_rejected": -2.4956464767456055,
      "logps/policy_chosen": -58.71564865112305,
      "logps/policy_rejected": -627.035888671875,
      "logps/reference_chosen": -215.49732971191406,
      "logps/reference_rejected": -227.08786010742188,
      "loss": 0.2068,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5678167343139648,
      "rewards/margins": 5.567296981811523,
      "rewards/rejected": -3.9994797706604004,
      "step": 433,
      "tokens/chosen_response_len_mean": 307.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 276.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.35,
      "grad_norm": 3.8229512032067072,
      "learning_rate": 4.1168049313971044e-07,
      "logits/policy_chosen": -1.222996473312378,
      "logits/policy_rejected": -1.5602505207061768,
      "logits/reference_chosen": -2.6328537464141846,
      "logits/reference_rejected": -2.2551283836364746,
      "logps/policy_chosen": -117.59001159667969,
      "logps/policy_rejected": -424.58990478515625,
      "logps/reference_chosen": -304.99359130859375,
      "logps/reference_rejected": -170.0227813720703,
      "loss": 0.1745,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8740357160568237,
      "rewards/margins": 4.41970682144165,
      "rewards/rejected": -2.545670986175537,
      "step": 434,
      "tokens/chosen_response_len_mean": 285.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 141.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.35,
      "grad_norm": 5.849826855827791,
      "learning_rate": 4.1114451843856157e-07,
      "logits/policy_chosen": -1.7831515073776245,
      "logits/policy_rejected": -2.1642847061157227,
      "logits/reference_chosen": -2.7312891483306885,
      "logits/reference_rejected": -2.752143383026123,
      "logps/policy_chosen": -82.5667953491211,
      "logps/policy_rejected": -596.3550415039062,
      "logps/reference_chosen": -212.68727111816406,
      "logps/reference_rejected": -242.5646514892578,
      "loss": 0.2724,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3012046813964844,
      "rewards/margins": 4.839108467102051,
      "rewards/rejected": -3.5379040241241455,
      "step": 435,
      "tokens/chosen_response_len_mean": 398.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 225.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.35,
      "grad_norm": 4.967666219878217,
      "learning_rate": 4.106072735879642e-07,
      "logits/policy_chosen": -1.4188954830169678,
      "logits/policy_rejected": -1.4325001239776611,
      "logits/reference_chosen": -2.6980528831481934,
      "logits/reference_rejected": -2.5964839458465576,
      "logps/policy_chosen": -41.39575958251953,
      "logps/policy_rejected": -402.8433837890625,
      "logps/reference_chosen": -118.13224792480469,
      "logps/reference_rejected": -169.9378662109375,
      "loss": 0.2193,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.7673648595809937,
      "rewards/margins": 3.0964198112487793,
      "rewards/rejected": -2.329055070877075,
      "step": 436,
      "tokens/chosen_response_len_mean": 212.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 213.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.35,
      "grad_norm": 2.7162493902101286,
      "learning_rate": 4.1006876282250984e-07,
      "logits/policy_chosen": -1.466094732284546,
      "logits/policy_rejected": -1.6965906620025635,
      "logits/reference_chosen": -2.997114658355713,
      "logits/reference_rejected": -2.4590818881988525,
      "logps/policy_chosen": -89.01886749267578,
      "logps/policy_rejected": -685.9882202148438,
      "logps/reference_chosen": -331.4296875,
      "logps/reference_rejected": -406.5826110839844,
      "loss": 0.1821,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4241080284118652,
      "rewards/margins": 5.218163967132568,
      "rewards/rejected": -2.794055700302124,
      "step": 437,
      "tokens/chosen_response_len_mean": 447.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 390.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.35,
      "grad_norm": 3.002567622989937,
      "learning_rate": 4.0952899038676846e-07,
      "logits/policy_chosen": -1.919247031211853,
      "logits/policy_rejected": -1.9389680624008179,
      "logits/reference_chosen": -2.785936117172241,
      "logits/reference_rejected": -2.8493332862854004,
      "logps/policy_chosen": -38.87525177001953,
      "logps/policy_rejected": -104.11477661132812,
      "logps/reference_chosen": -83.54544067382812,
      "logps/reference_rejected": -100.99596405029297,
      "loss": 0.2881,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.44670188426971436,
      "rewards/margins": 0.4778900742530823,
      "rewards/rejected": -0.03118819370865822,
      "step": 438,
      "tokens/chosen_response_len_mean": 132.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 130.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.35,
      "grad_norm": 2.8151324350429867,
      "learning_rate": 4.0898796053525433e-07,
      "logits/policy_chosen": -1.4973105192184448,
      "logits/policy_rejected": -1.5697115659713745,
      "logits/reference_chosen": -3.18088960647583,
      "logits/reference_rejected": -3.056142568588257,
      "logps/policy_chosen": -45.63398742675781,
      "logps/policy_rejected": -481.3449401855469,
      "logps/reference_chosen": -163.41641235351562,
      "logps/reference_rejected": -209.1478729248047,
      "loss": 0.1977,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1778240203857422,
      "rewards/margins": 3.899794578552246,
      "rewards/rejected": -2.721970319747925,
      "step": 439,
      "tokens/chosen_response_len_mean": 273.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 241.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.35,
      "grad_norm": 1.5282334958119317,
      "learning_rate": 4.0844567753239274e-07,
      "logits/policy_chosen": -1.8370198011398315,
      "logits/policy_rejected": -1.961785078048706,
      "logits/reference_chosen": -2.621945381164551,
      "logits/reference_rejected": -2.461028575897217,
      "logps/policy_chosen": -51.12299728393555,
      "logps/policy_rejected": -252.86175537109375,
      "logps/reference_chosen": -126.06745910644531,
      "logps/reference_rejected": -104.03255462646484,
      "loss": 0.1958,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.7494446635246277,
      "rewards/margins": 2.237736701965332,
      "rewards/rejected": -1.4882919788360596,
      "step": 440,
      "tokens/chosen_response_len_mean": 132.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 85.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.35,
      "grad_norm": 2.9568899869246725,
      "learning_rate": 4.0790214565248646e-07,
      "logits/policy_chosen": -2.175638437271118,
      "logits/policy_rejected": -2.2474286556243896,
      "logits/reference_chosen": -3.1316123008728027,
      "logits/reference_rejected": -3.0298709869384766,
      "logps/policy_chosen": -59.25941467285156,
      "logps/policy_rejected": -751.8880615234375,
      "logps/reference_chosen": -253.11935424804688,
      "logps/reference_rejected": -319.33355712890625,
      "loss": 0.1966,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9385993480682373,
      "rewards/margins": 6.264143943786621,
      "rewards/rejected": -4.325545310974121,
      "step": 441,
      "tokens/chosen_response_len_mean": 315.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 296.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.36,
      "grad_norm": 4.1969511500967975,
      "learning_rate": 4.07357369179682e-07,
      "logits/policy_chosen": -0.6922111511230469,
      "logits/policy_rejected": -0.9226256608963013,
      "logits/reference_chosen": -2.4857916831970215,
      "logits/reference_rejected": -1.7259654998779297,
      "logps/policy_chosen": -62.29807662963867,
      "logps/policy_rejected": -315.15863037109375,
      "logps/reference_chosen": -194.74822998046875,
      "logps/reference_rejected": -133.52804565429688,
      "loss": 0.2108,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.324501633644104,
      "rewards/margins": 3.1408071517944336,
      "rewards/rejected": -1.8163057565689087,
      "step": 442,
      "tokens/chosen_response_len_mean": 240.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 122.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.36,
      "grad_norm": 2.377949338984507,
      "learning_rate": 4.0681135240793573e-07,
      "logits/policy_chosen": -1.7226147651672363,
      "logits/policy_rejected": -1.8669655323028564,
      "logits/reference_chosen": -2.9326863288879395,
      "logits/reference_rejected": -2.767451763153076,
      "logps/policy_chosen": -62.14042663574219,
      "logps/policy_rejected": -972.4740600585938,
      "logps/reference_chosen": -256.1126708984375,
      "logps/reference_rejected": -380.34942626953125,
      "loss": 0.2178,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9397224187850952,
      "rewards/margins": 7.860968589782715,
      "rewards/rejected": -5.921245574951172,
      "step": 443,
      "tokens/chosen_response_len_mean": 339.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 282.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.36,
      "grad_norm": 2.58448109714579,
      "learning_rate": 4.062640996409801e-07,
      "logits/policy_chosen": -1.7527000904083252,
      "logits/policy_rejected": -2.007575750350952,
      "logits/reference_chosen": -2.902820587158203,
      "logits/reference_rejected": -2.2178595066070557,
      "logps/policy_chosen": -58.31782913208008,
      "logps/policy_rejected": -388.484130859375,
      "logps/reference_chosen": -192.4158935546875,
      "logps/reference_rejected": -138.38531494140625,
      "loss": 0.2186,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3409807682037354,
      "rewards/margins": 3.8419687747955322,
      "rewards/rejected": -2.500988006591797,
      "step": 444,
      "tokens/chosen_response_len_mean": 268.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 130.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.36,
      "grad_norm": 5.002394103035561,
      "learning_rate": 4.057156151922898e-07,
      "logits/policy_chosen": -1.6491831541061401,
      "logits/policy_rejected": -1.6232179403305054,
      "logits/reference_chosen": -2.821261405944824,
      "logits/reference_rejected": -2.8910410404205322,
      "logps/policy_chosen": -101.18916320800781,
      "logps/policy_rejected": -1515.281982421875,
      "logps/reference_chosen": -461.17987060546875,
      "logps/reference_rejected": -736.8016357421875,
      "loss": 0.253,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.5999069213867188,
      "rewards/margins": 11.384710311889648,
      "rewards/rejected": -7.7848029136657715,
      "step": 445,
      "tokens/chosen_response_len_mean": 496.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 517.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.36,
      "grad_norm": 2.691158731758319,
      "learning_rate": 4.0516590338504777e-07,
      "logits/policy_chosen": -1.4098029136657715,
      "logits/policy_rejected": -1.5383237600326538,
      "logits/reference_chosen": -2.8628320693969727,
      "logits/reference_rejected": -2.2458231449127197,
      "logps/policy_chosen": -47.287330627441406,
      "logps/policy_rejected": -487.77374267578125,
      "logps/reference_chosen": -230.512451171875,
      "logps/reference_rejected": -174.41436767578125,
      "loss": 0.0554,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8322510719299316,
      "rewards/margins": 4.96584415435791,
      "rewards/rejected": -3.1335935592651367,
      "step": 446,
      "tokens/chosen_response_len_mean": 401.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 460.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.36,
      "grad_norm": 4.102688182580578,
      "learning_rate": 4.0461496855211084e-07,
      "logits/policy_chosen": -1.7874503135681152,
      "logits/policy_rejected": -1.8912441730499268,
      "logits/reference_chosen": -2.991851329803467,
      "logits/reference_rejected": -2.955643653869629,
      "logps/policy_chosen": -92.59574127197266,
      "logps/policy_rejected": -1307.9232177734375,
      "logps/reference_chosen": -372.2900390625,
      "logps/reference_rejected": -586.1259765625,
      "loss": 0.1847,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 2.796942949295044,
      "rewards/margins": 10.014915466308594,
      "rewards/rejected": -7.217972755432129,
      "step": 447,
      "tokens/chosen_response_len_mean": 440.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 425.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.36,
      "grad_norm": 4.110237060170118,
      "learning_rate": 4.040628150359761e-07,
      "logits/policy_chosen": -1.8245118856430054,
      "logits/policy_rejected": -1.9212603569030762,
      "logits/reference_chosen": -2.7836415767669678,
      "logits/reference_rejected": -2.6452181339263916,
      "logps/policy_chosen": -39.00062561035156,
      "logps/policy_rejected": -263.56488037109375,
      "logps/reference_chosen": -134.78646850585938,
      "logps/reference_rejected": -115.96138000488281,
      "loss": 0.3047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9578583240509033,
      "rewards/margins": 2.4338929653167725,
      "rewards/rejected": -1.4760349988937378,
      "step": 448,
      "tokens/chosen_response_len_mean": 299.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 230.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.36,
      "grad_norm": 3.419279324631656,
      "learning_rate": 4.0350944718874604e-07,
      "logits/policy_chosen": -1.8202662467956543,
      "logits/policy_rejected": -1.7790930271148682,
      "logits/reference_chosen": -2.6695165634155273,
      "logits/reference_rejected": -2.713456869125366,
      "logps/policy_chosen": -25.7647705078125,
      "logps/policy_rejected": -198.35360717773438,
      "logps/reference_chosen": -48.652671813964844,
      "logps/reference_rejected": -86.2638931274414,
      "loss": 0.2704,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22887898981571198,
      "rewards/margins": 1.3497761487960815,
      "rewards/rejected": -1.1208972930908203,
      "step": 449,
      "tokens/chosen_response_len_mean": 55.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 63.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.36,
      "grad_norm": 5.454203764009997,
      "learning_rate": 4.029548693720948e-07,
      "logits/policy_chosen": -1.6439151763916016,
      "logits/policy_rejected": -1.7532329559326172,
      "logits/reference_chosen": -2.898176431655884,
      "logits/reference_rejected": -2.827488422393799,
      "logps/policy_chosen": -137.31057739257812,
      "logps/policy_rejected": -794.82275390625,
      "logps/reference_chosen": -352.7179870605469,
      "logps/reference_rejected": -371.3848571777344,
      "loss": 0.193,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.154073715209961,
      "rewards/margins": 6.388452529907227,
      "rewards/rejected": -4.234378814697266,
      "step": 450,
      "tokens/chosen_response_len_mean": 407.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 326.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.36,
      "grad_norm": 4.535167605363691,
      "learning_rate": 4.023990859572335e-07,
      "logits/policy_chosen": -1.5814235210418701,
      "logits/policy_rejected": -1.666548728942871,
      "logits/reference_chosen": -2.7445616722106934,
      "logits/reference_rejected": -2.505018472671509,
      "logps/policy_chosen": -73.30876922607422,
      "logps/policy_rejected": -592.4307861328125,
      "logps/reference_chosen": -298.4370422363281,
      "logps/reference_rejected": -263.11077880859375,
      "loss": 0.2234,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.2512824535369873,
      "rewards/margins": 5.544482231140137,
      "rewards/rejected": -3.2932000160217285,
      "step": 451,
      "tokens/chosen_response_len_mean": 315.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 209.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.36,
      "grad_norm": 2.9113858691064998,
      "learning_rate": 4.0184210132487573e-07,
      "logits/policy_chosen": -1.4564967155456543,
      "logits/policy_rejected": -1.8521862030029297,
      "logits/reference_chosen": -3.072540283203125,
      "logits/reference_rejected": -2.0113308429718018,
      "logps/policy_chosen": -79.07356262207031,
      "logps/policy_rejected": -423.0850830078125,
      "logps/reference_chosen": -300.52764892578125,
      "logps/reference_rejected": -174.76361083984375,
      "loss": 0.0689,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.214540958404541,
      "rewards/margins": 4.697755336761475,
      "rewards/rejected": -2.4832143783569336,
      "step": 452,
      "tokens/chosen_response_len_mean": 355.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 177.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.36,
      "grad_norm": 4.966399014491544,
      "learning_rate": 4.012839198652033e-07,
      "logits/policy_chosen": -0.4901418685913086,
      "logits/policy_rejected": -0.44909435510635376,
      "logits/reference_chosen": -1.8819324970245361,
      "logits/reference_rejected": -1.395154595375061,
      "logps/policy_chosen": -170.2413787841797,
      "logps/policy_rejected": -325.10675048828125,
      "logps/reference_chosen": -306.1954345703125,
      "logps/reference_rejected": -221.98388671875,
      "loss": 0.1846,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.359540343284607,
      "rewards/margins": 2.3907690048217773,
      "rewards/rejected": -1.0312285423278809,
      "step": 453,
      "tokens/chosen_response_len_mean": 255.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 230.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.36,
      "grad_norm": 3.8484515457423596,
      "learning_rate": 4.0072454597783143e-07,
      "logits/policy_chosen": -1.986307144165039,
      "logits/policy_rejected": -2.048461675643921,
      "logits/reference_chosen": -3.2192866802215576,
      "logits/reference_rejected": -2.2684943675994873,
      "logps/policy_chosen": -84.06011962890625,
      "logps/policy_rejected": -799.325439453125,
      "logps/reference_chosen": -253.83985900878906,
      "logps/reference_rejected": -289.5534362792969,
      "loss": 0.1204,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6977972984313965,
      "rewards/margins": 6.795517921447754,
      "rewards/rejected": -5.097720146179199,
      "step": 454,
      "tokens/chosen_response_len_mean": 275.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 215.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.37,
      "grad_norm": 3.290938107627499,
      "learning_rate": 4.001639840717741e-07,
      "logits/policy_chosen": -2.003429412841797,
      "logits/policy_rejected": -2.224893808364868,
      "logits/reference_chosen": -3.2483534812927246,
      "logits/reference_rejected": -2.5375115871429443,
      "logps/policy_chosen": -53.70265197753906,
      "logps/policy_rejected": -711.6553344726562,
      "logps/reference_chosen": -248.81216430664062,
      "logps/reference_rejected": -261.5429992675781,
      "loss": 0.1318,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9510952234268188,
      "rewards/margins": 6.452218532562256,
      "rewards/rejected": -4.501123428344727,
      "step": 455,
      "tokens/chosen_response_len_mean": 300.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 165.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.37,
      "grad_norm": 3.8241212421334305,
      "learning_rate": 3.9960223856540923e-07,
      "logits/policy_chosen": -1.7832884788513184,
      "logits/policy_rejected": -1.8550889492034912,
      "logits/reference_chosen": -2.966344118118286,
      "logits/reference_rejected": -2.901747703552246,
      "logps/policy_chosen": -62.340301513671875,
      "logps/policy_rejected": -603.2691040039062,
      "logps/reference_chosen": -274.39215087890625,
      "logps/reference_rejected": -291.37420654296875,
      "loss": 0.1291,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.120518684387207,
      "rewards/margins": 5.239466667175293,
      "rewards/rejected": -3.118948221206665,
      "step": 456,
      "tokens/chosen_response_len_mean": 404.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 349.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.37,
      "grad_norm": 2.071368254173307,
      "learning_rate": 3.990393138864442e-07,
      "logits/policy_chosen": -1.3390485048294067,
      "logits/policy_rejected": -1.6212348937988281,
      "logits/reference_chosen": -3.0200870037078857,
      "logits/reference_rejected": -2.1834700107574463,
      "logps/policy_chosen": -103.87483215332031,
      "logps/policy_rejected": -1101.960693359375,
      "logps/reference_chosen": -464.3850402832031,
      "logps/reference_rejected": -516.58056640625,
      "loss": 0.1412,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.605102062225342,
      "rewards/margins": 9.458904266357422,
      "rewards/rejected": -5.8538007736206055,
      "step": 457,
      "tokens/chosen_response_len_mean": 683.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 451.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.37,
      "grad_norm": 3.924787435424885,
      "learning_rate": 3.984752144718803e-07,
      "logits/policy_chosen": -1.393798828125,
      "logits/policy_rejected": -1.890100359916687,
      "logits/reference_chosen": -2.881308078765869,
      "logits/reference_rejected": -1.485945701599121,
      "logps/policy_chosen": -128.1219940185547,
      "logps/policy_rejected": -279.1759033203125,
      "logps/reference_chosen": -203.0387420654297,
      "logps/reference_rejected": -110.58848571777344,
      "loss": 0.2824,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.749167263507843,
      "rewards/margins": 2.4350414276123047,
      "rewards/rejected": -1.6858742237091064,
      "step": 458,
      "tokens/chosen_response_len_mean": 251.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 85.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.37,
      "grad_norm": 4.928485028475111,
      "learning_rate": 3.979099447679785e-07,
      "logits/policy_chosen": -0.9617516994476318,
      "logits/policy_rejected": -1.3755125999450684,
      "logits/reference_chosen": -2.7406558990478516,
      "logits/reference_rejected": -2.277865409851074,
      "logps/policy_chosen": -77.129638671875,
      "logps/policy_rejected": -495.1766357421875,
      "logps/reference_chosen": -291.79388427734375,
      "logps/reference_rejected": -241.28622436523438,
      "loss": 0.2311,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1466422080993652,
      "rewards/margins": 4.685546875,
      "rewards/rejected": -2.5389044284820557,
      "step": 459,
      "tokens/chosen_response_len_mean": 335.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 156.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.37,
      "grad_norm": 2.464226743730991,
      "learning_rate": 3.973435092302239e-07,
      "logits/policy_chosen": -1.5250427722930908,
      "logits/policy_rejected": -1.736707329750061,
      "logits/reference_chosen": -2.7931671142578125,
      "logits/reference_rejected": -2.5833699703216553,
      "logps/policy_chosen": -88.23294830322266,
      "logps/policy_rejected": -879.3582763671875,
      "logps/reference_chosen": -262.9145202636719,
      "logps/reference_rejected": -367.1168212890625,
      "loss": 0.2028,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.746815800666809,
      "rewards/margins": 6.869230270385742,
      "rewards/rejected": -5.122414588928223,
      "step": 460,
      "tokens/chosen_response_len_mean": 377.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 342.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.37,
      "grad_norm": 3.6644172415396112,
      "learning_rate": 3.967759123232907e-07,
      "logits/policy_chosen": -2.0651230812072754,
      "logits/policy_rejected": -1.9665887355804443,
      "logits/reference_chosen": -2.667119026184082,
      "logits/reference_rejected": -2.8886525630950928,
      "logps/policy_chosen": -23.002052307128906,
      "logps/policy_rejected": -165.80038452148438,
      "logps/reference_chosen": -41.391563415527344,
      "logps/reference_rejected": -88.01963806152344,
      "loss": 0.3572,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18389514088630676,
      "rewards/margins": 0.9617025852203369,
      "rewards/rejected": -0.7778074741363525,
      "step": 461,
      "tokens/chosen_response_len_mean": 27.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 42.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.37,
      "grad_norm": 2.4488125940397194,
      "learning_rate": 3.9620715852100715e-07,
      "logits/policy_chosen": -1.6889033317565918,
      "logits/policy_rejected": -1.8014506101608276,
      "logits/reference_chosen": -3.140888214111328,
      "logits/reference_rejected": -2.8778910636901855,
      "logps/policy_chosen": -77.96212768554688,
      "logps/policy_rejected": -1032.3880615234375,
      "logps/reference_chosen": -358.6717529296875,
      "logps/reference_rejected": -438.1993103027344,
      "loss": 0.1901,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.807096481323242,
      "rewards/margins": 8.748984336853027,
      "rewards/rejected": -5.941887855529785,
      "step": 462,
      "tokens/chosen_response_len_mean": 680.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 526.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.37,
      "grad_norm": 2.663854864984055,
      "learning_rate": 3.956372523063204e-07,
      "logits/policy_chosen": -1.5669915676116943,
      "logits/policy_rejected": -1.866358995437622,
      "logits/reference_chosen": -3.0896379947662354,
      "logits/reference_rejected": -2.3229081630706787,
      "logps/policy_chosen": -81.29017639160156,
      "logps/policy_rejected": -697.92138671875,
      "logps/reference_chosen": -336.5562438964844,
      "logps/reference_rejected": -314.7755432128906,
      "loss": 0.1552,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5526607036590576,
      "rewards/margins": 6.384119987487793,
      "rewards/rejected": -3.831458806991577,
      "step": 463,
      "tokens/chosen_response_len_mean": 449.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 300.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.37,
      "grad_norm": 12.700627493187222,
      "learning_rate": 3.9506619817126056e-07,
      "logits/policy_chosen": -1.2608709335327148,
      "logits/policy_rejected": -1.5062246322631836,
      "logits/reference_chosen": -2.2167999744415283,
      "logits/reference_rejected": -2.4979875087738037,
      "logps/policy_chosen": -125.39148712158203,
      "logps/policy_rejected": -516.367919921875,
      "logps/reference_chosen": -297.6036071777344,
      "logps/reference_rejected": -249.89381408691406,
      "loss": 0.0553,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7221214771270752,
      "rewards/margins": 4.386862277984619,
      "rewards/rejected": -2.664740800857544,
      "step": 464,
      "tokens/chosen_response_len_mean": 303.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 167.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.37,
      "grad_norm": 2.4324731098817622,
      "learning_rate": 3.9449400061690594e-07,
      "logits/policy_chosen": -0.47676318883895874,
      "logits/policy_rejected": -1.4113003015518188,
      "logits/reference_chosen": -2.079223871231079,
      "logits/reference_rejected": -2.296034097671509,
      "logps/policy_chosen": -143.30014038085938,
      "logps/policy_rejected": -445.83807373046875,
      "logps/reference_chosen": -177.93850708007812,
      "logps/reference_rejected": -213.76071166992188,
      "loss": 0.075,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3463835120201111,
      "rewards/margins": 2.6671571731567383,
      "rewards/rejected": -2.3207736015319824,
      "step": 465,
      "tokens/chosen_response_len_mean": 332.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 180.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.37,
      "grad_norm": 3.5348733862576407,
      "learning_rate": 3.939206641533475e-07,
      "logits/policy_chosen": -1.178583025932312,
      "logits/policy_rejected": -1.573470115661621,
      "logits/reference_chosen": -2.8996262550354004,
      "logits/reference_rejected": -2.4363341331481934,
      "logps/policy_chosen": -158.576416015625,
      "logps/policy_rejected": -996.4195556640625,
      "logps/reference_chosen": -364.0356140136719,
      "logps/reference_rejected": -377.8266906738281,
      "loss": 0.15,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0545918941497803,
      "rewards/margins": 8.240521430969238,
      "rewards/rejected": -6.185929298400879,
      "step": 466,
      "tokens/chosen_response_len_mean": 370.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 227.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.38,
      "grad_norm": 3.80645873651521,
      "learning_rate": 3.933461932996528e-07,
      "logits/policy_chosen": -1.7989884614944458,
      "logits/policy_rejected": -1.9539071321487427,
      "logits/reference_chosen": -3.1026721000671387,
      "logits/reference_rejected": -2.5945231914520264,
      "logps/policy_chosen": -55.29502868652344,
      "logps/policy_rejected": -611.16650390625,
      "logps/reference_chosen": -234.65228271484375,
      "logps/reference_rejected": -240.26974487304688,
      "loss": 0.2324,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7935724258422852,
      "rewards/margins": 5.50253963470459,
      "rewards/rejected": -3.7089672088623047,
      "step": 467,
      "tokens/chosen_response_len_mean": 267.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 172.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.38,
      "grad_norm": 5.601714702757003,
      "learning_rate": 3.927705925838309e-07,
      "logits/policy_chosen": -2.009681224822998,
      "logits/policy_rejected": -2.0441019535064697,
      "logits/reference_chosen": -2.758538007736206,
      "logits/reference_rejected": -2.7094216346740723,
      "logps/policy_chosen": -49.910987854003906,
      "logps/policy_rejected": -468.62139892578125,
      "logps/reference_chosen": -155.70806884765625,
      "logps/reference_rejected": -184.36940002441406,
      "loss": 0.2229,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.0579708814620972,
      "rewards/margins": 3.9004905223846436,
      "rewards/rejected": -2.842519760131836,
      "step": 468,
      "tokens/chosen_response_len_mean": 177.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 140.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.38,
      "grad_norm": 3.4405115980261343,
      "learning_rate": 3.921938665427966e-07,
      "logits/policy_chosen": -1.5700072050094604,
      "logits/policy_rejected": -1.9501850605010986,
      "logits/reference_chosen": -2.924121618270874,
      "logits/reference_rejected": -2.757909059524536,
      "logps/policy_chosen": -99.33882141113281,
      "logps/policy_rejected": -1161.767578125,
      "logps/reference_chosen": -398.44720458984375,
      "logps/reference_rejected": -431.28643798828125,
      "loss": 0.2094,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9910836219787598,
      "rewards/margins": 10.295894622802734,
      "rewards/rejected": -7.304811000823975,
      "step": 469,
      "tokens/chosen_response_len_mean": 650.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 338.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.38,
      "grad_norm": 4.502600871920184,
      "learning_rate": 3.9161601972233437e-07,
      "logits/policy_chosen": -1.610199213027954,
      "logits/policy_rejected": -1.7848241329193115,
      "logits/reference_chosen": -2.870377540588379,
      "logits/reference_rejected": -2.9776086807250977,
      "logps/policy_chosen": -51.68815231323242,
      "logps/policy_rejected": -801.9058837890625,
      "logps/reference_chosen": -202.98548889160156,
      "logps/reference_rejected": -303.7187805175781,
      "loss": 0.2187,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5129733085632324,
      "rewards/margins": 6.494843482971191,
      "rewards/rejected": -4.981870174407959,
      "step": 470,
      "tokens/chosen_response_len_mean": 341.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 286.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.38,
      "grad_norm": 5.01154291999628,
      "learning_rate": 3.9103705667706287e-07,
      "logits/policy_chosen": -0.8790886402130127,
      "logits/policy_rejected": -1.3873319625854492,
      "logits/reference_chosen": -2.4265692234039307,
      "logits/reference_rejected": -1.5936481952667236,
      "logps/policy_chosen": -183.01113891601562,
      "logps/policy_rejected": -424.90667724609375,
      "logps/reference_chosen": -290.137939453125,
      "logps/reference_rejected": -181.65017700195312,
      "loss": 0.1933,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.071268081665039,
      "rewards/margins": 3.503833293914795,
      "rewards/rejected": -2.432565212249756,
      "step": 471,
      "tokens/chosen_response_len_mean": 255.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 113.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.38,
      "grad_norm": 2.5674800397325486,
      "learning_rate": 3.9045698197039877e-07,
      "logits/policy_chosen": -2.203275680541992,
      "logits/policy_rejected": -2.3401906490325928,
      "logits/reference_chosen": -3.04363751411438,
      "logits/reference_rejected": -2.5827834606170654,
      "logps/policy_chosen": -42.52135467529297,
      "logps/policy_rejected": -329.11602783203125,
      "logps/reference_chosen": -139.37042236328125,
      "logps/reference_rejected": -113.49201965332031,
      "loss": 0.2074,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.9684905409812927,
      "rewards/margins": 3.1247305870056152,
      "rewards/rejected": -2.1562399864196777,
      "step": 472,
      "tokens/chosen_response_len_mean": 136.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 55.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.38,
      "grad_norm": 3.3968645336056555,
      "learning_rate": 3.89875800174521e-07,
      "logits/policy_chosen": -1.6618294715881348,
      "logits/policy_rejected": -1.7981846332550049,
      "logits/reference_chosen": -2.8354909420013428,
      "logits/reference_rejected": -2.7583560943603516,
      "logps/policy_chosen": -71.3835220336914,
      "logps/policy_rejected": -1316.426513671875,
      "logps/reference_chosen": -329.5763854980469,
      "logps/reference_rejected": -527.76025390625,
      "loss": 0.1865,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5819284915924072,
      "rewards/margins": 10.46859073638916,
      "rewards/rejected": -7.886661529541016,
      "step": 473,
      "tokens/chosen_response_len_mean": 538.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 465.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.38,
      "grad_norm": 3.9075067558401018,
      "learning_rate": 3.8929351587033456e-07,
      "logits/policy_chosen": -1.3083927631378174,
      "logits/policy_rejected": -1.4738872051239014,
      "logits/reference_chosen": -3.0947024822235107,
      "logits/reference_rejected": -2.663092851638794,
      "logps/policy_chosen": -101.74766540527344,
      "logps/policy_rejected": -1513.1650390625,
      "logps/reference_chosen": -490.8614501953125,
      "logps/reference_rejected": -573.277587890625,
      "loss": 0.1146,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.8911373615264893,
      "rewards/margins": 13.29001235961914,
      "rewards/rejected": -9.398874282836914,
      "step": 474,
      "tokens/chosen_response_len_mean": 435.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 374.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.38,
      "grad_norm": 4.398476527635467,
      "learning_rate": 3.887101336474346e-07,
      "logits/policy_chosen": -1.64131498336792,
      "logits/policy_rejected": -1.772996425628662,
      "logits/reference_chosen": -2.9844939708709717,
      "logits/reference_rejected": -2.950493335723877,
      "logps/policy_chosen": -119.42963409423828,
      "logps/policy_rejected": -845.1961669921875,
      "logps/reference_chosen": -365.9910888671875,
      "logps/reference_rejected": -374.59222412109375,
      "loss": 0.1625,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4656143188476562,
      "rewards/margins": 7.171652793884277,
      "rewards/rejected": -4.706038951873779,
      "step": 475,
      "tokens/chosen_response_len_mean": 477.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 350.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.38,
      "grad_norm": 2.9717841454385745,
      "learning_rate": 3.8812565810407006e-07,
      "logits/policy_chosen": -1.994938850402832,
      "logits/policy_rejected": -2.0392134189605713,
      "logits/reference_chosen": -2.7937960624694824,
      "logits/reference_rejected": -2.75999116897583,
      "logps/policy_chosen": -30.649547576904297,
      "logps/policy_rejected": -341.9468688964844,
      "logps/reference_chosen": -95.49700164794922,
      "logps/reference_rejected": -93.1899642944336,
      "loss": 0.1698,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6484744548797607,
      "rewards/margins": 3.1360433101654053,
      "rewards/rejected": -2.4875688552856445,
      "step": 476,
      "tokens/chosen_response_len_mean": 154.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 77.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.38,
      "grad_norm": 3.4479086300544006,
      "learning_rate": 3.8754009384710734e-07,
      "logits/policy_chosen": -1.6349786520004272,
      "logits/policy_rejected": -1.5870938301086426,
      "logits/reference_chosen": -2.8305823802948,
      "logits/reference_rejected": -2.701176166534424,
      "logps/policy_chosen": -83.12035369873047,
      "logps/policy_rejected": -966.974853515625,
      "logps/reference_chosen": -385.94769287109375,
      "logps/reference_rejected": -465.5050048828125,
      "loss": 0.0731,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.028273344039917,
      "rewards/margins": 8.04297161102295,
      "rewards/rejected": -5.0146989822387695,
      "step": 477,
      "tokens/chosen_response_len_mean": 554.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 486.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.38,
      "grad_norm": 4.734485385728398,
      "learning_rate": 3.8695344549199444e-07,
      "logits/policy_chosen": -1.5670480728149414,
      "logits/policy_rejected": -1.646657943725586,
      "logits/reference_chosen": -2.9678804874420166,
      "logits/reference_rejected": -2.6930649280548096,
      "logps/policy_chosen": -72.26788330078125,
      "logps/policy_rejected": -587.336181640625,
      "logps/reference_chosen": -359.11077880859375,
      "logps/reference_rejected": -303.3057861328125,
      "loss": 0.1475,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8684287071228027,
      "rewards/margins": 5.708732604980469,
      "rewards/rejected": -2.840303659439087,
      "step": 478,
      "tokens/chosen_response_len_mean": 475.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 259.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.39,
      "grad_norm": 5.015003391618428,
      "learning_rate": 3.8636571766272404e-07,
      "logits/policy_chosen": -1.8944305181503296,
      "logits/policy_rejected": -2.025644302368164,
      "logits/reference_chosen": -2.9225046634674072,
      "logits/reference_rejected": -2.8332769870758057,
      "logps/policy_chosen": -68.69515991210938,
      "logps/policy_rejected": -746.20361328125,
      "logps/reference_chosen": -285.18524169921875,
      "logps/reference_rejected": -280.99456787109375,
      "loss": 0.16,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.164900779724121,
      "rewards/margins": 6.816990852355957,
      "rewards/rejected": -4.652090072631836,
      "step": 479,
      "tokens/chosen_response_len_mean": 327.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 235.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.39,
      "grad_norm": 4.980844425169706,
      "learning_rate": 3.8577691499179726e-07,
      "logits/policy_chosen": -1.2482839822769165,
      "logits/policy_rejected": -1.657585859298706,
      "logits/reference_chosen": -2.9377005100250244,
      "logits/reference_rejected": -2.3877174854278564,
      "logps/policy_chosen": -98.79377746582031,
      "logps/policy_rejected": -1050.2646484375,
      "logps/reference_chosen": -442.6528625488281,
      "logps/reference_rejected": -522.2037353515625,
      "loss": 0.1132,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4385905265808105,
      "rewards/margins": 8.719200134277344,
      "rewards/rejected": -5.280609130859375,
      "step": 480,
      "tokens/chosen_response_len_mean": 535.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 396.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.39,
      "grad_norm": 4.276618624323445,
      "learning_rate": 3.8518704212018745e-07,
      "logits/policy_chosen": -1.902390718460083,
      "logits/policy_rejected": -2.1558570861816406,
      "logits/reference_chosen": -3.0612549781799316,
      "logits/reference_rejected": -2.291801691055298,
      "logps/policy_chosen": -102.15428161621094,
      "logps/policy_rejected": -528.06884765625,
      "logps/reference_chosen": -317.2443542480469,
      "logps/reference_rejected": -216.9208526611328,
      "loss": 0.1954,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.150900363922119,
      "rewards/margins": 5.2623796463012695,
      "rewards/rejected": -3.1114795207977295,
      "step": 481,
      "tokens/chosen_response_len_mean": 461.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 146.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.39,
      "grad_norm": 4.360160729481662,
      "learning_rate": 3.8459610369730313e-07,
      "logits/policy_chosen": -1.8097548484802246,
      "logits/policy_rejected": -2.004084825515747,
      "logits/reference_chosen": -2.7691895961761475,
      "logits/reference_rejected": -2.938284397125244,
      "logps/policy_chosen": -51.07239532470703,
      "logps/policy_rejected": -561.343505859375,
      "logps/reference_chosen": -162.84552001953125,
      "logps/reference_rejected": -237.24717712402344,
      "loss": 0.1637,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1177313327789307,
      "rewards/margins": 4.358695030212402,
      "rewards/rejected": -3.2409634590148926,
      "step": 482,
      "tokens/chosen_response_len_mean": 309.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 192.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.39,
      "grad_norm": 2.6113270293799125,
      "learning_rate": 3.8400410438095144e-07,
      "logits/policy_chosen": -1.657280445098877,
      "logits/policy_rejected": -1.6796960830688477,
      "logits/reference_chosen": -2.998448133468628,
      "logits/reference_rejected": -2.9614603519439697,
      "logps/policy_chosen": -27.643503189086914,
      "logps/policy_rejected": -342.7109375,
      "logps/reference_chosen": -69.10589599609375,
      "logps/reference_rejected": -143.43035888671875,
      "loss": 0.1851,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.41462385654449463,
      "rewards/margins": 2.4074296951293945,
      "rewards/rejected": -1.9928058385849,
      "step": 483,
      "tokens/chosen_response_len_mean": 88.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 94.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.39,
      "grad_norm": 2.0866243181816535,
      "learning_rate": 3.834110488373018e-07,
      "logits/policy_chosen": -1.4216290712356567,
      "logits/policy_rejected": -1.4997227191925049,
      "logits/reference_chosen": -2.9512441158294678,
      "logits/reference_rejected": -2.516376256942749,
      "logps/policy_chosen": -86.3067626953125,
      "logps/policy_rejected": -825.5755004882812,
      "logps/reference_chosen": -271.3060302734375,
      "logps/reference_rejected": -354.2067565917969,
      "loss": 0.046,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8499928712844849,
      "rewards/margins": 6.5636796951293945,
      "rewards/rejected": -4.713686943054199,
      "step": 484,
      "tokens/chosen_response_len_mean": 313.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 275.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.39,
      "grad_norm": 3.6515336877406837,
      "learning_rate": 3.828169417408488e-07,
      "logits/policy_chosen": -1.7070186138153076,
      "logits/policy_rejected": -1.7828279733657837,
      "logits/reference_chosen": -3.059636354446411,
      "logits/reference_rejected": -2.539207935333252,
      "logps/policy_chosen": -84.38409423828125,
      "logps/policy_rejected": -1105.31494140625,
      "logps/reference_chosen": -358.4156494140625,
      "logps/reference_rejected": -483.3223571777344,
      "loss": 0.2483,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7403154373168945,
      "rewards/margins": 8.960241317749023,
      "rewards/rejected": -6.219926357269287,
      "step": 485,
      "tokens/chosen_response_len_mean": 340.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 306.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.39,
      "grad_norm": 5.128696876081537,
      "learning_rate": 3.822217877743752e-07,
      "logits/policy_chosen": -0.8503623604774475,
      "logits/policy_rejected": -1.1568849086761475,
      "logits/reference_chosen": -2.3078320026397705,
      "logits/reference_rejected": -2.1441264152526855,
      "logps/policy_chosen": -108.78406524658203,
      "logps/policy_rejected": -533.9822387695312,
      "logps/reference_chosen": -301.215576171875,
      "logps/reference_rejected": -222.34962463378906,
      "loss": 0.1655,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.9243148565292358,
      "rewards/margins": 5.0406413078308105,
      "rewards/rejected": -3.116326332092285,
      "step": 486,
      "tokens/chosen_response_len_mean": 504.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 268.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.39,
      "grad_norm": 3.8124759860924042,
      "learning_rate": 3.8162559162891556e-07,
      "logits/policy_chosen": -1.6079539060592651,
      "logits/policy_rejected": -1.8466084003448486,
      "logits/reference_chosen": -2.7364659309387207,
      "logits/reference_rejected": -2.465275287628174,
      "logps/policy_chosen": -97.786865234375,
      "logps/policy_rejected": -495.4726867675781,
      "logps/reference_chosen": -263.3196105957031,
      "logps/reference_rejected": -205.1041259765625,
      "loss": 0.1481,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.655327320098877,
      "rewards/margins": 4.5590128898620605,
      "rewards/rejected": -2.903686046600342,
      "step": 487,
      "tokens/chosen_response_len_mean": 250.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 131.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.39,
      "grad_norm": 2.993519140592352,
      "learning_rate": 3.810283580037188e-07,
      "logits/policy_chosen": -1.4010628461837769,
      "logits/policy_rejected": -1.682344675064087,
      "logits/reference_chosen": -2.4431915283203125,
      "logits/reference_rejected": -1.503739356994629,
      "logps/policy_chosen": -109.23246002197266,
      "logps/policy_rejected": -501.57757568359375,
      "logps/reference_chosen": -271.9234619140625,
      "logps/reference_rejected": -158.89590454101562,
      "loss": 0.2142,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6269102096557617,
      "rewards/margins": 5.0537261962890625,
      "rewards/rejected": -3.42681622505188,
      "step": 488,
      "tokens/chosen_response_len_mean": 279.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 87.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.39,
      "grad_norm": 3.718208594139943,
      "learning_rate": 3.8043009160621135e-07,
      "logits/policy_chosen": -1.4066287279129028,
      "logits/policy_rejected": -1.593829870223999,
      "logits/reference_chosen": -2.863757371902466,
      "logits/reference_rejected": -2.361283779144287,
      "logps/policy_chosen": -81.95440673828125,
      "logps/policy_rejected": -575.0250244140625,
      "logps/reference_chosen": -281.31939697265625,
      "logps/reference_rejected": -243.4657745361328,
      "loss": 0.193,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9936497211456299,
      "rewards/margins": 5.30924129486084,
      "rewards/rejected": -3.315591812133789,
      "step": 489,
      "tokens/chosen_response_len_mean": 285.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 196.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.39,
      "grad_norm": 3.2842577999556952,
      "learning_rate": 3.798307971519598e-07,
      "logits/policy_chosen": -1.192153811454773,
      "logits/policy_rejected": -1.3026520013809204,
      "logits/reference_chosen": -2.8523921966552734,
      "logits/reference_rejected": -2.5833401679992676,
      "logps/policy_chosen": -79.40185546875,
      "logps/policy_rejected": -931.162353515625,
      "logps/reference_chosen": -322.69110107421875,
      "logps/reference_rejected": -357.1807861328125,
      "loss": 0.1422,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.432892084121704,
      "rewards/margins": 8.172707557678223,
      "rewards/rejected": -5.739815711975098,
      "step": 490,
      "tokens/chosen_response_len_mean": 425.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 379.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.39,
      "grad_norm": 1.827399607143504,
      "learning_rate": 3.7923047936463427e-07,
      "logits/policy_chosen": -2.1025638580322266,
      "logits/policy_rejected": -2.2601938247680664,
      "logits/reference_chosen": -2.9096992015838623,
      "logits/reference_rejected": -2.2563729286193848,
      "logps/policy_chosen": -47.24481964111328,
      "logps/policy_rejected": -158.30300903320312,
      "logps/reference_chosen": -86.63211059570312,
      "logps/reference_rejected": -60.00082778930664,
      "loss": 0.1561,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3938729166984558,
      "rewards/margins": 1.3768947124481201,
      "rewards/rejected": -0.9830217957496643,
      "step": 491,
      "tokens/chosen_response_len_mean": 72.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 29.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.4,
      "grad_norm": 3.4366010010517205,
      "learning_rate": 3.786291429759707e-07,
      "logits/policy_chosen": -1.7485054731369019,
      "logits/policy_rejected": -1.8896783590316772,
      "logits/reference_chosen": -3.1098830699920654,
      "logits/reference_rejected": -2.685762643814087,
      "logps/policy_chosen": -95.95683288574219,
      "logps/policy_rejected": -788.707763671875,
      "logps/reference_chosen": -385.95404052734375,
      "logps/reference_rejected": -337.0655517578125,
      "loss": 0.146,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8999717235565186,
      "rewards/margins": 7.416394233703613,
      "rewards/rejected": -4.516422271728516,
      "step": 492,
      "tokens/chosen_response_len_mean": 431.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 269.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.4,
      "grad_norm": 6.714052625280003,
      "learning_rate": 3.780267927257335e-07,
      "logits/policy_chosen": -1.5334389209747314,
      "logits/policy_rejected": -1.3842865228652954,
      "logits/reference_chosen": -2.078615188598633,
      "logits/reference_rejected": -2.1927413940429688,
      "logps/policy_chosen": -50.05555725097656,
      "logps/policy_rejected": -287.3623352050781,
      "logps/reference_chosen": -109.44884490966797,
      "logps/reference_rejected": -187.74757385253906,
      "loss": 0.2035,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.5939328074455261,
      "rewards/margins": 1.5900804996490479,
      "rewards/rejected": -0.996147632598877,
      "step": 493,
      "tokens/chosen_response_len_mean": 268.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 163.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.4,
      "grad_norm": 3.8244783432225526,
      "learning_rate": 3.7742343336167874e-07,
      "logits/policy_chosen": -1.4070253372192383,
      "logits/policy_rejected": -1.5500389337539673,
      "logits/reference_chosen": -3.1651623249053955,
      "logits/reference_rejected": -2.8918333053588867,
      "logps/policy_chosen": -85.66355895996094,
      "logps/policy_rejected": -1053.1829833984375,
      "logps/reference_chosen": -337.708251953125,
      "logps/reference_rejected": -420.5155029296875,
      "loss": 0.165,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.520447015762329,
      "rewards/margins": 8.847121238708496,
      "rewards/rejected": -6.326674461364746,
      "step": 494,
      "tokens/chosen_response_len_mean": 392.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 315.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.4,
      "grad_norm": 4.610006618867372,
      "learning_rate": 3.7681906963951614e-07,
      "logits/policy_chosen": -2.5103185176849365,
      "logits/policy_rejected": -2.620328664779663,
      "logits/reference_chosen": -3.0773396492004395,
      "logits/reference_rejected": -2.9915406703948975,
      "logps/policy_chosen": -28.558101654052734,
      "logps/policy_rejected": -334.2321472167969,
      "logps/reference_chosen": -76.20562744140625,
      "logps/reference_rejected": -100.6245346069336,
      "loss": 0.2474,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4764752686023712,
      "rewards/margins": 2.812551498413086,
      "rewards/rejected": -2.336076021194458,
      "step": 495,
      "tokens/chosen_response_len_mean": 103.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 68.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.4,
      "grad_norm": 3.447109039467778,
      "learning_rate": 3.7621370632287197e-07,
      "logits/policy_chosen": -1.655297040939331,
      "logits/policy_rejected": -1.9512938261032104,
      "logits/reference_chosen": -3.064889907836914,
      "logits/reference_rejected": -2.318901777267456,
      "logps/policy_chosen": -44.43120574951172,
      "logps/policy_rejected": -547.6224365234375,
      "logps/reference_chosen": -157.44891357421875,
      "logps/reference_rejected": -142.62307739257812,
      "loss": 0.1368,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1301771402359009,
      "rewards/margins": 5.180170059204102,
      "rewards/rejected": -4.049993515014648,
      "step": 496,
      "tokens/chosen_response_len_mean": 247.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 122.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.4,
      "grad_norm": 5.889236731822492,
      "learning_rate": 3.756073481832511e-07,
      "logits/policy_chosen": -1.1531667709350586,
      "logits/policy_rejected": -1.4263137578964233,
      "logits/reference_chosen": -2.8856821060180664,
      "logits/reference_rejected": -2.5042450428009033,
      "logps/policy_chosen": -77.25579071044922,
      "logps/policy_rejected": -765.1267700195312,
      "logps/reference_chosen": -304.6671142578125,
      "logps/reference_rejected": -336.7586364746094,
      "loss": 0.1166,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.274113416671753,
      "rewards/margins": 6.557794570922852,
      "rewards/rejected": -4.283681392669678,
      "step": 497,
      "tokens/chosen_response_len_mean": 362.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 327.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.4,
      "grad_norm": 2.7355377340919853,
      "learning_rate": 3.75e-07,
      "logits/policy_chosen": -1.4582428932189941,
      "logits/policy_rejected": -1.5357778072357178,
      "logits/reference_chosen": -2.959364414215088,
      "logits/reference_rejected": -2.400367021560669,
      "logps/policy_chosen": -76.64922332763672,
      "logps/policy_rejected": -920.2921142578125,
      "logps/reference_chosen": -372.42529296875,
      "logps/reference_rejected": -377.87445068359375,
      "loss": 0.1447,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.957760810852051,
      "rewards/margins": 8.381937026977539,
      "rewards/rejected": -5.424176216125488,
      "step": 498,
      "tokens/chosen_response_len_mean": 586.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 326.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.4,
      "grad_norm": 6.894094134514474,
      "learning_rate": 3.743916665602684e-07,
      "logits/policy_chosen": -1.6466257572174072,
      "logits/policy_rejected": -1.89170241355896,
      "logits/reference_chosen": -2.906830310821533,
      "logits/reference_rejected": -2.6794936656951904,
      "logps/policy_chosen": -93.32026672363281,
      "logps/policy_rejected": -810.1947021484375,
      "logps/reference_chosen": -363.33795166015625,
      "logps/reference_rejected": -380.83770751953125,
      "loss": 0.1762,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.700176239013672,
      "rewards/margins": 6.993746757507324,
      "rewards/rejected": -4.293570518493652,
      "step": 499,
      "tokens/chosen_response_len_mean": 440.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 271.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.4,
      "grad_norm": 1.6707406161768406,
      "learning_rate": 3.7378235265897216e-07,
      "logits/policy_chosen": -1.580689549446106,
      "logits/policy_rejected": -1.6722397804260254,
      "logits/reference_chosen": -3.0592634677886963,
      "logits/reference_rejected": -2.7667362689971924,
      "logps/policy_chosen": -90.38396453857422,
      "logps/policy_rejected": -1193.93994140625,
      "logps/reference_chosen": -309.0954284667969,
      "logps/reference_rejected": -435.341552734375,
      "loss": 0.0546,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1871144771575928,
      "rewards/margins": 9.77309799194336,
      "rewards/rejected": -7.585983753204346,
      "step": 500,
      "tokens/chosen_response_len_mean": 378.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 362.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.4,
      "grad_norm": 1.5601338590754457,
      "learning_rate": 3.7317206309875495e-07,
      "logits/policy_chosen": -1.835053563117981,
      "logits/policy_rejected": -1.890053153038025,
      "logits/reference_chosen": -2.997572422027588,
      "logits/reference_rejected": -3.104527711868286,
      "logps/policy_chosen": -55.43461608886719,
      "logps/policy_rejected": -984.8824462890625,
      "logps/reference_chosen": -248.0032501220703,
      "logps/reference_rejected": -415.5852966308594,
      "loss": 0.1339,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.9256861209869385,
      "rewards/margins": 7.618658065795898,
      "rewards/rejected": -5.692971229553223,
      "step": 501,
      "tokens/chosen_response_len_mean": 301.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 301.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.4,
      "grad_norm": 1.8753011461653843,
      "learning_rate": 3.725608026899507e-07,
      "logits/policy_chosen": -1.5234193801879883,
      "logits/policy_rejected": -1.5929675102233887,
      "logits/reference_chosen": -2.8223395347595215,
      "logits/reference_rejected": -2.629556179046631,
      "logps/policy_chosen": -65.57670593261719,
      "logps/policy_rejected": -715.5125122070312,
      "logps/reference_chosen": -244.7671356201172,
      "logps/reference_rejected": -272.6716613769531,
      "loss": 0.1198,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.791904091835022,
      "rewards/margins": 6.220312118530273,
      "rewards/rejected": -4.428408145904541,
      "step": 502,
      "tokens/chosen_response_len_mean": 526.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 506.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.4,
      "grad_norm": 2.9345771370552924,
      "learning_rate": 3.7194857625054564e-07,
      "logits/policy_chosen": -1.3543603420257568,
      "logits/policy_rejected": -1.5678825378417969,
      "logits/reference_chosen": -2.8275258541107178,
      "logits/reference_rejected": -2.7801454067230225,
      "logps/policy_chosen": -94.03852844238281,
      "logps/policy_rejected": -1714.405029296875,
      "logps/reference_chosen": -428.21563720703125,
      "logps/reference_rejected": -655.2080688476562,
      "loss": 0.1188,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.341771125793457,
      "rewards/margins": 13.933740615844727,
      "rewards/rejected": -10.59196949005127,
      "step": 503,
      "tokens/chosen_response_len_mean": 635.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 490.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.41,
      "grad_norm": 3.491568081601307,
      "learning_rate": 3.7133538860614036e-07,
      "logits/policy_chosen": -2.119919776916504,
      "logits/policy_rejected": -2.169619560241699,
      "logits/reference_chosen": -2.9648609161376953,
      "logits/reference_rejected": -2.9082226753234863,
      "logps/policy_chosen": -31.060192108154297,
      "logps/policy_rejected": -423.8539123535156,
      "logps/reference_chosen": -107.59683990478516,
      "logps/reference_rejected": -157.83209228515625,
      "loss": 0.2238,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7653664946556091,
      "rewards/margins": 3.4255850315093994,
      "rewards/rejected": -2.6602182388305664,
      "step": 504,
      "tokens/chosen_response_len_mean": 198.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 145.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.41,
      "grad_norm": 12.061457613639613,
      "learning_rate": 3.707212445899116e-07,
      "logits/policy_chosen": -1.343079924583435,
      "logits/policy_rejected": -1.4780192375183105,
      "logits/reference_chosen": -3.0048511028289795,
      "logits/reference_rejected": -2.7388644218444824,
      "logps/policy_chosen": -121.62698364257812,
      "logps/policy_rejected": -1530.34912109375,
      "logps/reference_chosen": -471.6476745605469,
      "logps/reference_rejected": -584.0636596679688,
      "loss": 0.2833,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.500206708908081,
      "rewards/margins": 12.96306324005127,
      "rewards/rejected": -9.46285629272461,
      "step": 505,
      "tokens/chosen_response_len_mean": 566.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 469.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.41,
      "grad_norm": 4.255352016281462,
      "learning_rate": 3.701061490425745e-07,
      "logits/policy_chosen": -2.0763936042785645,
      "logits/policy_rejected": -2.0667333602905273,
      "logits/reference_chosen": -2.8928892612457275,
      "logits/reference_rejected": -3.013897180557251,
      "logps/policy_chosen": -55.27082061767578,
      "logps/policy_rejected": -785.6943359375,
      "logps/reference_chosen": -276.63946533203125,
      "logps/reference_rejected": -299.0478515625,
      "loss": 0.1983,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.213686227798462,
      "rewards/margins": 7.080151081085205,
      "rewards/rejected": -4.866464614868164,
      "step": 506,
      "tokens/chosen_response_len_mean": 318.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 272.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.41,
      "grad_norm": 3.7506489510098033,
      "learning_rate": 3.6949010681234385e-07,
      "logits/policy_chosen": -1.602917194366455,
      "logits/policy_rejected": -1.8732863664627075,
      "logits/reference_chosen": -3.0403642654418945,
      "logits/reference_rejected": -2.624617576599121,
      "logps/policy_chosen": -62.29011535644531,
      "logps/policy_rejected": -774.29541015625,
      "logps/reference_chosen": -316.345947265625,
      "logps/reference_rejected": -306.0225830078125,
      "loss": 0.1253,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.540558338165283,
      "rewards/margins": 7.223285675048828,
      "rewards/rejected": -4.682727813720703,
      "step": 507,
      "tokens/chosen_response_len_mean": 415.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 258.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.41,
      "grad_norm": 1.7604278097608879,
      "learning_rate": 3.688731227548965e-07,
      "logits/policy_chosen": -1.6100629568099976,
      "logits/policy_rejected": -1.8206270933151245,
      "logits/reference_chosen": -3.0545573234558105,
      "logits/reference_rejected": -2.526207447052002,
      "logps/policy_chosen": -86.55546569824219,
      "logps/policy_rejected": -1087.854736328125,
      "logps/reference_chosen": -384.8187255859375,
      "logps/reference_rejected": -388.63616943359375,
      "loss": 0.1412,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.982632637023926,
      "rewards/margins": 9.97481918334961,
      "rewards/rejected": -6.992185592651367,
      "step": 508,
      "tokens/chosen_response_len_mean": 402.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 263.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.41,
      "grad_norm": 5.447535232212929,
      "learning_rate": 3.682552017333328e-07,
      "logits/policy_chosen": -1.7877931594848633,
      "logits/policy_rejected": -1.949285864830017,
      "logits/reference_chosen": -3.0602922439575195,
      "logits/reference_rejected": -2.3765697479248047,
      "logps/policy_chosen": -40.384971618652344,
      "logps/policy_rejected": -438.9278564453125,
      "logps/reference_chosen": -167.47283935546875,
      "logps/reference_rejected": -140.21371459960938,
      "loss": 0.1559,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.270878791809082,
      "rewards/margins": 4.258019924163818,
      "rewards/rejected": -2.9871413707733154,
      "step": 509,
      "tokens/chosen_response_len_mean": 210.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 80.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.41,
      "grad_norm": 5.54720148549507,
      "learning_rate": 3.6763634861813835e-07,
      "logits/policy_chosen": -2.035360336303711,
      "logits/policy_rejected": -1.4416533708572388,
      "logits/reference_chosen": -2.898369789123535,
      "logits/reference_rejected": -2.6384010314941406,
      "logps/policy_chosen": -34.5697135925293,
      "logps/policy_rejected": -183.6455841064453,
      "logps/reference_chosen": -41.313148498535156,
      "logps/reference_rejected": -94.95040893554688,
      "loss": 0.2277,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.06743432581424713,
      "rewards/margins": 0.9543862342834473,
      "rewards/rejected": -0.8869518637657166,
      "step": 510,
      "tokens/chosen_response_len_mean": 28.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 30.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.41,
      "grad_norm": 3.964798649874343,
      "learning_rate": 3.6701656828714534e-07,
      "logits/policy_chosen": -1.8111273050308228,
      "logits/policy_rejected": -2.0848844051361084,
      "logits/reference_chosen": -2.9827027320861816,
      "logits/reference_rejected": -2.8648905754089355,
      "logps/policy_chosen": -65.8554916381836,
      "logps/policy_rejected": -453.29095458984375,
      "logps/reference_chosen": -269.0480041503906,
      "logps/reference_rejected": -191.38449096679688,
      "loss": 0.1639,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0319249629974365,
      "rewards/margins": 4.650989532470703,
      "rewards/rejected": -2.6190643310546875,
      "step": 511,
      "tokens/chosen_response_len_mean": 403.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 183.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.41,
      "grad_norm": 2.7946482910354136,
      "learning_rate": 3.6639586562549464e-07,
      "logits/policy_chosen": -1.3696837425231934,
      "logits/policy_rejected": -1.44891357421875,
      "logits/reference_chosen": -2.5541605949401855,
      "logits/reference_rejected": -1.68296217918396,
      "logps/policy_chosen": -73.59698486328125,
      "logps/policy_rejected": -639.5834350585938,
      "logps/reference_chosen": -208.5489959716797,
      "logps/reference_rejected": -225.13343811035156,
      "loss": 0.1363,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.349519968032837,
      "rewards/margins": 5.494019985198975,
      "rewards/rejected": -4.144499778747559,
      "step": 512,
      "tokens/chosen_response_len_mean": 161.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 112.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.41,
      "grad_norm": 1.2560051200148832,
      "learning_rate": 3.6577424552559667e-07,
      "logits/policy_chosen": -2.173801898956299,
      "logits/policy_rejected": -2.212505340576172,
      "logits/reference_chosen": -2.971432685852051,
      "logits/reference_rejected": -2.931748867034912,
      "logps/policy_chosen": -42.875030517578125,
      "logps/policy_rejected": -451.9052734375,
      "logps/reference_chosen": -136.7771453857422,
      "logps/reference_rejected": -175.2373046875,
      "loss": 0.1189,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.939021110534668,
      "rewards/margins": 3.7057008743286133,
      "rewards/rejected": -2.7666797637939453,
      "step": 513,
      "tokens/chosen_response_len_mean": 283.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 192.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.41,
      "grad_norm": 2.1854568749272425,
      "learning_rate": 3.651517128870933e-07,
      "logits/policy_chosen": -1.6797782182693481,
      "logits/policy_rejected": -1.7808456420898438,
      "logits/reference_chosen": -3.1624789237976074,
      "logits/reference_rejected": -3.007197380065918,
      "logps/policy_chosen": -91.08135223388672,
      "logps/policy_rejected": -1569.0712890625,
      "logps/reference_chosen": -459.3399963378906,
      "logps/reference_rejected": -548.77001953125,
      "loss": 0.2284,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 3.682586193084717,
      "rewards/margins": 13.885599136352539,
      "rewards/rejected": -10.203012466430664,
      "step": 514,
      "tokens/chosen_response_len_mean": 566.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 427.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.41,
      "grad_norm": 4.694103421697667,
      "learning_rate": 3.6452827261681905e-07,
      "logits/policy_chosen": -1.5570166110992432,
      "logits/policy_rejected": -1.8741226196289062,
      "logits/reference_chosen": -2.7946345806121826,
      "logits/reference_rejected": -2.6603751182556152,
      "logps/policy_chosen": -102.12774658203125,
      "logps/policy_rejected": -781.8934326171875,
      "logps/reference_chosen": -413.195068359375,
      "logps/reference_rejected": -280.4862060546875,
      "loss": 0.1318,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.110673189163208,
      "rewards/margins": 8.12474536895752,
      "rewards/rejected": -5.014072418212891,
      "step": 515,
      "tokens/chosen_response_len_mean": 557.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 257.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.41,
      "grad_norm": 6.056850986954676,
      "learning_rate": 3.639039296287626e-07,
      "logits/policy_chosen": -1.9515093564987183,
      "logits/policy_rejected": -1.9906623363494873,
      "logits/reference_chosen": -2.848447799682617,
      "logits/reference_rejected": -2.8842453956604004,
      "logps/policy_chosen": -52.22584533691406,
      "logps/policy_rejected": -414.292236328125,
      "logps/reference_chosen": -122.67705535888672,
      "logps/reference_rejected": -188.04884338378906,
      "loss": 0.2483,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7045120596885681,
      "rewards/margins": 2.9669461250305176,
      "rewards/rejected": -2.2624340057373047,
      "step": 516,
      "tokens/chosen_response_len_mean": 188.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 201.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.42,
      "grad_norm": 11.95464782987471,
      "learning_rate": 3.632786888440276e-07,
      "logits/policy_chosen": -1.120755672454834,
      "logits/policy_rejected": -1.3444613218307495,
      "logits/reference_chosen": -2.315535068511963,
      "logits/reference_rejected": -1.9943441152572632,
      "logps/policy_chosen": -161.4275665283203,
      "logps/policy_rejected": -299.89044189453125,
      "logps/reference_chosen": -162.89154052734375,
      "logps/reference_rejected": -136.308837890625,
      "loss": 0.2366,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.014639586210250854,
      "rewards/margins": 1.6504554748535156,
      "rewards/rejected": -1.635815978050232,
      "step": 517,
      "tokens/chosen_response_len_mean": 170.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 93.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.42,
      "grad_norm": 2.7229706023132185,
      "learning_rate": 3.626525551907943e-07,
      "logits/policy_chosen": -1.5620253086090088,
      "logits/policy_rejected": -1.9047589302062988,
      "logits/reference_chosen": -2.9676084518432617,
      "logits/reference_rejected": -2.906032085418701,
      "logps/policy_chosen": -76.9676513671875,
      "logps/policy_rejected": -1110.067138671875,
      "logps/reference_chosen": -370.51678466796875,
      "logps/reference_rejected": -493.9027404785156,
      "loss": 0.1732,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9354913234710693,
      "rewards/margins": 9.097135543823242,
      "rewards/rejected": -6.1616435050964355,
      "step": 518,
      "tokens/chosen_response_len_mean": 536.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 418.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.42,
      "grad_norm": 2.9703047044421966,
      "learning_rate": 3.6202553360428073e-07,
      "logits/policy_chosen": -1.7502543926239014,
      "logits/policy_rejected": -1.7094565629959106,
      "logits/reference_chosen": -2.921245813369751,
      "logits/reference_rejected": -2.6325058937072754,
      "logps/policy_chosen": -76.24668884277344,
      "logps/policy_rejected": -555.4454345703125,
      "logps/reference_chosen": -241.67083740234375,
      "logps/reference_rejected": -261.66253662109375,
      "loss": 0.148,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6542415618896484,
      "rewards/margins": 4.592070579528809,
      "rewards/rejected": -2.937828779220581,
      "step": 519,
      "tokens/chosen_response_len_mean": 252.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 184.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.42,
      "grad_norm": 4.884698463902552,
      "learning_rate": 3.613976290267036e-07,
      "logits/policy_chosen": -1.9753375053405762,
      "logits/policy_rejected": -1.881452202796936,
      "logits/reference_chosen": -2.7441306114196777,
      "logits/reference_rejected": -2.6066412925720215,
      "logps/policy_chosen": -51.04777908325195,
      "logps/policy_rejected": -328.26580810546875,
      "logps/reference_chosen": -214.8020782470703,
      "logps/reference_rejected": -210.0876922607422,
      "loss": 0.178,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.637542963027954,
      "rewards/margins": 2.819324016571045,
      "rewards/rejected": -1.1817810535430908,
      "step": 520,
      "tokens/chosen_response_len_mean": 246.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 196.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.42,
      "grad_norm": 3.0762379085174483,
      "learning_rate": 3.607688464072391e-07,
      "logits/policy_chosen": -2.251037120819092,
      "logits/policy_rejected": -2.2533721923828125,
      "logits/reference_chosen": -2.8437530994415283,
      "logits/reference_rejected": -2.9152119159698486,
      "logps/policy_chosen": -20.297204971313477,
      "logps/policy_rejected": -200.4241943359375,
      "logps/reference_chosen": -22.331905364990234,
      "logps/reference_rejected": -76.84808349609375,
      "loss": 0.3692,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.02034701034426689,
      "rewards/margins": 1.256108045578003,
      "rewards/rejected": -1.235761046409607,
      "step": 521,
      "tokens/chosen_response_len_mean": 50.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 60.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.42,
      "grad_norm": 2.5982426720006315,
      "learning_rate": 3.601391907019846e-07,
      "logits/policy_chosen": -1.851576328277588,
      "logits/policy_rejected": -1.9099481105804443,
      "logits/reference_chosen": -2.8298428058624268,
      "logits/reference_rejected": -2.821462392807007,
      "logps/policy_chosen": -36.91337966918945,
      "logps/policy_rejected": -641.5531616210938,
      "logps/reference_chosen": -145.30227661132812,
      "logps/reference_rejected": -243.65882873535156,
      "loss": 0.1578,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0838890075683594,
      "rewards/margins": 5.062832832336426,
      "rewards/rejected": -3.9789438247680664,
      "step": 522,
      "tokens/chosen_response_len_mean": 200.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 199.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.42,
      "grad_norm": 1.8896404887187468,
      "learning_rate": 3.5950866687391904e-07,
      "logits/policy_chosen": -1.6377341747283936,
      "logits/policy_rejected": -1.7143683433532715,
      "logits/reference_chosen": -3.051149845123291,
      "logits/reference_rejected": -3.0039615631103516,
      "logps/policy_chosen": -100.38955688476562,
      "logps/policy_rejected": -1112.8839111328125,
      "logps/reference_chosen": -462.146240234375,
      "logps/reference_rejected": -452.32257080078125,
      "loss": 0.1453,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.6175670623779297,
      "rewards/margins": 10.22317886352539,
      "rewards/rejected": -6.605612754821777,
      "step": 523,
      "tokens/chosen_response_len_mean": 533.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 426.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.42,
      "grad_norm": 3.85893339649937,
      "learning_rate": 3.588772798928639e-07,
      "logits/policy_chosen": -1.7455567121505737,
      "logits/policy_rejected": -1.070192575454712,
      "logits/reference_chosen": -2.819988250732422,
      "logits/reference_rejected": -2.4435606002807617,
      "logps/policy_chosen": -96.84449768066406,
      "logps/policy_rejected": -773.0043334960938,
      "logps/reference_chosen": -285.88970947265625,
      "logps/reference_rejected": -374.89764404296875,
      "loss": 0.1507,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8904521465301514,
      "rewards/margins": 5.871518611907959,
      "rewards/rejected": -3.981066942214966,
      "step": 524,
      "tokens/chosen_response_len_mean": 242.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 518.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.42,
      "grad_norm": 4.331211793409003,
      "learning_rate": 3.5824503473544404e-07,
      "logits/policy_chosen": -2.2218847274780273,
      "logits/policy_rejected": -2.1853442192077637,
      "logits/reference_chosen": -2.5698864459991455,
      "logits/reference_rejected": -3.0179224014282227,
      "logps/policy_chosen": -63.2376708984375,
      "logps/policy_rejected": -813.5751953125,
      "logps/reference_chosen": -264.51507568359375,
      "logps/reference_rejected": -337.98345947265625,
      "loss": 0.2322,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0127739906311035,
      "rewards/margins": 6.768691062927246,
      "rewards/rejected": -4.755917072296143,
      "step": 525,
      "tokens/chosen_response_len_mean": 248.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 227.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.42,
      "grad_norm": 2.35262408959955,
      "learning_rate": 3.576119363850486e-07,
      "logits/policy_chosen": -0.8806135058403015,
      "logits/policy_rejected": -1.4104909896850586,
      "logits/reference_chosen": -2.685579538345337,
      "logits/reference_rejected": -2.5388383865356445,
      "logps/policy_chosen": -143.42222595214844,
      "logps/policy_rejected": -1747.0230712890625,
      "logps/reference_chosen": -675.3695678710938,
      "logps/reference_rejected": -733.340087890625,
      "loss": 0.1871,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 5.319473743438721,
      "rewards/margins": 15.456302642822266,
      "rewards/rejected": -10.136829376220703,
      "step": 526,
      "tokens/chosen_response_len_mean": 767.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 535.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.42,
      "grad_norm": 2.777012028581217,
      "learning_rate": 3.569779898317916e-07,
      "logits/policy_chosen": -1.0484815835952759,
      "logits/policy_rejected": -1.383299708366394,
      "logits/reference_chosen": -2.7294254302978516,
      "logits/reference_rejected": -2.43225359916687,
      "logps/policy_chosen": -80.23799133300781,
      "logps/policy_rejected": -357.5889892578125,
      "logps/reference_chosen": -263.91094970703125,
      "logps/reference_rejected": -181.80218505859375,
      "loss": 0.1579,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8367295265197754,
      "rewards/margins": 3.594597339630127,
      "rewards/rejected": -1.757867693901062,
      "step": 527,
      "tokens/chosen_response_len_mean": 227.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 310.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.42,
      "grad_norm": 4.126765018045025,
      "learning_rate": 3.5634320007247266e-07,
      "logits/policy_chosen": -1.517538070678711,
      "logits/policy_rejected": -1.6733059883117676,
      "logits/reference_chosen": -2.977997303009033,
      "logits/reference_rejected": -2.5001871585845947,
      "logps/policy_chosen": -81.98285675048828,
      "logps/policy_rejected": -884.9750366210938,
      "logps/reference_chosen": -323.5049133300781,
      "logps/reference_rejected": -331.4570007324219,
      "loss": 0.1536,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4152204990386963,
      "rewards/margins": 7.950399875640869,
      "rewards/rejected": -5.53518009185791,
      "step": 528,
      "tokens/chosen_response_len_mean": 334.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 258.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.43,
      "grad_norm": 4.3140614212184,
      "learning_rate": 3.5570757211053746e-07,
      "logits/policy_chosen": -1.7487778663635254,
      "logits/policy_rejected": -1.9494415521621704,
      "logits/reference_chosen": -2.7526111602783203,
      "logits/reference_rejected": -2.7956008911132812,
      "logps/policy_chosen": -75.49141693115234,
      "logps/policy_rejected": -642.2836303710938,
      "logps/reference_chosen": -223.76815795898438,
      "logps/reference_rejected": -289.1028747558594,
      "loss": 0.2311,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4827674627304077,
      "rewards/margins": 5.014575004577637,
      "rewards/rejected": -3.5318078994750977,
      "step": 529,
      "tokens/chosen_response_len_mean": 403.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 282.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.43,
      "grad_norm": 3.2208819689580346,
      "learning_rate": 3.5507111095603864e-07,
      "logits/policy_chosen": -1.433916449546814,
      "logits/policy_rejected": -1.5399501323699951,
      "logits/reference_chosen": -2.9575793743133545,
      "logits/reference_rejected": -2.827263832092285,
      "logps/policy_chosen": -71.69790649414062,
      "logps/policy_rejected": -963.1092529296875,
      "logps/reference_chosen": -357.26605224609375,
      "logps/reference_rejected": -355.61376953125,
      "loss": 0.1052,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8556814193725586,
      "rewards/margins": 8.930635452270508,
      "rewards/rejected": -6.074954986572266,
      "step": 530,
      "tokens/chosen_response_len_mean": 459.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 357.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.43,
      "grad_norm": 4.734767784216982,
      "learning_rate": 3.5443382162559616e-07,
      "logits/policy_chosen": -1.489007830619812,
      "logits/policy_rejected": -1.7155470848083496,
      "logits/reference_chosen": -2.6674938201904297,
      "logits/reference_rejected": -2.629805564880371,
      "logps/policy_chosen": -72.16195678710938,
      "logps/policy_rejected": -1126.726806640625,
      "logps/reference_chosen": -293.7249450683594,
      "logps/reference_rejected": -388.15594482421875,
      "loss": 0.1458,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.215629816055298,
      "rewards/margins": 9.601338386535645,
      "rewards/rejected": -7.385707855224609,
      "step": 531,
      "tokens/chosen_response_len_mean": 492.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 288.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.43,
      "grad_norm": 4.062770893161743,
      "learning_rate": 3.537957091423573e-07,
      "logits/policy_chosen": -1.6025749444961548,
      "logits/policy_rejected": -1.8184247016906738,
      "logits/reference_chosen": -2.8992581367492676,
      "logits/reference_rejected": -2.863769292831421,
      "logps/policy_chosen": -72.07038116455078,
      "logps/policy_rejected": -534.27880859375,
      "logps/reference_chosen": -346.4698181152344,
      "logps/reference_rejected": -317.3559265136719,
      "loss": 0.1768,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7439944744110107,
      "rewards/margins": 4.9132232666015625,
      "rewards/rejected": -2.1692287921905518,
      "step": 532,
      "tokens/chosen_response_len_mean": 494.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 393.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.43,
      "grad_norm": 3.0965215425461112,
      "learning_rate": 3.531567785359578e-07,
      "logits/policy_chosen": -1.337218999862671,
      "logits/policy_rejected": -1.681422233581543,
      "logits/reference_chosen": -2.8761448860168457,
      "logits/reference_rejected": -2.548964500427246,
      "logps/policy_chosen": -104.7605972290039,
      "logps/policy_rejected": -1222.03125,
      "logps/reference_chosen": -401.869384765625,
      "logps/reference_rejected": -439.6533203125,
      "loss": 0.0585,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.97108793258667,
      "rewards/margins": 10.794868469238281,
      "rewards/rejected": -7.823780059814453,
      "step": 533,
      "tokens/chosen_response_len_mean": 620.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 326.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.43,
      "grad_norm": 3.4215397082628933,
      "learning_rate": 3.5251703484248197e-07,
      "logits/policy_chosen": -2.181457042694092,
      "logits/policy_rejected": -2.2668938636779785,
      "logits/reference_chosen": -3.1640498638153076,
      "logits/reference_rejected": -3.120495319366455,
      "logps/policy_chosen": -51.236385345458984,
      "logps/policy_rejected": -749.4049072265625,
      "logps/reference_chosen": -239.9842071533203,
      "logps/reference_rejected": -316.8190002441406,
      "loss": 0.1635,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8874783515930176,
      "rewards/margins": 6.213336944580078,
      "rewards/rejected": -4.325859069824219,
      "step": 534,
      "tokens/chosen_response_len_mean": 351.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 305.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.43,
      "grad_norm": 2.4924709433925214,
      "learning_rate": 3.5187648310442275e-07,
      "logits/policy_chosen": -1.5884766578674316,
      "logits/policy_rejected": -1.7490499019622803,
      "logits/reference_chosen": -2.9638869762420654,
      "logits/reference_rejected": -2.017496347427368,
      "logps/policy_chosen": -70.10679626464844,
      "logps/policy_rejected": -757.836181640625,
      "logps/reference_chosen": -307.5505676269531,
      "logps/reference_rejected": -270.78619384765625,
      "loss": 0.048,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.3744373321533203,
      "rewards/margins": 7.244937419891357,
      "rewards/rejected": -4.870500087738037,
      "step": 535,
      "tokens/chosen_response_len_mean": 401.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 258.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.43,
      "grad_norm": 6.78416162381088,
      "learning_rate": 3.512351283706419e-07,
      "logits/policy_chosen": -1.5578632354736328,
      "logits/policy_rejected": -2.071242332458496,
      "logits/reference_chosen": -2.528616189956665,
      "logits/reference_rejected": -2.0685698986053467,
      "logps/policy_chosen": -111.4025650024414,
      "logps/policy_rejected": -205.73956298828125,
      "logps/reference_chosen": -196.06881713867188,
      "logps/reference_rejected": -94.46316528320312,
      "loss": 0.2365,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8466625213623047,
      "rewards/margins": 1.9594265222549438,
      "rewards/rejected": -1.1127640008926392,
      "step": 536,
      "tokens/chosen_response_len_mean": 199.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 56.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.43,
      "grad_norm": 4.660378488387965,
      "learning_rate": 3.5059297569633097e-07,
      "logits/policy_chosen": -1.8261831998825073,
      "logits/policy_rejected": -1.908783197402954,
      "logits/reference_chosen": -3.044473886489868,
      "logits/reference_rejected": -2.850213050842285,
      "logps/policy_chosen": -51.796966552734375,
      "logps/policy_rejected": -732.04638671875,
      "logps/reference_chosen": -249.64366149902344,
      "logps/reference_rejected": -291.7880554199219,
      "loss": 0.2116,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9784669876098633,
      "rewards/margins": 6.3810505867004395,
      "rewards/rejected": -4.402584075927734,
      "step": 537,
      "tokens/chosen_response_len_mean": 324.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 335.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.43,
      "grad_norm": 3.054106127740481,
      "learning_rate": 3.499500301429704e-07,
      "logits/policy_chosen": -1.2564502954483032,
      "logits/policy_rejected": -1.4578382968902588,
      "logits/reference_chosen": -2.663343906402588,
      "logits/reference_rejected": -2.322813034057617,
      "logps/policy_chosen": -85.20794677734375,
      "logps/policy_rejected": -1228.8956298828125,
      "logps/reference_chosen": -390.68182373046875,
      "logps/reference_rejected": -474.4229431152344,
      "loss": 0.1993,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.054738998413086,
      "rewards/margins": 10.599466323852539,
      "rewards/rejected": -7.544726848602295,
      "step": 538,
      "tokens/chosen_response_len_mean": 494.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 411.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.43,
      "grad_norm": 7.6059828448808044,
      "learning_rate": 3.493062967782906e-07,
      "logits/policy_chosen": -0.7277638912200928,
      "logits/policy_rejected": -0.8370018005371094,
      "logits/reference_chosen": -2.0911202430725098,
      "logits/reference_rejected": -1.8740336894989014,
      "logps/policy_chosen": -103.84000396728516,
      "logps/policy_rejected": -831.5616455078125,
      "logps/reference_chosen": -346.20648193359375,
      "logps/reference_rejected": -328.0775146484375,
      "loss": 0.1528,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4236645698547363,
      "rewards/margins": 7.458505630493164,
      "rewards/rejected": -5.0348405838012695,
      "step": 539,
      "tokens/chosen_response_len_mean": 461.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 337.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.43,
      "grad_norm": 8.525267798728615,
      "learning_rate": 3.486617806762312e-07,
      "logits/policy_chosen": -0.5032968521118164,
      "logits/policy_rejected": -1.4690107107162476,
      "logits/reference_chosen": -2.4321441650390625,
      "logits/reference_rejected": -2.2443180084228516,
      "logps/policy_chosen": -72.79258728027344,
      "logps/policy_rejected": -526.5192260742188,
      "logps/reference_chosen": -149.5489501953125,
      "logps/reference_rejected": -212.32266235351562,
      "loss": 0.1606,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7675636410713196,
      "rewards/margins": 3.909529209136963,
      "rewards/rejected": -3.141965866088867,
      "step": 540,
      "tokens/chosen_response_len_mean": 526.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 200.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.43,
      "grad_norm": 4.1846759440066155,
      "learning_rate": 3.480164869169015e-07,
      "logits/policy_chosen": -1.1990872621536255,
      "logits/policy_rejected": -1.3610371351242065,
      "logits/reference_chosen": -2.9251720905303955,
      "logits/reference_rejected": -2.4389283657073975,
      "logps/policy_chosen": -67.39421081542969,
      "logps/policy_rejected": -1102.8447265625,
      "logps/reference_chosen": -322.8033752441406,
      "logps/reference_rejected": -450.9704284667969,
      "loss": 0.1251,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.554091453552246,
      "rewards/margins": 9.072834014892578,
      "rewards/rejected": -6.518742561340332,
      "step": 541,
      "tokens/chosen_response_len_mean": 437.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 404.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.44,
      "grad_norm": 3.4834496334196907,
      "learning_rate": 3.473704205865404e-07,
      "logits/policy_chosen": -1.9992713928222656,
      "logits/policy_rejected": -2.140188217163086,
      "logits/reference_chosen": -3.0482730865478516,
      "logits/reference_rejected": -2.758751630783081,
      "logps/policy_chosen": -31.598953247070312,
      "logps/policy_rejected": -498.78118896484375,
      "logps/reference_chosen": -114.72731018066406,
      "logps/reference_rejected": -151.8946533203125,
      "loss": 0.2659,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.8312835693359375,
      "rewards/margins": 4.3001484870910645,
      "rewards/rejected": -3.468865156173706,
      "step": 542,
      "tokens/chosen_response_len_mean": 159.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 135.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.44,
      "grad_norm": 2.212606925320298,
      "learning_rate": 3.4672358677747625e-07,
      "logits/policy_chosen": -1.3873353004455566,
      "logits/policy_rejected": -1.5324995517730713,
      "logits/reference_chosen": -3.0306363105773926,
      "logits/reference_rejected": -2.1667964458465576,
      "logps/policy_chosen": -103.60790252685547,
      "logps/policy_rejected": -1208.731201171875,
      "logps/reference_chosen": -449.542236328125,
      "logps/reference_rejected": -515.6892700195312,
      "loss": 0.0792,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.459343671798706,
      "rewards/margins": 10.389762878417969,
      "rewards/rejected": -6.930418968200684,
      "step": 543,
      "tokens/chosen_response_len_mean": 564.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 469.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.44,
      "grad_norm": 2.8688314854304138,
      "learning_rate": 3.460759905880866e-07,
      "logits/policy_chosen": -1.4521598815917969,
      "logits/policy_rejected": -1.5567327737808228,
      "logits/reference_chosen": -2.6564087867736816,
      "logits/reference_rejected": -2.7792372703552246,
      "logps/policy_chosen": -65.16630554199219,
      "logps/policy_rejected": -918.9058227539062,
      "logps/reference_chosen": -256.8858337402344,
      "logps/reference_rejected": -348.1164855957031,
      "loss": 0.1628,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.917195200920105,
      "rewards/margins": 7.625089168548584,
      "rewards/rejected": -5.707893371582031,
      "step": 544,
      "tokens/chosen_response_len_mean": 356.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 320.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.44,
      "grad_norm": 3.943022352216706,
      "learning_rate": 3.454276371227583e-07,
      "logits/policy_chosen": -1.2155468463897705,
      "logits/policy_rejected": -1.6266111135482788,
      "logits/reference_chosen": -2.840851306915283,
      "logits/reference_rejected": -1.8044722080230713,
      "logps/policy_chosen": -62.91926574707031,
      "logps/policy_rejected": -384.3448486328125,
      "logps/reference_chosen": -226.05123901367188,
      "logps/reference_rejected": -133.25270080566406,
      "loss": 0.0916,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6313196420669556,
      "rewards/margins": 4.142241477966309,
      "rewards/rejected": -2.5109214782714844,
      "step": 545,
      "tokens/chosen_response_len_mean": 218.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 80.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.44,
      "grad_norm": 3.627490010579694,
      "learning_rate": 3.4477853149184685e-07,
      "logits/policy_chosen": -1.5325663089752197,
      "logits/policy_rejected": -1.6806870698928833,
      "logits/reference_chosen": -2.869450092315674,
      "logits/reference_rejected": -2.7260353565216064,
      "logps/policy_chosen": -82.05609130859375,
      "logps/policy_rejected": -1196.469482421875,
      "logps/reference_chosen": -392.67578125,
      "logps/reference_rejected": -487.3499450683594,
      "loss": 0.127,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.106196880340576,
      "rewards/margins": 10.197393417358398,
      "rewards/rejected": -7.091196060180664,
      "step": 546,
      "tokens/chosen_response_len_mean": 403.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 424.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.44,
      "grad_norm": 0.7804414379698411,
      "learning_rate": 3.4412867881163647e-07,
      "logits/policy_chosen": -1.7521880865097046,
      "logits/policy_rejected": -2.07790470123291,
      "logits/reference_chosen": -2.941479206085205,
      "logits/reference_rejected": -2.718097686767578,
      "logps/policy_chosen": -57.52407455444336,
      "logps/policy_rejected": -713.60693359375,
      "logps/reference_chosen": -230.14796447753906,
      "logps/reference_rejected": -235.18714904785156,
      "loss": 0.0889,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 1.7262389659881592,
      "rewards/margins": 6.51043701171875,
      "rewards/rejected": -4.784197807312012,
      "step": 547,
      "tokens/chosen_response_len_mean": 313.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 193.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.44,
      "grad_norm": 4.318963335125819,
      "learning_rate": 3.4347808420429983e-07,
      "logits/policy_chosen": -1.4977705478668213,
      "logits/policy_rejected": -2.0579535961151123,
      "logits/reference_chosen": -2.85351300239563,
      "logits/reference_rejected": -2.360363483428955,
      "logps/policy_chosen": -68.09393310546875,
      "logps/policy_rejected": -729.8070678710938,
      "logps/reference_chosen": -348.97772216796875,
      "logps/reference_rejected": -246.0881805419922,
      "loss": 0.1897,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.808837890625,
      "rewards/margins": 7.646027565002441,
      "rewards/rejected": -4.837189197540283,
      "step": 548,
      "tokens/chosen_response_len_mean": 429.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 146.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.44,
      "grad_norm": 5.084275121944622,
      "learning_rate": 3.428267527978572e-07,
      "logits/policy_chosen": -1.1892602443695068,
      "logits/policy_rejected": -1.1026020050048828,
      "logits/reference_chosen": -2.2982800006866455,
      "logits/reference_rejected": -2.147494077682495,
      "logps/policy_chosen": -132.86444091796875,
      "logps/policy_rejected": -1389.264892578125,
      "logps/reference_chosen": -328.26373291015625,
      "logps/reference_rejected": -649.0513305664062,
      "loss": 0.1469,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9539929628372192,
      "rewards/margins": 9.35612964630127,
      "rewards/rejected": -7.402136325836182,
      "step": 549,
      "tokens/chosen_response_len_mean": 302.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 337.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.44,
      "grad_norm": 3.334428861517296,
      "learning_rate": 3.4217468972613664e-07,
      "logits/policy_chosen": -1.2716220617294312,
      "logits/policy_rejected": -1.8428655862808228,
      "logits/reference_chosen": -2.549618721008301,
      "logits/reference_rejected": -2.3221817016601562,
      "logps/policy_chosen": -104.29700469970703,
      "logps/policy_rejected": -890.2571411132812,
      "logps/reference_chosen": -274.2265319824219,
      "logps/reference_rejected": -251.96420288085938,
      "loss": 0.1143,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6992954015731812,
      "rewards/margins": 8.08222484588623,
      "rewards/rejected": -6.382929801940918,
      "step": 550,
      "tokens/chosen_response_len_mean": 330.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 180.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.44,
      "grad_norm": 4.41139083624512,
      "learning_rate": 3.4152190012873293e-07,
      "logits/policy_chosen": -2.13014554977417,
      "logits/policy_rejected": -2.149657964706421,
      "logits/reference_chosen": -3.209202766418457,
      "logits/reference_rejected": -2.981388568878174,
      "logps/policy_chosen": -28.465259552001953,
      "logps/policy_rejected": -466.3083801269531,
      "logps/reference_chosen": -141.8134002685547,
      "logps/reference_rejected": -196.63211059570312,
      "loss": 0.2468,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1334813833236694,
      "rewards/margins": 3.830244302749634,
      "rewards/rejected": -2.696762800216675,
      "step": 551,
      "tokens/chosen_response_len_mean": 228.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 190.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.44,
      "grad_norm": 2.099908908503231,
      "learning_rate": 3.408683891509676e-07,
      "logits/policy_chosen": -1.3221943378448486,
      "logits/policy_rejected": -1.345398187637329,
      "logits/reference_chosen": -2.5638017654418945,
      "logits/reference_rejected": -2.4357707500457764,
      "logps/policy_chosen": -93.87046813964844,
      "logps/policy_rejected": -758.2496337890625,
      "logps/reference_chosen": -381.4332580566406,
      "logps/reference_rejected": -368.74755859375,
      "loss": 0.097,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.8756279945373535,
      "rewards/margins": 6.77064847946167,
      "rewards/rejected": -3.8950207233428955,
      "step": 552,
      "tokens/chosen_response_len_mean": 451.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 293.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.44,
      "grad_norm": 4.86129928709054,
      "learning_rate": 3.402141619438481e-07,
      "logits/policy_chosen": -1.3816745281219482,
      "logits/policy_rejected": -1.6162673234939575,
      "logits/reference_chosen": -3.17985200881958,
      "logits/reference_rejected": -2.3312668800354004,
      "logps/policy_chosen": -106.40277099609375,
      "logps/policy_rejected": -942.629150390625,
      "logps/reference_chosen": -552.941650390625,
      "logps/reference_rejected": -361.9104309082031,
      "loss": 0.1473,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 4.465388774871826,
      "rewards/margins": 10.272576332092285,
      "rewards/rejected": -5.807187080383301,
      "step": 553,
      "tokens/chosen_response_len_mean": 547.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 279.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.45,
      "grad_norm": 4.8879952962699775,
      "learning_rate": 3.395592236640271e-07,
      "logits/policy_chosen": -1.2729833126068115,
      "logits/policy_rejected": -1.55487859249115,
      "logits/reference_chosen": -2.8120789527893066,
      "logits/reference_rejected": -2.658278465270996,
      "logps/policy_chosen": -103.52959442138672,
      "logps/policy_rejected": -777.389892578125,
      "logps/reference_chosen": -442.6094970703125,
      "logps/reference_rejected": -439.10894775390625,
      "loss": 0.1893,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.390798568725586,
      "rewards/margins": 6.773608207702637,
      "rewards/rejected": -3.3828094005584717,
      "step": 554,
      "tokens/chosen_response_len_mean": 477.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 338.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.45,
      "grad_norm": 1.5060066913035608,
      "learning_rate": 3.3890357947376216e-07,
      "logits/policy_chosen": -0.9478991627693176,
      "logits/policy_rejected": -1.2323962450027466,
      "logits/reference_chosen": -2.575098991394043,
      "logits/reference_rejected": -2.5877485275268555,
      "logps/policy_chosen": -76.81658935546875,
      "logps/policy_rejected": -733.058349609375,
      "logps/reference_chosen": -336.38763427734375,
      "logps/reference_rejected": -301.5869445800781,
      "loss": 0.0939,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.5957107543945312,
      "rewards/margins": 6.91042423248291,
      "rewards/rejected": -4.314713478088379,
      "step": 555,
      "tokens/chosen_response_len_mean": 316.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 241.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.45,
      "grad_norm": 2.2952233428985993,
      "learning_rate": 3.3824723454087466e-07,
      "logits/policy_chosen": -2.273240327835083,
      "logits/policy_rejected": -2.196145534515381,
      "logits/reference_chosen": -2.9893598556518555,
      "logits/reference_rejected": -2.8801779747009277,
      "logps/policy_chosen": -49.751075744628906,
      "logps/policy_rejected": -437.60345458984375,
      "logps/reference_chosen": -178.29608154296875,
      "logps/reference_rejected": -173.517822265625,
      "loss": 0.1492,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2854499816894531,
      "rewards/margins": 3.9263062477111816,
      "rewards/rejected": -2.6408560276031494,
      "step": 556,
      "tokens/chosen_response_len_mean": 164.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 125.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.45,
      "grad_norm": 3.6563082959344984,
      "learning_rate": 3.375901940387095e-07,
      "logits/policy_chosen": -1.857072353363037,
      "logits/policy_rejected": -1.9487085342407227,
      "logits/reference_chosen": -2.9948766231536865,
      "logits/reference_rejected": -2.9093077182769775,
      "logps/policy_chosen": -51.00886535644531,
      "logps/policy_rejected": -1076.950927734375,
      "logps/reference_chosen": -271.2162170410156,
      "logps/reference_rejected": -428.81280517578125,
      "loss": 0.1001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.202073574066162,
      "rewards/margins": 8.683454513549805,
      "rewards/rejected": -6.481380462646484,
      "step": 557,
      "tokens/chosen_response_len_mean": 512.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 450.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.45,
      "grad_norm": 5.966839389680779,
      "learning_rate": 3.3693246314609405e-07,
      "logits/policy_chosen": -1.799519658088684,
      "logits/policy_rejected": -1.884425163269043,
      "logits/reference_chosen": -2.5967020988464355,
      "logits/reference_rejected": -2.655183792114258,
      "logps/policy_chosen": -75.13899230957031,
      "logps/policy_rejected": -493.25543212890625,
      "logps/reference_chosen": -187.2158660888672,
      "logps/reference_rejected": -269.1056213378906,
      "loss": 0.1943,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.120768666267395,
      "rewards/margins": 3.3622663021087646,
      "rewards/rejected": -2.241497755050659,
      "step": 558,
      "tokens/chosen_response_len_mean": 243.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 304.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.45,
      "grad_norm": 2.362372912667138,
      "learning_rate": 3.3627404704729705e-07,
      "logits/policy_chosen": -1.9872490167617798,
      "logits/policy_rejected": -2.284727096557617,
      "logits/reference_chosen": -3.1214911937713623,
      "logits/reference_rejected": -2.2349143028259277,
      "logps/policy_chosen": -63.00129699707031,
      "logps/policy_rejected": -732.4915771484375,
      "logps/reference_chosen": -241.21514892578125,
      "logps/reference_rejected": -271.373046875,
      "loss": 0.0322,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7821385860443115,
      "rewards/margins": 6.393324851989746,
      "rewards/rejected": -4.611185550689697,
      "step": 559,
      "tokens/chosen_response_len_mean": 331.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 149.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.45,
      "grad_norm": 4.171078952463684,
      "learning_rate": 3.3561495093198853e-07,
      "logits/policy_chosen": -2.051931381225586,
      "logits/policy_rejected": -2.1366114616394043,
      "logits/reference_chosen": -2.9525604248046875,
      "logits/reference_rejected": -2.8414297103881836,
      "logps/policy_chosen": -76.86653137207031,
      "logps/policy_rejected": -667.3883056640625,
      "logps/reference_chosen": -323.14581298828125,
      "logps/reference_rejected": -302.495361328125,
      "loss": 0.1182,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.4627926349639893,
      "rewards/margins": 6.111721515655518,
      "rewards/rejected": -3.6489288806915283,
      "step": 560,
      "tokens/chosen_response_len_mean": 368.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 254.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.45,
      "grad_norm": 5.346338548620434,
      "learning_rate": 3.349551799951982e-07,
      "logits/policy_chosen": -1.6533997058868408,
      "logits/policy_rejected": -1.9753875732421875,
      "logits/reference_chosen": -2.9736459255218506,
      "logits/reference_rejected": -1.8335206508636475,
      "logps/policy_chosen": -97.565673828125,
      "logps/policy_rejected": -1323.7308349609375,
      "logps/reference_chosen": -469.5448303222656,
      "logps/reference_rejected": -511.6222839355469,
      "loss": 0.2207,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.7197916507720947,
      "rewards/margins": 11.840876579284668,
      "rewards/rejected": -8.121084213256836,
      "step": 561,
      "tokens/chosen_response_len_mean": 480.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 301.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.45,
      "grad_norm": 3.9887993874835153,
      "learning_rate": 3.342947394372748e-07,
      "logits/policy_chosen": -1.3492289781570435,
      "logits/policy_rejected": -1.6602319478988647,
      "logits/reference_chosen": -2.579238176345825,
      "logits/reference_rejected": -2.7436940670013428,
      "logps/policy_chosen": -95.8171157836914,
      "logps/policy_rejected": -857.45068359375,
      "logps/reference_chosen": -257.5375671386719,
      "logps/reference_rejected": -306.32049560546875,
      "loss": 0.1844,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.6172046661376953,
      "rewards/margins": 7.128506183624268,
      "rewards/rejected": -5.5113019943237305,
      "step": 562,
      "tokens/chosen_response_len_mean": 362.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 238.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.45,
      "grad_norm": 7.7967556197022985,
      "learning_rate": 3.336336344638451e-07,
      "logits/policy_chosen": -2.023709297180176,
      "logits/policy_rejected": -2.149707555770874,
      "logits/reference_chosen": -3.021341562271118,
      "logits/reference_rejected": -2.852924346923828,
      "logps/policy_chosen": -36.027626037597656,
      "logps/policy_rejected": -528.3580322265625,
      "logps/reference_chosen": -136.70562744140625,
      "logps/reference_rejected": -183.865478515625,
      "loss": 0.159,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0067799091339111,
      "rewards/margins": 4.451704978942871,
      "rewards/rejected": -3.444925308227539,
      "step": 563,
      "tokens/chosen_response_len_mean": 209.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 165.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.45,
      "grad_norm": 3.4940008793323725,
      "learning_rate": 3.329718702857725e-07,
      "logits/policy_chosen": -1.681413173675537,
      "logits/policy_rejected": -1.9605016708374023,
      "logits/reference_chosen": -2.742757797241211,
      "logits/reference_rejected": -2.6325631141662598,
      "logps/policy_chosen": -35.40129470825195,
      "logps/policy_rejected": -374.8088684082031,
      "logps/reference_chosen": -140.6432647705078,
      "logps/reference_rejected": -126.66143798828125,
      "loss": 0.1466,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.052419662475586,
      "rewards/margins": 3.5338940620422363,
      "rewards/rejected": -2.4814743995666504,
      "step": 564,
      "tokens/chosen_response_len_mean": 336.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 109.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.45,
      "grad_norm": 3.8421532020108424,
      "learning_rate": 3.323094521191169e-07,
      "logits/policy_chosen": -1.8261432647705078,
      "logits/policy_rejected": -2.011218309402466,
      "logits/reference_chosen": -2.932382822036743,
      "logits/reference_rejected": -2.2946062088012695,
      "logps/policy_chosen": -34.13972091674805,
      "logps/policy_rejected": -295.70648193359375,
      "logps/reference_chosen": -102.51885223388672,
      "logps/reference_rejected": -123.9435806274414,
      "loss": 0.2042,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6837913393974304,
      "rewards/margins": 2.4014205932617188,
      "rewards/rejected": -1.717629313468933,
      "step": 565,
      "tokens/chosen_response_len_mean": 122.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 63.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.45,
      "grad_norm": 1.763383100320544,
      "learning_rate": 3.3164638518509247e-07,
      "logits/policy_chosen": -1.8183183670043945,
      "logits/policy_rejected": -1.9676923751831055,
      "logits/reference_chosen": -2.9041481018066406,
      "logits/reference_rejected": -2.5814497470855713,
      "logps/policy_chosen": -70.70805358886719,
      "logps/policy_rejected": -785.9116821289062,
      "logps/reference_chosen": -337.09552001953125,
      "logps/reference_rejected": -348.2403259277344,
      "loss": 0.0697,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.663874387741089,
      "rewards/margins": 7.040587425231934,
      "rewards/rejected": -4.376713275909424,
      "step": 566,
      "tokens/chosen_response_len_mean": 409.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 374.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.46,
      "grad_norm": 3.044154747717625,
      "learning_rate": 3.309826747100271e-07,
      "logits/policy_chosen": -1.2097326517105103,
      "logits/policy_rejected": -1.195814847946167,
      "logits/reference_chosen": -2.543266534805298,
      "logits/reference_rejected": -2.23286509513855,
      "logps/policy_chosen": -92.23149871826172,
      "logps/policy_rejected": -1401.49169921875,
      "logps/reference_chosen": -402.17816162109375,
      "logps/reference_rejected": -493.5354309082031,
      "loss": 0.2147,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.099466562271118,
      "rewards/margins": 12.179028511047363,
      "rewards/rejected": -9.07956314086914,
      "step": 567,
      "tokens/chosen_response_len_mean": 520.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 467.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.46,
      "grad_norm": 4.410760829629216,
      "learning_rate": 3.303183259253213e-07,
      "logits/policy_chosen": -0.8860348463058472,
      "logits/policy_rejected": -1.001037359237671,
      "logits/reference_chosen": -2.3485236167907715,
      "logits/reference_rejected": -1.8914377689361572,
      "logps/policy_chosen": -89.13920593261719,
      "logps/policy_rejected": -532.52294921875,
      "logps/reference_chosen": -203.27642822265625,
      "logps/reference_rejected": -223.0554962158203,
      "loss": 0.1796,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1413720846176147,
      "rewards/margins": 4.23604679107666,
      "rewards/rejected": -3.0946743488311768,
      "step": 568,
      "tokens/chosen_response_len_mean": 243.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 179.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.46,
      "grad_norm": 1.552396994351016,
      "learning_rate": 3.296533440674066e-07,
      "logits/policy_chosen": -1.5108449459075928,
      "logits/policy_rejected": -1.6747286319732666,
      "logits/reference_chosen": -2.9551920890808105,
      "logits/reference_rejected": -2.360665798187256,
      "logps/policy_chosen": -102.8811264038086,
      "logps/policy_rejected": -753.2782592773438,
      "logps/reference_chosen": -381.5318908691406,
      "logps/reference_rejected": -324.1732177734375,
      "loss": 0.0119,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.7865076065063477,
      "rewards/margins": 7.077558517456055,
      "rewards/rejected": -4.291049957275391,
      "step": 569,
      "tokens/chosen_response_len_mean": 508.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 296.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.46,
      "grad_norm": 3.590001813627757,
      "learning_rate": 3.2898773437770445e-07,
      "logits/policy_chosen": -1.4371052980422974,
      "logits/policy_rejected": -1.5018061399459839,
      "logits/reference_chosen": -2.731325626373291,
      "logits/reference_rejected": -2.3503198623657227,
      "logps/policy_chosen": -123.0985107421875,
      "logps/policy_rejected": -596.8621215820312,
      "logps/reference_chosen": -243.1231689453125,
      "logps/reference_rejected": -256.818359375,
      "loss": 0.1762,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2002465724945068,
      "rewards/margins": 4.60068416595459,
      "rewards/rejected": -3.400438070297241,
      "step": 570,
      "tokens/chosen_response_len_mean": 264.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 231.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.46,
      "grad_norm": 7.919750161902402,
      "learning_rate": 3.2832150210258497e-07,
      "logits/policy_chosen": -1.5049259662628174,
      "logits/policy_rejected": -1.7426738739013672,
      "logits/reference_chosen": -2.7650415897369385,
      "logits/reference_rejected": -2.447714328765869,
      "logps/policy_chosen": -42.87800598144531,
      "logps/policy_rejected": -331.63720703125,
      "logps/reference_chosen": -154.05686950683594,
      "logps/reference_rejected": -152.7490234375,
      "loss": 0.1791,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.1117886304855347,
      "rewards/margins": 2.900670289993286,
      "rewards/rejected": -1.7888814210891724,
      "step": 571,
      "tokens/chosen_response_len_mean": 377.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 332.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.46,
      "grad_norm": 1.917555444007843,
      "learning_rate": 3.276546524933254e-07,
      "logits/policy_chosen": -1.7504053115844727,
      "logits/policy_rejected": -1.953632116317749,
      "logits/reference_chosen": -3.143728733062744,
      "logits/reference_rejected": -2.263713836669922,
      "logps/policy_chosen": -40.50579833984375,
      "logps/policy_rejected": -423.501953125,
      "logps/reference_chosen": -148.3907928466797,
      "logps/reference_rejected": -147.25958251953125,
      "loss": 0.0617,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.0788499116897583,
      "rewards/margins": 3.841273307800293,
      "rewards/rejected": -2.7624237537384033,
      "step": 572,
      "tokens/chosen_response_len_mean": 145.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 95.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.46,
      "grad_norm": 4.073751976946772,
      "learning_rate": 3.26987190806069e-07,
      "logits/policy_chosen": -2.2442588806152344,
      "logits/policy_rejected": -2.3009047508239746,
      "logits/reference_chosen": -2.8314037322998047,
      "logits/reference_rejected": -2.860301971435547,
      "logps/policy_chosen": -36.16373825073242,
      "logps/policy_rejected": -422.2955017089844,
      "logps/reference_chosen": -116.51701354980469,
      "logps/reference_rejected": -146.45513916015625,
      "loss": 0.2421,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.8035327196121216,
      "rewards/margins": 3.561936140060425,
      "rewards/rejected": -2.7584035396575928,
      "step": 573,
      "tokens/chosen_response_len_mean": 134.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 122.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.46,
      "grad_norm": 7.46200218393278,
      "learning_rate": 3.2631912230178346e-07,
      "logits/policy_chosen": -1.7136214971542358,
      "logits/policy_rejected": -1.9581921100616455,
      "logits/reference_chosen": -2.6153814792633057,
      "logits/reference_rejected": -2.5992605686187744,
      "logps/policy_chosen": -37.279937744140625,
      "logps/policy_rejected": -387.2350158691406,
      "logps/reference_chosen": -181.6397247314453,
      "logps/reference_rejected": -154.2986297607422,
      "loss": 0.1878,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4435977935791016,
      "rewards/margins": 3.7729616165161133,
      "rewards/rejected": -2.3293638229370117,
      "step": 574,
      "tokens/chosen_response_len_mean": 297.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 168.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.46,
      "grad_norm": 2.6169423458233707,
      "learning_rate": 3.256504522462192e-07,
      "logits/policy_chosen": -1.1096845865249634,
      "logits/policy_rejected": -1.3256019353866577,
      "logits/reference_chosen": -2.7639667987823486,
      "logits/reference_rejected": -2.4644720554351807,
      "logps/policy_chosen": -46.98308181762695,
      "logps/policy_rejected": -1111.00634765625,
      "logps/reference_chosen": -224.19473266601562,
      "logps/reference_rejected": -377.4888916015625,
      "loss": 0.1213,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7721165418624878,
      "rewards/margins": 9.107290267944336,
      "rewards/rejected": -7.335173606872559,
      "step": 575,
      "tokens/chosen_response_len_mean": 501.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 464.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.46,
      "grad_norm": 2.590884360662341,
      "learning_rate": 3.2498118590986834e-07,
      "logits/policy_chosen": -1.511143684387207,
      "logits/policy_rejected": -1.683800220489502,
      "logits/reference_chosen": -3.0181705951690674,
      "logits/reference_rejected": -2.4551892280578613,
      "logps/policy_chosen": -79.88076782226562,
      "logps/policy_rejected": -674.153076171875,
      "logps/reference_chosen": -353.041259765625,
      "logps/reference_rejected": -278.8591613769531,
      "loss": 0.1522,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.731604814529419,
      "rewards/margins": 6.684543609619141,
      "rewards/rejected": -3.9529387950897217,
      "step": 576,
      "tokens/chosen_response_len_mean": 438.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 309.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.46,
      "grad_norm": 4.698246145712581,
      "learning_rate": 3.243113285679229e-07,
      "logits/policy_chosen": -1.7451937198638916,
      "logits/policy_rejected": -2.0770750045776367,
      "logits/reference_chosen": -2.9442076683044434,
      "logits/reference_rejected": -2.5817737579345703,
      "logps/policy_chosen": -72.82100677490234,
      "logps/policy_rejected": -719.95068359375,
      "logps/reference_chosen": -280.47760009765625,
      "logps/reference_rejected": -286.4196472167969,
      "loss": 0.2206,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.076565742492676,
      "rewards/margins": 6.411876678466797,
      "rewards/rejected": -4.335310459136963,
      "step": 577,
      "tokens/chosen_response_len_mean": 438.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 251.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.46,
      "grad_norm": 4.28564428090641,
      "learning_rate": 3.236408855002332e-07,
      "logits/policy_chosen": -1.7372032403945923,
      "logits/policy_rejected": -1.841402292251587,
      "logits/reference_chosen": -2.99750018119812,
      "logits/reference_rejected": -2.673139810562134,
      "logps/policy_chosen": -65.78880310058594,
      "logps/policy_rejected": -780.2632446289062,
      "logps/reference_chosen": -245.94114685058594,
      "logps/reference_rejected": -289.25726318359375,
      "loss": 0.1362,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.8015235662460327,
      "rewards/margins": 6.711583137512207,
      "rewards/rejected": -4.910059452056885,
      "step": 578,
      "tokens/chosen_response_len_mean": 260.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 207.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.47,
      "grad_norm": 1.4802108177613418,
      "learning_rate": 3.229698619912661e-07,
      "logits/policy_chosen": -1.4670802354812622,
      "logits/policy_rejected": -1.5387755632400513,
      "logits/reference_chosen": -2.9978156089782715,
      "logits/reference_rejected": -2.9016990661621094,
      "logps/policy_chosen": -85.59793090820312,
      "logps/policy_rejected": -1163.843017578125,
      "logps/reference_chosen": -286.76727294921875,
      "logps/reference_rejected": -479.8865051269531,
      "loss": 0.0515,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.011693239212036,
      "rewards/margins": 8.85125732421875,
      "rewards/rejected": -6.839564323425293,
      "step": 579,
      "tokens/chosen_response_len_mean": 355.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 346.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.47,
      "grad_norm": 3.0129455363570234,
      "learning_rate": 3.222982633300637e-07,
      "logits/policy_chosen": -1.9126781225204468,
      "logits/policy_rejected": -1.9057201147079468,
      "logits/reference_chosen": -2.8861560821533203,
      "logits/reference_rejected": -2.677542209625244,
      "logps/policy_chosen": -66.1485824584961,
      "logps/policy_rejected": -712.9755249023438,
      "logps/reference_chosen": -232.5531768798828,
      "logps/reference_rejected": -276.8454895019531,
      "loss": 0.1144,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.664045810699463,
      "rewards/margins": 6.025345802307129,
      "rewards/rejected": -4.361299991607666,
      "step": 580,
      "tokens/chosen_response_len_mean": 284.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 148.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.47,
      "grad_norm": 0.7564855145783937,
      "learning_rate": 3.216260948102017e-07,
      "logits/policy_chosen": -1.4139569997787476,
      "logits/policy_rejected": -1.798340082168579,
      "logits/reference_chosen": -2.9086437225341797,
      "logits/reference_rejected": -2.6370975971221924,
      "logps/policy_chosen": -77.29658508300781,
      "logps/policy_rejected": -930.476806640625,
      "logps/reference_chosen": -295.6878967285156,
      "logps/reference_rejected": -320.7557678222656,
      "loss": 0.0083,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.183912754058838,
      "rewards/margins": 8.281122207641602,
      "rewards/rejected": -6.097209930419922,
      "step": 581,
      "tokens/chosen_response_len_mean": 308.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 198.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.47,
      "grad_norm": 4.192069450021518,
      "learning_rate": 3.20953361729747e-07,
      "logits/policy_chosen": -1.7661616802215576,
      "logits/policy_rejected": -1.8929604291915894,
      "logits/reference_chosen": -3.025322914123535,
      "logits/reference_rejected": -2.8024351596832275,
      "logps/policy_chosen": -36.95729064941406,
      "logps/policy_rejected": -384.001708984375,
      "logps/reference_chosen": -184.52584838867188,
      "logps/reference_rejected": -187.4014892578125,
      "loss": 0.2237,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.4756855964660645,
      "rewards/margins": 3.44168758392334,
      "rewards/rejected": -1.9660018682479858,
      "step": 582,
      "tokens/chosen_response_len_mean": 320.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 242.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.47,
      "grad_norm": 4.241728418846266,
      "learning_rate": 3.202800693912166e-07,
      "logits/policy_chosen": -1.3410980701446533,
      "logits/policy_rejected": -1.4757648706436157,
      "logits/reference_chosen": -2.9654150009155273,
      "logits/reference_rejected": -2.7540764808654785,
      "logps/policy_chosen": -144.69093322753906,
      "logps/policy_rejected": -1453.03662109375,
      "logps/reference_chosen": -494.4051513671875,
      "logps/reference_rejected": -574.343017578125,
      "loss": 0.0669,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4971423149108887,
      "rewards/margins": 12.284077644348145,
      "rewards/rejected": -8.786934852600098,
      "step": 583,
      "tokens/chosen_response_len_mean": 563.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 440.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.47,
      "grad_norm": 1.297196589662984,
      "learning_rate": 3.1960622310153574e-07,
      "logits/policy_chosen": -2.0523805618286133,
      "logits/policy_rejected": -2.1051578521728516,
      "logits/reference_chosen": -2.9966893196105957,
      "logits/reference_rejected": -2.197732925415039,
      "logps/policy_chosen": -76.13396453857422,
      "logps/policy_rejected": -339.5893859863281,
      "logps/reference_chosen": -282.0647277832031,
      "logps/reference_rejected": -157.03732299804688,
      "loss": 0.1621,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.05930757522583,
      "rewards/margins": 3.8848283290863037,
      "rewards/rejected": -1.825520634651184,
      "step": 584,
      "tokens/chosen_response_len_mean": 228.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 136.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.47,
      "grad_norm": 3.4724373678734235,
      "learning_rate": 3.189318281719959e-07,
      "logits/policy_chosen": -2.0443167686462402,
      "logits/policy_rejected": -2.1436047554016113,
      "logits/reference_chosen": -3.0281822681427,
      "logits/reference_rejected": -2.9765584468841553,
      "logps/policy_chosen": -42.10350036621094,
      "logps/policy_rejected": -614.7108154296875,
      "logps/reference_chosen": -169.55819702148438,
      "logps/reference_rejected": -230.19625854492188,
      "loss": 0.1371,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.27454674243927,
      "rewards/margins": 5.119692802429199,
      "rewards/rejected": -3.8451459407806396,
      "step": 585,
      "tokens/chosen_response_len_mean": 216.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 155.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.47,
      "grad_norm": 4.355134165960404,
      "learning_rate": 3.1825688991821255e-07,
      "logits/policy_chosen": -1.6265398263931274,
      "logits/policy_rejected": -1.7407760620117188,
      "logits/reference_chosen": -3.0388336181640625,
      "logits/reference_rejected": -2.481330394744873,
      "logps/policy_chosen": -75.90835571289062,
      "logps/policy_rejected": -830.4935302734375,
      "logps/reference_chosen": -295.05670166015625,
      "logps/reference_rejected": -263.3392028808594,
      "loss": 0.1052,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.191483497619629,
      "rewards/margins": 7.8630266189575195,
      "rewards/rejected": -5.671543121337891,
      "step": 586,
      "tokens/chosen_response_len_mean": 294.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 169.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.47,
      "grad_norm": 1.2730967059804688,
      "learning_rate": 3.175814136600843e-07,
      "logits/policy_chosen": -1.7666873931884766,
      "logits/policy_rejected": -2.092139959335327,
      "logits/reference_chosen": -2.9999003410339355,
      "logits/reference_rejected": -2.3804709911346436,
      "logps/policy_chosen": -73.14275360107422,
      "logps/policy_rejected": -836.7254028320312,
      "logps/reference_chosen": -328.9959716796875,
      "logps/reference_rejected": -307.24664306640625,
      "loss": 0.044,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.558532238006592,
      "rewards/margins": 7.8533196449279785,
      "rewards/rejected": -5.294787406921387,
      "step": 587,
      "tokens/chosen_response_len_mean": 494.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 317.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.47,
      "grad_norm": 2.164629369643303,
      "learning_rate": 3.169054047217499e-07,
      "logits/policy_chosen": -1.9113683700561523,
      "logits/policy_rejected": -1.9219505786895752,
      "logits/reference_chosen": -2.758012056350708,
      "logits/reference_rejected": -2.284219741821289,
      "logps/policy_chosen": -72.52899932861328,
      "logps/policy_rejected": -1016.8125610351562,
      "logps/reference_chosen": -311.89178466796875,
      "logps/reference_rejected": -365.1423034667969,
      "loss": 0.162,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.393627882003784,
      "rewards/margins": 8.910329818725586,
      "rewards/rejected": -6.516702651977539,
      "step": 588,
      "tokens/chosen_response_len_mean": 417.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 362.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.47,
      "grad_norm": 4.320165924841784,
      "learning_rate": 3.162288684315468e-07,
      "logits/policy_chosen": -1.4018032550811768,
      "logits/policy_rejected": -1.1438647508621216,
      "logits/reference_chosen": -2.482426643371582,
      "logits/reference_rejected": -2.366238594055176,
      "logps/policy_chosen": -155.1798553466797,
      "logps/policy_rejected": -920.1234741210938,
      "logps/reference_chosen": -446.79522705078125,
      "logps/reference_rejected": -401.5038146972656,
      "loss": 0.1815,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9161536693573,
      "rewards/margins": 8.102350234985352,
      "rewards/rejected": -5.186196327209473,
      "step": 589,
      "tokens/chosen_response_len_mean": 427.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 269.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.47,
      "grad_norm": 3.974888417339266,
      "learning_rate": 3.1555181012196935e-07,
      "logits/policy_chosen": -1.5848742723464966,
      "logits/policy_rejected": -0.9532491564750671,
      "logits/reference_chosen": -2.9342503547668457,
      "logits/reference_rejected": -2.49821138381958,
      "logps/policy_chosen": -143.2293701171875,
      "logps/policy_rejected": -718.4899291992188,
      "logps/reference_chosen": -451.502685546875,
      "logps/reference_rejected": -265.6712341308594,
      "loss": 0.1339,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.082733154296875,
      "rewards/margins": 7.610919952392578,
      "rewards/rejected": -4.528186798095703,
      "step": 590,
      "tokens/chosen_response_len_mean": 489.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 462.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.48,
      "grad_norm": 2.348918454888301,
      "learning_rate": 3.14874235129626e-07,
      "logits/policy_chosen": -1.56516695022583,
      "logits/policy_rejected": -1.7976508140563965,
      "logits/reference_chosen": -3.065791130065918,
      "logits/reference_rejected": -2.7460622787475586,
      "logps/policy_chosen": -93.74554443359375,
      "logps/policy_rejected": -1566.837158203125,
      "logps/reference_chosen": -461.492919921875,
      "logps/reference_rejected": -544.7694702148438,
      "loss": 0.1511,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.677473306655884,
      "rewards/margins": 13.898149490356445,
      "rewards/rejected": -10.22067642211914,
      "step": 591,
      "tokens/chosen_response_len_mean": 532.0,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 417.75,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.48,
      "grad_norm": 3.844890642334795,
      "learning_rate": 3.141961487951981e-07,
      "logits/policy_chosen": -1.9106504917144775,
      "logits/policy_rejected": -1.9894771575927734,
      "logits/reference_chosen": -2.8962202072143555,
      "logits/reference_rejected": -2.722996711730957,
      "logps/policy_chosen": -53.63368225097656,
      "logps/policy_rejected": -660.487060546875,
      "logps/reference_chosen": -205.3843231201172,
      "logps/reference_rejected": -242.3226318359375,
      "loss": 0.135,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5175063610076904,
      "rewards/margins": 5.699150562286377,
      "rewards/rejected": -4.181644439697266,
      "step": 592,
      "tokens/chosen_response_len_mean": 213.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 154.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.48,
      "grad_norm": 4.373455537933952,
      "learning_rate": 3.1351755646339716e-07,
      "logits/policy_chosen": -1.4870643615722656,
      "logits/policy_rejected": -1.428262710571289,
      "logits/reference_chosen": -2.8044776916503906,
      "logits/reference_rejected": -2.699432373046875,
      "logps/policy_chosen": -76.64315795898438,
      "logps/policy_rejected": -872.3397216796875,
      "logps/reference_chosen": -386.992919921875,
      "logps/reference_rejected": -363.94488525390625,
      "loss": 0.0649,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.1034975051879883,
      "rewards/margins": 8.187445640563965,
      "rewards/rejected": -5.083948135375977,
      "step": 593,
      "tokens/chosen_response_len_mean": 548.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 342.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.48,
      "grad_norm": 2.6488918938644628,
      "learning_rate": 3.1283846348292326e-07,
      "logits/policy_chosen": -1.5703134536743164,
      "logits/policy_rejected": -1.4918897151947021,
      "logits/reference_chosen": -2.7862539291381836,
      "logits/reference_rejected": -2.4407873153686523,
      "logps/policy_chosen": -90.49102020263672,
      "logps/policy_rejected": -1093.0537109375,
      "logps/reference_chosen": -436.50030517578125,
      "logps/reference_rejected": -444.57440185546875,
      "loss": 0.0593,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 3.4600930213928223,
      "rewards/margins": 9.94488525390625,
      "rewards/rejected": -6.484792709350586,
      "step": 594,
      "tokens/chosen_response_len_mean": 476.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 353.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.48,
      "grad_norm": 2.5152603738136916,
      "learning_rate": 3.1215887520642236e-07,
      "logits/policy_chosen": -2.194831371307373,
      "logits/policy_rejected": -2.0193140506744385,
      "logits/reference_chosen": -2.626875877380371,
      "logits/reference_rejected": -2.893083095550537,
      "logps/policy_chosen": -77.49217224121094,
      "logps/policy_rejected": -1361.74267578125,
      "logps/reference_chosen": -277.9424133300781,
      "logps/reference_rejected": -665.537353515625,
      "loss": 0.1515,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.004502296447754,
      "rewards/margins": 8.966554641723633,
      "rewards/rejected": -6.962051868438721,
      "step": 595,
      "tokens/chosen_response_len_mean": 283.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 300.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.48,
      "grad_norm": 11.306568930341445,
      "learning_rate": 3.114787969904446e-07,
      "logits/policy_chosen": -1.5930339097976685,
      "logits/policy_rejected": -1.6910613775253296,
      "logits/reference_chosen": -3.1339163780212402,
      "logits/reference_rejected": -2.85372257232666,
      "logps/policy_chosen": -135.07546997070312,
      "logps/policy_rejected": -1294.70703125,
      "logps/reference_chosen": -446.10650634765625,
      "logps/reference_rejected": -537.7215576171875,
      "loss": 0.1922,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 3.1103103160858154,
      "rewards/margins": 10.680166244506836,
      "rewards/rejected": -7.569855690002441,
      "step": 596,
      "tokens/chosen_response_len_mean": 509.75,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 476.0,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.48,
      "grad_norm": 3.841800598546111,
      "learning_rate": 3.1079823419540177e-07,
      "logits/policy_chosen": -1.6570217609405518,
      "logits/policy_rejected": -1.749629259109497,
      "logits/reference_chosen": -3.01853346824646,
      "logits/reference_rejected": -2.4536404609680176,
      "logps/policy_chosen": -69.32925415039062,
      "logps/policy_rejected": -1030.8250732421875,
      "logps/reference_chosen": -360.5860595703125,
      "logps/reference_rejected": -427.4230041503906,
      "loss": 0.0745,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.9125680923461914,
      "rewards/margins": 8.946588516235352,
      "rewards/rejected": -6.03402042388916,
      "step": 597,
      "tokens/chosen_response_len_mean": 353.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 300.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.48,
      "grad_norm": 2.597533442527042,
      "learning_rate": 3.1011719218552484e-07,
      "logits/policy_chosen": -1.1671286821365356,
      "logits/policy_rejected": -1.1593250036239624,
      "logits/reference_chosen": -2.2973759174346924,
      "logits/reference_rejected": -2.1064136028289795,
      "logps/policy_chosen": -60.38359832763672,
      "logps/policy_rejected": -559.268798828125,
      "logps/reference_chosen": -195.4320068359375,
      "logps/reference_rejected": -228.5808868408203,
      "loss": 0.2483,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3504841327667236,
      "rewards/margins": 4.657362937927246,
      "rewards/rejected": -3.3068785667419434,
      "step": 598,
      "tokens/chosen_response_len_mean": 284.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 259.5,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.48,
      "grad_norm": 5.037365720866584,
      "learning_rate": 3.0943567632882257e-07,
      "logits/policy_chosen": -1.1204264163970947,
      "logits/policy_rejected": -1.524148941040039,
      "logits/reference_chosen": -2.8448486328125,
      "logits/reference_rejected": -1.551837682723999,
      "logps/policy_chosen": -59.946258544921875,
      "logps/policy_rejected": -860.8857421875,
      "logps/reference_chosen": -252.06268310546875,
      "logps/reference_rejected": -275.23046875,
      "loss": 0.2051,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.9211641550064087,
      "rewards/margins": 7.777716636657715,
      "rewards/rejected": -5.856552600860596,
      "step": 599,
      "tokens/chosen_response_len_mean": 293.25,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 167.25,
      "tokens/rejected_response_len_std": 0.0
    },
    {
      "epoch": 0.48,
      "grad_norm": 3.3012099850998053,
      "learning_rate": 3.0875369199703804e-07,
      "logits/policy_chosen": -1.5149235725402832,
      "logits/policy_rejected": -1.7855583429336548,
      "logits/reference_chosen": -2.490311622619629,
      "logits/reference_rejected": -2.4275195598602295,
      "logps/policy_chosen": -81.25556182861328,
      "logps/policy_rejected": -806.9634399414062,
      "logps/reference_chosen": -342.1833801269531,
      "logps/reference_rejected": -279.183349609375,
      "loss": 0.2025,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.6092779636383057,
      "rewards/margins": 7.887078285217285,
      "rewards/rejected": -5.2778000831604,
      "step": 600,
      "tokens/chosen_response_len_mean": 401.5,
      "tokens/chosen_response_len_std": 0.0,
      "tokens/rejected_response_len_mean": 256.0,
      "tokens/rejected_response_len_std": 0.0
    }
  ],
  "logging_steps": 1,
  "max_steps": 1244,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}