diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,9221 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.3215434083601286, + "eval_steps": 100, + "global_step": 400, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "grad_norm": 2.5493941690669297, + "learning_rate": 4e-09, + "logits/policy_chosen": -2.279445171356201, + "logits/policy_rejected": -2.070641279220581, + "logits/reference_chosen": -2.262451410293579, + "logits/reference_rejected": -1.700181484222412, + "logps/policy_chosen": -259.83294677734375, + "logps/policy_rejected": -282.3813171386719, + "logps/reference_chosen": -280.28363037109375, + "logps/reference_rejected": -290.83868408203125, + "loss": 0.6746, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.20450688898563385, + "rewards/margins": 0.11993316560983658, + "rewards/rejected": 0.08457372337579727, + "step": 1, + "tokens/chosen_response_len_mean": 528.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 444.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.0, + "grad_norm": 2.2490732615654547, + "learning_rate": 8e-09, + "logits/policy_chosen": -2.745072364807129, + "logits/policy_rejected": -2.607792854309082, + "logits/reference_chosen": -2.701054096221924, + "logits/reference_rejected": -2.529007911682129, + "logps/policy_chosen": -147.3030242919922, + "logps/policy_rejected": -197.99253845214844, + "logps/reference_chosen": -156.72518920898438, + "logps/reference_rejected": -206.69471740722656, + "loss": 0.6863, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.09422144293785095, + "rewards/margins": 0.0071996524930000305, + "rewards/rejected": 0.08702179044485092, + "step": 2, + "tokens/chosen_response_len_mean": 298.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 277.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.0, + "grad_norm": 2.2889093862107543, + "learning_rate": 1.2e-08, + "logits/policy_chosen": -2.882375478744507, + "logits/policy_rejected": -2.992877960205078, + "logits/reference_chosen": -2.748257637023926, + "logits/reference_rejected": -2.9701337814331055, + "logps/policy_chosen": -105.87107849121094, + "logps/policy_rejected": -148.20022583007812, + "logps/reference_chosen": -112.85089111328125, + "logps/reference_rejected": -155.7706298828125, + "loss": 0.6832, + "rewards/accuracies": 0.25, + "rewards/chosen": 0.06979808956384659, + "rewards/margins": -0.005905959755182266, + "rewards/rejected": 0.07570404559373856, + "step": 3, + "tokens/chosen_response_len_mean": 239.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 254.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.0, + "grad_norm": 2.2808153966501084, + "learning_rate": 1.6e-08, + "logits/policy_chosen": -2.9396884441375732, + "logits/policy_rejected": -2.9384472370147705, + "logits/reference_chosen": -2.9226865768432617, + "logits/reference_rejected": -2.9309639930725098, + "logps/policy_chosen": -123.63784790039062, + "logps/policy_rejected": -180.25296020507812, + "logps/reference_chosen": -129.93817138671875, + "logps/reference_rejected": -184.21334838867188, + "loss": 0.674, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.06300327926874161, + "rewards/margins": 0.023399492725729942, + "rewards/rejected": 0.039603784680366516, + "step": 4, + "tokens/chosen_response_len_mean": 298.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 297.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.0, + "grad_norm": 2.2660235876486974, + "learning_rate": 2e-08, + "logits/policy_chosen": -3.221891164779663, + "logits/policy_rejected": -2.8190152645111084, + "logits/reference_chosen": -3.207385778427124, + "logits/reference_rejected": -2.6983580589294434, + "logps/policy_chosen": -179.60354614257812, + "logps/policy_rejected": -206.77960205078125, + "logps/reference_chosen": -189.15216064453125, + "logps/reference_rejected": -209.81790161132812, + "loss": 0.6778, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.09548593312501907, + "rewards/margins": 0.06510297954082489, + "rewards/rejected": 0.03038296103477478, + "step": 5, + "tokens/chosen_response_len_mean": 440.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 232.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.0, + "grad_norm": 2.0558578581082307, + "learning_rate": 2.4e-08, + "logits/policy_chosen": -2.4811384677886963, + "logits/policy_rejected": -2.4067883491516113, + "logits/reference_chosen": -2.330796003341675, + "logits/reference_rejected": -2.4005446434020996, + "logps/policy_chosen": -148.8319091796875, + "logps/policy_rejected": -179.03482055664062, + "logps/reference_chosen": -160.83175659179688, + "logps/reference_rejected": -189.98275756835938, + "loss": 0.6989, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.11999829858541489, + "rewards/margins": 0.010519090108573437, + "rewards/rejected": 0.10947921127080917, + "step": 6, + "tokens/chosen_response_len_mean": 289.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 309.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.01, + "grad_norm": 2.5447256930451734, + "learning_rate": 2.8e-08, + "logits/policy_chosen": -2.2599525451660156, + "logits/policy_rejected": -2.190964698791504, + "logits/reference_chosen": -2.1052117347717285, + "logits/reference_rejected": -2.134941577911377, + "logps/policy_chosen": -19.338191986083984, + "logps/policy_rejected": -38.73615646362305, + "logps/reference_chosen": -20.665983200073242, + "logps/reference_rejected": -40.615394592285156, + "loss": 0.684, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.01327789481729269, + "rewards/margins": -0.005514495074748993, + "rewards/rejected": 0.018792390823364258, + "step": 7, + "tokens/chosen_response_len_mean": 34.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 47.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.01, + "grad_norm": 2.967658233666017, + "learning_rate": 3.2e-08, + "logits/policy_chosen": -2.4715514183044434, + "logits/policy_rejected": -2.4913477897644043, + "logits/reference_chosen": -2.421693801879883, + "logits/reference_rejected": -2.4608089923858643, + "logps/policy_chosen": -29.782058715820312, + "logps/policy_rejected": -64.6925048828125, + "logps/reference_chosen": -32.50696563720703, + "logps/reference_rejected": -66.7137222290039, + "loss": 0.6712, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.027249068021774292, + "rewards/margins": 0.00703694112598896, + "rewards/rejected": 0.020212125033140182, + "step": 8, + "tokens/chosen_response_len_mean": 40.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 48.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.01, + "grad_norm": 3.4470069507233316, + "learning_rate": 3.6e-08, + "logits/policy_chosen": -2.7727508544921875, + "logits/policy_rejected": -2.718796730041504, + "logits/reference_chosen": -2.676337957382202, + "logits/reference_rejected": -2.616457223892212, + "logps/policy_chosen": -108.26776123046875, + "logps/policy_rejected": -218.6373748779297, + "logps/reference_chosen": -118.67279815673828, + "logps/reference_rejected": -223.27325439453125, + "loss": 0.6796, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.1040503978729248, + "rewards/margins": 0.05769149586558342, + "rewards/rejected": 0.046358898282051086, + "step": 9, + "tokens/chosen_response_len_mean": 252.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 222.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.01, + "grad_norm": 2.304016636338041, + "learning_rate": 4e-08, + "logits/policy_chosen": -2.5566985607147217, + "logits/policy_rejected": -2.5395240783691406, + "logits/reference_chosen": -2.4825587272644043, + "logits/reference_rejected": -2.498563051223755, + "logps/policy_chosen": -168.69143676757812, + "logps/policy_rejected": -287.2906494140625, + "logps/reference_chosen": -181.2764129638672, + "logps/reference_rejected": -299.7208557128906, + "loss": 0.6885, + "rewards/accuracies": 0.25, + "rewards/chosen": 0.1258496791124344, + "rewards/margins": 0.001547420397400856, + "rewards/rejected": 0.12430226802825928, + "step": 10, + "tokens/chosen_response_len_mean": 518.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 503.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.01, + "grad_norm": 2.084799466815333, + "learning_rate": 4.4e-08, + "logits/policy_chosen": -2.7605199813842773, + "logits/policy_rejected": -2.7186694145202637, + "logits/reference_chosen": -2.7691569328308105, + "logits/reference_rejected": -2.7842984199523926, + "logps/policy_chosen": -145.88803100585938, + "logps/policy_rejected": -219.43760681152344, + "logps/reference_chosen": -155.90982055664062, + "logps/reference_rejected": -227.8662109375, + "loss": 0.6893, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.1002180427312851, + "rewards/margins": 0.015932004898786545, + "rewards/rejected": 0.08428603410720825, + "step": 11, + "tokens/chosen_response_len_mean": 604.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 448.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.01, + "grad_norm": 2.7628214791467736, + "learning_rate": 4.8e-08, + "logits/policy_chosen": -2.9070513248443604, + "logits/policy_rejected": -2.7778186798095703, + "logits/reference_chosen": -2.8774163722991943, + "logits/reference_rejected": -2.6446313858032227, + "logps/policy_chosen": -101.27095794677734, + "logps/policy_rejected": -136.12225341796875, + "logps/reference_chosen": -108.19136047363281, + "logps/reference_rejected": -140.3483123779297, + "loss": 0.6673, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.06920397281646729, + "rewards/margins": 0.026943188160657883, + "rewards/rejected": 0.0422607846558094, + "step": 12, + "tokens/chosen_response_len_mean": 339.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 269.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.01, + "grad_norm": 2.1795487992104348, + "learning_rate": 5.1999999999999996e-08, + "logits/policy_chosen": -2.4187445640563965, + "logits/policy_rejected": -2.845353364944458, + "logits/reference_chosen": -2.3343820571899414, + "logits/reference_rejected": -2.5970327854156494, + "logps/policy_chosen": -130.312255859375, + "logps/policy_rejected": -73.26902770996094, + "logps/reference_chosen": -134.85186767578125, + "logps/reference_rejected": -73.45825958251953, + "loss": 0.6772, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.045396123081445694, + "rewards/margins": 0.04350382089614868, + "rewards/rejected": 0.0018922993913292885, + "step": 13, + "tokens/chosen_response_len_mean": 177.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 101.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.01, + "grad_norm": 2.7157502561420026, + "learning_rate": 5.6e-08, + "logits/policy_chosen": -3.211674928665161, + "logits/policy_rejected": -3.0040595531463623, + "logits/reference_chosen": -3.2025675773620605, + "logits/reference_rejected": -3.0620803833007812, + "logps/policy_chosen": -195.99533081054688, + "logps/policy_rejected": -242.1226806640625, + "logps/reference_chosen": -208.40536499023438, + "logps/reference_rejected": -252.9475860595703, + "loss": 0.6781, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.12410032004117966, + "rewards/margins": 0.015851235017180443, + "rewards/rejected": 0.10824908316135406, + "step": 14, + "tokens/chosen_response_len_mean": 543.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 408.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.01, + "grad_norm": 3.50208846860952, + "learning_rate": 6e-08, + "logits/policy_chosen": -2.6727657318115234, + "logits/policy_rejected": -2.5919432640075684, + "logits/reference_chosen": -2.6382195949554443, + "logits/reference_rejected": -2.417813777923584, + "logps/policy_chosen": -170.3718719482422, + "logps/policy_rejected": -223.43052673339844, + "logps/reference_chosen": -191.32583618164062, + "logps/reference_rejected": -242.33273315429688, + "loss": 0.6686, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.209539532661438, + "rewards/margins": 0.020517464727163315, + "rewards/rejected": 0.18902206420898438, + "step": 15, + "tokens/chosen_response_len_mean": 431.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 403.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.01, + "grad_norm": 1.6207728056732267, + "learning_rate": 6.4e-08, + "logits/policy_chosen": -2.954475164413452, + "logits/policy_rejected": -2.8040027618408203, + "logits/reference_chosen": -2.9069364070892334, + "logits/reference_rejected": -2.8010973930358887, + "logps/policy_chosen": -54.05670166015625, + "logps/policy_rejected": -109.65800476074219, + "logps/reference_chosen": -55.35580825805664, + "logps/reference_rejected": -108.90874481201172, + "loss": 0.6824, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.012991041876375675, + "rewards/margins": 0.02048366516828537, + "rewards/rejected": -0.007492623291909695, + "step": 16, + "tokens/chosen_response_len_mean": 89.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 96.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.01, + "grad_norm": 2.409299627131989, + "learning_rate": 6.8e-08, + "logits/policy_chosen": -2.909968137741089, + "logits/policy_rejected": -2.84816837310791, + "logits/reference_chosen": -2.8810513019561768, + "logits/reference_rejected": -2.6653146743774414, + "logps/policy_chosen": -119.80767822265625, + "logps/policy_rejected": -267.8531799316406, + "logps/reference_chosen": -126.68125915527344, + "logps/reference_rejected": -275.761474609375, + "loss": 0.6995, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.06873579323291779, + "rewards/margins": -0.010347230359911919, + "rewards/rejected": 0.07908301800489426, + "step": 17, + "tokens/chosen_response_len_mean": 314.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 251.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.01, + "grad_norm": 2.703021449933624, + "learning_rate": 7.2e-08, + "logits/policy_chosen": -2.8886208534240723, + "logits/policy_rejected": -2.603667974472046, + "logits/reference_chosen": -2.8919739723205566, + "logits/reference_rejected": -2.336148262023926, + "logps/policy_chosen": -144.8170166015625, + "logps/policy_rejected": -144.6788330078125, + "logps/reference_chosen": -164.3677215576172, + "logps/reference_rejected": -156.78842163085938, + "loss": 0.679, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.1955069750547409, + "rewards/margins": 0.07441116869449615, + "rewards/rejected": 0.12109580636024475, + "step": 18, + "tokens/chosen_response_len_mean": 421.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 314.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.02, + "grad_norm": 1.9691382131351773, + "learning_rate": 7.599999999999999e-08, + "logits/policy_chosen": -2.766261339187622, + "logits/policy_rejected": -2.697242259979248, + "logits/reference_chosen": -2.710810899734497, + "logits/reference_rejected": -2.557218551635742, + "logps/policy_chosen": -150.92379760742188, + "logps/policy_rejected": -174.591064453125, + "logps/reference_chosen": -160.51756286621094, + "logps/reference_rejected": -185.50238037109375, + "loss": 0.6851, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.0959378108382225, + "rewards/margins": -0.013175351545214653, + "rewards/rejected": 0.1091131642460823, + "step": 19, + "tokens/chosen_response_len_mean": 263.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 253.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.02, + "grad_norm": 2.1003754961271675, + "learning_rate": 8e-08, + "logits/policy_chosen": -2.8774542808532715, + "logits/policy_rejected": -2.6342382431030273, + "logits/reference_chosen": -2.742877244949341, + "logits/reference_rejected": -2.4385361671447754, + "logps/policy_chosen": -104.96978759765625, + "logps/policy_rejected": -157.80657958984375, + "logps/reference_chosen": -107.06401062011719, + "logps/reference_rejected": -159.94366455078125, + "loss": 0.6774, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.02094227820634842, + "rewards/margins": -0.00042874296195805073, + "rewards/rejected": 0.02137102000415325, + "step": 20, + "tokens/chosen_response_len_mean": 111.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 124.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.02, + "grad_norm": 2.113555044323313, + "learning_rate": 8.4e-08, + "logits/policy_chosen": -3.1314477920532227, + "logits/policy_rejected": -2.8576135635375977, + "logits/reference_chosen": -3.1075565814971924, + "logits/reference_rejected": -2.6684985160827637, + "logps/policy_chosen": -121.34175109863281, + "logps/policy_rejected": -152.86807250976562, + "logps/reference_chosen": -127.80372619628906, + "logps/reference_rejected": -158.0865478515625, + "loss": 0.6994, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.06461974233388901, + "rewards/margins": 0.012434981763362885, + "rewards/rejected": 0.05218476057052612, + "step": 21, + "tokens/chosen_response_len_mean": 317.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 203.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.02, + "grad_norm": 2.8572997186380196, + "learning_rate": 8.8e-08, + "logits/policy_chosen": -3.023026466369629, + "logits/policy_rejected": -2.970913887023926, + "logits/reference_chosen": -2.9523911476135254, + "logits/reference_rejected": -2.8476057052612305, + "logps/policy_chosen": -123.84272003173828, + "logps/policy_rejected": -210.03341674804688, + "logps/reference_chosen": -129.73898315429688, + "logps/reference_rejected": -216.26409912109375, + "loss": 0.672, + "rewards/accuracies": 0.25, + "rewards/chosen": 0.05896264314651489, + "rewards/margins": -0.0033443691208958626, + "rewards/rejected": 0.06230701133608818, + "step": 22, + "tokens/chosen_response_len_mean": 241.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 224.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.02, + "grad_norm": 2.4344792354730034, + "learning_rate": 9.199999999999999e-08, + "logits/policy_chosen": -2.45721435546875, + "logits/policy_rejected": -2.4645514488220215, + "logits/reference_chosen": -2.429760694503784, + "logits/reference_rejected": -2.448065757751465, + "logps/policy_chosen": -107.37543487548828, + "logps/policy_rejected": -120.32754516601562, + "logps/reference_chosen": -113.06903839111328, + "logps/reference_rejected": -126.56575012207031, + "loss": 0.6748, + "rewards/accuracies": 0.25, + "rewards/chosen": 0.056935906410217285, + "rewards/margins": -0.005446167662739754, + "rewards/rejected": 0.06238207593560219, + "step": 23, + "tokens/chosen_response_len_mean": 314.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 315.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.02, + "grad_norm": 3.5341818462448504, + "learning_rate": 9.6e-08, + "logits/policy_chosen": -2.5186548233032227, + "logits/policy_rejected": -2.3946735858917236, + "logits/reference_chosen": -2.5037264823913574, + "logits/reference_rejected": -2.4204611778259277, + "logps/policy_chosen": -71.1528549194336, + "logps/policy_rejected": -96.15353393554688, + "logps/reference_chosen": -74.45181274414062, + "logps/reference_rejected": -93.63554382324219, + "loss": 0.6593, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.03298955038189888, + "rewards/margins": 0.05816952511668205, + "rewards/rejected": -0.025179967284202576, + "step": 24, + "tokens/chosen_response_len_mean": 247.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 155.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.02, + "grad_norm": 2.6672688539385514, + "learning_rate": 1e-07, + "logits/policy_chosen": -3.052779197692871, + "logits/policy_rejected": -2.9980740547180176, + "logits/reference_chosen": -3.0356476306915283, + "logits/reference_rejected": -2.9704558849334717, + "logps/policy_chosen": -158.24560546875, + "logps/policy_rejected": -246.05154418945312, + "logps/reference_chosen": -166.8022003173828, + "logps/reference_rejected": -252.77267456054688, + "loss": 0.6801, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.08556582033634186, + "rewards/margins": 0.01835446059703827, + "rewards/rejected": 0.06721135973930359, + "step": 25, + "tokens/chosen_response_len_mean": 302.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 309.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.02, + "grad_norm": 2.157863179356832, + "learning_rate": 1.0399999999999999e-07, + "logits/policy_chosen": -2.8244199752807617, + "logits/policy_rejected": -2.8272688388824463, + "logits/reference_chosen": -2.8120579719543457, + "logits/reference_rejected": -2.81868839263916, + "logps/policy_chosen": -188.75546264648438, + "logps/policy_rejected": -245.15794372558594, + "logps/reference_chosen": -201.83450317382812, + "logps/reference_rejected": -256.3631591796875, + "loss": 0.6824, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.13079024851322174, + "rewards/margins": 0.018737995997071266, + "rewards/rejected": 0.11205224692821503, + "step": 26, + "tokens/chosen_response_len_mean": 499.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 500.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.02, + "grad_norm": 2.6785946054818295, + "learning_rate": 1.0799999999999999e-07, + "logits/policy_chosen": -3.041701078414917, + "logits/policy_rejected": -3.0683014392852783, + "logits/reference_chosen": -3.021995782852173, + "logits/reference_rejected": -3.0257372856140137, + "logps/policy_chosen": -102.09321594238281, + "logps/policy_rejected": -163.72860717773438, + "logps/reference_chosen": -109.2179946899414, + "logps/reference_rejected": -167.15447998046875, + "loss": 0.6755, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.07124768942594528, + "rewards/margins": 0.036988936364650726, + "rewards/rejected": 0.03425874561071396, + "step": 27, + "tokens/chosen_response_len_mean": 243.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 237.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.02, + "grad_norm": 3.0106278359730614, + "learning_rate": 1.12e-07, + "logits/policy_chosen": -3.058267116546631, + "logits/policy_rejected": -3.0097880363464355, + "logits/reference_chosen": -3.000871181488037, + "logits/reference_rejected": -2.860621690750122, + "logps/policy_chosen": -177.05154418945312, + "logps/policy_rejected": -241.18173217773438, + "logps/reference_chosen": -187.62625122070312, + "logps/reference_rejected": -242.83314514160156, + "loss": 0.6642, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.10574702173471451, + "rewards/margins": 0.08923279494047165, + "rewards/rejected": 0.01651422120630741, + "step": 28, + "tokens/chosen_response_len_mean": 391.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 363.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.02, + "grad_norm": 1.9714081999119137, + "learning_rate": 1.16e-07, + "logits/policy_chosen": -2.7036781311035156, + "logits/policy_rejected": -2.706456422805786, + "logits/reference_chosen": -2.6821694374084473, + "logits/reference_rejected": -2.70880126953125, + "logps/policy_chosen": -50.51161193847656, + "logps/policy_rejected": -97.51376342773438, + "logps/reference_chosen": -53.98440170288086, + "logps/reference_rejected": -100.45523834228516, + "loss": 0.6839, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.03472791612148285, + "rewards/margins": 0.005313164554536343, + "rewards/rejected": 0.029414750635623932, + "step": 29, + "tokens/chosen_response_len_mean": 246.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 215.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.02, + "grad_norm": 3.3945073696477333, + "learning_rate": 1.2e-07, + "logits/policy_chosen": -2.679271936416626, + "logits/policy_rejected": -2.747288227081299, + "logits/reference_chosen": -2.648017406463623, + "logits/reference_rejected": -2.542226552963257, + "logps/policy_chosen": -181.09521484375, + "logps/policy_rejected": -349.05694580078125, + "logps/reference_chosen": -202.54962158203125, + "logps/reference_rejected": -355.8513488769531, + "loss": 0.6558, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.21454395353794098, + "rewards/margins": 0.14660006761550903, + "rewards/rejected": 0.06794389337301254, + "step": 30, + "tokens/chosen_response_len_mean": 309.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 268.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.02, + "grad_norm": 2.729481181496147, + "learning_rate": 1.24e-07, + "logits/policy_chosen": -3.1615757942199707, + "logits/policy_rejected": -2.618534564971924, + "logits/reference_chosen": -3.148972511291504, + "logits/reference_rejected": -2.28782320022583, + "logps/policy_chosen": -244.06692504882812, + "logps/policy_rejected": -395.2162780761719, + "logps/reference_chosen": -254.54090881347656, + "logps/reference_rejected": -397.2469177246094, + "loss": 0.6873, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.10473988205194473, + "rewards/margins": 0.08443356305360794, + "rewards/rejected": 0.020306318998336792, + "step": 31, + "tokens/chosen_response_len_mean": 410.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 269.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.03, + "grad_norm": 2.1927971055522133, + "learning_rate": 1.28e-07, + "logits/policy_chosen": -2.935199499130249, + "logits/policy_rejected": -2.862617254257202, + "logits/reference_chosen": -2.9165947437286377, + "logits/reference_rejected": -2.809278964996338, + "logps/policy_chosen": -156.64752197265625, + "logps/policy_rejected": -224.47006225585938, + "logps/reference_chosen": -176.1259765625, + "logps/reference_rejected": -239.154296875, + "loss": 0.6749, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.19478438794612885, + "rewards/margins": 0.04794217646121979, + "rewards/rejected": 0.14684221148490906, + "step": 32, + "tokens/chosen_response_len_mean": 545.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 527.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.03, + "grad_norm": 2.6358653856327257, + "learning_rate": 1.32e-07, + "logits/policy_chosen": -2.5262393951416016, + "logits/policy_rejected": -2.364823341369629, + "logits/reference_chosen": -2.506678342819214, + "logits/reference_rejected": -1.6685891151428223, + "logps/policy_chosen": -104.21517181396484, + "logps/policy_rejected": -126.63694763183594, + "logps/reference_chosen": -114.83543395996094, + "logps/reference_rejected": -136.51809692382812, + "loss": 0.6837, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.10620267689228058, + "rewards/margins": 0.007391175255179405, + "rewards/rejected": 0.09881150722503662, + "step": 33, + "tokens/chosen_response_len_mean": 448.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 339.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.03, + "grad_norm": 2.626247472893481, + "learning_rate": 1.36e-07, + "logits/policy_chosen": -3.0164244174957275, + "logits/policy_rejected": -2.670468330383301, + "logits/reference_chosen": -2.9480087757110596, + "logits/reference_rejected": -2.077363967895508, + "logps/policy_chosen": -144.32318115234375, + "logps/policy_rejected": -178.09408569335938, + "logps/reference_chosen": -154.3887939453125, + "logps/reference_rejected": -182.9420623779297, + "loss": 0.6748, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.1006559506058693, + "rewards/margins": 0.052176106721162796, + "rewards/rejected": 0.0484798438847065, + "step": 34, + "tokens/chosen_response_len_mean": 407.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 217.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.03, + "grad_norm": 2.296064376513374, + "learning_rate": 1.4e-07, + "logits/policy_chosen": -2.598402738571167, + "logits/policy_rejected": -2.6598832607269287, + "logits/reference_chosen": -2.553514003753662, + "logits/reference_rejected": -2.6580069065093994, + "logps/policy_chosen": -281.2328186035156, + "logps/policy_rejected": -179.6129608154297, + "logps/reference_chosen": -298.13836669921875, + "logps/reference_rejected": -186.5122528076172, + "loss": 0.6684, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.16905567049980164, + "rewards/margins": 0.10006258636713028, + "rewards/rejected": 0.06899307668209076, + "step": 35, + "tokens/chosen_response_len_mean": 377.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 248.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.03, + "grad_norm": 2.898376793238501, + "learning_rate": 1.44e-07, + "logits/policy_chosen": -2.7883753776550293, + "logits/policy_rejected": -2.6082558631896973, + "logits/reference_chosen": -2.780714273452759, + "logits/reference_rejected": -2.583864212036133, + "logps/policy_chosen": -105.63471984863281, + "logps/policy_rejected": -245.55908203125, + "logps/reference_chosen": -110.00314331054688, + "logps/reference_rejected": -244.04986572265625, + "loss": 0.6715, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.04368434101343155, + "rewards/margins": 0.05877665430307388, + "rewards/rejected": -0.015092315152287483, + "step": 36, + "tokens/chosen_response_len_mean": 183.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 284.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.03, + "grad_norm": 2.6935110955386, + "learning_rate": 1.4799999999999998e-07, + "logits/policy_chosen": -3.0653228759765625, + "logits/policy_rejected": -2.882253646850586, + "logits/reference_chosen": -3.0485243797302246, + "logits/reference_rejected": -2.6053519248962402, + "logps/policy_chosen": -186.7164764404297, + "logps/policy_rejected": -267.58123779296875, + "logps/reference_chosen": -204.80267333984375, + "logps/reference_rejected": -272.190185546875, + "loss": 0.6697, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.1808619499206543, + "rewards/margins": 0.13477247953414917, + "rewards/rejected": 0.046089474111795425, + "step": 37, + "tokens/chosen_response_len_mean": 433.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 321.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.03, + "grad_norm": 2.597922955352179, + "learning_rate": 1.5199999999999998e-07, + "logits/policy_chosen": -2.699632167816162, + "logits/policy_rejected": -2.7144150733947754, + "logits/reference_chosen": -2.658318519592285, + "logits/reference_rejected": -2.691765546798706, + "logps/policy_chosen": -135.453369140625, + "logps/policy_rejected": -242.96441650390625, + "logps/reference_chosen": -146.15704345703125, + "logps/reference_rejected": -248.74380493164062, + "loss": 0.6585, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.10703684389591217, + "rewards/margins": 0.049242909997701645, + "rewards/rejected": 0.057793937623500824, + "step": 38, + "tokens/chosen_response_len_mean": 375.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 387.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.03, + "grad_norm": 2.079049499920605, + "learning_rate": 1.56e-07, + "logits/policy_chosen": -3.077770233154297, + "logits/policy_rejected": -3.098449945449829, + "logits/reference_chosen": -3.0287766456604004, + "logits/reference_rejected": -3.0804648399353027, + "logps/policy_chosen": -12.474931716918945, + "logps/policy_rejected": -25.898088455200195, + "logps/reference_chosen": -12.915287971496582, + "logps/reference_rejected": -27.526098251342773, + "loss": 0.6793, + "rewards/accuracies": 0.25, + "rewards/chosen": 0.00440356507897377, + "rewards/margins": -0.011876537464559078, + "rewards/rejected": 0.016280101612210274, + "step": 39, + "tokens/chosen_response_len_mean": 23.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 25.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.03, + "grad_norm": 2.9270192927802197, + "learning_rate": 1.6e-07, + "logits/policy_chosen": -2.821016550064087, + "logits/policy_rejected": -2.7184388637542725, + "logits/reference_chosen": -2.5926756858825684, + "logits/reference_rejected": -2.7159690856933594, + "logps/policy_chosen": -115.9990005493164, + "logps/policy_rejected": -150.73153686523438, + "logps/reference_chosen": -120.78485107421875, + "logps/reference_rejected": -157.8815460205078, + "loss": 0.668, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.04785853251814842, + "rewards/margins": -0.02364155277609825, + "rewards/rejected": 0.07150008529424667, + "step": 40, + "tokens/chosen_response_len_mean": 179.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 168.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.03, + "grad_norm": 2.807071363383845, + "learning_rate": 1.64e-07, + "logits/policy_chosen": -2.6167349815368652, + "logits/policy_rejected": -2.5763139724731445, + "logits/reference_chosen": -2.513126850128174, + "logits/reference_rejected": -2.5481503009796143, + "logps/policy_chosen": -213.66064453125, + "logps/policy_rejected": -274.35003662109375, + "logps/reference_chosen": -240.58090209960938, + "logps/reference_rejected": -297.9310607910156, + "loss": 0.6847, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.26920264959335327, + "rewards/margins": 0.03339245170354843, + "rewards/rejected": 0.23581019043922424, + "step": 41, + "tokens/chosen_response_len_mean": 468.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 442.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.03, + "grad_norm": 3.1342954098759788, + "learning_rate": 1.68e-07, + "logits/policy_chosen": -2.98238205909729, + "logits/policy_rejected": -2.6742758750915527, + "logits/reference_chosen": -2.9786105155944824, + "logits/reference_rejected": -2.5011706352233887, + "logps/policy_chosen": -400.793701171875, + "logps/policy_rejected": -471.21826171875, + "logps/reference_chosen": -424.1178283691406, + "logps/reference_rejected": -484.6050720214844, + "loss": 0.6671, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.23324130475521088, + "rewards/margins": 0.09937307238578796, + "rewards/rejected": 0.1338682323694229, + "step": 42, + "tokens/chosen_response_len_mean": 593.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 479.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.03, + "grad_norm": 1.7422493551589444, + "learning_rate": 1.7199999999999998e-07, + "logits/policy_chosen": -2.763662338256836, + "logits/policy_rejected": -2.434535503387451, + "logits/reference_chosen": -2.710179567337036, + "logits/reference_rejected": -1.868179440498352, + "logps/policy_chosen": -77.93124389648438, + "logps/policy_rejected": -110.21360778808594, + "logps/reference_chosen": -86.2520751953125, + "logps/reference_rejected": -110.11550903320312, + "loss": 0.679, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.08320826292037964, + "rewards/margins": 0.08418922126293182, + "rewards/rejected": -0.0009809592738747597, + "step": 43, + "tokens/chosen_response_len_mean": 152.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 85.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.04, + "grad_norm": 3.138281711904061, + "learning_rate": 1.76e-07, + "logits/policy_chosen": -2.8771750926971436, + "logits/policy_rejected": -2.9175660610198975, + "logits/reference_chosen": -2.8088254928588867, + "logits/reference_rejected": -2.877462863922119, + "logps/policy_chosen": -37.14703369140625, + "logps/policy_rejected": -55.703121185302734, + "logps/reference_chosen": -38.25192642211914, + "logps/reference_rejected": -56.50023651123047, + "loss": 0.686, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.011048920452594757, + "rewards/margins": 0.0030777817592024803, + "rewards/rejected": 0.007971138693392277, + "step": 44, + "tokens/chosen_response_len_mean": 66.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 70.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.04, + "grad_norm": 3.284990775967774, + "learning_rate": 1.8e-07, + "logits/policy_chosen": -2.8581864833831787, + "logits/policy_rejected": -2.766713857650757, + "logits/reference_chosen": -2.8522939682006836, + "logits/reference_rejected": -2.7666430473327637, + "logps/policy_chosen": -165.7818603515625, + "logps/policy_rejected": -241.84002685546875, + "logps/reference_chosen": -181.072998046875, + "logps/reference_rejected": -250.3277587890625, + "loss": 0.6537, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.15291133522987366, + "rewards/margins": 0.06803382933139801, + "rewards/rejected": 0.08487750589847565, + "step": 45, + "tokens/chosen_response_len_mean": 560.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 473.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.04, + "grad_norm": 2.463717582717747, + "learning_rate": 1.8399999999999998e-07, + "logits/policy_chosen": -2.900191068649292, + "logits/policy_rejected": -2.9184107780456543, + "logits/reference_chosen": -2.8673532009124756, + "logits/reference_rejected": -2.892207145690918, + "logps/policy_chosen": -51.21653747558594, + "logps/policy_rejected": -107.85240936279297, + "logps/reference_chosen": -54.76094436645508, + "logps/reference_rejected": -117.80521392822266, + "loss": 0.695, + "rewards/accuracies": 0.0, + "rewards/chosen": 0.035444095730781555, + "rewards/margins": -0.06408397108316422, + "rewards/rejected": 0.09952806681394577, + "step": 46, + "tokens/chosen_response_len_mean": 109.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 108.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.04, + "grad_norm": 1.9738986795582658, + "learning_rate": 1.88e-07, + "logits/policy_chosen": -2.2502527236938477, + "logits/policy_rejected": -2.245668649673462, + "logits/reference_chosen": -2.177823781967163, + "logits/reference_rejected": -2.223409652709961, + "logps/policy_chosen": -37.820743560791016, + "logps/policy_rejected": -99.87039184570312, + "logps/reference_chosen": -44.7762451171875, + "logps/reference_rejected": -104.45420837402344, + "loss": 0.6825, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.06955502927303314, + "rewards/margins": 0.023716777563095093, + "rewards/rejected": 0.04583824798464775, + "step": 47, + "tokens/chosen_response_len_mean": 93.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 110.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.04, + "grad_norm": 2.8813658188320335, + "learning_rate": 1.92e-07, + "logits/policy_chosen": -2.6205151081085205, + "logits/policy_rejected": -2.5673413276672363, + "logits/reference_chosen": -2.5005288124084473, + "logits/reference_rejected": -2.561748504638672, + "logps/policy_chosen": -65.36790466308594, + "logps/policy_rejected": -167.23562622070312, + "logps/reference_chosen": -72.64520263671875, + "logps/reference_rejected": -172.6849365234375, + "loss": 0.6706, + "rewards/accuracies": 0.25, + "rewards/chosen": 0.07277299463748932, + "rewards/margins": 0.018280120566487312, + "rewards/rejected": 0.054492875933647156, + "step": 48, + "tokens/chosen_response_len_mean": 199.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 209.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.04, + "grad_norm": 2.862318181422901, + "learning_rate": 1.96e-07, + "logits/policy_chosen": -2.701411247253418, + "logits/policy_rejected": -2.8611505031585693, + "logits/reference_chosen": -2.2373385429382324, + "logits/reference_rejected": -2.845200777053833, + "logps/policy_chosen": -126.9096908569336, + "logps/policy_rejected": -269.68243408203125, + "logps/reference_chosen": -136.80958557128906, + "logps/reference_rejected": -277.586181640625, + "loss": 0.6796, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.09899893403053284, + "rewards/margins": 0.01996154710650444, + "rewards/rejected": 0.0790373906493187, + "step": 49, + "tokens/chosen_response_len_mean": 262.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 337.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.04, + "grad_norm": 2.324007938345189, + "learning_rate": 2e-07, + "logits/policy_chosen": -2.2418417930603027, + "logits/policy_rejected": -2.1697628498077393, + "logits/reference_chosen": -2.142324924468994, + "logits/reference_rejected": -2.1179537773132324, + "logps/policy_chosen": -136.041259765625, + "logps/policy_rejected": -266.0906677246094, + "logps/reference_chosen": -151.6305694580078, + "logps/reference_rejected": -273.1885986328125, + "loss": 0.6677, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.15589313209056854, + "rewards/margins": 0.08491363376379013, + "rewards/rejected": 0.07097949832677841, + "step": 50, + "tokens/chosen_response_len_mean": 526.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 469.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.04, + "grad_norm": 2.3504496124001744, + "learning_rate": 2.0399999999999997e-07, + "logits/policy_chosen": -2.899348497390747, + "logits/policy_rejected": -2.8138720989227295, + "logits/reference_chosen": -2.8722875118255615, + "logits/reference_rejected": -2.8029227256774902, + "logps/policy_chosen": -63.48197555541992, + "logps/policy_rejected": -101.47474670410156, + "logps/reference_chosen": -68.96565246582031, + "logps/reference_rejected": -108.34456634521484, + "loss": 0.6831, + "rewards/accuracies": 0.25, + "rewards/chosen": 0.05483681336045265, + "rewards/margins": -0.013861382380127907, + "rewards/rejected": 0.06869819760322571, + "step": 51, + "tokens/chosen_response_len_mean": 212.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 181.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.04, + "grad_norm": 2.4664067603984665, + "learning_rate": 2.0799999999999998e-07, + "logits/policy_chosen": -2.710914134979248, + "logits/policy_rejected": -2.6489503383636475, + "logits/reference_chosen": -2.6887543201446533, + "logits/reference_rejected": -2.565074920654297, + "logps/policy_chosen": -120.6403579711914, + "logps/policy_rejected": -180.31842041015625, + "logps/reference_chosen": -129.82321166992188, + "logps/reference_rejected": -187.31112670898438, + "loss": 0.6824, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.0918285921216011, + "rewards/margins": 0.021901624277234077, + "rewards/rejected": 0.06992696225643158, + "step": 52, + "tokens/chosen_response_len_mean": 340.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 269.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.04, + "grad_norm": 3.7734473565962108, + "learning_rate": 2.12e-07, + "logits/policy_chosen": -2.9076874256134033, + "logits/policy_rejected": -2.8471598625183105, + "logits/reference_chosen": -2.837496280670166, + "logits/reference_rejected": -2.783947467803955, + "logps/policy_chosen": -88.39242553710938, + "logps/policy_rejected": -232.25318908691406, + "logps/reference_chosen": -96.45068359375, + "logps/reference_rejected": -237.35009765625, + "loss": 0.6784, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.08058254420757294, + "rewards/margins": 0.02961355820298195, + "rewards/rejected": 0.05096898972988129, + "step": 53, + "tokens/chosen_response_len_mean": 195.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 194.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.04, + "grad_norm": 2.8319761915445367, + "learning_rate": 2.1599999999999998e-07, + "logits/policy_chosen": -3.144252300262451, + "logits/policy_rejected": -3.0141947269439697, + "logits/reference_chosen": -3.116751194000244, + "logits/reference_rejected": -2.8330681324005127, + "logps/policy_chosen": -132.089111328125, + "logps/policy_rejected": -225.1402587890625, + "logps/reference_chosen": -136.70138549804688, + "logps/reference_rejected": -224.64724731445312, + "loss": 0.6737, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.046122774481773376, + "rewards/margins": 0.05105280876159668, + "rewards/rejected": -0.004930033814162016, + "step": 54, + "tokens/chosen_response_len_mean": 299.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 257.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.04, + "grad_norm": 2.947847201514034, + "learning_rate": 2.1999999999999998e-07, + "logits/policy_chosen": -2.8887228965759277, + "logits/policy_rejected": -2.68259596824646, + "logits/reference_chosen": -2.864752769470215, + "logits/reference_rejected": -2.628319263458252, + "logps/policy_chosen": -135.53025817871094, + "logps/policy_rejected": -133.9132843017578, + "logps/reference_chosen": -145.9614715576172, + "logps/reference_rejected": -137.02508544921875, + "loss": 0.666, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.10431211441755295, + "rewards/margins": 0.07319390773773193, + "rewards/rejected": 0.031118206679821014, + "step": 55, + "tokens/chosen_response_len_mean": 349.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 196.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.05, + "grad_norm": 2.771906040366205, + "learning_rate": 2.24e-07, + "logits/policy_chosen": -2.231527328491211, + "logits/policy_rejected": -2.0439558029174805, + "logits/reference_chosen": -2.230081796646118, + "logits/reference_rejected": -2.080432176589966, + "logps/policy_chosen": -399.80303955078125, + "logps/policy_rejected": -446.4791259765625, + "logps/reference_chosen": -431.472412109375, + "logps/reference_rejected": -470.2456970214844, + "loss": 0.6709, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.31669357419013977, + "rewards/margins": 0.07902783155441284, + "rewards/rejected": 0.23766574263572693, + "step": 56, + "tokens/chosen_response_len_mean": 805.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 711.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.05, + "grad_norm": 1.7428598517814506, + "learning_rate": 2.28e-07, + "logits/policy_chosen": -2.7684319019317627, + "logits/policy_rejected": -2.71665096282959, + "logits/reference_chosen": -2.747512102127075, + "logits/reference_rejected": -2.690124034881592, + "logps/policy_chosen": -85.4046630859375, + "logps/policy_rejected": -111.96269226074219, + "logps/reference_chosen": -96.96040344238281, + "logps/reference_rejected": -121.33839416503906, + "loss": 0.6888, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.11555740237236023, + "rewards/margins": 0.021800296381115913, + "rewards/rejected": 0.09375710785388947, + "step": 57, + "tokens/chosen_response_len_mean": 261.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 248.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.05, + "grad_norm": 2.6040542171359817, + "learning_rate": 2.32e-07, + "logits/policy_chosen": -2.989000082015991, + "logits/policy_rejected": -2.8631539344787598, + "logits/reference_chosen": -2.9670627117156982, + "logits/reference_rejected": -2.5129096508026123, + "logps/policy_chosen": -87.29337310791016, + "logps/policy_rejected": -119.82539367675781, + "logps/reference_chosen": -95.52102661132812, + "logps/reference_rejected": -122.27404022216797, + "loss": 0.6657, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.08227652311325073, + "rewards/margins": 0.05779005587100983, + "rewards/rejected": 0.024486469104886055, + "step": 58, + "tokens/chosen_response_len_mean": 162.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 138.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.05, + "grad_norm": 3.249420407772776, + "learning_rate": 2.3599999999999997e-07, + "logits/policy_chosen": -2.9996423721313477, + "logits/policy_rejected": -2.974095582962036, + "logits/reference_chosen": -2.964367151260376, + "logits/reference_rejected": -2.953282356262207, + "logps/policy_chosen": -56.97121810913086, + "logps/policy_rejected": -141.20883178710938, + "logps/reference_chosen": -58.637413024902344, + "logps/reference_rejected": -143.08058166503906, + "loss": 0.679, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.0166619960218668, + "rewards/margins": -0.0020555637311190367, + "rewards/rejected": 0.018717560917139053, + "step": 59, + "tokens/chosen_response_len_mean": 121.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 127.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.05, + "grad_norm": 3.503891824715986, + "learning_rate": 2.4e-07, + "logits/policy_chosen": -3.207751512527466, + "logits/policy_rejected": -2.8613648414611816, + "logits/reference_chosen": -3.1973516941070557, + "logits/reference_rejected": -2.244464874267578, + "logps/policy_chosen": -115.4853286743164, + "logps/policy_rejected": -205.64161682128906, + "logps/reference_chosen": -123.15602111816406, + "logps/reference_rejected": -206.07244873046875, + "loss": 0.6713, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.0767069011926651, + "rewards/margins": 0.07239863276481628, + "rewards/rejected": 0.004308272153139114, + "step": 60, + "tokens/chosen_response_len_mean": 262.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 185.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.05, + "grad_norm": 2.987682806191956, + "learning_rate": 2.4399999999999996e-07, + "logits/policy_chosen": -2.65006947517395, + "logits/policy_rejected": -2.60097599029541, + "logits/reference_chosen": -2.6217522621154785, + "logits/reference_rejected": -2.507542610168457, + "logps/policy_chosen": -184.2285614013672, + "logps/policy_rejected": -184.65708923339844, + "logps/reference_chosen": -215.3209228515625, + "logps/reference_rejected": -187.43243408203125, + "loss": 0.6592, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.31092363595962524, + "rewards/margins": 0.2831703722476959, + "rewards/rejected": 0.027753248810768127, + "step": 61, + "tokens/chosen_response_len_mean": 344.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 292.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.05, + "grad_norm": 3.02187794129979, + "learning_rate": 2.48e-07, + "logits/policy_chosen": -3.102184534072876, + "logits/policy_rejected": -3.0169076919555664, + "logits/reference_chosen": -3.0980794429779053, + "logits/reference_rejected": -3.06693434715271, + "logps/policy_chosen": -180.20120239257812, + "logps/policy_rejected": -290.76556396484375, + "logps/reference_chosen": -186.42239379882812, + "logps/reference_rejected": -299.5263977050781, + "loss": 0.6663, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.06221204251050949, + "rewards/margins": -0.025396330282092094, + "rewards/rejected": 0.08760837465524673, + "step": 62, + "tokens/chosen_response_len_mean": 628.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 573.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.05, + "grad_norm": 2.2026851566699364, + "learning_rate": 2.52e-07, + "logits/policy_chosen": -2.9184327125549316, + "logits/policy_rejected": -2.7862141132354736, + "logits/reference_chosen": -2.9029111862182617, + "logits/reference_rejected": -2.562068462371826, + "logps/policy_chosen": -176.180908203125, + "logps/policy_rejected": -224.06822204589844, + "logps/reference_chosen": -191.051025390625, + "logps/reference_rejected": -231.65249633789062, + "loss": 0.682, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.14870098233222961, + "rewards/margins": 0.07285812497138977, + "rewards/rejected": 0.07584285736083984, + "step": 63, + "tokens/chosen_response_len_mean": 383.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 315.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.05, + "grad_norm": 2.6923985748881183, + "learning_rate": 2.56e-07, + "logits/policy_chosen": -3.1094112396240234, + "logits/policy_rejected": -3.059873104095459, + "logits/reference_chosen": -3.0696709156036377, + "logits/reference_rejected": -3.0277552604675293, + "logps/policy_chosen": -109.50044250488281, + "logps/policy_rejected": -227.6053009033203, + "logps/reference_chosen": -114.45265197753906, + "logps/reference_rejected": -232.44473266601562, + "loss": 0.6689, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.04952218383550644, + "rewards/margins": 0.0011278600431978703, + "rewards/rejected": 0.04839432239532471, + "step": 64, + "tokens/chosen_response_len_mean": 172.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 173.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.05, + "grad_norm": 2.3372174586928685, + "learning_rate": 2.6e-07, + "logits/policy_chosen": -2.9087581634521484, + "logits/policy_rejected": -2.973167657852173, + "logits/reference_chosen": -2.754256248474121, + "logits/reference_rejected": -2.943002223968506, + "logps/policy_chosen": -16.885406494140625, + "logps/policy_rejected": -58.0799560546875, + "logps/reference_chosen": -16.55327033996582, + "logps/reference_rejected": -58.852867126464844, + "loss": 0.6709, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.0033213470596820116, + "rewards/margins": -0.011050481349229813, + "rewards/rejected": 0.007729134988039732, + "step": 65, + "tokens/chosen_response_len_mean": 25.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 37.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.05, + "grad_norm": 1.8439881768995727, + "learning_rate": 2.64e-07, + "logits/policy_chosen": -3.2132272720336914, + "logits/policy_rejected": -3.156252861022949, + "logits/reference_chosen": -3.1969757080078125, + "logits/reference_rejected": -3.008265733718872, + "logps/policy_chosen": -89.50849914550781, + "logps/policy_rejected": -105.58460998535156, + "logps/reference_chosen": -96.482421875, + "logps/reference_rejected": -111.33866119384766, + "loss": 0.6855, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.06973925232887268, + "rewards/margins": 0.0121987946331501, + "rewards/rejected": 0.05754045769572258, + "step": 66, + "tokens/chosen_response_len_mean": 163.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 142.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.05, + "grad_norm": 1.9679864280666897, + "learning_rate": 2.68e-07, + "logits/policy_chosen": -2.443567991256714, + "logits/policy_rejected": -2.2462878227233887, + "logits/reference_chosen": -2.3940353393554688, + "logits/reference_rejected": -1.4334111213684082, + "logps/policy_chosen": -133.6657257080078, + "logps/policy_rejected": -126.65130615234375, + "logps/reference_chosen": -143.77218627929688, + "logps/reference_rejected": -130.96572875976562, + "loss": 0.6801, + "rewards/accuracies": 0.25, + "rewards/chosen": 0.10106448829174042, + "rewards/margins": 0.057920388877391815, + "rewards/rejected": 0.0431441031396389, + "step": 67, + "tokens/chosen_response_len_mean": 360.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 238.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.05, + "grad_norm": 2.3444172227352817, + "learning_rate": 2.72e-07, + "logits/policy_chosen": -2.447854518890381, + "logits/policy_rejected": -2.4554026126861572, + "logits/reference_chosen": -2.4224636554718018, + "logits/reference_rejected": -2.44649600982666, + "logps/policy_chosen": -116.48789978027344, + "logps/policy_rejected": -196.64881896972656, + "logps/reference_chosen": -129.35415649414062, + "logps/reference_rejected": -209.97300720214844, + "loss": 0.6714, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.12866270542144775, + "rewards/margins": -0.004579143598675728, + "rewards/rejected": 0.13324186205863953, + "step": 68, + "tokens/chosen_response_len_mean": 627.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 597.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.06, + "grad_norm": 1.9355500776493433, + "learning_rate": 2.7600000000000004e-07, + "logits/policy_chosen": -3.001335620880127, + "logits/policy_rejected": -2.8170900344848633, + "logits/reference_chosen": -2.9873673915863037, + "logits/reference_rejected": -2.8246562480926514, + "logps/policy_chosen": -122.1274642944336, + "logps/policy_rejected": -174.14083862304688, + "logps/reference_chosen": -129.572509765625, + "logps/reference_rejected": -175.02163696289062, + "loss": 0.6814, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.07445055991411209, + "rewards/margins": 0.06564265489578247, + "rewards/rejected": 0.008807908743619919, + "step": 69, + "tokens/chosen_response_len_mean": 316.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 212.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.06, + "grad_norm": 3.1880887553688613, + "learning_rate": 2.8e-07, + "logits/policy_chosen": -2.666834831237793, + "logits/policy_rejected": -2.602752208709717, + "logits/reference_chosen": -2.5575764179229736, + "logits/reference_rejected": -2.594496965408325, + "logps/policy_chosen": -144.6117401123047, + "logps/policy_rejected": -323.0421447753906, + "logps/reference_chosen": -156.17013549804688, + "logps/reference_rejected": -333.77972412109375, + "loss": 0.6725, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.11558418720960617, + "rewards/margins": 0.00820855051279068, + "rewards/rejected": 0.10737563669681549, + "step": 70, + "tokens/chosen_response_len_mean": 319.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 349.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.06, + "grad_norm": 2.476324701775898, + "learning_rate": 2.8399999999999995e-07, + "logits/policy_chosen": -2.8258767127990723, + "logits/policy_rejected": -2.808089017868042, + "logits/reference_chosen": -2.8227195739746094, + "logits/reference_rejected": -2.8310012817382812, + "logps/policy_chosen": -141.63214111328125, + "logps/policy_rejected": -178.0096893310547, + "logps/reference_chosen": -151.22640991210938, + "logps/reference_rejected": -182.14822387695312, + "loss": 0.6621, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.09594269841909409, + "rewards/margins": 0.054557494819164276, + "rewards/rejected": 0.04138519987463951, + "step": 71, + "tokens/chosen_response_len_mean": 486.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 302.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.06, + "grad_norm": 2.699352211419919, + "learning_rate": 2.88e-07, + "logits/policy_chosen": -2.597538709640503, + "logits/policy_rejected": -2.41672945022583, + "logits/reference_chosen": -2.5760765075683594, + "logits/reference_rejected": -1.9195555448532104, + "logps/policy_chosen": -247.6851043701172, + "logps/policy_rejected": -382.4797058105469, + "logps/reference_chosen": -263.0463562011719, + "logps/reference_rejected": -387.8023376464844, + "loss": 0.6657, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.15361259877681732, + "rewards/margins": 0.10038647800683975, + "rewards/rejected": 0.05322612449526787, + "step": 72, + "tokens/chosen_response_len_mean": 411.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 344.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.06, + "grad_norm": 2.5969468603738255, + "learning_rate": 2.9199999999999997e-07, + "logits/policy_chosen": -2.7702274322509766, + "logits/policy_rejected": -2.5614566802978516, + "logits/reference_chosen": -2.7665748596191406, + "logits/reference_rejected": -2.3978798389434814, + "logps/policy_chosen": -72.09918212890625, + "logps/policy_rejected": -68.87893676757812, + "logps/reference_chosen": -80.68479919433594, + "logps/reference_rejected": -67.986328125, + "loss": 0.6635, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.08585613965988159, + "rewards/margins": 0.0947822630405426, + "rewards/rejected": -0.008926120586693287, + "step": 73, + "tokens/chosen_response_len_mean": 205.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 90.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.06, + "grad_norm": 2.8994112666814575, + "learning_rate": 2.9599999999999995e-07, + "logits/policy_chosen": -2.9491050243377686, + "logits/policy_rejected": -2.9374337196350098, + "logits/reference_chosen": -2.924247980117798, + "logits/reference_rejected": -2.8732385635375977, + "logps/policy_chosen": -163.5004425048828, + "logps/policy_rejected": -186.22500610351562, + "logps/reference_chosen": -180.18331909179688, + "logps/reference_rejected": -202.66912841796875, + "loss": 0.6673, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.16682888567447662, + "rewards/margins": 0.00238775834441185, + "rewards/rejected": 0.16444112360477448, + "step": 74, + "tokens/chosen_response_len_mean": 365.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 331.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.06, + "grad_norm": 2.1648523975476306, + "learning_rate": 3e-07, + "logits/policy_chosen": -2.1604349613189697, + "logits/policy_rejected": -2.168158769607544, + "logits/reference_chosen": -2.1622910499572754, + "logits/reference_rejected": -2.1353628635406494, + "logps/policy_chosen": -88.18841552734375, + "logps/policy_rejected": -119.30603790283203, + "logps/reference_chosen": -93.11495971679688, + "logps/reference_rejected": -123.05696868896484, + "loss": 0.6881, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.04926542192697525, + "rewards/margins": 0.011756086722016335, + "rewards/rejected": 0.037509337067604065, + "step": 75, + "tokens/chosen_response_len_mean": 258.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 199.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.06, + "grad_norm": 2.403809139260171, + "learning_rate": 3.0399999999999997e-07, + "logits/policy_chosen": -2.830843925476074, + "logits/policy_rejected": -2.8032288551330566, + "logits/reference_chosen": -2.8209171295166016, + "logits/reference_rejected": -2.797828435897827, + "logps/policy_chosen": -69.43246459960938, + "logps/policy_rejected": -162.20603942871094, + "logps/reference_chosen": -74.48234558105469, + "logps/reference_rejected": -164.12376403808594, + "loss": 0.6658, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.05049885809421539, + "rewards/margins": 0.03132178634405136, + "rewards/rejected": 0.019177071750164032, + "step": 76, + "tokens/chosen_response_len_mean": 142.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 126.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.06, + "grad_norm": 2.854272678012172, + "learning_rate": 3.08e-07, + "logits/policy_chosen": -3.0018627643585205, + "logits/policy_rejected": -2.6590089797973633, + "logits/reference_chosen": -2.995105028152466, + "logits/reference_rejected": -2.385610580444336, + "logps/policy_chosen": -85.27000427246094, + "logps/policy_rejected": -132.92564392089844, + "logps/reference_chosen": -90.3302001953125, + "logps/reference_rejected": -134.35781860351562, + "loss": 0.6608, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.05060192570090294, + "rewards/margins": 0.03628016635775566, + "rewards/rejected": 0.014321764931082726, + "step": 77, + "tokens/chosen_response_len_mean": 363.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 320.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.06, + "grad_norm": 2.7505136356983133, + "learning_rate": 3.12e-07, + "logits/policy_chosen": -2.0338382720947266, + "logits/policy_rejected": -2.276106357574463, + "logits/reference_chosen": -2.0206451416015625, + "logits/reference_rejected": -2.060913562774658, + "logps/policy_chosen": -232.1044464111328, + "logps/policy_rejected": -146.2607879638672, + "logps/reference_chosen": -250.27952575683594, + "logps/reference_rejected": -149.79270935058594, + "loss": 0.6679, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.18175065517425537, + "rewards/margins": 0.14643150568008423, + "rewards/rejected": 0.03531915321946144, + "step": 78, + "tokens/chosen_response_len_mean": 521.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 307.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.06, + "grad_norm": 2.4952987336105665, + "learning_rate": 3.1599999999999997e-07, + "logits/policy_chosen": -2.8599331378936768, + "logits/policy_rejected": -2.8679845333099365, + "logits/reference_chosen": -2.7079713344573975, + "logits/reference_rejected": -2.4389915466308594, + "logps/policy_chosen": -148.4790496826172, + "logps/policy_rejected": -277.09405517578125, + "logps/reference_chosen": -156.32412719726562, + "logps/reference_rejected": -281.2509765625, + "loss": 0.6709, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.0784510001540184, + "rewards/margins": 0.03688175603747368, + "rewards/rejected": 0.041569244116544724, + "step": 79, + "tokens/chosen_response_len_mean": 276.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 336.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.06, + "grad_norm": 2.3479129695327297, + "learning_rate": 3.2e-07, + "logits/policy_chosen": -1.8025144338607788, + "logits/policy_rejected": -1.9184333086013794, + "logits/reference_chosen": -1.7402411699295044, + "logits/reference_rejected": -1.906538724899292, + "logps/policy_chosen": -79.56036376953125, + "logps/policy_rejected": -185.89906311035156, + "logps/reference_chosen": -88.7896499633789, + "logps/reference_rejected": -198.23004150390625, + "loss": 0.6921, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.09229286015033722, + "rewards/margins": -0.03101687878370285, + "rewards/rejected": 0.12330973893404007, + "step": 80, + "tokens/chosen_response_len_mean": 432.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 410.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.07, + "grad_norm": 1.4788277018262954, + "learning_rate": 3.24e-07, + "logits/policy_chosen": -2.7751684188842773, + "logits/policy_rejected": -2.7188775539398193, + "logits/reference_chosen": -2.774470567703247, + "logits/reference_rejected": -2.3951942920684814, + "logps/policy_chosen": -64.14806365966797, + "logps/policy_rejected": -91.78968048095703, + "logps/reference_chosen": -70.32818603515625, + "logps/reference_rejected": -92.31802368164062, + "loss": 0.6813, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.06180126219987869, + "rewards/margins": 0.05651785433292389, + "rewards/rejected": 0.005283408332616091, + "step": 81, + "tokens/chosen_response_len_mean": 345.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 316.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.07, + "grad_norm": 2.8935594664890503, + "learning_rate": 3.28e-07, + "logits/policy_chosen": -2.936018943786621, + "logits/policy_rejected": -2.9313597679138184, + "logits/reference_chosen": -2.9295578002929688, + "logits/reference_rejected": -2.9169669151306152, + "logps/policy_chosen": -141.27691650390625, + "logps/policy_rejected": -264.863037109375, + "logps/reference_chosen": -152.80865478515625, + "logps/reference_rejected": -274.9104919433594, + "loss": 0.6644, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.11531716585159302, + "rewards/margins": 0.014842674136161804, + "rewards/rejected": 0.10047448426485062, + "step": 82, + "tokens/chosen_response_len_mean": 394.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 390.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.07, + "grad_norm": 2.4649587565099678, + "learning_rate": 3.32e-07, + "logits/policy_chosen": -2.9249117374420166, + "logits/policy_rejected": -2.9090662002563477, + "logits/reference_chosen": -2.868709087371826, + "logits/reference_rejected": -2.898679256439209, + "logps/policy_chosen": -154.240234375, + "logps/policy_rejected": -297.08160400390625, + "logps/reference_chosen": -161.24609375, + "logps/reference_rejected": -297.5753479003906, + "loss": 0.6687, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.07005855441093445, + "rewards/margins": 0.06512109935283661, + "rewards/rejected": 0.004937455989420414, + "step": 83, + "tokens/chosen_response_len_mean": 334.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 309.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.07, + "grad_norm": 2.5355636198848175, + "learning_rate": 3.36e-07, + "logits/policy_chosen": -2.9488754272460938, + "logits/policy_rejected": -2.9289844036102295, + "logits/reference_chosen": -2.890922784805298, + "logits/reference_rejected": -2.870727062225342, + "logps/policy_chosen": -18.58037757873535, + "logps/policy_rejected": -47.89413833618164, + "logps/reference_chosen": -19.83108901977539, + "logps/reference_rejected": -48.526878356933594, + "loss": 0.6755, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.012507104314863682, + "rewards/margins": 0.006179675459861755, + "rewards/rejected": 0.006327427923679352, + "step": 84, + "tokens/chosen_response_len_mean": 27.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 32.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.07, + "grad_norm": 2.5319776077967746, + "learning_rate": 3.4000000000000003e-07, + "logits/policy_chosen": -2.7499167919158936, + "logits/policy_rejected": -2.690596103668213, + "logits/reference_chosen": -2.73750638961792, + "logits/reference_rejected": -2.6311354637145996, + "logps/policy_chosen": -123.83195495605469, + "logps/policy_rejected": -194.00137329101562, + "logps/reference_chosen": -139.81263732910156, + "logps/reference_rejected": -196.44784545898438, + "loss": 0.6523, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.15980683267116547, + "rewards/margins": 0.13534212112426758, + "rewards/rejected": 0.024464718997478485, + "step": 85, + "tokens/chosen_response_len_mean": 606.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 519.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.07, + "grad_norm": 2.007472503017741, + "learning_rate": 3.4399999999999996e-07, + "logits/policy_chosen": -2.9675540924072266, + "logits/policy_rejected": -2.9613142013549805, + "logits/reference_chosen": -2.9432828426361084, + "logits/reference_rejected": -2.9374613761901855, + "logps/policy_chosen": -12.885274887084961, + "logps/policy_rejected": -34.69200134277344, + "logps/reference_chosen": -13.591547012329102, + "logps/reference_rejected": -35.15093994140625, + "loss": 0.6775, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.007062721066176891, + "rewards/margins": 0.002473349217325449, + "rewards/rejected": 0.004589371383190155, + "step": 86, + "tokens/chosen_response_len_mean": 11.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 12.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.07, + "grad_norm": 2.3366851120609398, + "learning_rate": 3.4799999999999994e-07, + "logits/policy_chosen": -2.510186195373535, + "logits/policy_rejected": -2.3673720359802246, + "logits/reference_chosen": -2.5208425521850586, + "logits/reference_rejected": -2.307051658630371, + "logps/policy_chosen": -103.87987518310547, + "logps/policy_rejected": -118.88311767578125, + "logps/reference_chosen": -114.24214172363281, + "logps/reference_rejected": -118.43659973144531, + "loss": 0.664, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.10362273454666138, + "rewards/margins": 0.10808798670768738, + "rewards/rejected": -0.004465251229703426, + "step": 87, + "tokens/chosen_response_len_mean": 216.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 106.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.07, + "grad_norm": 2.4579695955138874, + "learning_rate": 3.52e-07, + "logits/policy_chosen": -2.9871153831481934, + "logits/policy_rejected": -2.8764257431030273, + "logits/reference_chosen": -2.9515204429626465, + "logits/reference_rejected": -2.883835792541504, + "logps/policy_chosen": -151.91807556152344, + "logps/policy_rejected": -141.33587646484375, + "logps/reference_chosen": -162.7379608154297, + "logps/reference_rejected": -147.37442016601562, + "loss": 0.6695, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.10819891095161438, + "rewards/margins": 0.04781348258256912, + "rewards/rejected": 0.06038542836904526, + "step": 88, + "tokens/chosen_response_len_mean": 299.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 223.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.07, + "grad_norm": 2.2124030442195703, + "learning_rate": 3.5599999999999996e-07, + "logits/policy_chosen": -3.097362995147705, + "logits/policy_rejected": -2.821688652038574, + "logits/reference_chosen": -3.0792651176452637, + "logits/reference_rejected": -2.6497914791107178, + "logps/policy_chosen": -106.85523986816406, + "logps/policy_rejected": -142.70628356933594, + "logps/reference_chosen": -114.98326873779297, + "logps/reference_rejected": -145.11463928222656, + "loss": 0.6769, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.08128029853105545, + "rewards/margins": 0.05719655007123947, + "rewards/rejected": 0.02408374845981598, + "step": 89, + "tokens/chosen_response_len_mean": 288.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 241.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.07, + "grad_norm": 2.726548509097342, + "learning_rate": 3.6e-07, + "logits/policy_chosen": -2.911972761154175, + "logits/policy_rejected": -2.7545974254608154, + "logits/reference_chosen": -2.8914873600006104, + "logits/reference_rejected": -2.4686713218688965, + "logps/policy_chosen": -90.4947509765625, + "logps/policy_rejected": -144.89512634277344, + "logps/reference_chosen": -100.72172546386719, + "logps/reference_rejected": -144.84951782226562, + "loss": 0.6578, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.1022697389125824, + "rewards/margins": 0.10272593796253204, + "rewards/rejected": -0.0004562041722238064, + "step": 90, + "tokens/chosen_response_len_mean": 254.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 112.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.07, + "grad_norm": 2.2900988215150644, + "learning_rate": 3.64e-07, + "logits/policy_chosen": -2.8383259773254395, + "logits/policy_rejected": -2.824490547180176, + "logits/reference_chosen": -2.8273422718048096, + "logits/reference_rejected": -2.8102381229400635, + "logps/policy_chosen": -132.3096466064453, + "logps/policy_rejected": -264.598388671875, + "logps/reference_chosen": -139.56736755371094, + "logps/reference_rejected": -264.87725830078125, + "loss": 0.6825, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.07257718592882156, + "rewards/margins": 0.0697886273264885, + "rewards/rejected": 0.002788563258945942, + "step": 91, + "tokens/chosen_response_len_mean": 395.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 397.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.07, + "grad_norm": 2.8545865680492692, + "learning_rate": 3.6799999999999996e-07, + "logits/policy_chosen": -2.7706384658813477, + "logits/policy_rejected": -2.5063834190368652, + "logits/reference_chosen": -2.7504565715789795, + "logits/reference_rejected": -1.929320216178894, + "logps/policy_chosen": -126.52993774414062, + "logps/policy_rejected": -134.23519897460938, + "logps/reference_chosen": -138.69195556640625, + "logps/reference_rejected": -138.32545471191406, + "loss": 0.6536, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.12162025272846222, + "rewards/margins": 0.08071767538785934, + "rewards/rejected": 0.040902577340602875, + "step": 92, + "tokens/chosen_response_len_mean": 315.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 201.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.07, + "grad_norm": 2.284769139553738, + "learning_rate": 3.72e-07, + "logits/policy_chosen": -2.872082233428955, + "logits/policy_rejected": -2.6889710426330566, + "logits/reference_chosen": -2.8072874546051025, + "logits/reference_rejected": -2.4844841957092285, + "logps/policy_chosen": -74.13471984863281, + "logps/policy_rejected": -116.40203857421875, + "logps/reference_chosen": -80.3674545288086, + "logps/reference_rejected": -114.1785888671875, + "loss": 0.6602, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.06232735142111778, + "rewards/margins": 0.08456189930438995, + "rewards/rejected": -0.022234544157981873, + "step": 93, + "tokens/chosen_response_len_mean": 162.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 95.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.08, + "grad_norm": 1.898688154588326, + "learning_rate": 3.76e-07, + "logits/policy_chosen": -2.469783306121826, + "logits/policy_rejected": -2.3272128105163574, + "logits/reference_chosen": -2.425753593444824, + "logits/reference_rejected": -2.3546535968780518, + "logps/policy_chosen": -158.62905883789062, + "logps/policy_rejected": -291.726318359375, + "logps/reference_chosen": -168.30911254882812, + "logps/reference_rejected": -291.2076110839844, + "loss": 0.6805, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.09680052101612091, + "rewards/margins": 0.10198744386434555, + "rewards/rejected": -0.005186920985579491, + "step": 94, + "tokens/chosen_response_len_mean": 359.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 302.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.08, + "grad_norm": 2.9949420570538363, + "learning_rate": 3.7999999999999996e-07, + "logits/policy_chosen": -2.8909621238708496, + "logits/policy_rejected": -2.906135082244873, + "logits/reference_chosen": -2.792299747467041, + "logits/reference_rejected": -2.848114252090454, + "logps/policy_chosen": -50.312339782714844, + "logps/policy_rejected": -105.15270233154297, + "logps/reference_chosen": -52.856605529785156, + "logps/reference_rejected": -108.09819030761719, + "loss": 0.6483, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.02544267475605011, + "rewards/margins": -0.004012217745184898, + "rewards/rejected": 0.029454896226525307, + "step": 95, + "tokens/chosen_response_len_mean": 116.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 120.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.08, + "grad_norm": 2.4241955868213405, + "learning_rate": 3.84e-07, + "logits/policy_chosen": -2.789196014404297, + "logits/policy_rejected": -2.543649911880493, + "logits/reference_chosen": -2.7888166904449463, + "logits/reference_rejected": -2.5362277030944824, + "logps/policy_chosen": -127.26165771484375, + "logps/policy_rejected": -104.05661010742188, + "logps/reference_chosen": -139.64187622070312, + "logps/reference_rejected": -108.60116577148438, + "loss": 0.6656, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.12380214780569077, + "rewards/margins": 0.07835659384727478, + "rewards/rejected": 0.04544556140899658, + "step": 96, + "tokens/chosen_response_len_mean": 381.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 272.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.08, + "grad_norm": 3.4704291601036683, + "learning_rate": 3.88e-07, + "logits/policy_chosen": -2.9072158336639404, + "logits/policy_rejected": -2.845478057861328, + "logits/reference_chosen": -2.882009983062744, + "logits/reference_rejected": -2.943215847015381, + "logps/policy_chosen": -199.15065002441406, + "logps/policy_rejected": -345.61346435546875, + "logps/reference_chosen": -208.939697265625, + "logps/reference_rejected": -351.9114990234375, + "loss": 0.6532, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.09789040684700012, + "rewards/margins": 0.0349101684987545, + "rewards/rejected": 0.06298023462295532, + "step": 97, + "tokens/chosen_response_len_mean": 443.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 334.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.08, + "grad_norm": 2.413585232959013, + "learning_rate": 3.92e-07, + "logits/policy_chosen": -2.581110954284668, + "logits/policy_rejected": -2.5821545124053955, + "logits/reference_chosen": -2.6214118003845215, + "logits/reference_rejected": -2.6232800483703613, + "logps/policy_chosen": -66.4002914428711, + "logps/policy_rejected": -92.362060546875, + "logps/reference_chosen": -73.42770385742188, + "logps/reference_rejected": -96.90080261230469, + "loss": 0.6682, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.07027411460876465, + "rewards/margins": 0.02488674223423004, + "rewards/rejected": 0.0453873835504055, + "step": 98, + "tokens/chosen_response_len_mean": 369.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 353.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.08, + "grad_norm": 2.566883896226423, + "learning_rate": 3.96e-07, + "logits/policy_chosen": -3.13887357711792, + "logits/policy_rejected": -3.1172709465026855, + "logits/reference_chosen": -3.127457618713379, + "logits/reference_rejected": -3.1038241386413574, + "logps/policy_chosen": -131.76913452148438, + "logps/policy_rejected": -274.8529052734375, + "logps/reference_chosen": -142.9169158935547, + "logps/reference_rejected": -277.9893798828125, + "loss": 0.68, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.11147774755954742, + "rewards/margins": 0.08011295646429062, + "rewards/rejected": 0.031364794820547104, + "step": 99, + "tokens/chosen_response_len_mean": 291.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 296.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.08, + "grad_norm": 2.7837569692907294, + "learning_rate": 4e-07, + "logits/policy_chosen": -3.046888589859009, + "logits/policy_rejected": -2.9924631118774414, + "logits/reference_chosen": -3.0203585624694824, + "logits/reference_rejected": -2.9802844524383545, + "logps/policy_chosen": -96.66795349121094, + "logps/policy_rejected": -151.02667236328125, + "logps/reference_chosen": -106.40546417236328, + "logps/reference_rejected": -156.30604553222656, + "loss": 0.6494, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.09737517684698105, + "rewards/margins": 0.04458136856555939, + "rewards/rejected": 0.05279380828142166, + "step": 100, + "tokens/chosen_response_len_mean": 246.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 213.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.08, + "grad_norm": 2.5533090774871385, + "learning_rate": 4.04e-07, + "logits/policy_chosen": -3.0745935440063477, + "logits/policy_rejected": -2.9924702644348145, + "logits/reference_chosen": -3.0464026927948, + "logits/reference_rejected": -2.9301576614379883, + "logps/policy_chosen": -130.01454162597656, + "logps/policy_rejected": -273.28997802734375, + "logps/reference_chosen": -138.4945831298828, + "logps/reference_rejected": -277.6722106933594, + "loss": 0.6659, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.08480045944452286, + "rewards/margins": 0.04097801819443703, + "rewards/rejected": 0.04382243752479553, + "step": 101, + "tokens/chosen_response_len_mean": 351.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 294.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.08, + "grad_norm": 2.658053156359227, + "learning_rate": 4.0799999999999995e-07, + "logits/policy_chosen": -2.9965174198150635, + "logits/policy_rejected": -2.756721019744873, + "logits/reference_chosen": -3.0305070877075195, + "logits/reference_rejected": -2.5150346755981445, + "logps/policy_chosen": -150.61041259765625, + "logps/policy_rejected": -145.7916717529297, + "logps/reference_chosen": -169.15542602539062, + "logps/reference_rejected": -147.83485412597656, + "loss": 0.6388, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.18545001745224, + "rewards/margins": 0.16501817107200623, + "rewards/rejected": 0.02043186128139496, + "step": 102, + "tokens/chosen_response_len_mean": 276.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 211.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.08, + "grad_norm": 1.8527975992911931, + "learning_rate": 4.12e-07, + "logits/policy_chosen": -2.4530792236328125, + "logits/policy_rejected": -2.4156646728515625, + "logits/reference_chosen": -2.4479713439941406, + "logits/reference_rejected": -2.3904917240142822, + "logps/policy_chosen": -173.00030517578125, + "logps/policy_rejected": -199.19415283203125, + "logps/reference_chosen": -189.02093505859375, + "logps/reference_rejected": -209.84201049804688, + "loss": 0.6666, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.16020633280277252, + "rewards/margins": 0.05372770130634308, + "rewards/rejected": 0.10647863149642944, + "step": 103, + "tokens/chosen_response_len_mean": 309.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 310.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.08, + "grad_norm": 2.5555937206717187, + "learning_rate": 4.1599999999999997e-07, + "logits/policy_chosen": -3.0911600589752197, + "logits/policy_rejected": -3.0214200019836426, + "logits/reference_chosen": -3.089872360229492, + "logits/reference_rejected": -2.882385730743408, + "logps/policy_chosen": -153.61936950683594, + "logps/policy_rejected": -227.18930053710938, + "logps/reference_chosen": -176.565673828125, + "logps/reference_rejected": -239.63970947265625, + "loss": 0.665, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.22946298122406006, + "rewards/margins": 0.10495913028717041, + "rewards/rejected": 0.12450385838747025, + "step": 104, + "tokens/chosen_response_len_mean": 368.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 301.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.08, + "grad_norm": 2.0081558824498096, + "learning_rate": 4.1999999999999995e-07, + "logits/policy_chosen": -2.958939790725708, + "logits/policy_rejected": -2.80196213722229, + "logits/reference_chosen": -2.944668769836426, + "logits/reference_rejected": -2.6392948627471924, + "logps/policy_chosen": -37.521568298339844, + "logps/policy_rejected": -74.32329559326172, + "logps/reference_chosen": -41.80109786987305, + "logps/reference_rejected": -72.32585906982422, + "loss": 0.664, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.0427953377366066, + "rewards/margins": 0.06276971846818924, + "rewards/rejected": -0.01997438445687294, + "step": 105, + "tokens/chosen_response_len_mean": 105.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 85.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.09, + "grad_norm": 2.7526269849819696, + "learning_rate": 4.24e-07, + "logits/policy_chosen": -3.100095272064209, + "logits/policy_rejected": -2.8839964866638184, + "logits/reference_chosen": -3.0812511444091797, + "logits/reference_rejected": -2.936655044555664, + "logps/policy_chosen": -207.67698669433594, + "logps/policy_rejected": -240.35739135742188, + "logps/reference_chosen": -223.25807189941406, + "logps/reference_rejected": -239.6046142578125, + "loss": 0.6494, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.15581092238426208, + "rewards/margins": 0.16333873569965363, + "rewards/rejected": -0.007527794688940048, + "step": 106, + "tokens/chosen_response_len_mean": 392.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 203.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.09, + "grad_norm": 1.9552389682760483, + "learning_rate": 4.2799999999999997e-07, + "logits/policy_chosen": -2.922211170196533, + "logits/policy_rejected": -2.94032621383667, + "logits/reference_chosen": -2.9038805961608887, + "logits/reference_rejected": -2.9326558113098145, + "logps/policy_chosen": -142.2540283203125, + "logps/policy_rejected": -238.67227172851562, + "logps/reference_chosen": -151.64273071289062, + "logps/reference_rejected": -239.12753295898438, + "loss": 0.6753, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.09388697892427444, + "rewards/margins": 0.08933433890342712, + "rewards/rejected": 0.004552632570266724, + "step": 107, + "tokens/chosen_response_len_mean": 347.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 352.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.09, + "grad_norm": 3.014699368900832, + "learning_rate": 4.3199999999999995e-07, + "logits/policy_chosen": -3.1775548458099365, + "logits/policy_rejected": -3.03210186958313, + "logits/reference_chosen": -3.1847496032714844, + "logits/reference_rejected": -2.531522750854492, + "logps/policy_chosen": -143.90025329589844, + "logps/policy_rejected": -291.92388916015625, + "logps/reference_chosen": -160.6278839111328, + "logps/reference_rejected": -297.986572265625, + "loss": 0.6498, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.16727623343467712, + "rewards/margins": 0.10664979368448257, + "rewards/rejected": 0.060626450926065445, + "step": 108, + "tokens/chosen_response_len_mean": 411.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 381.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.09, + "grad_norm": 3.3247511691725236, + "learning_rate": 4.36e-07, + "logits/policy_chosen": -3.0070884227752686, + "logits/policy_rejected": -3.006807565689087, + "logits/reference_chosen": -2.9760210514068604, + "logits/reference_rejected": -2.982029438018799, + "logps/policy_chosen": -117.77191925048828, + "logps/policy_rejected": -288.87689208984375, + "logps/reference_chosen": -130.60389709472656, + "logps/reference_rejected": -292.3191833496094, + "loss": 0.6517, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.12831971049308777, + "rewards/margins": 0.09389686584472656, + "rewards/rejected": 0.034422844648361206, + "step": 109, + "tokens/chosen_response_len_mean": 318.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 302.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.09, + "grad_norm": 3.158975765008515, + "learning_rate": 4.3999999999999997e-07, + "logits/policy_chosen": -2.50785756111145, + "logits/policy_rejected": -2.504612684249878, + "logits/reference_chosen": -2.5301709175109863, + "logits/reference_rejected": -2.5272464752197266, + "logps/policy_chosen": -136.28662109375, + "logps/policy_rejected": -200.89352416992188, + "logps/reference_chosen": -158.42433166503906, + "logps/reference_rejected": -212.5179443359375, + "loss": 0.638, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.22137700021266937, + "rewards/margins": 0.10513283312320709, + "rewards/rejected": 0.11624416708946228, + "step": 110, + "tokens/chosen_response_len_mean": 608.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 525.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.09, + "grad_norm": 2.8278528577842095, + "learning_rate": 4.44e-07, + "logits/policy_chosen": -2.725069999694824, + "logits/policy_rejected": -2.732088565826416, + "logits/reference_chosen": -2.6485838890075684, + "logits/reference_rejected": -2.76961612701416, + "logps/policy_chosen": -178.09320068359375, + "logps/policy_rejected": -237.05970764160156, + "logps/reference_chosen": -192.22482299804688, + "logps/reference_rejected": -248.2169647216797, + "loss": 0.6253, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.14131632447242737, + "rewards/margins": 0.029743948951363564, + "rewards/rejected": 0.11157237738370895, + "step": 111, + "tokens/chosen_response_len_mean": 589.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 556.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.09, + "grad_norm": 3.4191485129114714, + "learning_rate": 4.48e-07, + "logits/policy_chosen": -2.7914977073669434, + "logits/policy_rejected": -2.804826259613037, + "logits/reference_chosen": -2.7042627334594727, + "logits/reference_rejected": -2.640076160430908, + "logps/policy_chosen": -146.0531768798828, + "logps/policy_rejected": -232.5332794189453, + "logps/reference_chosen": -168.23060607910156, + "logps/reference_rejected": -237.49801635742188, + "loss": 0.6295, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.22177428007125854, + "rewards/margins": 0.172126904129982, + "rewards/rejected": 0.049647387117147446, + "step": 112, + "tokens/chosen_response_len_mean": 438.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 396.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.09, + "grad_norm": 2.331597318165628, + "learning_rate": 4.5199999999999997e-07, + "logits/policy_chosen": -3.1060757637023926, + "logits/policy_rejected": -2.961545467376709, + "logits/reference_chosen": -3.1052682399749756, + "logits/reference_rejected": -2.833801746368408, + "logps/policy_chosen": -127.66785430908203, + "logps/policy_rejected": -196.163330078125, + "logps/reference_chosen": -137.27505493164062, + "logps/reference_rejected": -198.14578247070312, + "loss": 0.6605, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.09607198089361191, + "rewards/margins": 0.07624746859073639, + "rewards/rejected": 0.01982451230287552, + "step": 113, + "tokens/chosen_response_len_mean": 314.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 291.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.09, + "grad_norm": 2.750538166446364, + "learning_rate": 4.56e-07, + "logits/policy_chosen": -2.7500100135803223, + "logits/policy_rejected": -2.681910514831543, + "logits/reference_chosen": -2.789179563522339, + "logits/reference_rejected": -2.3359477519989014, + "logps/policy_chosen": -85.98888397216797, + "logps/policy_rejected": -100.74612426757812, + "logps/reference_chosen": -98.14977264404297, + "logps/reference_rejected": -102.4841537475586, + "loss": 0.6408, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.12160897254943848, + "rewards/margins": 0.10422859340906143, + "rewards/rejected": 0.017380371689796448, + "step": 114, + "tokens/chosen_response_len_mean": 256.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 223.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.09, + "grad_norm": 2.5440555248438996, + "learning_rate": 4.6e-07, + "logits/policy_chosen": -2.971923351287842, + "logits/policy_rejected": -2.989212989807129, + "logits/reference_chosen": -2.9268057346343994, + "logits/reference_rejected": -2.919334650039673, + "logps/policy_chosen": -98.5408706665039, + "logps/policy_rejected": -166.144775390625, + "logps/reference_chosen": -109.1729965209961, + "logps/reference_rejected": -167.0085906982422, + "loss": 0.6505, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.10632127523422241, + "rewards/margins": 0.09768319875001907, + "rewards/rejected": 0.008638077415525913, + "step": 115, + "tokens/chosen_response_len_mean": 229.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 254.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.09, + "grad_norm": 2.4244019154261474, + "learning_rate": 4.64e-07, + "logits/policy_chosen": -2.8886287212371826, + "logits/policy_rejected": -2.922623872756958, + "logits/reference_chosen": -2.8570451736450195, + "logits/reference_rejected": -2.888643741607666, + "logps/policy_chosen": -113.93293762207031, + "logps/policy_rejected": -225.3817596435547, + "logps/reference_chosen": -120.10133361816406, + "logps/reference_rejected": -223.7104034423828, + "loss": 0.6579, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.06168397143483162, + "rewards/margins": 0.0783977136015892, + "rewards/rejected": -0.016713742166757584, + "step": 116, + "tokens/chosen_response_len_mean": 330.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 318.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.09, + "grad_norm": 2.400248188855275, + "learning_rate": 4.68e-07, + "logits/policy_chosen": -2.8977608680725098, + "logits/policy_rejected": -2.9003448486328125, + "logits/reference_chosen": -2.8988900184631348, + "logits/reference_rejected": -2.9036717414855957, + "logps/policy_chosen": -74.31845092773438, + "logps/policy_rejected": -92.14234924316406, + "logps/reference_chosen": -78.81231689453125, + "logps/reference_rejected": -93.07196807861328, + "loss": 0.6588, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.04493863508105278, + "rewards/margins": 0.035642486065626144, + "rewards/rejected": 0.00929614994674921, + "step": 117, + "tokens/chosen_response_len_mean": 275.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 274.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.09, + "grad_norm": 2.4323431386681595, + "learning_rate": 4.7199999999999994e-07, + "logits/policy_chosen": -2.6670236587524414, + "logits/policy_rejected": -2.518751621246338, + "logits/reference_chosen": -2.666407585144043, + "logits/reference_rejected": -2.5256845951080322, + "logps/policy_chosen": -92.44061279296875, + "logps/policy_rejected": -151.3938446044922, + "logps/reference_chosen": -100.54401397705078, + "logps/reference_rejected": -153.23507690429688, + "loss": 0.6474, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.08103407174348831, + "rewards/margins": 0.06262162327766418, + "rewards/rejected": 0.01841244101524353, + "step": 118, + "tokens/chosen_response_len_mean": 302.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 332.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.1, + "grad_norm": 2.9673553403256707, + "learning_rate": 4.76e-07, + "logits/policy_chosen": -2.9274580478668213, + "logits/policy_rejected": -2.8594419956207275, + "logits/reference_chosen": -2.9505772590637207, + "logits/reference_rejected": -2.850255012512207, + "logps/policy_chosen": -169.86106872558594, + "logps/policy_rejected": -316.6098327636719, + "logps/reference_chosen": -191.77130126953125, + "logps/reference_rejected": -317.0496520996094, + "loss": 0.6273, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.21910235285758972, + "rewards/margins": 0.21470384299755096, + "rewards/rejected": 0.004398513585329056, + "step": 119, + "tokens/chosen_response_len_mean": 431.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 374.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.1, + "grad_norm": 2.3505092556042304, + "learning_rate": 4.8e-07, + "logits/policy_chosen": -2.814655303955078, + "logits/policy_rejected": -2.5319275856018066, + "logits/reference_chosen": -2.8454208374023438, + "logits/reference_rejected": -2.315487861633301, + "logps/policy_chosen": -50.07554626464844, + "logps/policy_rejected": -110.84510040283203, + "logps/reference_chosen": -60.05486297607422, + "logps/reference_rejected": -106.57113647460938, + "loss": 0.6368, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.09979310631752014, + "rewards/margins": 0.14253273606300354, + "rewards/rejected": -0.0427396334707737, + "step": 120, + "tokens/chosen_response_len_mean": 267.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 187.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.1, + "grad_norm": 2.8274965378312586, + "learning_rate": 4.839999999999999e-07, + "logits/policy_chosen": -2.942972421646118, + "logits/policy_rejected": -2.8031089305877686, + "logits/reference_chosen": -2.961139678955078, + "logits/reference_rejected": -2.5472524166107178, + "logps/policy_chosen": -130.38365173339844, + "logps/policy_rejected": -191.11968994140625, + "logps/reference_chosen": -159.17897033691406, + "logps/reference_rejected": -199.572021484375, + "loss": 0.6282, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2879532277584076, + "rewards/margins": 0.2034299373626709, + "rewards/rejected": 0.0845232680439949, + "step": 121, + "tokens/chosen_response_len_mean": 420.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 307.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.1, + "grad_norm": 2.886588252399639, + "learning_rate": 4.879999999999999e-07, + "logits/policy_chosen": -2.853999376296997, + "logits/policy_rejected": -2.480708122253418, + "logits/reference_chosen": -2.710318088531494, + "logits/reference_rejected": -1.9466197490692139, + "logps/policy_chosen": -167.15240478515625, + "logps/policy_rejected": -201.3469696044922, + "logps/reference_chosen": -181.32119750976562, + "logps/reference_rejected": -195.66384887695312, + "loss": 0.6126, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.14168795943260193, + "rewards/margins": 0.1985190212726593, + "rewards/rejected": -0.056831058114767075, + "step": 122, + "tokens/chosen_response_len_mean": 326.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 201.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.1, + "grad_norm": 2.826829476556088, + "learning_rate": 4.92e-07, + "logits/policy_chosen": -2.465135335922241, + "logits/policy_rejected": -2.4350569248199463, + "logits/reference_chosen": -2.4818878173828125, + "logits/reference_rejected": -2.3954761028289795, + "logps/policy_chosen": -100.21925354003906, + "logps/policy_rejected": -125.87073516845703, + "logps/reference_chosen": -126.35533905029297, + "logps/reference_rejected": -144.03236389160156, + "loss": 0.6328, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.26136085391044617, + "rewards/margins": 0.07974465936422348, + "rewards/rejected": 0.1816161870956421, + "step": 123, + "tokens/chosen_response_len_mean": 426.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 408.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.1, + "grad_norm": 2.5289041457286774, + "learning_rate": 4.96e-07, + "logits/policy_chosen": -2.6869404315948486, + "logits/policy_rejected": -2.773239850997925, + "logits/reference_chosen": -2.653362512588501, + "logits/reference_rejected": -2.5821008682250977, + "logps/policy_chosen": -71.6850357055664, + "logps/policy_rejected": -108.95183563232422, + "logps/reference_chosen": -77.52404022216797, + "logps/reference_rejected": -107.73311614990234, + "loss": 0.6376, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.05839002877473831, + "rewards/margins": 0.07057718932628632, + "rewards/rejected": -0.012187155894935131, + "step": 124, + "tokens/chosen_response_len_mean": 229.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 211.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.1, + "grad_norm": 3.7452560936185804, + "learning_rate": 5e-07, + "logits/policy_chosen": -2.8474879264831543, + "logits/policy_rejected": -2.77992844581604, + "logits/reference_chosen": -2.8415565490722656, + "logits/reference_rejected": -2.658388376235962, + "logps/policy_chosen": -85.31348419189453, + "logps/policy_rejected": -100.18048095703125, + "logps/reference_chosen": -91.71619415283203, + "logps/reference_rejected": -100.39627838134766, + "loss": 0.6215, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.0640270784497261, + "rewards/margins": 0.06186922267079353, + "rewards/rejected": 0.00215785950422287, + "step": 125, + "tokens/chosen_response_len_mean": 91.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 71.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.1, + "grad_norm": 2.791877501388191, + "learning_rate": 4.999990147435196e-07, + "logits/policy_chosen": -2.772634506225586, + "logits/policy_rejected": -2.7663345336914062, + "logits/reference_chosen": -2.8182811737060547, + "logits/reference_rejected": -2.821171760559082, + "logps/policy_chosen": -114.60906982421875, + "logps/policy_rejected": -122.4944839477539, + "logps/reference_chosen": -141.17056274414062, + "logps/reference_rejected": -135.64431762695312, + "loss": 0.63, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2656148672103882, + "rewards/margins": 0.13411659002304077, + "rewards/rejected": 0.1314982771873474, + "step": 126, + "tokens/chosen_response_len_mean": 458.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 332.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.1, + "grad_norm": 2.692910890812567, + "learning_rate": 4.99996058981844e-07, + "logits/policy_chosen": -2.5161116123199463, + "logits/policy_rejected": -2.3703317642211914, + "logits/reference_chosen": -2.4154701232910156, + "logits/reference_rejected": -2.4023566246032715, + "logps/policy_chosen": -72.83512115478516, + "logps/policy_rejected": -136.28607177734375, + "logps/reference_chosen": -82.63104248046875, + "logps/reference_rejected": -140.04986572265625, + "loss": 0.6369, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.09795919060707092, + "rewards/margins": 0.06032119691371918, + "rewards/rejected": 0.037637993693351746, + "step": 127, + "tokens/chosen_response_len_mean": 416.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 410.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.1, + "grad_norm": 2.859419025019686, + "learning_rate": 4.999911327382711e-07, + "logits/policy_chosen": -2.0673351287841797, + "logits/policy_rejected": -1.9885450601577759, + "logits/reference_chosen": -2.1499083042144775, + "logits/reference_rejected": -1.6471858024597168, + "logps/policy_chosen": -57.06989669799805, + "logps/policy_rejected": -87.03865814208984, + "logps/reference_chosen": -69.13873291015625, + "logps/reference_rejected": -83.46507263183594, + "loss": 0.5998, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.12068840861320496, + "rewards/margins": 0.15642426908016205, + "rewards/rejected": -0.035735856741666794, + "step": 128, + "tokens/chosen_response_len_mean": 343.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 194.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.1, + "grad_norm": 2.8759886658807288, + "learning_rate": 4.999842360516294e-07, + "logits/policy_chosen": -3.1045093536376953, + "logits/policy_rejected": -3.1223738193511963, + "logits/reference_chosen": -3.09148907661438, + "logits/reference_rejected": -3.091776132583618, + "logps/policy_chosen": -40.253719329833984, + "logps/policy_rejected": -93.64241027832031, + "logps/reference_chosen": -43.66665267944336, + "logps/reference_rejected": -92.32077026367188, + "loss": 0.5883, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.03412933275103569, + "rewards/margins": 0.047345712780952454, + "rewards/rejected": -0.01321637723594904, + "step": 129, + "tokens/chosen_response_len_mean": 134.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 142.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.1, + "grad_norm": 2.396608033911008, + "learning_rate": 4.999753689762792e-07, + "logits/policy_chosen": -2.7669222354888916, + "logits/policy_rejected": -3.0244264602661133, + "logits/reference_chosen": -2.7182681560516357, + "logits/reference_rejected": -3.016852855682373, + "logps/policy_chosen": -49.25522994995117, + "logps/policy_rejected": -209.12026977539062, + "logps/reference_chosen": -58.14155197143555, + "logps/reference_rejected": -213.75338745117188, + "loss": 0.6532, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.08886321634054184, + "rewards/margins": 0.04253204166889191, + "rewards/rejected": 0.04633117839694023, + "step": 130, + "tokens/chosen_response_len_mean": 140.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 397.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.11, + "grad_norm": 2.8338085411833127, + "learning_rate": 4.999645315821111e-07, + "logits/policy_chosen": -2.177436590194702, + "logits/policy_rejected": -2.1923160552978516, + "logits/reference_chosen": -2.23976993560791, + "logits/reference_rejected": -2.227308511734009, + "logps/policy_chosen": -92.42753601074219, + "logps/policy_rejected": -104.93522644042969, + "logps/reference_chosen": -111.59609985351562, + "logps/reference_rejected": -120.66436004638672, + "loss": 0.6227, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.19168560206890106, + "rewards/margins": 0.03439423814415932, + "rewards/rejected": 0.15729136765003204, + "step": 131, + "tokens/chosen_response_len_mean": 428.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 393.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.11, + "grad_norm": 2.537615272647266, + "learning_rate": 4.99951723954546e-07, + "logits/policy_chosen": -2.6528267860412598, + "logits/policy_rejected": -2.7896065711975098, + "logits/reference_chosen": -2.3186991214752197, + "logits/reference_rejected": -2.7976412773132324, + "logps/policy_chosen": -35.68031311035156, + "logps/policy_rejected": -72.86119079589844, + "logps/reference_chosen": -42.128787994384766, + "logps/reference_rejected": -76.47801208496094, + "loss": 0.6365, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.06448473036289215, + "rewards/margins": 0.028316427022218704, + "rewards/rejected": 0.03616830334067345, + "step": 132, + "tokens/chosen_response_len_mean": 226.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 273.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.11, + "grad_norm": 2.952022770646515, + "learning_rate": 4.999369461945343e-07, + "logits/policy_chosen": -3.1463866233825684, + "logits/policy_rejected": -2.8932509422302246, + "logits/reference_chosen": -3.141604423522949, + "logits/reference_rejected": -2.444639205932617, + "logps/policy_chosen": -83.71105194091797, + "logps/policy_rejected": -113.13372802734375, + "logps/reference_chosen": -94.0230941772461, + "logps/reference_rejected": -108.04383850097656, + "loss": 0.5892, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.10312040150165558, + "rewards/margins": 0.15401925146579742, + "rewards/rejected": -0.05089886486530304, + "step": 133, + "tokens/chosen_response_len_mean": 249.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 178.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.11, + "grad_norm": 3.540843119893174, + "learning_rate": 4.999201984185552e-07, + "logits/policy_chosen": -3.0958125591278076, + "logits/policy_rejected": -3.042353868484497, + "logits/reference_chosen": -3.133866310119629, + "logits/reference_rejected": -3.0950450897216797, + "logps/policy_chosen": -236.39605712890625, + "logps/policy_rejected": -330.2846374511719, + "logps/reference_chosen": -259.0321350097656, + "logps/reference_rejected": -336.391357421875, + "loss": 0.5663, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.22636055946350098, + "rewards/margins": 0.16529357433319092, + "rewards/rejected": 0.06106698513031006, + "step": 134, + "tokens/chosen_response_len_mean": 717.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 639.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.11, + "grad_norm": 2.981444512825392, + "learning_rate": 4.999014807586154e-07, + "logits/policy_chosen": -2.3856396675109863, + "logits/policy_rejected": -2.1970813274383545, + "logits/reference_chosen": -2.4693095684051514, + "logits/reference_rejected": -1.5482537746429443, + "logps/policy_chosen": -142.9355926513672, + "logps/policy_rejected": -189.60543823242188, + "logps/reference_chosen": -164.72271728515625, + "logps/reference_rejected": -189.248046875, + "loss": 0.6065, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2178712785243988, + "rewards/margins": 0.22144514322280884, + "rewards/rejected": -0.003573855385184288, + "step": 135, + "tokens/chosen_response_len_mean": 339.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 276.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.11, + "grad_norm": 2.4024504350993654, + "learning_rate": 4.998807933622484e-07, + "logits/policy_chosen": -3.1550679206848145, + "logits/policy_rejected": -3.1557650566101074, + "logits/reference_chosen": -3.1761221885681152, + "logits/reference_rejected": -3.135535955429077, + "logps/policy_chosen": -118.20024871826172, + "logps/policy_rejected": -264.43560791015625, + "logps/reference_chosen": -130.25527954101562, + "logps/reference_rejected": -264.16741943359375, + "loss": 0.6336, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.12055033445358276, + "rewards/margins": 0.12323223054409027, + "rewards/rejected": -0.002681899815797806, + "step": 136, + "tokens/chosen_response_len_mean": 354.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 386.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.11, + "grad_norm": 2.8382305196380684, + "learning_rate": 4.998581363925135e-07, + "logits/policy_chosen": -2.2047266960144043, + "logits/policy_rejected": -2.433465003967285, + "logits/reference_chosen": -2.275531530380249, + "logits/reference_rejected": -2.245540142059326, + "logps/policy_chosen": -378.1270751953125, + "logps/policy_rejected": -344.7900085449219, + "logps/reference_chosen": -416.6529541015625, + "logps/reference_rejected": -347.785888671875, + "loss": 0.5804, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3852587938308716, + "rewards/margins": 0.3552997410297394, + "rewards/rejected": 0.02995903044939041, + "step": 137, + "tokens/chosen_response_len_mean": 584.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 427.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.11, + "grad_norm": 2.1385240477591636, + "learning_rate": 4.998335100279941e-07, + "logits/policy_chosen": -2.6462714672088623, + "logits/policy_rejected": -2.658045768737793, + "logits/reference_chosen": -2.729498863220215, + "logits/reference_rejected": -2.694502830505371, + "logps/policy_chosen": -93.70889282226562, + "logps/policy_rejected": -182.60643005371094, + "logps/reference_chosen": -114.53910827636719, + "logps/reference_rejected": -185.69065856933594, + "loss": 0.6218, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2083020955324173, + "rewards/margins": 0.17745977640151978, + "rewards/rejected": 0.030842319130897522, + "step": 138, + "tokens/chosen_response_len_mean": 282.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 247.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.11, + "grad_norm": 2.116685308038921, + "learning_rate": 4.998069144627964e-07, + "logits/policy_chosen": -2.870461940765381, + "logits/policy_rejected": -2.6627469062805176, + "logits/reference_chosen": -2.924255132675171, + "logits/reference_rejected": -2.6669623851776123, + "logps/policy_chosen": -149.49957275390625, + "logps/policy_rejected": -133.25009155273438, + "logps/reference_chosen": -191.32994079589844, + "logps/reference_rejected": -157.3734588623047, + "loss": 0.6358, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4183034896850586, + "rewards/margins": 0.1770700067281723, + "rewards/rejected": 0.2412334829568863, + "step": 139, + "tokens/chosen_response_len_mean": 457.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 308.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.11, + "grad_norm": 2.5870338720368387, + "learning_rate": 4.99778349906548e-07, + "logits/policy_chosen": -2.9523699283599854, + "logits/policy_rejected": -2.9597270488739014, + "logits/reference_chosen": -2.9815268516540527, + "logits/reference_rejected": -2.9721755981445312, + "logps/policy_chosen": -45.849361419677734, + "logps/policy_rejected": -106.37723541259766, + "logps/reference_chosen": -56.053375244140625, + "logps/reference_rejected": -102.900390625, + "loss": 0.6334, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.10204015672206879, + "rewards/margins": 0.13680851459503174, + "rewards/rejected": -0.03476836904883385, + "step": 140, + "tokens/chosen_response_len_mean": 174.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 184.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.11, + "grad_norm": 3.1171856643556626, + "learning_rate": 4.997478165843964e-07, + "logits/policy_chosen": -2.8171520233154297, + "logits/policy_rejected": -2.8751494884490967, + "logits/reference_chosen": -2.9145286083221436, + "logits/reference_rejected": -2.9242196083068848, + "logps/policy_chosen": -170.06459045410156, + "logps/policy_rejected": -296.7114562988281, + "logps/reference_chosen": -197.37327575683594, + "logps/reference_rejected": -302.43438720703125, + "loss": 0.5862, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.27308669686317444, + "rewards/margins": 0.21585765480995178, + "rewards/rejected": 0.057229042053222656, + "step": 141, + "tokens/chosen_response_len_mean": 416.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 422.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.11, + "grad_norm": 3.0084655845363835, + "learning_rate": 4.997153147370065e-07, + "logits/policy_chosen": -2.85317325592041, + "logits/policy_rejected": -2.8581879138946533, + "logits/reference_chosen": -2.7449471950531006, + "logits/reference_rejected": -2.9342892169952393, + "logps/policy_chosen": -95.86734008789062, + "logps/policy_rejected": -255.03668212890625, + "logps/reference_chosen": -114.49011993408203, + "logps/reference_rejected": -234.34010314941406, + "loss": 0.5691, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.18622779846191406, + "rewards/margins": 0.39319345355033875, + "rewards/rejected": -0.20696566998958588, + "step": 142, + "tokens/chosen_response_len_mean": 348.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 233.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.11, + "grad_norm": 2.8051570403665753, + "learning_rate": 4.996808446205598e-07, + "logits/policy_chosen": -2.9738688468933105, + "logits/policy_rejected": -2.8889176845550537, + "logits/reference_chosen": -3.072221517562866, + "logits/reference_rejected": -2.744046449661255, + "logps/policy_chosen": -106.87481689453125, + "logps/policy_rejected": -214.15478515625, + "logps/reference_chosen": -126.41868591308594, + "logps/reference_rejected": -216.4077606201172, + "loss": 0.6244, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.19543880224227905, + "rewards/margins": 0.17290906608104706, + "rewards/rejected": 0.022529736161231995, + "step": 143, + "tokens/chosen_response_len_mean": 340.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 310.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.12, + "grad_norm": 3.1099351541314864, + "learning_rate": 4.996444065067515e-07, + "logits/policy_chosen": -2.849621534347534, + "logits/policy_rejected": -2.8288486003875732, + "logits/reference_chosen": -2.9862475395202637, + "logits/reference_rejected": -2.9479100704193115, + "logps/policy_chosen": -331.7215576171875, + "logps/policy_rejected": -571.941650390625, + "logps/reference_chosen": -382.40533447265625, + "logps/reference_rejected": -549.6744995117188, + "loss": 0.5803, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5068381428718567, + "rewards/margins": 0.7295098304748535, + "rewards/rejected": -0.2226717472076416, + "step": 144, + "tokens/chosen_response_len_mean": 687.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 550.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.12, + "grad_norm": 2.712783580670268, + "learning_rate": 4.996060006827887e-07, + "logits/policy_chosen": -2.882990837097168, + "logits/policy_rejected": -2.8665995597839355, + "logits/reference_chosen": -2.9348299503326416, + "logits/reference_rejected": -2.9280238151550293, + "logps/policy_chosen": -75.93386840820312, + "logps/policy_rejected": -159.058837890625, + "logps/reference_chosen": -91.52436828613281, + "logps/reference_rejected": -146.97247314453125, + "loss": 0.5965, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.15590497851371765, + "rewards/margins": 0.2767685055732727, + "rewards/rejected": -0.12086354941129684, + "step": 145, + "tokens/chosen_response_len_mean": 248.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 191.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.12, + "grad_norm": 2.364187725655967, + "learning_rate": 4.995656274513881e-07, + "logits/policy_chosen": -3.0416619777679443, + "logits/policy_rejected": -2.995178699493408, + "logits/reference_chosen": -3.053309440612793, + "logits/reference_rejected": -3.0008325576782227, + "logps/policy_chosen": -90.75337982177734, + "logps/policy_rejected": -127.47672271728516, + "logps/reference_chosen": -102.95506286621094, + "logps/reference_rejected": -132.29486083984375, + "loss": 0.6275, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.1220167949795723, + "rewards/margins": 0.07383548468351364, + "rewards/rejected": 0.04818130284547806, + "step": 146, + "tokens/chosen_response_len_mean": 403.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 352.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.12, + "grad_norm": 3.163944044368287, + "learning_rate": 4.995232871307736e-07, + "logits/policy_chosen": -3.0712456703186035, + "logits/policy_rejected": -2.916374921798706, + "logits/reference_chosen": -3.115262746810913, + "logits/reference_rejected": -2.6205530166625977, + "logps/policy_chosen": -104.03935241699219, + "logps/policy_rejected": -156.13577270507812, + "logps/reference_chosen": -121.57315063476562, + "logps/reference_rejected": -153.22605895996094, + "loss": 0.583, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.17533791065216064, + "rewards/margins": 0.2044350802898407, + "rewards/rejected": -0.029097169637680054, + "step": 147, + "tokens/chosen_response_len_mean": 273.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 197.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.12, + "grad_norm": 2.8981364214624588, + "learning_rate": 4.994789800546737e-07, + "logits/policy_chosen": -2.519289970397949, + "logits/policy_rejected": -2.3818559646606445, + "logits/reference_chosen": -2.5594940185546875, + "logits/reference_rejected": -2.2251200675964355, + "logps/policy_chosen": -64.7489242553711, + "logps/policy_rejected": -90.21540832519531, + "logps/reference_chosen": -80.6169204711914, + "logps/reference_rejected": -92.92266845703125, + "loss": 0.5887, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.15867996215820312, + "rewards/margins": 0.13160738348960876, + "rewards/rejected": 0.027072593569755554, + "step": 148, + "tokens/chosen_response_len_mean": 252.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 195.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.12, + "grad_norm": 2.7118945452462073, + "learning_rate": 4.994327065723192e-07, + "logits/policy_chosen": -2.6516730785369873, + "logits/policy_rejected": -2.5235743522644043, + "logits/reference_chosen": -2.629504442214966, + "logits/reference_rejected": -2.4736945629119873, + "logps/policy_chosen": -80.69113159179688, + "logps/policy_rejected": -134.16526794433594, + "logps/reference_chosen": -103.33537292480469, + "logps/reference_rejected": -144.43687438964844, + "loss": 0.5924, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2264423370361328, + "rewards/margins": 0.12372627854347229, + "rewards/rejected": 0.10271605104207993, + "step": 149, + "tokens/chosen_response_len_mean": 244.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 244.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.12, + "grad_norm": 2.5381060398035205, + "learning_rate": 4.9938446704844e-07, + "logits/policy_chosen": -2.8819587230682373, + "logits/policy_rejected": -2.9721384048461914, + "logits/reference_chosen": -2.7422800064086914, + "logits/reference_rejected": -2.9950976371765137, + "logps/policy_chosen": -57.52130126953125, + "logps/policy_rejected": -120.23619079589844, + "logps/reference_chosen": -59.788352966308594, + "logps/reference_rejected": -115.03448486328125, + "loss": 0.619, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.022670526057481766, + "rewards/margins": 0.07468754798173904, + "rewards/rejected": -0.05201702564954758, + "step": 150, + "tokens/chosen_response_len_mean": 145.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 157.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.12, + "grad_norm": 2.8034120172472052, + "learning_rate": 4.993342618632626e-07, + "logits/policy_chosen": -3.0852291584014893, + "logits/policy_rejected": -2.92305850982666, + "logits/reference_chosen": -3.162454128265381, + "logits/reference_rejected": -2.4117138385772705, + "logps/policy_chosen": -101.7428207397461, + "logps/policy_rejected": -242.9844207763672, + "logps/reference_chosen": -119.85108184814453, + "logps/reference_rejected": -215.4344940185547, + "loss": 0.5702, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.18108254671096802, + "rewards/margins": 0.45658186078071594, + "rewards/rejected": -0.2754993140697479, + "step": 151, + "tokens/chosen_response_len_mean": 318.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 234.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.12, + "grad_norm": 2.9459543515337776, + "learning_rate": 4.992820914125069e-07, + "logits/policy_chosen": -2.8646183013916016, + "logits/policy_rejected": -2.754851818084717, + "logits/reference_chosen": -3.0292770862579346, + "logits/reference_rejected": -2.5228896141052246, + "logps/policy_chosen": -150.56875610351562, + "logps/policy_rejected": -277.60125732421875, + "logps/reference_chosen": -179.77772521972656, + "logps/reference_rejected": -275.3694152832031, + "loss": 0.586, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.2920897901058197, + "rewards/margins": 0.31440818309783936, + "rewards/rejected": -0.022318381816148758, + "step": 152, + "tokens/chosen_response_len_mean": 543.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 491.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.12, + "grad_norm": 2.9972427731212954, + "learning_rate": 4.99227956107383e-07, + "logits/policy_chosen": -2.133497714996338, + "logits/policy_rejected": -2.275707721710205, + "logits/reference_chosen": -2.4100184440612793, + "logits/reference_rejected": -2.27435040473938, + "logps/policy_chosen": -149.89268493652344, + "logps/policy_rejected": -225.88043212890625, + "logps/reference_chosen": -195.48631286621094, + "logps/reference_rejected": -232.33531188964844, + "loss": 0.5712, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4559362828731537, + "rewards/margins": 0.3913875222206116, + "rewards/rejected": 0.06454875320196152, + "step": 153, + "tokens/chosen_response_len_mean": 551.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 501.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.12, + "grad_norm": 2.6455429598648053, + "learning_rate": 4.991718563745882e-07, + "logits/policy_chosen": -3.022681951522827, + "logits/policy_rejected": -3.033928155899048, + "logits/reference_chosen": -3.0507426261901855, + "logits/reference_rejected": -3.0070152282714844, + "logps/policy_chosen": -17.34947395324707, + "logps/policy_rejected": -55.760353088378906, + "logps/reference_chosen": -20.67547607421875, + "logps/reference_rejected": -50.72275924682617, + "loss": 0.5857, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.03326002135872841, + "rewards/margins": 0.08363597095012665, + "rewards/rejected": -0.05037594214081764, + "step": 154, + "tokens/chosen_response_len_mean": 28.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 28.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.12, + "grad_norm": 2.188503580126624, + "learning_rate": 4.991137926563035e-07, + "logits/policy_chosen": -2.8026483058929443, + "logits/policy_rejected": -2.840829372406006, + "logits/reference_chosen": -2.866769790649414, + "logits/reference_rejected": -2.868910789489746, + "logps/policy_chosen": -45.941871643066406, + "logps/policy_rejected": -90.47376251220703, + "logps/reference_chosen": -53.47924041748047, + "logps/reference_rejected": -84.99751281738281, + "loss": 0.6102, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.07537368685007095, + "rewards/margins": 0.1301361620426178, + "rewards/rejected": -0.054762475192546844, + "step": 155, + "tokens/chosen_response_len_mean": 173.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 133.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.13, + "grad_norm": 2.5667127080808974, + "learning_rate": 4.990537654101903e-07, + "logits/policy_chosen": -2.980593681335449, + "logits/policy_rejected": -3.0731048583984375, + "logits/reference_chosen": -3.066528797149658, + "logits/reference_rejected": -3.1092565059661865, + "logps/policy_chosen": -92.3929672241211, + "logps/policy_rejected": -223.2591552734375, + "logps/reference_chosen": -117.49671936035156, + "logps/reference_rejected": -224.93527221679688, + "loss": 0.5677, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.25103750824928284, + "rewards/margins": 0.2342762053012848, + "rewards/rejected": 0.016761288046836853, + "step": 156, + "tokens/chosen_response_len_mean": 291.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 321.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.13, + "grad_norm": 2.742556033679143, + "learning_rate": 4.989917751093862e-07, + "logits/policy_chosen": -2.9195590019226074, + "logits/policy_rejected": -2.7180957794189453, + "logits/reference_chosen": -3.0530753135681152, + "logits/reference_rejected": -2.1471331119537354, + "logps/policy_chosen": -86.05413818359375, + "logps/policy_rejected": -171.83856201171875, + "logps/reference_chosen": -108.33873748779297, + "logps/reference_rejected": -158.24143981933594, + "loss": 0.5624, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.22284595668315887, + "rewards/margins": 0.35881713032722473, + "rewards/rejected": -0.13597117364406586, + "step": 157, + "tokens/chosen_response_len_mean": 300.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 193.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.13, + "grad_norm": 2.320383809339318, + "learning_rate": 4.989278222425021e-07, + "logits/policy_chosen": -2.9507193565368652, + "logits/policy_rejected": -3.0587496757507324, + "logits/reference_chosen": -3.0194993019104004, + "logits/reference_rejected": -3.0736992359161377, + "logps/policy_chosen": -103.41101837158203, + "logps/policy_rejected": -312.6053466796875, + "logps/reference_chosen": -126.41258239746094, + "logps/reference_rejected": -292.84515380859375, + "loss": 0.5961, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2300155907869339, + "rewards/margins": 0.4276174306869507, + "rewards/rejected": -0.19760185480117798, + "step": 158, + "tokens/chosen_response_len_mean": 338.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 361.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.13, + "grad_norm": 2.516259723460725, + "learning_rate": 4.988619073136179e-07, + "logits/policy_chosen": -2.578339099884033, + "logits/policy_rejected": -2.6243929862976074, + "logits/reference_chosen": -2.651563882827759, + "logits/reference_rejected": -2.7075228691101074, + "logps/policy_chosen": -60.65949249267578, + "logps/policy_rejected": -124.75245666503906, + "logps/reference_chosen": -77.07334899902344, + "logps/reference_rejected": -121.59002685546875, + "loss": 0.5969, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.16413861513137817, + "rewards/margins": 0.19576290249824524, + "rewards/rejected": -0.031624287366867065, + "step": 159, + "tokens/chosen_response_len_mean": 344.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 347.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.13, + "grad_norm": 2.715123355655463, + "learning_rate": 4.987940308422783e-07, + "logits/policy_chosen": -2.0114169120788574, + "logits/policy_rejected": -2.0045886039733887, + "logits/reference_chosen": -2.092721700668335, + "logits/reference_rejected": -1.7612866163253784, + "logps/policy_chosen": -84.94512176513672, + "logps/policy_rejected": -133.75323486328125, + "logps/reference_chosen": -107.3111801147461, + "logps/reference_rejected": -128.45538330078125, + "loss": 0.5549, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.22366058826446533, + "rewards/margins": 0.27663907408714294, + "rewards/rejected": -0.05297844856977463, + "step": 160, + "tokens/chosen_response_len_mean": 425.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 374.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.13, + "grad_norm": 2.6648898364230735, + "learning_rate": 4.987241933634892e-07, + "logits/policy_chosen": -2.6987576484680176, + "logits/policy_rejected": -2.7210419178009033, + "logits/reference_chosen": -2.804107189178467, + "logits/reference_rejected": -2.7485485076904297, + "logps/policy_chosen": -101.68658447265625, + "logps/policy_rejected": -234.5952606201172, + "logps/reference_chosen": -123.14704132080078, + "logps/reference_rejected": -228.36183166503906, + "loss": 0.5769, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.21460454165935516, + "rewards/margins": 0.276938796043396, + "rewards/rejected": -0.06233424320816994, + "step": 161, + "tokens/chosen_response_len_mean": 409.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 407.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.13, + "grad_norm": 2.82572590776649, + "learning_rate": 4.986523954277134e-07, + "logits/policy_chosen": -2.5539088249206543, + "logits/policy_rejected": -2.5690407752990723, + "logits/reference_chosen": -2.634115695953369, + "logits/reference_rejected": -2.615056037902832, + "logps/policy_chosen": -37.4378662109375, + "logps/policy_rejected": -93.33854675292969, + "logps/reference_chosen": -45.8214225769043, + "logps/reference_rejected": -89.81768798828125, + "loss": 0.5852, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.08383554965257645, + "rewards/margins": 0.11904413998126984, + "rewards/rejected": -0.03520859405398369, + "step": 162, + "tokens/chosen_response_len_mean": 228.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 240.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.13, + "grad_norm": 2.7170854054640863, + "learning_rate": 4.985786376008658e-07, + "logits/policy_chosen": -2.820416212081909, + "logits/policy_rejected": -2.79477858543396, + "logits/reference_chosen": -2.9494552612304688, + "logits/reference_rejected": -2.837876081466675, + "logps/policy_chosen": -71.54603576660156, + "logps/policy_rejected": -156.01953125, + "logps/reference_chosen": -90.15393829345703, + "logps/reference_rejected": -154.65646362304688, + "loss": 0.5657, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.1860790103673935, + "rewards/margins": 0.1997096687555313, + "rewards/rejected": -0.01363067515194416, + "step": 163, + "tokens/chosen_response_len_mean": 302.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 296.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.13, + "grad_norm": 2.3740765633517396, + "learning_rate": 4.985029204643094e-07, + "logits/policy_chosen": -2.747786045074463, + "logits/policy_rejected": -2.7663800716400146, + "logits/reference_chosen": -2.8246467113494873, + "logits/reference_rejected": -2.6436727046966553, + "logps/policy_chosen": -58.324737548828125, + "logps/policy_rejected": -223.62765502929688, + "logps/reference_chosen": -85.9714126586914, + "logps/reference_rejected": -203.6089630126953, + "loss": 0.6018, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.27646681666374207, + "rewards/margins": 0.4766535758972168, + "rewards/rejected": -0.20018675923347473, + "step": 164, + "tokens/chosen_response_len_mean": 223.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 216.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.13, + "grad_norm": 2.188806416183682, + "learning_rate": 4.984252446148508e-07, + "logits/policy_chosen": -2.701019287109375, + "logits/policy_rejected": -2.729661226272583, + "logits/reference_chosen": -2.8306493759155273, + "logits/reference_rejected": -2.8457367420196533, + "logps/policy_chosen": -154.78518676757812, + "logps/policy_rejected": -214.3829345703125, + "logps/reference_chosen": -194.75637817382812, + "logps/reference_rejected": -246.5928497314453, + "loss": 0.6206, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.39971184730529785, + "rewards/margins": 0.07761266827583313, + "rewards/rejected": 0.3220991790294647, + "step": 165, + "tokens/chosen_response_len_mean": 429.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 445.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.13, + "grad_norm": 2.8889287147059104, + "learning_rate": 4.983456106647348e-07, + "logits/policy_chosen": -2.861962080001831, + "logits/policy_rejected": -2.8529515266418457, + "logits/reference_chosen": -3.064021587371826, + "logits/reference_rejected": -2.927248001098633, + "logps/policy_chosen": -146.7970428466797, + "logps/policy_rejected": -283.18389892578125, + "logps/reference_chosen": -181.90396118164062, + "logps/reference_rejected": -277.1955871582031, + "loss": 0.5642, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.35106930136680603, + "rewards/margins": 0.4109525978565216, + "rewards/rejected": -0.05988328531384468, + "step": 166, + "tokens/chosen_response_len_mean": 468.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 421.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.13, + "grad_norm": 2.9841019270826292, + "learning_rate": 4.982640192416404e-07, + "logits/policy_chosen": -2.6893157958984375, + "logits/policy_rejected": -2.681230068206787, + "logits/reference_chosen": -2.948234796524048, + "logits/reference_rejected": -2.248135566711426, + "logps/policy_chosen": -122.96820831298828, + "logps/policy_rejected": -197.66404724121094, + "logps/reference_chosen": -155.029541015625, + "logps/reference_rejected": -194.3126678466797, + "loss": 0.5642, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.32061344385147095, + "rewards/margins": 0.35412728786468506, + "rewards/rejected": -0.0335138775408268, + "step": 167, + "tokens/chosen_response_len_mean": 452.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 366.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.14, + "grad_norm": 2.766687221527344, + "learning_rate": 4.981804709886755e-07, + "logits/policy_chosen": -2.4212305545806885, + "logits/policy_rejected": -2.4506514072418213, + "logits/reference_chosen": -2.6806509494781494, + "logits/reference_rejected": -2.608292579650879, + "logps/policy_chosen": -60.471797943115234, + "logps/policy_rejected": -120.82171630859375, + "logps/reference_chosen": -92.31079864501953, + "logps/reference_rejected": -130.82656860351562, + "loss": 0.54, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.31838998198509216, + "rewards/margins": 0.21834133565425873, + "rewards/rejected": 0.10004864633083344, + "step": 168, + "tokens/chosen_response_len_mean": 324.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 310.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.14, + "grad_norm": 2.1828651372724637, + "learning_rate": 4.980949665643719e-07, + "logits/policy_chosen": -2.7441906929016113, + "logits/policy_rejected": -2.7218408584594727, + "logits/reference_chosen": -2.8971827030181885, + "logits/reference_rejected": -2.860405445098877, + "logps/policy_chosen": -92.7531509399414, + "logps/policy_rejected": -210.39797973632812, + "logps/reference_chosen": -123.75934600830078, + "logps/reference_rejected": -200.2626190185547, + "loss": 0.5755, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.3100619614124298, + "rewards/margins": 0.41141554713249207, + "rewards/rejected": -0.10135360062122345, + "step": 169, + "tokens/chosen_response_len_mean": 517.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 465.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.14, + "grad_norm": 2.9989447327164425, + "learning_rate": 4.980075066426795e-07, + "logits/policy_chosen": -1.6698166131973267, + "logits/policy_rejected": -1.7434771060943604, + "logits/reference_chosen": -1.803000569343567, + "logits/reference_rejected": -1.8658090829849243, + "logps/policy_chosen": -61.05796813964844, + "logps/policy_rejected": -89.74996948242188, + "logps/reference_chosen": -77.35304260253906, + "logps/reference_rejected": -98.12295532226562, + "loss": 0.5561, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.1629507839679718, + "rewards/margins": 0.07922087609767914, + "rewards/rejected": 0.08372990787029266, + "step": 170, + "tokens/chosen_response_len_mean": 243.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 245.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.14, + "grad_norm": 2.5554431416025243, + "learning_rate": 4.979180919129624e-07, + "logits/policy_chosen": -2.85784912109375, + "logits/policy_rejected": -2.838977098464966, + "logits/reference_chosen": -2.8757684230804443, + "logits/reference_rejected": -2.8608899116516113, + "logps/policy_chosen": -53.759185791015625, + "logps/policy_rejected": -90.618408203125, + "logps/reference_chosen": -62.647789001464844, + "logps/reference_rejected": -86.81078338623047, + "loss": 0.5702, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.08888600766658783, + "rewards/margins": 0.1269623190164566, + "rewards/rejected": -0.03807631880044937, + "step": 171, + "tokens/chosen_response_len_mean": 207.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 209.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.14, + "grad_norm": 2.7869612955855776, + "learning_rate": 4.978267230799918e-07, + "logits/policy_chosen": -2.0658278465270996, + "logits/policy_rejected": -2.2496557235717773, + "logits/reference_chosen": -2.1524839401245117, + "logits/reference_rejected": -2.3013737201690674, + "logps/policy_chosen": -86.49990844726562, + "logps/policy_rejected": -197.741943359375, + "logps/reference_chosen": -109.14804077148438, + "logps/reference_rejected": -182.33616638183594, + "loss": 0.5549, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.22648128867149353, + "rewards/margins": 0.380539208650589, + "rewards/rejected": -0.15405790507793427, + "step": 172, + "tokens/chosen_response_len_mean": 443.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 439.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.14, + "grad_norm": 2.9395952260087506, + "learning_rate": 4.977334008639417e-07, + "logits/policy_chosen": -2.605189800262451, + "logits/policy_rejected": -2.5856776237487793, + "logits/reference_chosen": -2.778407335281372, + "logits/reference_rejected": -2.793548583984375, + "logps/policy_chosen": -112.27513122558594, + "logps/policy_rejected": -266.2890625, + "logps/reference_chosen": -140.7950439453125, + "logps/reference_rejected": -241.72708129882812, + "loss": 0.5051, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.28519904613494873, + "rewards/margins": 0.5308188796043396, + "rewards/rejected": -0.24561986327171326, + "step": 173, + "tokens/chosen_response_len_mean": 462.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 293.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.14, + "grad_norm": 2.7423154795634153, + "learning_rate": 4.976381260003826e-07, + "logits/policy_chosen": -2.7179572582244873, + "logits/policy_rejected": -2.756556510925293, + "logits/reference_chosen": -2.8246583938598633, + "logits/reference_rejected": -2.793865203857422, + "logps/policy_chosen": -80.221923828125, + "logps/policy_rejected": -204.06829833984375, + "logps/reference_chosen": -96.3402328491211, + "logps/reference_rejected": -181.1507568359375, + "loss": 0.5916, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.16118302941322327, + "rewards/margins": 0.3903582692146301, + "rewards/rejected": -0.22917523980140686, + "step": 174, + "tokens/chosen_response_len_mean": 307.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 307.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.14, + "grad_norm": 2.8390846969584946, + "learning_rate": 4.97540899240276e-07, + "logits/policy_chosen": -2.6149544715881348, + "logits/policy_rejected": -2.7186009883880615, + "logits/reference_chosen": -2.755988597869873, + "logits/reference_rejected": -2.8059897422790527, + "logps/policy_chosen": -83.78113555908203, + "logps/policy_rejected": -158.60336303710938, + "logps/reference_chosen": -105.53045654296875, + "logps/reference_rejected": -161.6735076904297, + "loss": 0.5707, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2174932360649109, + "rewards/margins": 0.18679174780845642, + "rewards/rejected": 0.030701473355293274, + "step": 175, + "tokens/chosen_response_len_mean": 227.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 221.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.14, + "grad_norm": 1.928131402745428, + "learning_rate": 4.974417213499681e-07, + "logits/policy_chosen": -2.97554874420166, + "logits/policy_rejected": -3.0169177055358887, + "logits/reference_chosen": -3.1351237297058105, + "logits/reference_rejected": -3.054442882537842, + "logps/policy_chosen": -155.18731689453125, + "logps/policy_rejected": -306.79449462890625, + "logps/reference_chosen": -198.6287841796875, + "logps/reference_rejected": -296.349609375, + "loss": 0.5884, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4344147741794586, + "rewards/margins": 0.5388633608818054, + "rewards/rejected": -0.1044485792517662, + "step": 176, + "tokens/chosen_response_len_mean": 399.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 380.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.14, + "grad_norm": 2.5891349823773604, + "learning_rate": 4.973405931111844e-07, + "logits/policy_chosen": -2.475595474243164, + "logits/policy_rejected": -2.5474050045013428, + "logits/reference_chosen": -2.574971914291382, + "logits/reference_rejected": -2.7303850650787354, + "logps/policy_chosen": -60.937068939208984, + "logps/policy_rejected": -196.1728515625, + "logps/reference_chosen": -93.98703002929688, + "logps/reference_rejected": -193.26068115234375, + "loss": 0.5721, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.330499529838562, + "rewards/margins": 0.3596211075782776, + "rewards/rejected": -0.029121577739715576, + "step": 177, + "tokens/chosen_response_len_mean": 277.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 290.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.14, + "grad_norm": 3.186008328730442, + "learning_rate": 4.972375153210227e-07, + "logits/policy_chosen": -2.73168683052063, + "logits/policy_rejected": -2.80218505859375, + "logits/reference_chosen": -2.8586461544036865, + "logits/reference_rejected": -2.856407642364502, + "logps/policy_chosen": -41.751312255859375, + "logps/policy_rejected": -121.0931396484375, + "logps/reference_chosen": -52.108253479003906, + "logps/reference_rejected": -104.97093200683594, + "loss": 0.5198, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.1035693883895874, + "rewards/margins": 0.26479148864746094, + "rewards/rejected": -0.16122210025787354, + "step": 178, + "tokens/chosen_response_len_mean": 131.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 131.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.14, + "grad_norm": 2.4038166772062186, + "learning_rate": 4.971324887919477e-07, + "logits/policy_chosen": -2.768160820007324, + "logits/policy_rejected": -2.2585337162017822, + "logits/reference_chosen": -2.8858625888824463, + "logits/reference_rejected": -2.4065427780151367, + "logps/policy_chosen": -67.46320343017578, + "logps/policy_rejected": -281.3670654296875, + "logps/reference_chosen": -94.11170196533203, + "logps/reference_rejected": -263.22698974609375, + "loss": 0.5596, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.26648497581481934, + "rewards/margins": 0.4478859305381775, + "rewards/rejected": -0.18140093982219696, + "step": 179, + "tokens/chosen_response_len_mean": 239.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 333.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.14, + "grad_norm": 2.530913530757402, + "learning_rate": 4.970255143517838e-07, + "logits/policy_chosen": -2.6580893993377686, + "logits/policy_rejected": -2.717331886291504, + "logits/reference_chosen": -2.90189790725708, + "logits/reference_rejected": -2.8899142742156982, + "logps/policy_chosen": -129.58033752441406, + "logps/policy_rejected": -165.34884643554688, + "logps/reference_chosen": -179.34756469726562, + "logps/reference_rejected": -170.3519287109375, + "loss": 0.5522, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4976721405982971, + "rewards/margins": 0.4476412534713745, + "rewards/rejected": 0.050030872225761414, + "step": 180, + "tokens/chosen_response_len_mean": 500.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 318.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.15, + "grad_norm": 2.633909183828016, + "learning_rate": 4.969165928437091e-07, + "logits/policy_chosen": -2.570159912109375, + "logits/policy_rejected": -2.5334911346435547, + "logits/reference_chosen": -2.760227680206299, + "logits/reference_rejected": -2.3865246772766113, + "logps/policy_chosen": -55.27775955200195, + "logps/policy_rejected": -119.49053955078125, + "logps/reference_chosen": -72.7205581665039, + "logps/reference_rejected": -111.53352355957031, + "loss": 0.5199, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.17442797124385834, + "rewards/margins": 0.2539982497692108, + "rewards/rejected": -0.07957027852535248, + "step": 181, + "tokens/chosen_response_len_mean": 307.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 267.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.15, + "grad_norm": 2.3888599137710167, + "learning_rate": 4.968057251262486e-07, + "logits/policy_chosen": -2.6059675216674805, + "logits/policy_rejected": -2.395353078842163, + "logits/reference_chosen": -2.9607644081115723, + "logits/reference_rejected": -1.8757669925689697, + "logps/policy_chosen": -74.94976043701172, + "logps/policy_rejected": -134.52859497070312, + "logps/reference_chosen": -111.65763092041016, + "logps/reference_rejected": -96.27473449707031, + "loss": 0.5536, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.36707866191864014, + "rewards/margins": 0.7496173977851868, + "rewards/rejected": -0.38253867626190186, + "step": 182, + "tokens/chosen_response_len_mean": 351.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 148.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.15, + "grad_norm": 2.826021624132104, + "learning_rate": 4.966929120732675e-07, + "logits/policy_chosen": -2.3233590126037598, + "logits/policy_rejected": -2.4008846282958984, + "logits/reference_chosen": -2.5622141361236572, + "logits/reference_rejected": -2.381047248840332, + "logps/policy_chosen": -185.09976196289062, + "logps/policy_rejected": -336.95941162109375, + "logps/reference_chosen": -236.7534637451172, + "logps/reference_rejected": -320.8805847167969, + "loss": 0.5087, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.516537070274353, + "rewards/margins": 0.6773253679275513, + "rewards/rejected": -0.16078826785087585, + "step": 183, + "tokens/chosen_response_len_mean": 441.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 384.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.15, + "grad_norm": 2.26864738127188, + "learning_rate": 4.965781545739638e-07, + "logits/policy_chosen": -2.7422099113464355, + "logits/policy_rejected": -2.7613723278045654, + "logits/reference_chosen": -2.9637043476104736, + "logits/reference_rejected": -2.921086549758911, + "logps/policy_chosen": -45.85551452636719, + "logps/policy_rejected": -118.8583984375, + "logps/reference_chosen": -65.04806518554688, + "logps/reference_rejected": -119.62251281738281, + "loss": 0.5676, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.191925510764122, + "rewards/margins": 0.1842842549085617, + "rewards/rejected": 0.007641255855560303, + "step": 184, + "tokens/chosen_response_len_mean": 266.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 242.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.15, + "grad_norm": 2.8182732561665023, + "learning_rate": 4.964614535328625e-07, + "logits/policy_chosen": -2.523998975753784, + "logits/policy_rejected": -2.4280190467834473, + "logits/reference_chosen": -2.717611789703369, + "logits/reference_rejected": -2.5818982124328613, + "logps/policy_chosen": -250.7300262451172, + "logps/policy_rejected": -131.51202392578125, + "logps/reference_chosen": -304.0659484863281, + "logps/reference_rejected": -136.93946838378906, + "loss": 0.5284, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.533359169960022, + "rewards/margins": 0.47908464074134827, + "rewards/rejected": 0.0542745515704155, + "step": 185, + "tokens/chosen_response_len_mean": 653.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 300.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.15, + "grad_norm": 2.658487252908494, + "learning_rate": 4.963428098698069e-07, + "logits/policy_chosen": -2.8848061561584473, + "logits/policy_rejected": -2.887396812438965, + "logits/reference_chosen": -3.0770082473754883, + "logits/reference_rejected": -3.012545347213745, + "logps/policy_chosen": -26.844032287597656, + "logps/policy_rejected": -64.93582916259766, + "logps/reference_chosen": -37.33853530883789, + "logps/reference_rejected": -57.781166076660156, + "loss": 0.5617, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.10494504123926163, + "rewards/margins": 0.17649173736572266, + "rewards/rejected": -0.07154669612646103, + "step": 186, + "tokens/chosen_response_len_mean": 101.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 99.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.15, + "grad_norm": 3.192921534395214, + "learning_rate": 4.962222245199527e-07, + "logits/policy_chosen": -2.5711615085601807, + "logits/policy_rejected": -2.556881904602051, + "logits/reference_chosen": -2.9153711795806885, + "logits/reference_rejected": -2.4689133167266846, + "logps/policy_chosen": -79.51646423339844, + "logps/policy_rejected": -212.90431213378906, + "logps/reference_chosen": -108.06434631347656, + "logps/reference_rejected": -189.78848266601562, + "loss": 0.5117, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.285478800535202, + "rewards/margins": 0.5166370868682861, + "rewards/rejected": -0.2311583161354065, + "step": 187, + "tokens/chosen_response_len_mean": 368.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 330.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.15, + "grad_norm": 3.185300778929063, + "learning_rate": 4.960996984337598e-07, + "logits/policy_chosen": -2.322800397872925, + "logits/policy_rejected": -2.180086851119995, + "logits/reference_chosen": -2.647416830062866, + "logits/reference_rejected": -2.404383420944214, + "logps/policy_chosen": -88.6377944946289, + "logps/policy_rejected": -154.1780242919922, + "logps/reference_chosen": -146.73565673828125, + "logps/reference_rejected": -158.4337921142578, + "loss": 0.4879, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5809786319732666, + "rewards/margins": 0.5384209156036377, + "rewards/rejected": 0.042557742446660995, + "step": 188, + "tokens/chosen_response_len_mean": 496.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 375.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.15, + "grad_norm": 3.018235573859185, + "learning_rate": 4.959752325769852e-07, + "logits/policy_chosen": -2.469785213470459, + "logits/policy_rejected": -2.580235481262207, + "logits/reference_chosen": -2.680108070373535, + "logits/reference_rejected": -2.8124053478240967, + "logps/policy_chosen": -57.80915069580078, + "logps/policy_rejected": -210.04135131835938, + "logps/reference_chosen": -90.7098617553711, + "logps/reference_rejected": -191.3103485107422, + "loss": 0.5062, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.329007089138031, + "rewards/margins": 0.5163170099258423, + "rewards/rejected": -0.18730990588665009, + "step": 189, + "tokens/chosen_response_len_mean": 332.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 348.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.15, + "grad_norm": 3.1505949518170437, + "learning_rate": 4.958488279306753e-07, + "logits/policy_chosen": -2.3762245178222656, + "logits/policy_rejected": -2.466752767562866, + "logits/reference_chosen": -2.847898483276367, + "logits/reference_rejected": -2.2204389572143555, + "logps/policy_chosen": -104.57975006103516, + "logps/policy_rejected": -250.35414123535156, + "logps/reference_chosen": -159.56100463867188, + "logps/reference_rejected": -225.86146545410156, + "loss": 0.522, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5498126149177551, + "rewards/margins": 0.7947396039962769, + "rewards/rejected": -0.24492698907852173, + "step": 190, + "tokens/chosen_response_len_mean": 512.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 435.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.15, + "grad_norm": 3.056135297935867, + "learning_rate": 4.957204854911578e-07, + "logits/policy_chosen": -2.485416889190674, + "logits/policy_rejected": -2.3778135776519775, + "logits/reference_chosen": -2.856623411178589, + "logits/reference_rejected": -2.3928959369659424, + "logps/policy_chosen": -83.4710693359375, + "logps/policy_rejected": -267.17791748046875, + "logps/reference_chosen": -124.20071411132812, + "logps/reference_rejected": -209.21670532226562, + "loss": 0.5036, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.40729638934135437, + "rewards/margins": 0.9869081377983093, + "rewards/rejected": -0.5796117782592773, + "step": 191, + "tokens/chosen_response_len_mean": 528.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 410.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.15, + "grad_norm": 2.503765131980355, + "learning_rate": 4.955902062700349e-07, + "logits/policy_chosen": -2.3622069358825684, + "logits/policy_rejected": -2.4439258575439453, + "logits/reference_chosen": -2.7395248413085938, + "logits/reference_rejected": -2.6338257789611816, + "logps/policy_chosen": -76.7771224975586, + "logps/policy_rejected": -262.30841064453125, + "logps/reference_chosen": -117.23094940185547, + "logps/reference_rejected": -224.7442626953125, + "loss": 0.5445, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.40453827381134033, + "rewards/margins": 0.7801797986030579, + "rewards/rejected": -0.37564149498939514, + "step": 192, + "tokens/chosen_response_len_mean": 359.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 367.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.16, + "grad_norm": 2.9703660280881734, + "learning_rate": 4.954579912941737e-07, + "logits/policy_chosen": -2.622279644012451, + "logits/policy_rejected": -2.6470251083374023, + "logits/reference_chosen": -2.853696823120117, + "logits/reference_rejected": -2.840925931930542, + "logps/policy_chosen": -38.201744079589844, + "logps/policy_rejected": -138.85470581054688, + "logps/reference_chosen": -63.35798645019531, + "logps/reference_rejected": -113.46359252929688, + "loss": 0.4864, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2515623867511749, + "rewards/margins": 0.5054734349250793, + "rewards/rejected": -0.25391101837158203, + "step": 193, + "tokens/chosen_response_len_mean": 157.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 169.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.16, + "grad_norm": 2.9246186619159347, + "learning_rate": 4.953238416057e-07, + "logits/policy_chosen": -2.605665683746338, + "logits/policy_rejected": -2.5566539764404297, + "logits/reference_chosen": -2.9360122680664062, + "logits/reference_rejected": -2.068471908569336, + "logps/policy_chosen": -84.89468383789062, + "logps/policy_rejected": -218.09048461914062, + "logps/reference_chosen": -133.76991271972656, + "logps/reference_rejected": -183.8456573486328, + "loss": 0.4881, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.48875218629837036, + "rewards/margins": 0.8312005996704102, + "rewards/rejected": -0.3424483835697174, + "step": 194, + "tokens/chosen_response_len_mean": 360.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 238.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.16, + "grad_norm": 2.80153859644754, + "learning_rate": 4.951877582619881e-07, + "logits/policy_chosen": -2.8731517791748047, + "logits/policy_rejected": -2.975919246673584, + "logits/reference_chosen": -3.0668981075286865, + "logits/reference_rejected": -3.1014294624328613, + "logps/policy_chosen": -24.165613174438477, + "logps/policy_rejected": -61.2584342956543, + "logps/reference_chosen": -23.939205169677734, + "logps/reference_rejected": -43.215232849121094, + "loss": 0.5741, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.002264055423438549, + "rewards/margins": 0.1781679093837738, + "rewards/rejected": -0.18043196201324463, + "step": 195, + "tokens/chosen_response_len_mean": 62.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 63.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.16, + "grad_norm": 2.7527879974178955, + "learning_rate": 4.950497423356542e-07, + "logits/policy_chosen": -2.9940450191497803, + "logits/policy_rejected": -2.710885763168335, + "logits/reference_chosen": -3.169459581375122, + "logits/reference_rejected": -2.7128515243530273, + "logps/policy_chosen": -120.05692291259766, + "logps/policy_rejected": -105.24632263183594, + "logps/reference_chosen": -160.91629028320312, + "logps/reference_rejected": -78.08905029296875, + "loss": 0.5115, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4085935354232788, + "rewards/margins": 0.6801662445068359, + "rewards/rejected": -0.27157267928123474, + "step": 196, + "tokens/chosen_response_len_mean": 332.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 53.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.16, + "grad_norm": 2.473954328567208, + "learning_rate": 4.949097949145471e-07, + "logits/policy_chosen": -2.994095802307129, + "logits/policy_rejected": -2.9911117553710938, + "logits/reference_chosen": -3.0934770107269287, + "logits/reference_rejected": -3.05942440032959, + "logps/policy_chosen": -54.1805419921875, + "logps/policy_rejected": -158.96043395996094, + "logps/reference_chosen": -74.52861785888672, + "logps/reference_rejected": -128.6375732421875, + "loss": 0.4544, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2034807801246643, + "rewards/margins": 0.5067094564437866, + "rewards/rejected": -0.3032286465167999, + "step": 197, + "tokens/chosen_response_len_mean": 182.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 165.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.16, + "grad_norm": 2.6028329307586944, + "learning_rate": 4.947679171017394e-07, + "logits/policy_chosen": -2.7241291999816895, + "logits/policy_rejected": -2.8773388862609863, + "logits/reference_chosen": -2.9862842559814453, + "logits/reference_rejected": -2.9953157901763916, + "logps/policy_chosen": -56.624752044677734, + "logps/policy_rejected": -255.8826141357422, + "logps/reference_chosen": -101.90193176269531, + "logps/reference_rejected": -203.60231018066406, + "loss": 0.491, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4527718722820282, + "rewards/margins": 0.9755749106407166, + "rewards/rejected": -0.5228029489517212, + "step": 198, + "tokens/chosen_response_len_mean": 225.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 217.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.16, + "grad_norm": 2.8264702439738474, + "learning_rate": 4.946241100155197e-07, + "logits/policy_chosen": -2.6785213947296143, + "logits/policy_rejected": -2.800018310546875, + "logits/reference_chosen": -2.9413294792175293, + "logits/reference_rejected": -2.913431406021118, + "logps/policy_chosen": -38.42972183227539, + "logps/policy_rejected": -164.37850952148438, + "logps/reference_chosen": -65.64104461669922, + "logps/reference_rejected": -128.12234497070312, + "loss": 0.4973, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.27211323380470276, + "rewards/margins": 0.6346749067306519, + "rewards/rejected": -0.3625616431236267, + "step": 199, + "tokens/chosen_response_len_mean": 312.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 278.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.16, + "grad_norm": 3.0030443321611644, + "learning_rate": 4.944783747893825e-07, + "logits/policy_chosen": -2.727503538131714, + "logits/policy_rejected": -2.6386661529541016, + "logits/reference_chosen": -2.9862632751464844, + "logits/reference_rejected": -2.1630351543426514, + "logps/policy_chosen": -84.61045837402344, + "logps/policy_rejected": -191.14212036132812, + "logps/reference_chosen": -125.34469604492188, + "logps/reference_rejected": -150.1243896484375, + "loss": 0.4714, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4073423147201538, + "rewards/margins": 0.8175195455551147, + "rewards/rejected": -0.41017720103263855, + "step": 200, + "tokens/chosen_response_len_mean": 264.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 164.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.16, + "grad_norm": 2.7775892532799955, + "learning_rate": 4.943307125720206e-07, + "logits/policy_chosen": -2.4299516677856445, + "logits/policy_rejected": -2.5677711963653564, + "logits/reference_chosen": -2.907986640930176, + "logits/reference_rejected": -2.6046524047851562, + "logps/policy_chosen": -107.82955169677734, + "logps/policy_rejected": -325.7413635253906, + "logps/reference_chosen": -176.93951416015625, + "logps/reference_rejected": -240.94747924804688, + "loss": 0.425, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.6910995841026306, + "rewards/margins": 1.5390385389328003, + "rewards/rejected": -0.8479388952255249, + "step": 201, + "tokens/chosen_response_len_mean": 608.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 298.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.16, + "grad_norm": 2.8271865552266267, + "learning_rate": 4.941811245273153e-07, + "logits/policy_chosen": -2.6319420337677, + "logits/policy_rejected": -2.5893003940582275, + "logits/reference_chosen": -3.000640392303467, + "logits/reference_rejected": -2.3574352264404297, + "logps/policy_chosen": -64.42079162597656, + "logps/policy_rejected": -202.20693969726562, + "logps/reference_chosen": -102.7632064819336, + "logps/reference_rejected": -181.3625030517578, + "loss": 0.3885, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3834240138530731, + "rewards/margins": 0.5918683409690857, + "rewards/rejected": -0.20844432711601257, + "step": 202, + "tokens/chosen_response_len_mean": 263.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 237.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.16, + "grad_norm": 2.804030062412289, + "learning_rate": 4.940296118343273e-07, + "logits/policy_chosen": -2.741957187652588, + "logits/policy_rejected": -2.7859504222869873, + "logits/reference_chosen": -2.956801414489746, + "logits/reference_rejected": -2.9367804527282715, + "logps/policy_chosen": -72.28400421142578, + "logps/policy_rejected": -123.73040771484375, + "logps/reference_chosen": -100.23980712890625, + "logps/reference_rejected": -129.1441650390625, + "loss": 0.5117, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2795579731464386, + "rewards/margins": 0.22542047500610352, + "rewards/rejected": 0.05413751304149628, + "step": 203, + "tokens/chosen_response_len_mean": 226.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 221.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.16, + "grad_norm": 3.1378106559973666, + "learning_rate": 4.938761756872874e-07, + "logits/policy_chosen": -2.6754446029663086, + "logits/policy_rejected": -2.5880160331726074, + "logits/reference_chosen": -3.173189163208008, + "logits/reference_rejected": -2.022594451904297, + "logps/policy_chosen": -59.6561393737793, + "logps/policy_rejected": -180.80789184570312, + "logps/reference_chosen": -116.20298767089844, + "logps/reference_rejected": -130.52818298339844, + "loss": 0.4056, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5654685497283936, + "rewards/margins": 1.0682657957077026, + "rewards/rejected": -0.5027973055839539, + "step": 204, + "tokens/chosen_response_len_mean": 284.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 165.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.16, + "grad_norm": 3.1932802336332298, + "learning_rate": 4.937208172955876e-07, + "logits/policy_chosen": -2.549724578857422, + "logits/policy_rejected": -2.6621100902557373, + "logits/reference_chosen": -2.8741047382354736, + "logits/reference_rejected": -2.9152588844299316, + "logps/policy_chosen": -46.50154113769531, + "logps/policy_rejected": -164.41668701171875, + "logps/reference_chosen": -80.38411712646484, + "logps/reference_rejected": -138.61923217773438, + "loss": 0.47, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3388257622718811, + "rewards/margins": 0.5968001484870911, + "rewards/rejected": -0.25797438621520996, + "step": 205, + "tokens/chosen_response_len_mean": 209.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 216.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.17, + "grad_norm": 3.3325761570853016, + "learning_rate": 4.935635378837702e-07, + "logits/policy_chosen": -2.4038610458374023, + "logits/policy_rejected": -2.402017831802368, + "logits/reference_chosen": -2.797759771347046, + "logits/reference_rejected": -2.5829825401306152, + "logps/policy_chosen": -44.91532516479492, + "logps/policy_rejected": -160.6805419921875, + "logps/reference_chosen": -74.13213348388672, + "logps/reference_rejected": -132.3360595703125, + "loss": 0.4302, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2921680212020874, + "rewards/margins": 0.5756129026412964, + "rewards/rejected": -0.2834448516368866, + "step": 206, + "tokens/chosen_response_len_mean": 232.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 251.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.17, + "grad_norm": 2.7368863153844414, + "learning_rate": 4.934043386915202e-07, + "logits/policy_chosen": -2.5967376232147217, + "logits/policy_rejected": -2.661456823348999, + "logits/reference_chosen": -2.750797748565674, + "logits/reference_rejected": -2.847586154937744, + "logps/policy_chosen": -31.504817962646484, + "logps/policy_rejected": -81.93124389648438, + "logps/reference_chosen": -47.813560485839844, + "logps/reference_rejected": -76.41636657714844, + "loss": 0.4935, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.16308742761611938, + "rewards/margins": 0.21823616325855255, + "rewards/rejected": -0.05514874309301376, + "step": 207, + "tokens/chosen_response_len_mean": 135.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 144.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.17, + "grad_norm": 3.2289580903320956, + "learning_rate": 4.932432209736538e-07, + "logits/policy_chosen": -2.779569149017334, + "logits/policy_rejected": -2.719834089279175, + "logits/reference_chosen": -3.0510177612304688, + "logits/reference_rejected": -2.049421787261963, + "logps/policy_chosen": -39.49280548095703, + "logps/policy_rejected": -97.85432434082031, + "logps/reference_chosen": -61.321197509765625, + "logps/reference_rejected": -68.41065979003906, + "loss": 0.4074, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.21828393638134003, + "rewards/margins": 0.5127205848693848, + "rewards/rejected": -0.29443657398223877, + "step": 208, + "tokens/chosen_response_len_mean": 127.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 47.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.17, + "grad_norm": 3.6234390581200833, + "learning_rate": 4.93080186000109e-07, + "logits/policy_chosen": -2.0586435794830322, + "logits/policy_rejected": -2.2156870365142822, + "logits/reference_chosen": -2.455575466156006, + "logits/reference_rejected": -2.4480810165405273, + "logps/policy_chosen": -179.0718994140625, + "logps/policy_rejected": -132.7694854736328, + "logps/reference_chosen": -197.98556518554688, + "logps/reference_rejected": -119.65494537353516, + "loss": 0.4632, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.18913674354553223, + "rewards/margins": 0.32028234004974365, + "rewards/rejected": -0.13114558160305023, + "step": 209, + "tokens/chosen_response_len_mean": 428.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 281.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.17, + "grad_norm": 3.033273237981518, + "learning_rate": 4.92915235055936e-07, + "logits/policy_chosen": -2.756727933883667, + "logits/policy_rejected": -2.7047367095947266, + "logits/reference_chosen": -3.014847755432129, + "logits/reference_rejected": -2.7174530029296875, + "logps/policy_chosen": -55.59083938598633, + "logps/policy_rejected": -117.91790008544922, + "logps/reference_chosen": -91.90667724609375, + "logps/reference_rejected": -112.02029418945312, + "loss": 0.4147, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.36315834522247314, + "rewards/margins": 0.4221343994140625, + "rewards/rejected": -0.05897604674100876, + "step": 210, + "tokens/chosen_response_len_mean": 255.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 197.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.17, + "grad_norm": 2.632232149269225, + "learning_rate": 4.927483694412868e-07, + "logits/policy_chosen": -2.6325759887695312, + "logits/policy_rejected": -2.4173927307128906, + "logits/reference_chosen": -2.4960875511169434, + "logits/reference_rejected": -2.534782648086548, + "logps/policy_chosen": -24.59671974182129, + "logps/policy_rejected": -98.01979064941406, + "logps/reference_chosen": -36.76423645019531, + "logps/reference_rejected": -77.22918701171875, + "loss": 0.4171, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.12167516350746155, + "rewards/margins": 0.32958126068115234, + "rewards/rejected": -0.2079060971736908, + "step": 211, + "tokens/chosen_response_len_mean": 114.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 95.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.17, + "grad_norm": 2.6677296030035, + "learning_rate": 4.925795904714048e-07, + "logits/policy_chosen": -2.6325550079345703, + "logits/policy_rejected": -2.7918920516967773, + "logits/reference_chosen": -3.0179286003112793, + "logits/reference_rejected": -2.9879252910614014, + "logps/policy_chosen": -68.28926849365234, + "logps/policy_rejected": -233.2633514404297, + "logps/reference_chosen": -110.06537628173828, + "logps/reference_rejected": -193.7027587890625, + "loss": 0.3524, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.41776108741760254, + "rewards/margins": 0.8133671283721924, + "rewards/rejected": -0.39560604095458984, + "step": 212, + "tokens/chosen_response_len_mean": 270.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 280.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.17, + "grad_norm": 3.4599652097860303, + "learning_rate": 4.924088994766144e-07, + "logits/policy_chosen": -2.2819485664367676, + "logits/policy_rejected": -2.3868868350982666, + "logits/reference_chosen": -2.75758695602417, + "logits/reference_rejected": -2.2176506519317627, + "logps/policy_chosen": -94.18050384521484, + "logps/policy_rejected": -299.33758544921875, + "logps/reference_chosen": -170.72991943359375, + "logps/reference_rejected": -266.3306579589844, + "loss": 0.4024, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.765494167804718, + "rewards/margins": 1.0955636501312256, + "rewards/rejected": -0.3300694227218628, + "step": 213, + "tokens/chosen_response_len_mean": 548.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 511.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.17, + "grad_norm": 2.821041192679951, + "learning_rate": 4.922362978023111e-07, + "logits/policy_chosen": -2.624894618988037, + "logits/policy_rejected": -2.621846914291382, + "logits/reference_chosen": -3.108778476715088, + "logits/reference_rejected": -3.036085367202759, + "logps/policy_chosen": -128.86549377441406, + "logps/policy_rejected": -284.0617370605469, + "logps/reference_chosen": -219.0366973876953, + "logps/reference_rejected": -296.7759094238281, + "loss": 0.4239, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.9017120003700256, + "rewards/margins": 0.7745702266693115, + "rewards/rejected": 0.12714174389839172, + "step": 214, + "tokens/chosen_response_len_mean": 571.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 543.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.17, + "grad_norm": 3.0002891083329857, + "learning_rate": 4.9206178680895e-07, + "logits/policy_chosen": -1.7852914333343506, + "logits/policy_rejected": -1.8987267017364502, + "logits/reference_chosen": -1.9375059604644775, + "logits/reference_rejected": -2.050326108932495, + "logps/policy_chosen": -52.338626861572266, + "logps/policy_rejected": -93.70428466796875, + "logps/reference_chosen": -71.64360046386719, + "logps/reference_rejected": -91.83578491210938, + "loss": 0.4021, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.19304978847503662, + "rewards/margins": 0.2117348462343216, + "rewards/rejected": -0.01868506893515587, + "step": 215, + "tokens/chosen_response_len_mean": 359.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 364.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.17, + "grad_norm": 3.157069335949432, + "learning_rate": 4.918853678720361e-07, + "logits/policy_chosen": -2.6262574195861816, + "logits/policy_rejected": -2.659903049468994, + "logits/reference_chosen": -2.9374420642852783, + "logits/reference_rejected": -2.76753568649292, + "logps/policy_chosen": -38.7560920715332, + "logps/policy_rejected": -142.21218872070312, + "logps/reference_chosen": -69.5650634765625, + "logps/reference_rejected": -97.97706604003906, + "loss": 0.5087, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3080897629261017, + "rewards/margins": 0.7504409551620483, + "rewards/rejected": -0.44235122203826904, + "step": 216, + "tokens/chosen_response_len_mean": 248.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 202.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.17, + "grad_norm": 2.8169264822166338, + "learning_rate": 4.917070423821125e-07, + "logits/policy_chosen": -2.4803824424743652, + "logits/policy_rejected": -2.511946439743042, + "logits/reference_chosen": -2.749699592590332, + "logits/reference_rejected": -2.704967975616455, + "logps/policy_chosen": -83.62620544433594, + "logps/policy_rejected": -296.3905029296875, + "logps/reference_chosen": -138.813720703125, + "logps/reference_rejected": -214.48797607421875, + "loss": 0.3658, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5518751740455627, + "rewards/margins": 1.3709005117416382, + "rewards/rejected": -0.8190253376960754, + "step": 217, + "tokens/chosen_response_len_mean": 493.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 406.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.18, + "grad_norm": 2.7162895030534195, + "learning_rate": 4.915268117447497e-07, + "logits/policy_chosen": -2.778090476989746, + "logits/policy_rejected": -2.7868547439575195, + "logits/reference_chosen": -3.147437572479248, + "logits/reference_rejected": -2.9738471508026123, + "logps/policy_chosen": -58.98277282714844, + "logps/policy_rejected": -193.54931640625, + "logps/reference_chosen": -103.57978057861328, + "logps/reference_rejected": -130.34754943847656, + "loss": 0.4043, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4459700584411621, + "rewards/margins": 1.077987790107727, + "rewards/rejected": -0.6320176720619202, + "step": 218, + "tokens/chosen_response_len_mean": 233.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 185.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.18, + "grad_norm": 3.6458168828683926, + "learning_rate": 4.913446773805354e-07, + "logits/policy_chosen": -2.333178997039795, + "logits/policy_rejected": -2.3620800971984863, + "logits/reference_chosen": -3.10478138923645, + "logits/reference_rejected": -2.8959479331970215, + "logps/policy_chosen": -90.79602813720703, + "logps/policy_rejected": -286.6520690917969, + "logps/reference_chosen": -163.10195922851562, + "logps/reference_rejected": -245.46189880371094, + "loss": 0.4028, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.723059356212616, + "rewards/margins": 1.1349611282348633, + "rewards/rejected": -0.4119018018245697, + "step": 219, + "tokens/chosen_response_len_mean": 469.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 442.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.18, + "grad_norm": 3.233765475571934, + "learning_rate": 4.911606407250617e-07, + "logits/policy_chosen": -2.2644901275634766, + "logits/policy_rejected": -2.4729533195495605, + "logits/reference_chosen": -2.8000075817108154, + "logits/reference_rejected": -2.8166446685791016, + "logps/policy_chosen": -30.913768768310547, + "logps/policy_rejected": -142.3131561279297, + "logps/reference_chosen": -51.407657623291016, + "logps/reference_rejected": -95.70589447021484, + "loss": 0.4707, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2049388736486435, + "rewards/margins": 0.6710115671157837, + "rewards/rejected": -0.4660727083683014, + "step": 220, + "tokens/chosen_response_len_mean": 272.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 233.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.18, + "grad_norm": 2.816944510019336, + "learning_rate": 4.909747032289152e-07, + "logits/policy_chosen": -1.9074110984802246, + "logits/policy_rejected": -2.0913877487182617, + "logits/reference_chosen": -2.375129461288452, + "logits/reference_rejected": -2.261173725128174, + "logps/policy_chosen": -131.19876098632812, + "logps/policy_rejected": -256.4100646972656, + "logps/reference_chosen": -193.19882202148438, + "logps/reference_rejected": -205.95828247070312, + "loss": 0.4362, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.6200005412101746, + "rewards/margins": 1.1245182752609253, + "rewards/rejected": -0.5045177936553955, + "step": 221, + "tokens/chosen_response_len_mean": 388.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 280.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.18, + "grad_norm": 3.102088864340547, + "learning_rate": 4.90786866357665e-07, + "logits/policy_chosen": -2.299764633178711, + "logits/policy_rejected": -2.4341142177581787, + "logits/reference_chosen": -3.09466290473938, + "logits/reference_rejected": -2.8344757556915283, + "logps/policy_chosen": -104.12731170654297, + "logps/policy_rejected": -403.4098205566406, + "logps/reference_chosen": -205.33453369140625, + "logps/reference_rejected": -297.8888854980469, + "loss": 0.4303, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.0120720863342285, + "rewards/margins": 2.067281484603882, + "rewards/rejected": -1.0552091598510742, + "step": 222, + "tokens/chosen_response_len_mean": 487.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 350.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.18, + "grad_norm": 2.6980157978791506, + "learning_rate": 4.905971315918508e-07, + "logits/policy_chosen": -2.2736315727233887, + "logits/policy_rejected": -2.342026472091675, + "logits/reference_chosen": -2.8474254608154297, + "logits/reference_rejected": -2.5709283351898193, + "logps/policy_chosen": -85.25898742675781, + "logps/policy_rejected": -367.0793151855469, + "logps/reference_chosen": -168.6109161376953, + "logps/reference_rejected": -258.70306396484375, + "loss": 0.3805, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.8335192203521729, + "rewards/margins": 1.9172817468643188, + "rewards/rejected": -1.0837624073028564, + "step": 223, + "tokens/chosen_response_len_mean": 459.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 381.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.18, + "grad_norm": 3.023484025148734, + "learning_rate": 4.904055004269721e-07, + "logits/policy_chosen": -2.454890251159668, + "logits/policy_rejected": -2.5622761249542236, + "logits/reference_chosen": -2.9128506183624268, + "logits/reference_rejected": -2.8558335304260254, + "logps/policy_chosen": -36.11943435668945, + "logps/policy_rejected": -159.4558563232422, + "logps/reference_chosen": -75.32926177978516, + "logps/reference_rejected": -124.79375457763672, + "loss": 0.3956, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.39209815859794617, + "rewards/margins": 0.7387192845344543, + "rewards/rejected": -0.3466210961341858, + "step": 224, + "tokens/chosen_response_len_mean": 181.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 169.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.18, + "grad_norm": 2.879025667733998, + "learning_rate": 4.902119743734755e-07, + "logits/policy_chosen": -2.664403200149536, + "logits/policy_rejected": -2.587775230407715, + "logits/reference_chosen": -3.1453912258148193, + "logits/reference_rejected": -2.816845417022705, + "logps/policy_chosen": -49.29490661621094, + "logps/policy_rejected": -94.93446350097656, + "logps/reference_chosen": -71.66400146484375, + "logps/reference_rejected": -87.74375915527344, + "loss": 0.3954, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.22369101643562317, + "rewards/margins": 0.2955980896949768, + "rewards/rejected": -0.07190708816051483, + "step": 225, + "tokens/chosen_response_len_mean": 198.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 179.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.18, + "grad_norm": 3.246164993624403, + "learning_rate": 4.900165549567435e-07, + "logits/policy_chosen": -2.3957905769348145, + "logits/policy_rejected": -2.423283100128174, + "logits/reference_chosen": -2.991285800933838, + "logits/reference_rejected": -2.787295341491699, + "logps/policy_chosen": -88.65385437011719, + "logps/policy_rejected": -120.2602767944336, + "logps/reference_chosen": -112.840087890625, + "logps/reference_rejected": -75.72503662109375, + "loss": 0.3168, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.24186235666275024, + "rewards/margins": 0.6872147917747498, + "rewards/rejected": -0.44535237550735474, + "step": 226, + "tokens/chosen_response_len_mean": 422.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 400.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.18, + "grad_norm": 3.0799697537609085, + "learning_rate": 4.898192437170821e-07, + "logits/policy_chosen": -2.530318021774292, + "logits/policy_rejected": -2.645883083343506, + "logits/reference_chosen": -3.1499884128570557, + "logits/reference_rejected": -2.5675787925720215, + "logps/policy_chosen": -54.94190216064453, + "logps/policy_rejected": -261.4326171875, + "logps/reference_chosen": -134.1338348388672, + "logps/reference_rejected": -190.62583923339844, + "loss": 0.3325, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.7919193506240845, + "rewards/margins": 1.4999868869781494, + "rewards/rejected": -0.7080676555633545, + "step": 227, + "tokens/chosen_response_len_mean": 255.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 204.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.18, + "grad_norm": 2.659841913817482, + "learning_rate": 4.896200422097086e-07, + "logits/policy_chosen": -1.9145750999450684, + "logits/policy_rejected": -2.0923149585723877, + "logits/reference_chosen": -2.8488292694091797, + "logits/reference_rejected": -2.7205710411071777, + "logps/policy_chosen": -107.40567016601562, + "logps/policy_rejected": -427.5948486328125, + "logps/reference_chosen": -223.86032104492188, + "logps/reference_rejected": -329.6167297363281, + "loss": 0.1916, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.1645466089248657, + "rewards/margins": 2.1443278789520264, + "rewards/rejected": -0.9797812700271606, + "step": 228, + "tokens/chosen_response_len_mean": 517.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 439.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.18, + "grad_norm": 3.234230812286525, + "learning_rate": 4.894189520047397e-07, + "logits/policy_chosen": -2.2693915367126465, + "logits/policy_rejected": -2.3958592414855957, + "logits/reference_chosen": -3.0886495113372803, + "logits/reference_rejected": -2.8636574745178223, + "logps/policy_chosen": -51.258445739746094, + "logps/policy_rejected": -378.75201416015625, + "logps/reference_chosen": -118.88182830810547, + "logps/reference_rejected": -246.41397094726562, + "loss": 0.3282, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.6762338280677795, + "rewards/margins": 1.9996143579483032, + "rewards/rejected": -1.3233805894851685, + "step": 229, + "tokens/chosen_response_len_mean": 271.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 266.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.18, + "grad_norm": 3.457038560417765, + "learning_rate": 4.892159746871788e-07, + "logits/policy_chosen": -2.3888025283813477, + "logits/policy_rejected": -2.411865234375, + "logits/reference_chosen": -3.154548168182373, + "logits/reference_rejected": -2.1464414596557617, + "logps/policy_chosen": -57.5999870300293, + "logps/policy_rejected": -173.8509063720703, + "logps/reference_chosen": -131.91171264648438, + "logps/reference_rejected": -113.2002182006836, + "loss": 0.3683, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.7431172728538513, + "rewards/margins": 1.3496241569519043, + "rewards/rejected": -0.6065068244934082, + "step": 230, + "tokens/chosen_response_len_mean": 285.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 94.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.19, + "grad_norm": 3.453784172007788, + "learning_rate": 4.890111118569036e-07, + "logits/policy_chosen": -2.0798964500427246, + "logits/policy_rejected": -2.2078676223754883, + "logits/reference_chosen": -2.8997390270233154, + "logits/reference_rejected": -2.4869751930236816, + "logps/policy_chosen": -75.02999877929688, + "logps/policy_rejected": -229.92823791503906, + "logps/reference_chosen": -143.95648193359375, + "logps/reference_rejected": -181.41172790527344, + "loss": 0.327, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.6892647743225098, + "rewards/margins": 1.1744297742843628, + "rewards/rejected": -0.485164999961853, + "step": 231, + "tokens/chosen_response_len_mean": 280.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 238.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.19, + "grad_norm": 3.239627424882494, + "learning_rate": 4.888043651286537e-07, + "logits/policy_chosen": -2.5193958282470703, + "logits/policy_rejected": -2.452767848968506, + "logits/reference_chosen": -3.070887565612793, + "logits/reference_rejected": -2.4067721366882324, + "logps/policy_chosen": -44.861671447753906, + "logps/policy_rejected": -245.7529296875, + "logps/reference_chosen": -78.50507354736328, + "logps/reference_rejected": -147.85055541992188, + "loss": 0.3734, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.336434006690979, + "rewards/margins": 1.315457820892334, + "rewards/rejected": -0.979023814201355, + "step": 232, + "tokens/chosen_response_len_mean": 251.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 159.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.19, + "grad_norm": 3.085937654519379, + "learning_rate": 4.885957361320172e-07, + "logits/policy_chosen": -1.9149789810180664, + "logits/policy_rejected": -2.088989734649658, + "logits/reference_chosen": -2.9695346355438232, + "logits/reference_rejected": -2.6794815063476562, + "logps/policy_chosen": -73.91265869140625, + "logps/policy_rejected": -403.6729431152344, + "logps/reference_chosen": -209.8995819091797, + "logps/reference_rejected": -327.79486083984375, + "loss": 0.2662, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.359869360923767, + "rewards/margins": 2.118650197982788, + "rewards/rejected": -0.7587809562683105, + "step": 233, + "tokens/chosen_response_len_mean": 403.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 360.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.19, + "grad_norm": 2.7768750875070762, + "learning_rate": 4.88385226511419e-07, + "logits/policy_chosen": -1.917398452758789, + "logits/policy_rejected": -2.089282989501953, + "logits/reference_chosen": -3.025069236755371, + "logits/reference_rejected": -2.1790575981140137, + "logps/policy_chosen": -98.6364974975586, + "logps/policy_rejected": -710.46142578125, + "logps/reference_chosen": -247.2069854736328, + "logps/reference_rejected": -409.91046142578125, + "loss": 0.2487, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.485705018043518, + "rewards/margins": 4.491214752197266, + "rewards/rejected": -3.005509853363037, + "step": 234, + "tokens/chosen_response_len_mean": 516.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 393.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.19, + "grad_norm": 2.6357031102373694, + "learning_rate": 4.881728379261067e-07, + "logits/policy_chosen": -2.5067009925842285, + "logits/policy_rejected": -2.6151890754699707, + "logits/reference_chosen": -2.879830837249756, + "logits/reference_rejected": -2.289306879043579, + "logps/policy_chosen": -28.403209686279297, + "logps/policy_rejected": -94.16014099121094, + "logps/reference_chosen": -50.78627014160156, + "logps/reference_rejected": -61.18941116333008, + "loss": 0.3383, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.22383059561252594, + "rewards/margins": 0.553537905216217, + "rewards/rejected": -0.3297072649002075, + "step": 235, + "tokens/chosen_response_len_mean": 130.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 70.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.19, + "grad_norm": 2.5416162412883967, + "learning_rate": 4.879585720501382e-07, + "logits/policy_chosen": -2.079270362854004, + "logits/policy_rejected": -2.1965620517730713, + "logits/reference_chosen": -2.69492769241333, + "logits/reference_rejected": -2.7009551525115967, + "logps/policy_chosen": -47.36531448364258, + "logps/policy_rejected": -210.86045837402344, + "logps/reference_chosen": -109.4851303100586, + "logps/reference_rejected": -157.570556640625, + "loss": 0.4301, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.6211981177330017, + "rewards/margins": 1.1540971994400024, + "rewards/rejected": -0.5328990817070007, + "step": 236, + "tokens/chosen_response_len_mean": 382.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 342.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.19, + "grad_norm": 3.3640626360746606, + "learning_rate": 4.877424305723682e-07, + "logits/policy_chosen": -2.3050661087036133, + "logits/policy_rejected": -2.362180709838867, + "logits/reference_chosen": -2.8899943828582764, + "logits/reference_rejected": -2.970350503921509, + "logps/policy_chosen": -58.130821228027344, + "logps/policy_rejected": -187.88409423828125, + "logps/reference_chosen": -105.74044036865234, + "logps/reference_rejected": -162.71900939941406, + "loss": 0.3679, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4760962128639221, + "rewards/margins": 0.7277471423149109, + "rewards/rejected": -0.2516508996486664, + "step": 237, + "tokens/chosen_response_len_mean": 321.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 332.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.19, + "grad_norm": 3.092634828507096, + "learning_rate": 4.87524415196435e-07, + "logits/policy_chosen": -1.7353723049163818, + "logits/policy_rejected": -1.8133890628814697, + "logits/reference_chosen": -2.299503803253174, + "logits/reference_rejected": -1.7435356378555298, + "logps/policy_chosen": -25.773319244384766, + "logps/policy_rejected": -84.91059875488281, + "logps/reference_chosen": -49.502906799316406, + "logps/reference_rejected": -59.85688781738281, + "loss": 0.4402, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.23729589581489563, + "rewards/margins": 0.4878329634666443, + "rewards/rejected": -0.25053703784942627, + "step": 238, + "tokens/chosen_response_len_mean": 180.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 131.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.19, + "grad_norm": 2.30111282627728, + "learning_rate": 4.873045276407472e-07, + "logits/policy_chosen": -2.3006367683410645, + "logits/policy_rejected": -2.373878240585327, + "logits/reference_chosen": -3.0563242435455322, + "logits/reference_rejected": -2.862274408340454, + "logps/policy_chosen": -60.457550048828125, + "logps/policy_rejected": -362.2609558105469, + "logps/reference_chosen": -138.99404907226562, + "logps/reference_rejected": -211.34681701660156, + "loss": 0.263, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.7853649258613586, + "rewards/margins": 2.294506072998047, + "rewards/rejected": -1.5091413259506226, + "step": 239, + "tokens/chosen_response_len_mean": 323.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 253.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.19, + "grad_norm": 3.3647443577329184, + "learning_rate": 4.870827696384697e-07, + "logits/policy_chosen": -1.7883644104003906, + "logits/policy_rejected": -1.9111793041229248, + "logits/reference_chosen": -2.7533793449401855, + "logits/reference_rejected": -2.8058459758758545, + "logps/policy_chosen": -63.6192512512207, + "logps/policy_rejected": -472.13812255859375, + "logps/reference_chosen": -182.55494689941406, + "logps/reference_rejected": -324.2348327636719, + "loss": 0.4491, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.189356803894043, + "rewards/margins": 2.6683895587921143, + "rewards/rejected": -1.4790325164794922, + "step": 240, + "tokens/chosen_response_len_mean": 425.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 412.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.19, + "grad_norm": 2.8850200026957507, + "learning_rate": 4.868591429375109e-07, + "logits/policy_chosen": -2.5384299755096436, + "logits/policy_rejected": -2.5679569244384766, + "logits/reference_chosen": -3.137737274169922, + "logits/reference_rejected": -2.9568300247192383, + "logps/policy_chosen": -52.663124084472656, + "logps/policy_rejected": -150.413818359375, + "logps/reference_chosen": -119.31211853027344, + "logps/reference_rejected": -144.43154907226562, + "loss": 0.3981, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.6664900183677673, + "rewards/margins": 0.7263126373291016, + "rewards/rejected": -0.059822604060173035, + "step": 241, + "tokens/chosen_response_len_mean": 247.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 232.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.19, + "grad_norm": 2.740311113779816, + "learning_rate": 4.866336493005077e-07, + "logits/policy_chosen": -2.3799917697906494, + "logits/policy_rejected": -2.3980116844177246, + "logits/reference_chosen": -2.8105556964874268, + "logits/reference_rejected": -2.789691925048828, + "logps/policy_chosen": -86.99842834472656, + "logps/policy_rejected": -245.59523010253906, + "logps/reference_chosen": -164.35971069335938, + "logps/reference_rejected": -204.79449462890625, + "loss": 0.3237, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.7736127376556396, + "rewards/margins": 1.1816203594207764, + "rewards/rejected": -0.4080074727535248, + "step": 242, + "tokens/chosen_response_len_mean": 234.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 228.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.2, + "grad_norm": 3.5286650522231846, + "learning_rate": 4.86406290504813e-07, + "logits/policy_chosen": -2.302743434906006, + "logits/policy_rejected": -2.3800129890441895, + "logits/reference_chosen": -3.100501537322998, + "logits/reference_rejected": -2.502018690109253, + "logps/policy_chosen": -56.89078140258789, + "logps/policy_rejected": -293.7001953125, + "logps/reference_chosen": -124.03472900390625, + "logps/reference_rejected": -186.7720489501953, + "loss": 0.262, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.671439528465271, + "rewards/margins": 1.7407207489013672, + "rewards/rejected": -1.0692812204360962, + "step": 243, + "tokens/chosen_response_len_mean": 347.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 276.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.2, + "grad_norm": 3.590713146645003, + "learning_rate": 4.861770683424803e-07, + "logits/policy_chosen": -2.072890043258667, + "logits/policy_rejected": -1.9427573680877686, + "logits/reference_chosen": -2.6935362815856934, + "logits/reference_rejected": -2.673877000808716, + "logps/policy_chosen": -45.185890197753906, + "logps/policy_rejected": -246.34381103515625, + "logps/reference_chosen": -113.1274642944336, + "logps/reference_rejected": -189.79290771484375, + "loss": 0.365, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.6794158220291138, + "rewards/margins": 1.2449246644973755, + "rewards/rejected": -0.5655089616775513, + "step": 244, + "tokens/chosen_response_len_mean": 282.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 369.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.2, + "grad_norm": 2.610774413059978, + "learning_rate": 4.859459846202507e-07, + "logits/policy_chosen": -2.30108380317688, + "logits/policy_rejected": -2.4250240325927734, + "logits/reference_chosen": -3.1003952026367188, + "logits/reference_rejected": -2.8910207748413086, + "logps/policy_chosen": -113.18220520019531, + "logps/policy_rejected": -571.716064453125, + "logps/reference_chosen": -273.4241943359375, + "logps/reference_rejected": -353.7633056640625, + "loss": 0.3468, + "rewards/accuracies": 0.75, + "rewards/chosen": 1.6024198532104492, + "rewards/margins": 3.781947135925293, + "rewards/rejected": -2.1795272827148438, + "step": 245, + "tokens/chosen_response_len_mean": 495.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 352.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.2, + "grad_norm": 3.1327030609878674, + "learning_rate": 4.857130411595381e-07, + "logits/policy_chosen": -1.9849697351455688, + "logits/policy_rejected": -2.2542386054992676, + "logits/reference_chosen": -2.8174877166748047, + "logits/reference_rejected": -2.6481382846832275, + "logps/policy_chosen": -65.47968292236328, + "logps/policy_rejected": -309.6473083496094, + "logps/reference_chosen": -125.56417846679688, + "logps/reference_rejected": -215.66690063476562, + "loss": 0.2941, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.6008449792861938, + "rewards/margins": 1.5406490564346313, + "rewards/rejected": -0.9398040771484375, + "step": 246, + "tokens/chosen_response_len_mean": 314.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 219.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.2, + "grad_norm": 2.8949654648587106, + "learning_rate": 4.854782397964148e-07, + "logits/policy_chosen": -2.4038028717041016, + "logits/policy_rejected": -2.535879373550415, + "logits/reference_chosen": -3.110623836517334, + "logits/reference_rejected": -2.892681360244751, + "logps/policy_chosen": -42.45046615600586, + "logps/policy_rejected": -223.6653594970703, + "logps/reference_chosen": -108.52606201171875, + "logps/reference_rejected": -122.96241760253906, + "loss": 0.1793, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.6607558727264404, + "rewards/margins": 1.667785406112671, + "rewards/rejected": -1.0070295333862305, + "step": 247, + "tokens/chosen_response_len_mean": 224.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 148.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.2, + "grad_norm": 3.2467096251740397, + "learning_rate": 4.852415823815975e-07, + "logits/policy_chosen": -2.0340566635131836, + "logits/policy_rejected": -2.007965087890625, + "logits/reference_chosen": -2.2954137325286865, + "logits/reference_rejected": -2.1136422157287598, + "logps/policy_chosen": -48.099082946777344, + "logps/policy_rejected": -218.53590393066406, + "logps/reference_chosen": -83.52349090576172, + "logps/reference_rejected": -118.22250366210938, + "loss": 0.3545, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3542441725730896, + "rewards/margins": 1.3573782444000244, + "rewards/rejected": -1.0031341314315796, + "step": 248, + "tokens/chosen_response_len_mean": 159.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 128.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.2, + "grad_norm": 2.4511654553576294, + "learning_rate": 4.850030707804322e-07, + "logits/policy_chosen": -2.421999454498291, + "logits/policy_rejected": -2.4670658111572266, + "logits/reference_chosen": -3.0133814811706543, + "logits/reference_rejected": -2.3799855709075928, + "logps/policy_chosen": -53.002376556396484, + "logps/policy_rejected": -286.0945739746094, + "logps/reference_chosen": -161.1949462890625, + "logps/reference_rejected": -157.0173797607422, + "loss": 0.3471, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.081925630569458, + "rewards/margins": 2.372697353363037, + "rewards/rejected": -1.2907719612121582, + "step": 249, + "tokens/chosen_response_len_mean": 254.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 189.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.2, + "grad_norm": 3.2467420827995683, + "learning_rate": 4.847627068728794e-07, + "logits/policy_chosen": -2.1038780212402344, + "logits/policy_rejected": -2.1949315071105957, + "logits/reference_chosen": -3.1234359741210938, + "logits/reference_rejected": -2.4281110763549805, + "logps/policy_chosen": -33.008514404296875, + "logps/policy_rejected": -167.2099151611328, + "logps/reference_chosen": -75.09925842285156, + "logps/reference_rejected": -86.3980484008789, + "loss": 0.3442, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4209074079990387, + "rewards/margins": 1.2290260791778564, + "rewards/rejected": -0.8081187009811401, + "step": 250, + "tokens/chosen_response_len_mean": 105.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 71.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.2, + "grad_norm": 2.2929712454735145, + "learning_rate": 4.845204925535003e-07, + "logits/policy_chosen": -1.7899479866027832, + "logits/policy_rejected": -1.9353580474853516, + "logits/reference_chosen": -2.701301336288452, + "logits/reference_rejected": -2.7325639724731445, + "logps/policy_chosen": -50.73257827758789, + "logps/policy_rejected": -341.6130676269531, + "logps/reference_chosen": -127.89730834960938, + "logps/reference_rejected": -177.73052978515625, + "loss": 0.1825, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.7716473340988159, + "rewards/margins": 2.4104726314544678, + "rewards/rejected": -1.638825535774231, + "step": 251, + "tokens/chosen_response_len_mean": 542.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 416.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.2, + "grad_norm": 2.718487080105349, + "learning_rate": 4.842764297314406e-07, + "logits/policy_chosen": -1.9718434810638428, + "logits/policy_rejected": -2.002476692199707, + "logits/reference_chosen": -2.571845293045044, + "logits/reference_rejected": -2.6133267879486084, + "logps/policy_chosen": -41.810699462890625, + "logps/policy_rejected": -177.89076232910156, + "logps/reference_chosen": -95.01820373535156, + "logps/reference_rejected": -145.2048797607422, + "loss": 0.2304, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5320751070976257, + "rewards/margins": 0.8589338064193726, + "rewards/rejected": -0.3268587589263916, + "step": 252, + "tokens/chosen_response_len_mean": 206.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 208.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.2, + "grad_norm": 3.7180834260951325, + "learning_rate": 4.840305203304159e-07, + "logits/policy_chosen": -2.030735969543457, + "logits/policy_rejected": -2.1171207427978516, + "logits/reference_chosen": -2.9690611362457275, + "logits/reference_rejected": -2.7168192863464355, + "logps/policy_chosen": -41.81312561035156, + "logps/policy_rejected": -201.77508544921875, + "logps/reference_chosen": -100.09971618652344, + "logps/reference_rejected": -119.85955047607422, + "loss": 0.3607, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.5828658938407898, + "rewards/margins": 1.4020211696624756, + "rewards/rejected": -0.8191554546356201, + "step": 253, + "tokens/chosen_response_len_mean": 273.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 156.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.2, + "grad_norm": 2.6023063627206815, + "learning_rate": 4.837827662886971e-07, + "logits/policy_chosen": -2.1987662315368652, + "logits/policy_rejected": -2.3438453674316406, + "logits/reference_chosen": -3.212616443634033, + "logits/reference_rejected": -3.013723134994507, + "logps/policy_chosen": -59.346351623535156, + "logps/policy_rejected": -576.636962890625, + "logps/reference_chosen": -173.44329833984375, + "logps/reference_rejected": -289.7512512207031, + "loss": 0.2857, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.1409695148468018, + "rewards/margins": 4.009826183319092, + "rewards/rejected": -2.86885666847229, + "step": 254, + "tokens/chosen_response_len_mean": 327.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 266.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.2, + "grad_norm": 2.968765579985845, + "learning_rate": 4.835331695590943e-07, + "logits/policy_chosen": -2.362076759338379, + "logits/policy_rejected": -2.3499341011047363, + "logits/reference_chosen": -3.076436996459961, + "logits/reference_rejected": -2.8915157318115234, + "logps/policy_chosen": -24.552675247192383, + "logps/policy_rejected": -53.35011291503906, + "logps/reference_chosen": -56.392234802246094, + "logps/reference_rejected": -45.72980499267578, + "loss": 0.3399, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.31839558482170105, + "rewards/margins": 0.39459866285324097, + "rewards/rejected": -0.07620306313037872, + "step": 255, + "tokens/chosen_response_len_mean": 143.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 110.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.21, + "grad_norm": 2.8370570975666873, + "learning_rate": 4.832817321089418e-07, + "logits/policy_chosen": -2.4410159587860107, + "logits/policy_rejected": -2.4411826133728027, + "logits/reference_chosen": -2.7271342277526855, + "logits/reference_rejected": -2.732917308807373, + "logps/policy_chosen": -31.23749351501465, + "logps/policy_rejected": -98.78245544433594, + "logps/reference_chosen": -43.81713104248047, + "logps/reference_rejected": -72.64688873291016, + "loss": 0.4067, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.1257963329553604, + "rewards/margins": 0.3871520459651947, + "rewards/rejected": -0.2613556981086731, + "step": 256, + "tokens/chosen_response_len_mean": 45.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 46.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.21, + "grad_norm": 2.7797560376214, + "learning_rate": 4.830284559200827e-07, + "logits/policy_chosen": -1.952653408050537, + "logits/policy_rejected": -2.1609013080596924, + "logits/reference_chosen": -3.012723922729492, + "logits/reference_rejected": -2.739423990249634, + "logps/policy_chosen": -57.349365234375, + "logps/policy_rejected": -257.4586486816406, + "logps/reference_chosen": -145.12765502929688, + "logps/reference_rejected": -142.13612365722656, + "loss": 0.2769, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.8777830004692078, + "rewards/margins": 2.03100848197937, + "rewards/rejected": -1.153225302696228, + "step": 257, + "tokens/chosen_response_len_mean": 332.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 215.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.21, + "grad_norm": 2.6056834165338865, + "learning_rate": 4.827733429888531e-07, + "logits/policy_chosen": -2.5333569049835205, + "logits/policy_rejected": -2.632535934448242, + "logits/reference_chosen": -3.0343687534332275, + "logits/reference_rejected": -2.7431583404541016, + "logps/policy_chosen": -51.503787994384766, + "logps/policy_rejected": -205.5330810546875, + "logps/reference_chosen": -112.60284423828125, + "logps/reference_rejected": -109.25039672851562, + "loss": 0.2703, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.6109905242919922, + "rewards/margins": 1.573817253112793, + "rewards/rejected": -0.962826669216156, + "step": 258, + "tokens/chosen_response_len_mean": 209.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 112.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.21, + "grad_norm": 2.7017642438408416, + "learning_rate": 4.825163953260663e-07, + "logits/policy_chosen": -2.209869146347046, + "logits/policy_rejected": -2.233635902404785, + "logits/reference_chosen": -3.0763533115386963, + "logits/reference_rejected": -2.9993386268615723, + "logps/policy_chosen": -72.18954467773438, + "logps/policy_rejected": -188.30348205566406, + "logps/reference_chosen": -163.45166015625, + "logps/reference_rejected": -172.4595947265625, + "loss": 0.2949, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.9126211404800415, + "rewards/margins": 1.0710599422454834, + "rewards/rejected": -0.1584388017654419, + "step": 259, + "tokens/chosen_response_len_mean": 309.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 269.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.21, + "grad_norm": 2.7221262346156205, + "learning_rate": 4.82257614956997e-07, + "logits/policy_chosen": -1.0971046686172485, + "logits/policy_rejected": -1.240858793258667, + "logits/reference_chosen": -2.244776964187622, + "logits/reference_rejected": -1.5502854585647583, + "logps/policy_chosen": -91.92208862304688, + "logps/policy_rejected": -343.6435241699219, + "logps/reference_chosen": -204.17926025390625, + "logps/reference_rejected": -210.630615234375, + "loss": 0.2794, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.122571587562561, + "rewards/margins": 2.452700614929199, + "rewards/rejected": -1.3301290273666382, + "step": 260, + "tokens/chosen_response_len_mean": 590.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 508.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.21, + "grad_norm": 3.8028582614963775, + "learning_rate": 4.819970039213655e-07, + "logits/policy_chosen": -2.1693973541259766, + "logits/policy_rejected": -2.2979447841644287, + "logits/reference_chosen": -3.102466106414795, + "logits/reference_rejected": -2.547724723815918, + "logps/policy_chosen": -59.068519592285156, + "logps/policy_rejected": -445.1409606933594, + "logps/reference_chosen": -156.0995635986328, + "logps/reference_rejected": -305.90802001953125, + "loss": 0.3756, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.9703104496002197, + "rewards/margins": 2.362639904022217, + "rewards/rejected": -1.3923293352127075, + "step": 261, + "tokens/chosen_response_len_mean": 229.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 209.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.21, + "grad_norm": 3.239958730128674, + "learning_rate": 4.817345642733218e-07, + "logits/policy_chosen": -2.0956246852874756, + "logits/policy_rejected": -2.182451009750366, + "logits/reference_chosen": -3.003354072570801, + "logits/reference_rejected": -2.506141424179077, + "logps/policy_chosen": -42.89665603637695, + "logps/policy_rejected": -128.68064880371094, + "logps/reference_chosen": -79.88959503173828, + "logps/reference_rejected": -72.20051574707031, + "loss": 0.2773, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.36992937326431274, + "rewards/margins": 0.9347306489944458, + "rewards/rejected": -0.5648012757301331, + "step": 262, + "tokens/chosen_response_len_mean": 154.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 111.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.21, + "grad_norm": 2.7801336555461136, + "learning_rate": 4.814702980814284e-07, + "logits/policy_chosen": -2.0152645111083984, + "logits/policy_rejected": -2.148892879486084, + "logits/reference_chosen": -3.0190117359161377, + "logits/reference_rejected": -2.243680238723755, + "logps/policy_chosen": -57.687110900878906, + "logps/policy_rejected": -464.7051696777344, + "logps/reference_chosen": -180.52468872070312, + "logps/reference_rejected": -230.62245178222656, + "loss": 0.1516, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.2283756732940674, + "rewards/margins": 3.5692026615142822, + "rewards/rejected": -2.340827226638794, + "step": 263, + "tokens/chosen_response_len_mean": 365.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 254.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.21, + "grad_norm": 2.66395215116379, + "learning_rate": 4.812042074286453e-07, + "logits/policy_chosen": -2.0517807006835938, + "logits/policy_rejected": -2.1628949642181396, + "logits/reference_chosen": -3.143606662750244, + "logits/reference_rejected": -2.7492570877075195, + "logps/policy_chosen": -114.81196594238281, + "logps/policy_rejected": -593.2320556640625, + "logps/reference_chosen": -350.7387390136719, + "logps/reference_rejected": -264.07171630859375, + "loss": 0.3429, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.3592677116394043, + "rewards/margins": 5.650871276855469, + "rewards/rejected": -3.2916030883789062, + "step": 264, + "tokens/chosen_response_len_mean": 589.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 265.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.21, + "grad_norm": 2.579532955538129, + "learning_rate": 4.809362944123128e-07, + "logits/policy_chosen": -2.0034396648406982, + "logits/policy_rejected": -2.1076416969299316, + "logits/reference_chosen": -2.9764785766601562, + "logits/reference_rejected": -2.718217611312866, + "logps/policy_chosen": -56.3231086730957, + "logps/policy_rejected": -294.1711730957031, + "logps/reference_chosen": -122.71052551269531, + "logps/reference_rejected": -174.21401977539062, + "loss": 0.3059, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.6638741493225098, + "rewards/margins": 1.8634452819824219, + "rewards/rejected": -1.1995712518692017, + "step": 265, + "tokens/chosen_response_len_mean": 196.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 169.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.21, + "grad_norm": 2.6877602739497, + "learning_rate": 4.806665611441354e-07, + "logits/policy_chosen": -2.2234950065612793, + "logits/policy_rejected": -2.2985267639160156, + "logits/reference_chosen": -3.1664934158325195, + "logits/reference_rejected": -2.4548802375793457, + "logps/policy_chosen": -53.821807861328125, + "logps/policy_rejected": -135.544921875, + "logps/reference_chosen": -155.88006591796875, + "logps/reference_rejected": -98.04533386230469, + "loss": 0.3466, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.020582675933838, + "rewards/margins": 1.3955786228179932, + "rewards/rejected": -0.3749959468841553, + "step": 266, + "tokens/chosen_response_len_mean": 369.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 267.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.21, + "grad_norm": 2.691839396443011, + "learning_rate": 4.803950097501643e-07, + "logits/policy_chosen": -2.3689355850219727, + "logits/policy_rejected": -2.406802177429199, + "logits/reference_chosen": -3.110931158065796, + "logits/reference_rejected": -2.629725694656372, + "logps/policy_chosen": -61.78421401977539, + "logps/policy_rejected": -293.3619079589844, + "logps/reference_chosen": -205.92092895507812, + "logps/reference_rejected": -133.5921630859375, + "loss": 0.3107, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.4413671493530273, + "rewards/margins": 3.039064407348633, + "rewards/rejected": -1.597697377204895, + "step": 267, + "tokens/chosen_response_len_mean": 334.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 111.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.22, + "grad_norm": 4.332732574683176, + "learning_rate": 4.80121642370782e-07, + "logits/policy_chosen": -1.435086965560913, + "logits/policy_rejected": -1.555901288986206, + "logits/reference_chosen": -2.4242351055145264, + "logits/reference_rejected": -2.2076950073242188, + "logps/policy_chosen": -122.86267852783203, + "logps/policy_rejected": -370.72705078125, + "logps/reference_chosen": -256.2296142578125, + "logps/reference_rejected": -221.7090301513672, + "loss": 0.2893, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.3336691856384277, + "rewards/margins": 2.8238492012023926, + "rewards/rejected": -1.4901798963546753, + "step": 268, + "tokens/chosen_response_len_mean": 474.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 229.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.22, + "grad_norm": 3.679173265554199, + "learning_rate": 4.798464611606842e-07, + "logits/policy_chosen": -1.9900972843170166, + "logits/policy_rejected": -0.7595923542976379, + "logits/reference_chosen": -2.307075023651123, + "logits/reference_rejected": -2.1424033641815186, + "logps/policy_chosen": -39.70943832397461, + "logps/policy_rejected": -101.0633544921875, + "logps/reference_chosen": -61.08944320678711, + "logps/reference_rejected": -77.70144653320312, + "loss": 0.3654, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.21380005776882172, + "rewards/margins": 0.44741907715797424, + "rewards/rejected": -0.23361904919147491, + "step": 269, + "tokens/chosen_response_len_mean": 73.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 361.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.22, + "grad_norm": 2.860165012641077, + "learning_rate": 4.795694682888635e-07, + "logits/policy_chosen": -1.8247418403625488, + "logits/policy_rejected": -2.0296616554260254, + "logits/reference_chosen": -2.9217374324798584, + "logits/reference_rejected": -2.3353545665740967, + "logps/policy_chosen": -60.13949966430664, + "logps/policy_rejected": -374.595947265625, + "logps/reference_chosen": -168.12413024902344, + "logps/reference_rejected": -216.00479125976562, + "loss": 0.1644, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.0798462629318237, + "rewards/margins": 2.665757656097412, + "rewards/rejected": -1.5859116315841675, + "step": 270, + "tokens/chosen_response_len_mean": 366.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 259.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.22, + "grad_norm": 2.8914101024190764, + "learning_rate": 4.79290665938592e-07, + "logits/policy_chosen": -2.393129587173462, + "logits/policy_rejected": -2.532593250274658, + "logits/reference_chosen": -3.0000436305999756, + "logits/reference_rejected": -2.605870246887207, + "logps/policy_chosen": -32.41455841064453, + "logps/policy_rejected": -120.82582092285156, + "logps/reference_chosen": -80.18354034423828, + "logps/reference_rejected": -52.557701110839844, + "loss": 0.2656, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.4776897728443146, + "rewards/margins": 1.1603708267211914, + "rewards/rejected": -0.6826812028884888, + "step": 271, + "tokens/chosen_response_len_mean": 264.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 82.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.22, + "grad_norm": 1.9537358053696072, + "learning_rate": 4.790100563074044e-07, + "logits/policy_chosen": -1.9999793767929077, + "logits/policy_rejected": -2.0931291580200195, + "logits/reference_chosen": -2.969376564025879, + "logits/reference_rejected": -1.7754515409469604, + "logps/policy_chosen": -88.11581420898438, + "logps/policy_rejected": -398.7203369140625, + "logps/reference_chosen": -281.721435546875, + "logps/reference_rejected": -166.79583740234375, + "loss": 0.3137, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.936056137084961, + "rewards/margins": 4.255301475524902, + "rewards/rejected": -2.3192451000213623, + "step": 272, + "tokens/chosen_response_len_mean": 415.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 262.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.22, + "grad_norm": 2.7544872780237926, + "learning_rate": 4.787276416070804e-07, + "logits/policy_chosen": -2.0469934940338135, + "logits/policy_rejected": -1.6853868961334229, + "logits/reference_chosen": -2.1769094467163086, + "logits/reference_rejected": -2.2352547645568848, + "logps/policy_chosen": -33.57738494873047, + "logps/policy_rejected": -220.69619750976562, + "logps/reference_chosen": -67.25520324707031, + "logps/reference_rejected": -119.21714782714844, + "loss": 0.298, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.3367781937122345, + "rewards/margins": 1.351568579673767, + "rewards/rejected": -1.014790415763855, + "step": 273, + "tokens/chosen_response_len_mean": 89.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 110.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.22, + "grad_norm": 2.311066363535201, + "learning_rate": 4.784434240636271e-07, + "logits/policy_chosen": -1.7351186275482178, + "logits/policy_rejected": -1.8542585372924805, + "logits/reference_chosen": -3.0118846893310547, + "logits/reference_rejected": -2.1115365028381348, + "logps/policy_chosen": -82.1336898803711, + "logps/policy_rejected": -439.58245849609375, + "logps/reference_chosen": -232.67691040039062, + "logps/reference_rejected": -256.78057861328125, + "loss": 0.1728, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.50543212890625, + "rewards/margins": 3.3334507942199707, + "rewards/rejected": -1.8280186653137207, + "step": 274, + "tokens/chosen_response_len_mean": 609.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 465.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.22, + "grad_norm": 2.9956815155001992, + "learning_rate": 4.78157405917262e-07, + "logits/policy_chosen": -1.8693939447402954, + "logits/policy_rejected": -1.9391303062438965, + "logits/reference_chosen": -2.969177484512329, + "logits/reference_rejected": -2.4460105895996094, + "logps/policy_chosen": -86.36649322509766, + "logps/policy_rejected": -566.4467163085938, + "logps/reference_chosen": -261.6522216796875, + "logps/reference_rejected": -263.3449401855469, + "loss": 0.2451, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.7528573274612427, + "rewards/margins": 4.783874988555908, + "rewards/rejected": -3.031017780303955, + "step": 275, + "tokens/chosen_response_len_mean": 339.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 251.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.22, + "grad_norm": 3.2803570213469144, + "learning_rate": 4.778695894223952e-07, + "logits/policy_chosen": -2.3628909587860107, + "logits/policy_rejected": -2.3530797958374023, + "logits/reference_chosen": -2.9668712615966797, + "logits/reference_rejected": -2.966824531555176, + "logps/policy_chosen": -54.248233795166016, + "logps/policy_rejected": -327.43170166015625, + "logps/reference_chosen": -146.84291076660156, + "logps/reference_rejected": -196.168212890625, + "loss": 0.3313, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.9259467124938965, + "rewards/margins": 2.238581418991089, + "rewards/rejected": -1.3126345872879028, + "step": 276, + "tokens/chosen_response_len_mean": 289.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 287.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.22, + "grad_norm": 3.3737296079999517, + "learning_rate": 4.775799768476108e-07, + "logits/policy_chosen": -1.623067855834961, + "logits/policy_rejected": -1.66438627243042, + "logits/reference_chosen": -2.50834321975708, + "logits/reference_rejected": -2.3877673149108887, + "logps/policy_chosen": -32.76015853881836, + "logps/policy_rejected": -140.15492248535156, + "logps/reference_chosen": -57.18016052246094, + "logps/reference_rejected": -86.19461822509766, + "loss": 0.2575, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.24420003592967987, + "rewards/margins": 0.7838031053543091, + "rewards/rejected": -0.5396031141281128, + "step": 277, + "tokens/chosen_response_len_mean": 176.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 170.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.22, + "grad_norm": 9.199685663575346, + "learning_rate": 4.772885704756505e-07, + "logits/policy_chosen": -1.915112018585205, + "logits/policy_rejected": -2.189150333404541, + "logits/reference_chosen": -3.0652871131896973, + "logits/reference_rejected": -2.8179969787597656, + "logps/policy_chosen": -88.47130584716797, + "logps/policy_rejected": -441.1356506347656, + "logps/reference_chosen": -312.25732421875, + "logps/reference_rejected": -313.57989501953125, + "loss": 0.3432, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.2378599643707275, + "rewards/margins": 3.5134174823760986, + "rewards/rejected": -1.2755577564239502, + "step": 278, + "tokens/chosen_response_len_mean": 416.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 258.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.22, + "grad_norm": 2.568407460041794, + "learning_rate": 4.769953726033942e-07, + "logits/policy_chosen": -2.126152515411377, + "logits/policy_rejected": -2.1976869106292725, + "logits/reference_chosen": -2.918933391571045, + "logits/reference_rejected": -2.9181439876556396, + "logps/policy_chosen": -55.46541213989258, + "logps/policy_rejected": -393.147705078125, + "logps/reference_chosen": -175.48330688476562, + "logps/reference_rejected": -261.3435363769531, + "loss": 0.1778, + "rewards/accuracies": 0.75, + "rewards/chosen": 1.200179100036621, + "rewards/margins": 2.518220901489258, + "rewards/rejected": -1.3180416822433472, + "step": 279, + "tokens/chosen_response_len_mean": 215.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 210.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.23, + "grad_norm": 3.1302891031635984, + "learning_rate": 4.767003855418429e-07, + "logits/policy_chosen": -2.0603606700897217, + "logits/policy_rejected": -2.1878933906555176, + "logits/reference_chosen": -3.049166679382324, + "logits/reference_rejected": -2.4211201667785645, + "logps/policy_chosen": -70.95120239257812, + "logps/policy_rejected": -435.4041442871094, + "logps/reference_chosen": -203.48745727539062, + "logps/reference_rejected": -209.47271728515625, + "loss": 0.1805, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.3253624439239502, + "rewards/margins": 3.5846762657165527, + "rewards/rejected": -2.2593138217926025, + "step": 280, + "tokens/chosen_response_len_mean": 273.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 158.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.23, + "grad_norm": 3.0759421417192003, + "learning_rate": 4.764036116160997e-07, + "logits/policy_chosen": -1.9647924900054932, + "logits/policy_rejected": -2.0003933906555176, + "logits/reference_chosen": -3.1076905727386475, + "logits/reference_rejected": -2.8670225143432617, + "logps/policy_chosen": -52.48387145996094, + "logps/policy_rejected": -527.5159912109375, + "logps/reference_chosen": -158.17994689941406, + "logps/reference_rejected": -246.14735412597656, + "loss": 0.1958, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.0569607019424438, + "rewards/margins": 3.8706471920013428, + "rewards/rejected": -2.8136863708496094, + "step": 281, + "tokens/chosen_response_len_mean": 214.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 191.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.23, + "grad_norm": 2.7361755546386277, + "learning_rate": 4.761050531653523e-07, + "logits/policy_chosen": -1.5984692573547363, + "logits/policy_rejected": -1.8361363410949707, + "logits/reference_chosen": -2.7592179775238037, + "logits/reference_rejected": -2.4084267616271973, + "logps/policy_chosen": -80.7918930053711, + "logps/policy_rejected": -345.93499755859375, + "logps/reference_chosen": -228.01821899414062, + "logps/reference_rejected": -187.31529235839844, + "loss": 0.1742, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.4722633361816406, + "rewards/margins": 3.058460235595703, + "rewards/rejected": -1.586196780204773, + "step": 282, + "tokens/chosen_response_len_mean": 634.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 348.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.23, + "grad_norm": 3.522563546814339, + "learning_rate": 4.758047125428537e-07, + "logits/policy_chosen": -1.7348957061767578, + "logits/policy_rejected": -2.081566333770752, + "logits/reference_chosen": -2.989297389984131, + "logits/reference_rejected": -2.2824132442474365, + "logps/policy_chosen": -58.18089294433594, + "logps/policy_rejected": -389.5596923828125, + "logps/reference_chosen": -171.86676025390625, + "logps/reference_rejected": -148.6519775390625, + "loss": 0.2789, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.1368587017059326, + "rewards/margins": 3.54593563079834, + "rewards/rejected": -2.409076690673828, + "step": 283, + "tokens/chosen_response_len_mean": 379.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 131.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.23, + "grad_norm": 3.4857525347681495, + "learning_rate": 4.7550259211590436e-07, + "logits/policy_chosen": -2.155935764312744, + "logits/policy_rejected": -2.212632894515991, + "logits/reference_chosen": -3.005995750427246, + "logits/reference_rejected": -2.9473025798797607, + "logps/policy_chosen": -46.34202575683594, + "logps/policy_rejected": -374.5079345703125, + "logps/reference_chosen": -132.93014526367188, + "logps/reference_rejected": -190.07769775390625, + "loss": 0.2694, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.8658812642097473, + "rewards/margins": 2.710183620452881, + "rewards/rejected": -1.8443021774291992, + "step": 284, + "tokens/chosen_response_len_mean": 127.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 132.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.23, + "grad_norm": 2.146664751577214, + "learning_rate": 4.751986942658332e-07, + "logits/policy_chosen": -1.8320860862731934, + "logits/policy_rejected": -1.9627416133880615, + "logits/reference_chosen": -3.0087828636169434, + "logits/reference_rejected": -2.798800230026245, + "logps/policy_chosen": -99.31060028076172, + "logps/policy_rejected": -590.6198120117188, + "logps/reference_chosen": -292.62762451171875, + "logps/reference_rejected": -301.8166198730469, + "loss": 0.1743, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.9331700801849365, + "rewards/margins": 4.821201801300049, + "rewards/rejected": -2.8880317211151123, + "step": 285, + "tokens/chosen_response_len_mean": 672.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 380.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.23, + "grad_norm": 4.0068979390995665, + "learning_rate": 4.748930213879787e-07, + "logits/policy_chosen": -1.7841126918792725, + "logits/policy_rejected": -1.616112232208252, + "logits/reference_chosen": -2.8974661827087402, + "logits/reference_rejected": -2.3146543502807617, + "logps/policy_chosen": -83.1224365234375, + "logps/policy_rejected": -509.5625305175781, + "logps/reference_chosen": -248.04489135742188, + "logps/reference_rejected": -299.60675048828125, + "loss": 0.2229, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.6492245197296143, + "rewards/margins": 3.7487823963165283, + "rewards/rejected": -2.099557638168335, + "step": 286, + "tokens/chosen_response_len_mean": 312.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 268.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.23, + "grad_norm": 3.6884808286561905, + "learning_rate": 4.745855758916704e-07, + "logits/policy_chosen": -1.9492144584655762, + "logits/policy_rejected": -2.23884916305542, + "logits/reference_chosen": -2.644348382949829, + "logits/reference_rejected": -2.677292823791504, + "logps/policy_chosen": -167.10963439941406, + "logps/policy_rejected": -408.90594482421875, + "logps/reference_chosen": -245.54014587402344, + "logps/reference_rejected": -228.08700561523438, + "loss": 0.3101, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.784305214881897, + "rewards/margins": 2.592494487762451, + "rewards/rejected": -1.8081893920898438, + "step": 287, + "tokens/chosen_response_len_mean": 660.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 374.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.23, + "grad_norm": 2.9811598150339806, + "learning_rate": 4.7427636020020964e-07, + "logits/policy_chosen": -1.6351268291473389, + "logits/policy_rejected": -1.8411903381347656, + "logits/reference_chosen": -3.084428310394287, + "logits/reference_rejected": -2.840968132019043, + "logps/policy_chosen": -109.2936019897461, + "logps/policy_rejected": -708.4165649414062, + "logps/reference_chosen": -367.9963073730469, + "logps/reference_rejected": -379.46942138671875, + "loss": 0.1152, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.587027072906494, + "rewards/margins": 5.876498222351074, + "rewards/rejected": -3.289471387863159, + "step": 288, + "tokens/chosen_response_len_mean": 711.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 495.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.23, + "grad_norm": 4.220920032892386, + "learning_rate": 4.7396537675085046e-07, + "logits/policy_chosen": -1.8225438594818115, + "logits/policy_rejected": -1.9494044780731201, + "logits/reference_chosen": -2.518173933029175, + "logits/reference_rejected": -2.711627721786499, + "logps/policy_chosen": -47.87618637084961, + "logps/policy_rejected": -247.03065490722656, + "logps/reference_chosen": -69.38465881347656, + "logps/reference_rejected": -130.92787170410156, + "loss": 0.2582, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.21508470177650452, + "rewards/margins": 1.3761123418807983, + "rewards/rejected": -1.1610276699066162, + "step": 289, + "tokens/chosen_response_len_mean": 285.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 109.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.23, + "grad_norm": 3.2644411706471614, + "learning_rate": 4.7365262799478065e-07, + "logits/policy_chosen": -1.631312370300293, + "logits/policy_rejected": -1.8853410482406616, + "logits/reference_chosen": -2.863694906234741, + "logits/reference_rejected": -2.2361578941345215, + "logps/policy_chosen": -87.65226745605469, + "logps/policy_rejected": -368.45916748046875, + "logps/reference_chosen": -175.37078857421875, + "logps/reference_rejected": -180.3101043701172, + "loss": 0.3482, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.8771853446960449, + "rewards/margins": 2.758676052093506, + "rewards/rejected": -1.881490707397461, + "step": 290, + "tokens/chosen_response_len_mean": 238.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 142.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.23, + "grad_norm": 4.164740816732374, + "learning_rate": 4.733381163971021e-07, + "logits/policy_chosen": -2.0723226070404053, + "logits/policy_rejected": -2.1896250247955322, + "logits/reference_chosen": -2.9462733268737793, + "logits/reference_rejected": -2.904141902923584, + "logps/policy_chosen": -82.70597839355469, + "logps/policy_rejected": -706.9203491210938, + "logps/reference_chosen": -254.06658935546875, + "logps/reference_rejected": -371.26043701171875, + "loss": 0.2392, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.7136059999465942, + "rewards/margins": 5.070204734802246, + "rewards/rejected": -3.3565988540649414, + "step": 291, + "tokens/chosen_response_len_mean": 440.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 360.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.23, + "grad_norm": 2.591496235271577, + "learning_rate": 4.7302184443681135e-07, + "logits/policy_chosen": -1.7894690036773682, + "logits/policy_rejected": -1.8064680099487305, + "logits/reference_chosen": -2.837014675140381, + "logits/reference_rejected": -2.7575623989105225, + "logps/policy_chosen": -70.64473724365234, + "logps/policy_rejected": -504.9518737792969, + "logps/reference_chosen": -204.25482177734375, + "logps/reference_rejected": -285.4273376464844, + "loss": 0.1799, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.3361008167266846, + "rewards/margins": 3.531346321105957, + "rewards/rejected": -2.1952455043792725, + "step": 292, + "tokens/chosen_response_len_mean": 480.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 399.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.24, + "grad_norm": 3.3860324397417396, + "learning_rate": 4.7270381460678064e-07, + "logits/policy_chosen": -1.842910885810852, + "logits/policy_rejected": -1.6502610445022583, + "logits/reference_chosen": -2.704162120819092, + "logits/reference_rejected": -2.680164098739624, + "logps/policy_chosen": -49.48709487915039, + "logps/policy_rejected": -214.38595581054688, + "logps/reference_chosen": -82.64054107666016, + "logps/reference_rejected": -121.0648193359375, + "loss": 0.2579, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3315344452857971, + "rewards/margins": 1.264745831489563, + "rewards/rejected": -0.9332114458084106, + "step": 293, + "tokens/chosen_response_len_mean": 103.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 131.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.24, + "grad_norm": 3.188225158781283, + "learning_rate": 4.7238402941373743e-07, + "logits/policy_chosen": -1.6677745580673218, + "logits/policy_rejected": -1.6923861503601074, + "logits/reference_chosen": -2.790350914001465, + "logits/reference_rejected": -2.7842464447021484, + "logps/policy_chosen": -69.78219604492188, + "logps/policy_rejected": -374.1966552734375, + "logps/reference_chosen": -223.26382446289062, + "logps/reference_rejected": -258.4195861816406, + "loss": 0.2856, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.5348161458969116, + "rewards/margins": 2.692586898803711, + "rewards/rejected": -1.1577708721160889, + "step": 294, + "tokens/chosen_response_len_mean": 361.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 321.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.24, + "grad_norm": 2.8300164573379623, + "learning_rate": 4.7206249137824525e-07, + "logits/policy_chosen": -1.2584807872772217, + "logits/policy_rejected": -1.1313340663909912, + "logits/reference_chosen": -2.185845136642456, + "logits/reference_rejected": -1.8424296379089355, + "logps/policy_chosen": -100.97215270996094, + "logps/policy_rejected": -257.4559631347656, + "logps/reference_chosen": -234.6266326904297, + "logps/reference_rejected": -179.3731689453125, + "loss": 0.3429, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.3365448713302612, + "rewards/margins": 2.117372512817383, + "rewards/rejected": -0.7808278203010559, + "step": 295, + "tokens/chosen_response_len_mean": 382.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 457.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.24, + "grad_norm": 3.830130538309461, + "learning_rate": 4.717392030346835e-07, + "logits/policy_chosen": -2.0780117511749268, + "logits/policy_rejected": -2.133204698562622, + "logits/reference_chosen": -2.9950385093688965, + "logits/reference_rejected": -2.916698932647705, + "logps/policy_chosen": -83.94184875488281, + "logps/policy_rejected": -504.6825866699219, + "logps/reference_chosen": -269.20159912109375, + "logps/reference_rejected": -335.22650146484375, + "loss": 0.2967, + "rewards/accuracies": 0.75, + "rewards/chosen": 1.8525973558425903, + "rewards/margins": 3.5471582412719727, + "rewards/rejected": -1.6945607662200928, + "step": 296, + "tokens/chosen_response_len_mean": 454.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 379.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.24, + "grad_norm": 6.893378090531771, + "learning_rate": 4.714141669312277e-07, + "logits/policy_chosen": -1.9810442924499512, + "logits/policy_rejected": -2.0455687046051025, + "logits/reference_chosen": -2.7563416957855225, + "logits/reference_rejected": -2.617710590362549, + "logps/policy_chosen": -73.21089935302734, + "logps/policy_rejected": -340.0194091796875, + "logps/reference_chosen": -221.56581115722656, + "logps/reference_rejected": -196.06849670410156, + "loss": 0.3247, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.4835491180419922, + "rewards/margins": 2.923058032989502, + "rewards/rejected": -1.4395091533660889, + "step": 297, + "tokens/chosen_response_len_mean": 321.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 218.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.24, + "grad_norm": 2.56574676261905, + "learning_rate": 4.710873856298293e-07, + "logits/policy_chosen": -1.895401954650879, + "logits/policy_rejected": -1.9954261779785156, + "logits/reference_chosen": -2.9610021114349365, + "logits/reference_rejected": -2.7275171279907227, + "logps/policy_chosen": -93.83104705810547, + "logps/policy_rejected": -574.4371948242188, + "logps/reference_chosen": -263.5850524902344, + "logps/reference_rejected": -328.8235168457031, + "loss": 0.1995, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.6975396871566772, + "rewards/margins": 4.1536760330200195, + "rewards/rejected": -2.456136465072632, + "step": 298, + "tokens/chosen_response_len_mean": 543.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 433.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.24, + "grad_norm": 2.62583819131534, + "learning_rate": 4.7075886170619536e-07, + "logits/policy_chosen": -1.3762860298156738, + "logits/policy_rejected": -1.4836927652359009, + "logits/reference_chosen": -2.8365345001220703, + "logits/reference_rejected": -2.107300281524658, + "logps/policy_chosen": -109.9390869140625, + "logps/policy_rejected": -396.7261657714844, + "logps/reference_chosen": -285.54638671875, + "logps/reference_rejected": -202.1131591796875, + "loss": 0.2451, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.7560728788375854, + "rewards/margins": 3.7022030353546143, + "rewards/rejected": -1.9461302757263184, + "step": 299, + "tokens/chosen_response_len_mean": 378.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 207.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.24, + "grad_norm": 2.9338445441961096, + "learning_rate": 4.704285977497687e-07, + "logits/policy_chosen": -1.83700692653656, + "logits/policy_rejected": -2.106710433959961, + "logits/reference_chosen": -2.9238107204437256, + "logits/reference_rejected": -2.5823757648468018, + "logps/policy_chosen": -82.13780975341797, + "logps/policy_rejected": -493.58087158203125, + "logps/reference_chosen": -289.5302429199219, + "logps/reference_rejected": -208.1035919189453, + "loss": 0.113, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.0739243030548096, + "rewards/margins": 4.928697109222412, + "rewards/rejected": -2.8547730445861816, + "step": 300, + "tokens/chosen_response_len_mean": 445.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 201.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.24, + "grad_norm": 2.290878044608878, + "learning_rate": 4.700965963637067e-07, + "logits/policy_chosen": -1.2083768844604492, + "logits/policy_rejected": -1.3407108783721924, + "logits/reference_chosen": -2.4264414310455322, + "logits/reference_rejected": -2.0838944911956787, + "logps/policy_chosen": -87.56996154785156, + "logps/policy_rejected": -337.8468933105469, + "logps/reference_chosen": -196.71490478515625, + "logps/reference_rejected": -176.93084716796875, + "loss": 0.1417, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.0914493799209595, + "rewards/margins": 2.7006096839904785, + "rewards/rejected": -1.609160304069519, + "step": 301, + "tokens/chosen_response_len_mean": 252.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 133.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.24, + "grad_norm": 3.664882673854998, + "learning_rate": 4.6976286016486156e-07, + "logits/policy_chosen": -1.6306910514831543, + "logits/policy_rejected": -1.7238553762435913, + "logits/reference_chosen": -3.024641513824463, + "logits/reference_rejected": -2.9463772773742676, + "logps/policy_chosen": -103.36473846435547, + "logps/policy_rejected": -482.4529113769531, + "logps/reference_chosen": -300.7445068359375, + "logps/reference_rejected": -272.9915466308594, + "loss": 0.2465, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.9737976789474487, + "rewards/margins": 4.068410873413086, + "rewards/rejected": -2.0946133136749268, + "step": 302, + "tokens/chosen_response_len_mean": 537.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 378.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.24, + "grad_norm": 4.07962885346587, + "learning_rate": 4.694273917837593e-07, + "logits/policy_chosen": -1.718010663986206, + "logits/policy_rejected": -1.1793601512908936, + "logits/reference_chosen": -2.1970267295837402, + "logits/reference_rejected": -2.242309093475342, + "logps/policy_chosen": -89.3014144897461, + "logps/policy_rejected": -925.4991455078125, + "logps/reference_chosen": -293.8038330078125, + "logps/reference_rejected": -482.5627136230469, + "loss": 0.2865, + "rewards/accuracies": 0.75, + "rewards/chosen": 2.0450241565704346, + "rewards/margins": 6.47438907623291, + "rewards/rejected": -4.429364204406738, + "step": 303, + "tokens/chosen_response_len_mean": 358.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 365.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.24, + "grad_norm": 2.405932913361, + "learning_rate": 4.6909019386457927e-07, + "logits/policy_chosen": -1.6491061449050903, + "logits/policy_rejected": -1.6597315073013306, + "logits/reference_chosen": -2.341599464416504, + "logits/reference_rejected": -2.266160488128662, + "logps/policy_chosen": -68.28823852539062, + "logps/policy_rejected": -270.70416259765625, + "logps/reference_chosen": -157.44268798828125, + "logps/reference_rejected": -147.8431396484375, + "loss": 0.2139, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.8915444016456604, + "rewards/margins": 2.120154857635498, + "rewards/rejected": -1.2286103963851929, + "step": 304, + "tokens/chosen_response_len_mean": 237.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 197.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.25, + "grad_norm": 3.572341442607126, + "learning_rate": 4.687512690651327e-07, + "logits/policy_chosen": -2.2066917419433594, + "logits/policy_rejected": -2.2525765895843506, + "logits/reference_chosen": -2.9350485801696777, + "logits/reference_rejected": -2.8024892807006836, + "logps/policy_chosen": -59.40142822265625, + "logps/policy_rejected": -203.68032836914062, + "logps/reference_chosen": -145.8667449951172, + "logps/reference_rejected": -85.89591979980469, + "loss": 0.209, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.8646531701087952, + "rewards/margins": 2.042497158050537, + "rewards/rejected": -1.1778440475463867, + "step": 305, + "tokens/chosen_response_len_mean": 184.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 98.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.25, + "grad_norm": 3.124612250113926, + "learning_rate": 4.6841062005684255e-07, + "logits/policy_chosen": -1.8113189935684204, + "logits/policy_rejected": -1.8169225454330444, + "logits/reference_chosen": -2.8508493900299072, + "logits/reference_rejected": -2.9348199367523193, + "logps/policy_chosen": -35.2087516784668, + "logps/policy_rejected": -334.8651123046875, + "logps/reference_chosen": -112.86093139648438, + "logps/reference_rejected": -148.45632934570312, + "loss": 0.3251, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.7765218019485474, + "rewards/margins": 2.6406097412109375, + "rewards/rejected": -1.8640880584716797, + "step": 306, + "tokens/chosen_response_len_mean": 320.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 285.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.25, + "grad_norm": 3.32496586258652, + "learning_rate": 4.68068249524722e-07, + "logits/policy_chosen": -2.3824849128723145, + "logits/policy_rejected": -1.6881012916564941, + "logits/reference_chosen": -3.0569143295288086, + "logits/reference_rejected": -2.4748129844665527, + "logps/policy_chosen": -61.716796875, + "logps/policy_rejected": -166.7222137451172, + "logps/reference_chosen": -222.57785034179688, + "logps/reference_rejected": -93.1553955078125, + "loss": 0.2575, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.6086106300354004, + "rewards/margins": 2.344278573989868, + "rewards/rejected": -0.7356680631637573, + "step": 307, + "tokens/chosen_response_len_mean": 291.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 310.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.25, + "grad_norm": 3.3874944482736735, + "learning_rate": 4.6772416016735323e-07, + "logits/policy_chosen": -1.7318488359451294, + "logits/policy_rejected": -2.1800973415374756, + "logits/reference_chosen": -2.737339973449707, + "logits/reference_rejected": -2.64113712310791, + "logps/policy_chosen": -109.86978149414062, + "logps/policy_rejected": -449.2695007324219, + "logps/reference_chosen": -305.08203125, + "logps/reference_rejected": -229.10501098632812, + "loss": 0.1993, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.9521225690841675, + "rewards/margins": 4.1537675857543945, + "rewards/rejected": -2.2016448974609375, + "step": 308, + "tokens/chosen_response_len_mean": 614.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 206.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.25, + "grad_norm": 3.3558308202865, + "learning_rate": 4.673783546968665e-07, + "logits/policy_chosen": -2.2060866355895996, + "logits/policy_rejected": -2.37375545501709, + "logits/reference_chosen": -3.0307087898254395, + "logits/reference_rejected": -2.645644426345825, + "logps/policy_chosen": -47.22865676879883, + "logps/policy_rejected": -480.0763244628906, + "logps/reference_chosen": -145.3326873779297, + "logps/reference_rejected": -169.44834899902344, + "loss": 0.19, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.981040358543396, + "rewards/margins": 4.087320327758789, + "rewards/rejected": -3.1062798500061035, + "step": 309, + "tokens/chosen_response_len_mean": 243.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 131.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.25, + "grad_norm": 4.153117211006617, + "learning_rate": 4.6703083583891835e-07, + "logits/policy_chosen": -2.103739023208618, + "logits/policy_rejected": -2.222684383392334, + "logits/reference_chosen": -2.9488046169281006, + "logits/reference_rejected": -2.909597158432007, + "logps/policy_chosen": -72.56431579589844, + "logps/policy_rejected": -597.1136474609375, + "logps/reference_chosen": -229.9650115966797, + "logps/reference_rejected": -291.440673828125, + "loss": 0.2705, + "rewards/accuracies": 0.75, + "rewards/chosen": 1.5740069150924683, + "rewards/margins": 4.630736827850342, + "rewards/rejected": -3.056729793548584, + "step": 310, + "tokens/chosen_response_len_mean": 416.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 279.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.25, + "grad_norm": 6.474334517638787, + "learning_rate": 4.666816063326705e-07, + "logits/policy_chosen": -1.8240094184875488, + "logits/policy_rejected": -1.2601674795150757, + "logits/reference_chosen": -2.8085074424743652, + "logits/reference_rejected": -1.7981979846954346, + "logps/policy_chosen": -75.03584289550781, + "logps/policy_rejected": -454.2119140625, + "logps/reference_chosen": -269.4425964355469, + "logps/reference_rejected": -277.75634765625, + "loss": 0.2551, + "rewards/accuracies": 0.75, + "rewards/chosen": 1.9440672397613525, + "rewards/margins": 3.708622932434082, + "rewards/rejected": -1.7645556926727295, + "step": 311, + "tokens/chosen_response_len_mean": 437.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 564.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.25, + "grad_norm": 5.519032924912619, + "learning_rate": 4.66330668930768e-07, + "logits/policy_chosen": -1.546844244003296, + "logits/policy_rejected": -1.0349215269088745, + "logits/reference_chosen": -2.6315059661865234, + "logits/reference_rejected": -2.3254542350769043, + "logps/policy_chosen": -46.538272857666016, + "logps/policy_rejected": -340.2102355957031, + "logps/reference_chosen": -118.1043701171875, + "logps/reference_rejected": -194.15060424804688, + "loss": 0.2405, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.7156610488891602, + "rewards/margins": 2.1762571334838867, + "rewards/rejected": -1.4605962038040161, + "step": 312, + "tokens/chosen_response_len_mean": 286.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 542.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.25, + "grad_norm": 3.0150528597270467, + "learning_rate": 4.659780263993177e-07, + "logits/policy_chosen": -1.8767701387405396, + "logits/policy_rejected": -1.8039137125015259, + "logits/reference_chosen": -3.0502028465270996, + "logits/reference_rejected": -3.0671067237854004, + "logps/policy_chosen": -54.472354888916016, + "logps/policy_rejected": -321.30377197265625, + "logps/reference_chosen": -124.24363708496094, + "logps/reference_rejected": -180.5016326904297, + "loss": 0.3478, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.697712779045105, + "rewards/margins": 2.105734348297119, + "rewards/rejected": -1.4080214500427246, + "step": 313, + "tokens/chosen_response_len_mean": 180.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 186.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.25, + "grad_norm": 3.338552410487336, + "learning_rate": 4.656236815178662e-07, + "logits/policy_chosen": -1.8194024562835693, + "logits/policy_rejected": -1.8620460033416748, + "logits/reference_chosen": -2.952608585357666, + "logits/reference_rejected": -2.880305528640747, + "logps/policy_chosen": -64.1020736694336, + "logps/policy_rejected": -668.8197021484375, + "logps/reference_chosen": -191.9952392578125, + "logps/reference_rejected": -297.9425964355469, + "loss": 0.305, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.2789316177368164, + "rewards/margins": 4.987702369689941, + "rewards/rejected": -3.708770513534546, + "step": 314, + "tokens/chosen_response_len_mean": 302.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 252.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.25, + "grad_norm": 4.216599658137105, + "learning_rate": 4.652676370793783e-07, + "logits/policy_chosen": -1.9225913286209106, + "logits/policy_rejected": -2.21560001373291, + "logits/reference_chosen": -2.867231845855713, + "logits/reference_rejected": -2.1903316974639893, + "logps/policy_chosen": -87.3575439453125, + "logps/policy_rejected": -352.74395751953125, + "logps/reference_chosen": -222.33282470703125, + "logps/reference_rejected": -139.34376525878906, + "loss": 0.1772, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.3497529029846191, + "rewards/margins": 3.4837546348571777, + "rewards/rejected": -2.1340014934539795, + "step": 315, + "tokens/chosen_response_len_mean": 291.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 90.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.25, + "grad_norm": 1.9381348585009064, + "learning_rate": 4.649098958902148e-07, + "logits/policy_chosen": -1.4626785516738892, + "logits/policy_rejected": -1.6685293912887573, + "logits/reference_chosen": -3.0980098247528076, + "logits/reference_rejected": -2.3443450927734375, + "logps/policy_chosen": -132.43142700195312, + "logps/policy_rejected": -1093.05859375, + "logps/reference_chosen": -439.55596923828125, + "logps/reference_rejected": -496.19171142578125, + "loss": 0.1559, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.0712451934814453, + "rewards/margins": 9.039913177490234, + "rewards/rejected": -5.968668460845947, + "step": 316, + "tokens/chosen_response_len_mean": 614.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 437.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.25, + "grad_norm": 3.3208168275072727, + "learning_rate": 4.6455046077011017e-07, + "logits/policy_chosen": -1.6945469379425049, + "logits/policy_rejected": -1.6586346626281738, + "logits/reference_chosen": -2.797177314758301, + "logits/reference_rejected": -2.7550530433654785, + "logps/policy_chosen": -89.57418823242188, + "logps/policy_rejected": -376.0241394042969, + "logps/reference_chosen": -193.11834716796875, + "logps/reference_rejected": -237.541015625, + "loss": 0.1793, + "rewards/accuracies": 0.75, + "rewards/chosen": 1.035441517829895, + "rewards/margins": 2.4202725887298584, + "rewards/rejected": -1.384831190109253, + "step": 317, + "tokens/chosen_response_len_mean": 268.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 252.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.26, + "grad_norm": 4.834044743941225, + "learning_rate": 4.6418933455215076e-07, + "logits/policy_chosen": -1.7103971242904663, + "logits/policy_rejected": -1.777637004852295, + "logits/reference_chosen": -2.8807411193847656, + "logits/reference_rejected": -2.910388946533203, + "logps/policy_chosen": -43.051544189453125, + "logps/policy_rejected": -451.1328430175781, + "logps/reference_chosen": -98.37159729003906, + "logps/reference_rejected": -200.70199584960938, + "loss": 0.2363, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5532006025314331, + "rewards/margins": 3.057508945465088, + "rewards/rejected": -2.5043087005615234, + "step": 318, + "tokens/chosen_response_len_mean": 247.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 208.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.26, + "grad_norm": 3.0765900633406913, + "learning_rate": 4.63826520082752e-07, + "logits/policy_chosen": -1.9023985862731934, + "logits/policy_rejected": -2.087189197540283, + "logits/reference_chosen": -2.684725522994995, + "logits/reference_rejected": -2.7294716835021973, + "logps/policy_chosen": -29.559532165527344, + "logps/policy_rejected": -101.22508239746094, + "logps/reference_chosen": -34.04695129394531, + "logps/reference_rejected": -49.713741302490234, + "loss": 0.1553, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.044874198734760284, + "rewards/margins": 0.5599876046180725, + "rewards/rejected": -0.5151134133338928, + "step": 319, + "tokens/chosen_response_len_mean": 277.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 294.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.26, + "grad_norm": 3.1173897201990224, + "learning_rate": 4.6346202022163656e-07, + "logits/policy_chosen": -1.382790446281433, + "logits/policy_rejected": -1.732831597328186, + "logits/reference_chosen": -2.682234764099121, + "logits/reference_rejected": -2.60119891166687, + "logps/policy_chosen": -110.54884338378906, + "logps/policy_rejected": -942.2950439453125, + "logps/reference_chosen": -358.08880615234375, + "logps/reference_rejected": -484.7252197265625, + "loss": 0.2402, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.4753994941711426, + "rewards/margins": 7.051098346710205, + "rewards/rejected": -4.575698375701904, + "step": 320, + "tokens/chosen_response_len_mean": 697.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 564.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.26, + "grad_norm": 3.703781134184552, + "learning_rate": 4.6309583784181104e-07, + "logits/policy_chosen": -1.6343090534210205, + "logits/policy_rejected": -1.7306853532791138, + "logits/reference_chosen": -2.6760318279266357, + "logits/reference_rejected": -2.2841994762420654, + "logps/policy_chosen": -63.79367446899414, + "logps/policy_rejected": -231.47938537597656, + "logps/reference_chosen": -115.32276916503906, + "logps/reference_rejected": -110.79583740234375, + "loss": 0.276, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.5152909159660339, + "rewards/margins": 1.7221263647079468, + "rewards/rejected": -1.2068355083465576, + "step": 321, + "tokens/chosen_response_len_mean": 192.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 137.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.26, + "grad_norm": 3.5327692055358666, + "learning_rate": 4.62727975829544e-07, + "logits/policy_chosen": -2.3158786296844482, + "logits/policy_rejected": -2.3356070518493652, + "logits/reference_chosen": -2.933302402496338, + "logits/reference_rejected": -2.826026678085327, + "logps/policy_chosen": -32.73407745361328, + "logps/policy_rejected": -107.17605590820312, + "logps/reference_chosen": -40.871437072753906, + "logps/reference_rejected": -49.330963134765625, + "loss": 0.2464, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.08137363195419312, + "rewards/margins": 0.659824550151825, + "rewards/rejected": -0.5784509181976318, + "step": 322, + "tokens/chosen_response_len_mean": 34.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 32.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.26, + "grad_norm": 4.524743189821704, + "learning_rate": 4.6235843708434305e-07, + "logits/policy_chosen": -1.0855112075805664, + "logits/policy_rejected": -1.2527565956115723, + "logits/reference_chosen": -2.279548168182373, + "logits/reference_rejected": -2.0591039657592773, + "logps/policy_chosen": -65.47724914550781, + "logps/policy_rejected": -398.4112548828125, + "logps/reference_chosen": -202.6960906982422, + "logps/reference_rejected": -213.66448974609375, + "loss": 0.2768, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.3721883296966553, + "rewards/margins": 3.219656467437744, + "rewards/rejected": -1.8474678993225098, + "step": 323, + "tokens/chosen_response_len_mean": 237.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 194.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.26, + "grad_norm": 2.9172728498954816, + "learning_rate": 4.619872245189315e-07, + "logits/policy_chosen": -1.803147554397583, + "logits/policy_rejected": -2.121699333190918, + "logits/reference_chosen": -2.9548394680023193, + "logits/reference_rejected": -2.4536960124969482, + "logps/policy_chosen": -96.12008666992188, + "logps/policy_rejected": -595.5452270507812, + "logps/reference_chosen": -285.5267333984375, + "logps/reference_rejected": -236.10336303710938, + "loss": 0.2238, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.8940664529800415, + "rewards/margins": 5.488485336303711, + "rewards/rejected": -3.594418525695801, + "step": 324, + "tokens/chosen_response_len_mean": 443.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 240.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.26, + "grad_norm": 3.0831020027878946, + "learning_rate": 4.616143410592261e-07, + "logits/policy_chosen": -2.2690231800079346, + "logits/policy_rejected": -2.37300181388855, + "logits/reference_chosen": -3.101170539855957, + "logits/reference_rejected": -2.814063310623169, + "logps/policy_chosen": -75.43067932128906, + "logps/policy_rejected": -1028.3170166015625, + "logps/reference_chosen": -261.9538269042969, + "logps/reference_rejected": -396.73486328125, + "loss": 0.2074, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.8652312755584717, + "rewards/margins": 8.181053161621094, + "rewards/rejected": -6.315821647644043, + "step": 325, + "tokens/chosen_response_len_mean": 413.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 280.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.26, + "grad_norm": 2.9506911486463143, + "learning_rate": 4.6123978964431376e-07, + "logits/policy_chosen": -1.961487054824829, + "logits/policy_rejected": -2.289116144180298, + "logits/reference_chosen": -3.1111416816711426, + "logits/reference_rejected": -2.8734333515167236, + "logps/policy_chosen": -71.76238250732422, + "logps/policy_rejected": -479.92535400390625, + "logps/reference_chosen": -267.5050964355469, + "logps/reference_rejected": -202.55059814453125, + "loss": 0.2505, + "rewards/accuracies": 0.75, + "rewards/chosen": 1.9574271440505981, + "rewards/margins": 4.731174468994141, + "rewards/rejected": -2.773747682571411, + "step": 326, + "tokens/chosen_response_len_mean": 445.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 148.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.26, + "grad_norm": 2.819044425119029, + "learning_rate": 4.60863573226428e-07, + "logits/policy_chosen": -1.2217872142791748, + "logits/policy_rejected": -1.4427027702331543, + "logits/reference_chosen": -2.5383174419403076, + "logits/reference_rejected": -2.6582465171813965, + "logps/policy_chosen": -78.91648864746094, + "logps/policy_rejected": -792.6351318359375, + "logps/reference_chosen": -325.96051025390625, + "logps/reference_rejected": -492.7793884277344, + "loss": 0.1434, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.470440149307251, + "rewards/margins": 5.468997955322266, + "rewards/rejected": -2.9985575675964355, + "step": 327, + "tokens/chosen_response_len_mean": 698.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 698.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.26, + "grad_norm": 2.051994037783105, + "learning_rate": 4.604856947709262e-07, + "logits/policy_chosen": -1.699342966079712, + "logits/policy_rejected": -1.9920241832733154, + "logits/reference_chosen": -3.1137824058532715, + "logits/reference_rejected": -2.245004892349243, + "logps/policy_chosen": -108.17164611816406, + "logps/policy_rejected": -584.7960205078125, + "logps/reference_chosen": -348.4723205566406, + "logps/reference_rejected": -226.91238403320312, + "loss": 0.154, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.4030065536499023, + "rewards/margins": 5.981842517852783, + "rewards/rejected": -3.578835964202881, + "step": 328, + "tokens/chosen_response_len_mean": 482.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 178.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.26, + "grad_norm": 2.853440760910482, + "learning_rate": 4.601061572562659e-07, + "logits/policy_chosen": -1.1898789405822754, + "logits/policy_rejected": -1.331478476524353, + "logits/reference_chosen": -2.611237049102783, + "logits/reference_rejected": -2.666792869567871, + "logps/policy_chosen": -48.299530029296875, + "logps/policy_rejected": -434.9912109375, + "logps/reference_chosen": -154.5133056640625, + "logps/reference_rejected": -240.702392578125, + "loss": 0.2731, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.0621377229690552, + "rewards/margins": 3.00502610206604, + "rewards/rejected": -1.9428882598876953, + "step": 329, + "tokens/chosen_response_len_mean": 265.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 216.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.27, + "grad_norm": 3.1024957208965094, + "learning_rate": 4.5972496367398146e-07, + "logits/policy_chosen": -1.352826714515686, + "logits/policy_rejected": -1.392114520072937, + "logits/reference_chosen": -2.7036824226379395, + "logits/reference_rejected": -2.694230794906616, + "logps/policy_chosen": -61.405372619628906, + "logps/policy_rejected": -453.8320007324219, + "logps/reference_chosen": -191.30259704589844, + "logps/reference_rejected": -260.45123291015625, + "loss": 0.2595, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.298972249031067, + "rewards/margins": 3.2327802181243896, + "rewards/rejected": -1.933807611465454, + "step": 330, + "tokens/chosen_response_len_mean": 460.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 469.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.27, + "grad_norm": 2.523243805150809, + "learning_rate": 4.5934211702866046e-07, + "logits/policy_chosen": -1.6151251792907715, + "logits/policy_rejected": -1.6389597654342651, + "logits/reference_chosen": -2.9203569889068604, + "logits/reference_rejected": -2.905449390411377, + "logps/policy_chosen": -63.08237075805664, + "logps/policy_rejected": -630.6449584960938, + "logps/reference_chosen": -203.67691040039062, + "logps/reference_rejected": -291.34234619140625, + "loss": 0.1624, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.4059453010559082, + "rewards/margins": 4.798971176147461, + "rewards/rejected": -3.3930258750915527, + "step": 331, + "tokens/chosen_response_len_mean": 318.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 279.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.27, + "grad_norm": 3.2573268171576544, + "learning_rate": 4.5895762033791995e-07, + "logits/policy_chosen": -2.4760982990264893, + "logits/policy_rejected": -2.6151270866394043, + "logits/reference_chosen": -3.021172523498535, + "logits/reference_rejected": -2.504538059234619, + "logps/policy_chosen": -40.59739685058594, + "logps/policy_rejected": -89.36610412597656, + "logps/reference_chosen": -70.21237182617188, + "logps/reference_rejected": -43.146018981933594, + "loss": 0.3215, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.29614976048469543, + "rewards/margins": 0.7583505511283875, + "rewards/rejected": -0.4622008204460144, + "step": 332, + "tokens/chosen_response_len_mean": 64.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 32.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.27, + "grad_norm": 3.8626650481777554, + "learning_rate": 4.58571476632383e-07, + "logits/policy_chosen": -1.6957329511642456, + "logits/policy_rejected": -2.019444227218628, + "logits/reference_chosen": -2.8664603233337402, + "logits/reference_rejected": -2.8443779945373535, + "logps/policy_chosen": -86.0948257446289, + "logps/policy_rejected": -600.858154296875, + "logps/reference_chosen": -277.8157958984375, + "logps/reference_rejected": -353.20318603515625, + "loss": 0.2874, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.9172096252441406, + "rewards/margins": 4.393758773803711, + "rewards/rejected": -2.4765493869781494, + "step": 333, + "tokens/chosen_response_len_mean": 540.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 265.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.27, + "grad_norm": 4.40990215478422, + "learning_rate": 4.5818368895565405e-07, + "logits/policy_chosen": -1.576200008392334, + "logits/policy_rejected": -1.6361205577850342, + "logits/reference_chosen": -2.847994089126587, + "logits/reference_rejected": -2.3715221881866455, + "logps/policy_chosen": -62.154117584228516, + "logps/policy_rejected": -340.4329833984375, + "logps/reference_chosen": -182.93911743164062, + "logps/reference_rejected": -150.31068420410156, + "loss": 0.2179, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.2078500986099243, + "rewards/margins": 3.109072685241699, + "rewards/rejected": -1.9012229442596436, + "step": 334, + "tokens/chosen_response_len_mean": 285.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 197.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.27, + "grad_norm": 2.8983851836796353, + "learning_rate": 4.5779426036429587e-07, + "logits/policy_chosen": -2.055795907974243, + "logits/policy_rejected": -1.6440610885620117, + "logits/reference_chosen": -2.7132530212402344, + "logits/reference_rejected": -2.2304413318634033, + "logps/policy_chosen": -93.63249206542969, + "logps/policy_rejected": -186.17974853515625, + "logps/reference_chosen": -258.497314453125, + "logps/reference_rejected": -183.7268524169922, + "loss": 0.2936, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.6486481428146362, + "rewards/margins": 1.6731772422790527, + "rewards/rejected": -0.024528950452804565, + "step": 335, + "tokens/chosen_response_len_mean": 367.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 459.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.27, + "grad_norm": 2.3274853742981976, + "learning_rate": 4.574031939278047e-07, + "logits/policy_chosen": -1.8988397121429443, + "logits/policy_rejected": -1.9968366622924805, + "logits/reference_chosen": -3.2483341693878174, + "logits/reference_rejected": -2.948215961456299, + "logps/policy_chosen": -96.612548828125, + "logps/policy_rejected": -861.7078247070312, + "logps/reference_chosen": -347.1319885253906, + "logps/reference_rejected": -461.7752990722656, + "loss": 0.206, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.505194664001465, + "rewards/margins": 6.504519939422607, + "rewards/rejected": -3.9993255138397217, + "step": 336, + "tokens/chosen_response_len_mean": 526.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 455.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.27, + "grad_norm": 4.109692231324148, + "learning_rate": 4.570104927285864e-07, + "logits/policy_chosen": -1.5520960092544556, + "logits/policy_rejected": -1.7077003717422485, + "logits/reference_chosen": -2.795018196105957, + "logits/reference_rejected": -2.788074016571045, + "logps/policy_chosen": -82.21895599365234, + "logps/policy_rejected": -586.7863159179688, + "logps/reference_chosen": -284.3222351074219, + "logps/reference_rejected": -400.7400817871094, + "loss": 0.1851, + "rewards/accuracies": 0.75, + "rewards/chosen": 2.0210328102111816, + "rewards/margins": 3.8814949989318848, + "rewards/rejected": -1.8604620695114136, + "step": 337, + "tokens/chosen_response_len_mean": 517.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 480.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.27, + "grad_norm": 3.2891497747662277, + "learning_rate": 4.5661615986193243e-07, + "logits/policy_chosen": -2.0379586219787598, + "logits/policy_rejected": -2.2403860092163086, + "logits/reference_chosen": -2.9786267280578613, + "logits/reference_rejected": -2.831549882888794, + "logps/policy_chosen": -86.32127380371094, + "logps/policy_rejected": -617.3648071289062, + "logps/reference_chosen": -241.71507263183594, + "logps/reference_rejected": -268.22021484375, + "loss": 0.15, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.5539380311965942, + "rewards/margins": 5.045384407043457, + "rewards/rejected": -3.4914462566375732, + "step": 338, + "tokens/chosen_response_len_mean": 496.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 221.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.27, + "grad_norm": 2.5176895862702615, + "learning_rate": 4.562201984359946e-07, + "logits/policy_chosen": -1.5903446674346924, + "logits/policy_rejected": -1.8955085277557373, + "logits/reference_chosen": -2.9453320503234863, + "logits/reference_rejected": -2.7459821701049805, + "logps/policy_chosen": -67.94086456298828, + "logps/policy_rejected": -488.8743896484375, + "logps/reference_chosen": -216.46241760253906, + "logps/reference_rejected": -206.1934356689453, + "loss": 0.1801, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.485215425491333, + "rewards/margins": 4.31202507019043, + "rewards/rejected": -2.826809883117676, + "step": 339, + "tokens/chosen_response_len_mean": 474.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 204.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.27, + "grad_norm": 2.394686969217896, + "learning_rate": 4.558226115717616e-07, + "logits/policy_chosen": -1.1812870502471924, + "logits/policy_rejected": -1.297174334526062, + "logits/reference_chosen": -2.829561948776245, + "logits/reference_rejected": -2.5182604789733887, + "logps/policy_chosen": -114.76174926757812, + "logps/policy_rejected": -792.0388793945312, + "logps/reference_chosen": -383.6563415527344, + "logps/reference_rejected": -463.0151672363281, + "loss": 0.1879, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.688946008682251, + "rewards/margins": 5.979182720184326, + "rewards/rejected": -3.290236473083496, + "step": 340, + "tokens/chosen_response_len_mean": 546.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 485.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.27, + "grad_norm": 3.0673469023984423, + "learning_rate": 4.5542340240303367e-07, + "logits/policy_chosen": -1.7937376499176025, + "logits/policy_rejected": -1.815042495727539, + "logits/reference_chosen": -3.0137624740600586, + "logits/reference_rejected": -2.86102294921875, + "logps/policy_chosen": -52.81929016113281, + "logps/policy_rejected": -513.2135620117188, + "logps/reference_chosen": -155.88839721679688, + "logps/reference_rejected": -219.87347412109375, + "loss": 0.271, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.0306909084320068, + "rewards/margins": 3.9640917778015137, + "rewards/rejected": -2.933401107788086, + "step": 341, + "tokens/chosen_response_len_mean": 236.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 219.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.27, + "grad_norm": 1.9367186293576946, + "learning_rate": 4.5502257407639806e-07, + "logits/policy_chosen": -1.552323341369629, + "logits/policy_rejected": -1.8226666450500488, + "logits/reference_chosen": -3.2792892456054688, + "logits/reference_rejected": -2.0340218544006348, + "logps/policy_chosen": -97.57713317871094, + "logps/policy_rejected": -627.7135009765625, + "logps/reference_chosen": -344.95343017578125, + "logps/reference_rejected": -284.43548583984375, + "loss": 0.134, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.4737627506256104, + "rewards/margins": 5.906542778015137, + "rewards/rejected": -3.4327802658081055, + "step": 342, + "tokens/chosen_response_len_mean": 452.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 255.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.28, + "grad_norm": 5.549352474070261, + "learning_rate": 4.5462012975120453e-07, + "logits/policy_chosen": -1.2813889980316162, + "logits/policy_rejected": -1.7645294666290283, + "logits/reference_chosen": -2.8398776054382324, + "logits/reference_rejected": -2.596129894256592, + "logps/policy_chosen": -90.58872985839844, + "logps/policy_rejected": -583.3743896484375, + "logps/reference_chosen": -270.8271789550781, + "logps/reference_rejected": -242.07595825195312, + "loss": 0.1804, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.802384376525879, + "rewards/margins": 5.21536922454834, + "rewards/rejected": -3.412984848022461, + "step": 343, + "tokens/chosen_response_len_mean": 581.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 286.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.28, + "grad_norm": 1.9674810528357884, + "learning_rate": 4.542160725995401e-07, + "logits/policy_chosen": -2.343667984008789, + "logits/policy_rejected": -2.326040029525757, + "logits/reference_chosen": -2.737659454345703, + "logits/reference_rejected": -2.802224636077881, + "logps/policy_chosen": -28.079269409179688, + "logps/policy_rejected": -175.65020751953125, + "logps/reference_chosen": -27.471698760986328, + "logps/reference_rejected": -72.05661010742188, + "loss": 0.2642, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.006075695157051086, + "rewards/margins": 1.0298603773117065, + "rewards/rejected": -1.0359361171722412, + "step": 344, + "tokens/chosen_response_len_mean": 86.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 111.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.28, + "grad_norm": 5.451886317836004, + "learning_rate": 4.538104058062041e-07, + "logits/policy_chosen": -1.6862752437591553, + "logits/policy_rejected": -1.886472225189209, + "logits/reference_chosen": -2.9743881225585938, + "logits/reference_rejected": -2.8067820072174072, + "logps/policy_chosen": -62.13983917236328, + "logps/policy_rejected": -612.0286254882812, + "logps/reference_chosen": -192.025634765625, + "logps/reference_rejected": -268.3218688964844, + "loss": 0.3178, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.2988580465316772, + "rewards/margins": 4.735925674438477, + "rewards/rejected": -3.437067747116089, + "step": 345, + "tokens/chosen_response_len_mean": 359.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 269.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.28, + "grad_norm": 4.435509463597182, + "learning_rate": 4.5340313256868337e-07, + "logits/policy_chosen": -1.5953540802001953, + "logits/policy_rejected": -1.0541170835494995, + "logits/reference_chosen": -2.2541754245758057, + "logits/reference_rejected": -1.9815850257873535, + "logps/policy_chosen": -91.02217864990234, + "logps/policy_rejected": -648.81787109375, + "logps/reference_chosen": -300.9630432128906, + "logps/reference_rejected": -305.43585205078125, + "loss": 0.288, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.0994083881378174, + "rewards/margins": 5.533228874206543, + "rewards/rejected": -3.4338202476501465, + "step": 346, + "tokens/chosen_response_len_mean": 344.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 248.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.28, + "grad_norm": 4.398404543060366, + "learning_rate": 4.529942560971266e-07, + "logits/policy_chosen": -1.4266515970230103, + "logits/policy_rejected": -1.49228835105896, + "logits/reference_chosen": -2.7921342849731445, + "logits/reference_rejected": -2.7268803119659424, + "logps/policy_chosen": -120.46929931640625, + "logps/policy_rejected": -746.52880859375, + "logps/reference_chosen": -357.797607421875, + "logps/reference_rejected": -444.6593322753906, + "loss": 0.2106, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.3732829093933105, + "rewards/margins": 5.3919782638549805, + "rewards/rejected": -3.01869535446167, + "step": 347, + "tokens/chosen_response_len_mean": 441.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 367.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.28, + "grad_norm": 2.688781964809469, + "learning_rate": 4.5258377961431937e-07, + "logits/policy_chosen": -1.0297237634658813, + "logits/policy_rejected": -0.6758372187614441, + "logits/reference_chosen": -2.3045077323913574, + "logits/reference_rejected": -1.6970428228378296, + "logps/policy_chosen": -84.93526458740234, + "logps/policy_rejected": -675.788330078125, + "logps/reference_chosen": -266.78472900390625, + "logps/reference_rejected": -359.9700927734375, + "loss": 0.1233, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.818494439125061, + "rewards/margins": 4.976676940917969, + "rewards/rejected": -3.158182382583618, + "step": 348, + "tokens/chosen_response_len_mean": 318.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 289.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.28, + "grad_norm": 3.214059022448137, + "learning_rate": 4.5217170635565855e-07, + "logits/policy_chosen": -2.1661670207977295, + "logits/policy_rejected": -2.1412088871002197, + "logits/reference_chosen": -2.990678310394287, + "logits/reference_rejected": -2.691145181655884, + "logps/policy_chosen": -27.263290405273438, + "logps/policy_rejected": -180.78501892089844, + "logps/reference_chosen": -51.59008026123047, + "logps/reference_rejected": -96.38774108886719, + "loss": 0.2374, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.24326787889003754, + "rewards/margins": 1.0872406959533691, + "rewards/rejected": -0.8439728617668152, + "step": 349, + "tokens/chosen_response_len_mean": 106.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 89.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.28, + "grad_norm": 3.518670555830533, + "learning_rate": 4.5175803956912694e-07, + "logits/policy_chosen": -1.9499872922897339, + "logits/policy_rejected": -2.1933279037475586, + "logits/reference_chosen": -3.137690305709839, + "logits/reference_rejected": -2.8975977897644043, + "logps/policy_chosen": -66.71675109863281, + "logps/policy_rejected": -577.4403076171875, + "logps/reference_chosen": -220.90902709960938, + "logps/reference_rejected": -255.65286254882812, + "loss": 0.1443, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.5419225692749023, + "rewards/margins": 4.759797096252441, + "rewards/rejected": -3.217874526977539, + "step": 350, + "tokens/chosen_response_len_mean": 485.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 242.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.28, + "grad_norm": 3.699314909735531, + "learning_rate": 4.5134278251526766e-07, + "logits/policy_chosen": -1.4733799695968628, + "logits/policy_rejected": -1.4901491403579712, + "logits/reference_chosen": -2.594074249267578, + "logits/reference_rejected": -2.557436466217041, + "logps/policy_chosen": -65.0804443359375, + "logps/policy_rejected": -769.9380493164062, + "logps/reference_chosen": -200.39883422851562, + "logps/reference_rejected": -388.98785400390625, + "loss": 0.2727, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.3531838655471802, + "rewards/margins": 5.162685394287109, + "rewards/rejected": -3.8095014095306396, + "step": 351, + "tokens/chosen_response_len_mean": 433.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 425.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.28, + "grad_norm": 2.955343714828839, + "learning_rate": 4.509259384671582e-07, + "logits/policy_chosen": -1.5196211338043213, + "logits/policy_rejected": -1.566109299659729, + "logits/reference_chosen": -3.052445888519287, + "logits/reference_rejected": -2.340404510498047, + "logps/policy_chosen": -115.84571838378906, + "logps/policy_rejected": -830.0971069335938, + "logps/reference_chosen": -340.6599426269531, + "logps/reference_rejected": -353.662109375, + "loss": 0.175, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.2481420040130615, + "rewards/margins": 7.012492656707764, + "rewards/rejected": -4.764350414276123, + "step": 352, + "tokens/chosen_response_len_mean": 470.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 305.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.28, + "grad_norm": 3.500797998192056, + "learning_rate": 4.5050751071038514e-07, + "logits/policy_chosen": -2.069139003753662, + "logits/policy_rejected": -2.066579580307007, + "logits/reference_chosen": -3.0677695274353027, + "logits/reference_rejected": -3.032003402709961, + "logps/policy_chosen": -59.17474365234375, + "logps/policy_rejected": -444.245361328125, + "logps/reference_chosen": -161.52598571777344, + "logps/reference_rejected": -251.91793823242188, + "loss": 0.2833, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.0235124826431274, + "rewards/margins": 2.946786642074585, + "rewards/rejected": -1.9232741594314575, + "step": 353, + "tokens/chosen_response_len_mean": 381.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 321.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.28, + "grad_norm": 2.1150271947388437, + "learning_rate": 4.500875025430176e-07, + "logits/policy_chosen": -1.5964065790176392, + "logits/policy_rejected": -1.8535802364349365, + "logits/reference_chosen": -3.0790903568267822, + "logits/reference_rejected": -2.013641595840454, + "logps/policy_chosen": -91.34295654296875, + "logps/policy_rejected": -1013.7203369140625, + "logps/reference_chosen": -323.52752685546875, + "logps/reference_rejected": -508.953369140625, + "loss": 0.082, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.321845769882202, + "rewards/margins": 7.3695149421691895, + "rewards/rejected": -5.047668933868408, + "step": 354, + "tokens/chosen_response_len_mean": 414.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 346.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.29, + "grad_norm": 4.458337777869365, + "learning_rate": 4.496659172755818e-07, + "logits/policy_chosen": -1.3642287254333496, + "logits/policy_rejected": -1.410393476486206, + "logits/reference_chosen": -2.549731492996216, + "logits/reference_rejected": -2.4937071800231934, + "logps/policy_chosen": -111.19947814941406, + "logps/policy_rejected": -509.53564453125, + "logps/reference_chosen": -215.30552673339844, + "logps/reference_rejected": -308.2844543457031, + "loss": 0.3226, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.0410603284835815, + "rewards/margins": 3.053572416305542, + "rewards/rejected": -2.01251220703125, + "step": 355, + "tokens/chosen_response_len_mean": 330.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 318.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.29, + "grad_norm": 4.024586930018457, + "learning_rate": 4.4924275823103466e-07, + "logits/policy_chosen": -1.8358407020568848, + "logits/policy_rejected": -1.9370864629745483, + "logits/reference_chosen": -3.0807526111602783, + "logits/reference_rejected": -2.92172908782959, + "logps/policy_chosen": -74.08454132080078, + "logps/policy_rejected": -655.4462890625, + "logps/reference_chosen": -287.81585693359375, + "logps/reference_rejected": -331.1325378417969, + "loss": 0.1674, + "rewards/accuracies": 0.75, + "rewards/chosen": 2.137312889099121, + "rewards/margins": 5.380450248718262, + "rewards/rejected": -3.2431373596191406, + "step": 356, + "tokens/chosen_response_len_mean": 392.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 230.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.29, + "grad_norm": 2.5198509782294924, + "learning_rate": 4.4881802874473774e-07, + "logits/policy_chosen": -1.2551157474517822, + "logits/policy_rejected": -1.7098793983459473, + "logits/reference_chosen": -2.8631482124328613, + "logits/reference_rejected": -1.495772123336792, + "logps/policy_chosen": -56.709354400634766, + "logps/policy_rejected": -329.5363464355469, + "logps/reference_chosen": -180.67190551757812, + "logps/reference_rejected": -128.47109985351562, + "loss": 0.1685, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.2396254539489746, + "rewards/margins": 3.2502779960632324, + "rewards/rejected": -2.0106523036956787, + "step": 357, + "tokens/chosen_response_len_mean": 321.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 157.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.29, + "grad_norm": 2.7881215108282253, + "learning_rate": 4.4839173216443084e-07, + "logits/policy_chosen": -2.051104784011841, + "logits/policy_rejected": -2.1336636543273926, + "logits/reference_chosen": -2.95450758934021, + "logits/reference_rejected": -2.7968645095825195, + "logps/policy_chosen": -36.93220520019531, + "logps/policy_rejected": -377.7947082519531, + "logps/reference_chosen": -118.31900787353516, + "logps/reference_rejected": -182.75210571289062, + "loss": 0.1298, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.8138680458068848, + "rewards/margins": 2.764294147491455, + "rewards/rejected": -1.9504261016845703, + "step": 358, + "tokens/chosen_response_len_mean": 222.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 194.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.29, + "grad_norm": 2.9202638151581564, + "learning_rate": 4.479638718502056e-07, + "logits/policy_chosen": -2.800821304321289, + "logits/policy_rejected": -2.74906063079834, + "logits/reference_chosen": -2.9278526306152344, + "logits/reference_rejected": -3.074326515197754, + "logps/policy_chosen": -26.134109497070312, + "logps/policy_rejected": -101.59957122802734, + "logps/reference_chosen": -21.818992614746094, + "logps/reference_rejected": -37.89555358886719, + "loss": 0.245, + "rewards/accuracies": 1.0, + "rewards/chosen": -0.04315117001533508, + "rewards/margins": 0.5938889980316162, + "rewards/rejected": -0.6370401382446289, + "step": 359, + "tokens/chosen_response_len_mean": 14.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 28.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.29, + "grad_norm": 2.2402578949629, + "learning_rate": 4.475344511744794e-07, + "logits/policy_chosen": -1.5277884006500244, + "logits/policy_rejected": -1.1120015382766724, + "logits/reference_chosen": -3.0673985481262207, + "logits/reference_rejected": -1.979531168937683, + "logps/policy_chosen": -129.73883056640625, + "logps/policy_rejected": -1009.8583984375, + "logps/reference_chosen": -419.92962646484375, + "logps/reference_rejected": -410.1859130859375, + "loss": 0.131, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.9019076824188232, + "rewards/margins": 8.898632049560547, + "rewards/rejected": -5.996725082397461, + "step": 360, + "tokens/chosen_response_len_mean": 535.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 503.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.29, + "grad_norm": 2.2362340915683405, + "learning_rate": 4.471034735219682e-07, + "logits/policy_chosen": -1.8571796417236328, + "logits/policy_rejected": -2.0071451663970947, + "logits/reference_chosen": -3.0822551250457764, + "logits/reference_rejected": -2.8071236610412598, + "logps/policy_chosen": -113.39871215820312, + "logps/policy_rejected": -765.5164184570312, + "logps/reference_chosen": -314.39288330078125, + "logps/reference_rejected": -295.49212646484375, + "loss": 0.286, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.009941577911377, + "rewards/margins": 6.7101850509643555, + "rewards/rejected": -4.7002434730529785, + "step": 361, + "tokens/chosen_response_len_mean": 370.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 250.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.29, + "grad_norm": 2.9958059876475645, + "learning_rate": 4.4667094228966006e-07, + "logits/policy_chosen": -1.3339217901229858, + "logits/policy_rejected": -1.4387969970703125, + "logits/reference_chosen": -2.5392210483551025, + "logits/reference_rejected": -2.5316193103790283, + "logps/policy_chosen": -65.20072174072266, + "logps/policy_rejected": -141.90574645996094, + "logps/reference_chosen": -153.89601135253906, + "logps/reference_rejected": -161.5721893310547, + "loss": 0.3208, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.8869529962539673, + "rewards/margins": 0.6902884840965271, + "rewards/rejected": 0.19666451215744019, + "step": 362, + "tokens/chosen_response_len_mean": 182.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 179.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.29, + "grad_norm": 2.714261614183403, + "learning_rate": 4.462368608867888e-07, + "logits/policy_chosen": -2.0045056343078613, + "logits/policy_rejected": -2.136458158493042, + "logits/reference_chosen": -2.8513717651367188, + "logits/reference_rejected": -2.6678905487060547, + "logps/policy_chosen": -57.24700164794922, + "logps/policy_rejected": -487.1899719238281, + "logps/reference_chosen": -193.75621032714844, + "logps/reference_rejected": -158.42803955078125, + "loss": 0.2195, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.3650922775268555, + "rewards/margins": 4.652711868286133, + "rewards/rejected": -3.287619113922119, + "step": 363, + "tokens/chosen_response_len_mean": 301.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 134.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.29, + "grad_norm": 3.0348087146187934, + "learning_rate": 4.4580123273480643e-07, + "logits/policy_chosen": -1.2777620553970337, + "logits/policy_rejected": -1.6785014867782593, + "logits/reference_chosen": -2.842172622680664, + "logits/reference_rejected": -2.3908252716064453, + "logps/policy_chosen": -62.06177520751953, + "logps/policy_rejected": -431.913330078125, + "logps/reference_chosen": -187.41107177734375, + "logps/reference_rejected": -209.33456420898438, + "loss": 0.1306, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.2534929513931274, + "rewards/margins": 3.4792802333831787, + "rewards/rejected": -2.225787401199341, + "step": 364, + "tokens/chosen_response_len_mean": 385.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 249.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.29, + "grad_norm": 4.751862010403708, + "learning_rate": 4.453640612673566e-07, + "logits/policy_chosen": -1.219412088394165, + "logits/policy_rejected": -1.36615788936615, + "logits/reference_chosen": -2.3538596630096436, + "logits/reference_rejected": -2.0649256706237793, + "logps/policy_chosen": -101.12548828125, + "logps/policy_rejected": -373.8370361328125, + "logps/reference_chosen": -326.5352783203125, + "logps/reference_rejected": -239.3107147216797, + "loss": 0.1992, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.2540977001190186, + "rewards/margins": 3.599360942840576, + "rewards/rejected": -1.3452633619308472, + "step": 365, + "tokens/chosen_response_len_mean": 465.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 235.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.29, + "grad_norm": 3.4845170475354874, + "learning_rate": 4.4492534993024744e-07, + "logits/policy_chosen": -1.7572671175003052, + "logits/policy_rejected": -1.7631720304489136, + "logits/reference_chosen": -2.83882737159729, + "logits/reference_rejected": -2.7936973571777344, + "logps/policy_chosen": -79.20316314697266, + "logps/policy_rejected": -479.2203063964844, + "logps/reference_chosen": -240.270263671875, + "logps/reference_rejected": -315.88555908203125, + "loss": 0.2738, + "rewards/accuracies": 0.75, + "rewards/chosen": 1.610671043395996, + "rewards/margins": 3.2440185546875, + "rewards/rejected": -1.633347511291504, + "step": 366, + "tokens/chosen_response_len_mean": 446.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 421.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.3, + "grad_norm": 1.7048313796935655, + "learning_rate": 4.4448510218142466e-07, + "logits/policy_chosen": -1.9213192462921143, + "logits/policy_rejected": -1.650076150894165, + "logits/reference_chosen": -3.0253703594207764, + "logits/reference_rejected": -2.3654181957244873, + "logps/policy_chosen": -61.119239807128906, + "logps/policy_rejected": -502.1123962402344, + "logps/reference_chosen": -204.41744995117188, + "logps/reference_rejected": -221.98794555664062, + "loss": 0.1403, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.4329822063446045, + "rewards/margins": 4.234225749969482, + "rewards/rejected": -2.801244020462036, + "step": 367, + "tokens/chosen_response_len_mean": 211.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 156.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.3, + "grad_norm": 4.368110245343919, + "learning_rate": 4.440433214909436e-07, + "logits/policy_chosen": -1.6689025163650513, + "logits/policy_rejected": -0.940861701965332, + "logits/reference_chosen": -2.9102039337158203, + "logits/reference_rejected": -2.0719833374023438, + "logps/policy_chosen": -48.4640007019043, + "logps/policy_rejected": -306.5902099609375, + "logps/reference_chosen": -142.2967529296875, + "logps/reference_rejected": -147.8551025390625, + "loss": 0.2671, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.938327431678772, + "rewards/margins": 2.5256781578063965, + "rewards/rejected": -1.587350845336914, + "step": 368, + "tokens/chosen_response_len_mean": 255.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 382.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.3, + "grad_norm": 4.421616617478599, + "learning_rate": 4.4360001134094274e-07, + "logits/policy_chosen": -1.7381703853607178, + "logits/policy_rejected": -1.873031497001648, + "logits/reference_chosen": -2.868490695953369, + "logits/reference_rejected": -2.679164171218872, + "logps/policy_chosen": -37.81551742553711, + "logps/policy_rejected": -327.39471435546875, + "logps/reference_chosen": -100.18566131591797, + "logps/reference_rejected": -154.6254425048828, + "loss": 0.2761, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.6237015128135681, + "rewards/margins": 2.3513944149017334, + "rewards/rejected": -1.7276926040649414, + "step": 369, + "tokens/chosen_response_len_mean": 260.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 207.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.3, + "grad_norm": 3.55457857405225, + "learning_rate": 4.431551752256155e-07, + "logits/policy_chosen": -1.4202126264572144, + "logits/policy_rejected": -1.435547113418579, + "logits/reference_chosen": -2.81121563911438, + "logits/reference_rejected": -2.563999652862549, + "logps/policy_chosen": -81.54356384277344, + "logps/policy_rejected": -511.54022216796875, + "logps/reference_chosen": -295.8115539550781, + "logps/reference_rejected": -280.714111328125, + "loss": 0.0897, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.1426796913146973, + "rewards/margins": 4.4509406089782715, + "rewards/rejected": -2.308260917663574, + "step": 370, + "tokens/chosen_response_len_mean": 406.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 281.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.3, + "grad_norm": 3.6157257883700984, + "learning_rate": 4.4270881665118344e-07, + "logits/policy_chosen": -1.7662705183029175, + "logits/policy_rejected": -1.9252934455871582, + "logits/reference_chosen": -2.995553493499756, + "logits/reference_rejected": -2.469679594039917, + "logps/policy_chosen": -77.08085632324219, + "logps/policy_rejected": -437.26995849609375, + "logps/reference_chosen": -275.45654296875, + "logps/reference_rejected": -245.6436767578125, + "loss": 0.2254, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.9837571382522583, + "rewards/margins": 3.900019884109497, + "rewards/rejected": -1.9162627458572388, + "step": 371, + "tokens/chosen_response_len_mean": 352.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 245.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.3, + "grad_norm": 2.739985089644356, + "learning_rate": 4.422609391358677e-07, + "logits/policy_chosen": -1.491788387298584, + "logits/policy_rejected": -1.8629511594772339, + "logits/reference_chosen": -2.8367395401000977, + "logits/reference_rejected": -2.385693073272705, + "logps/policy_chosen": -93.3801498413086, + "logps/policy_rejected": -564.1484985351562, + "logps/reference_chosen": -333.2955322265625, + "logps/reference_rejected": -284.53338623046875, + "loss": 0.154, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.399153709411621, + "rewards/margins": 5.195304870605469, + "rewards/rejected": -2.7961511611938477, + "step": 372, + "tokens/chosen_response_len_mean": 537.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 220.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.3, + "grad_norm": 2.8326056798898325, + "learning_rate": 4.418115462098623e-07, + "logits/policy_chosen": -1.6880661249160767, + "logits/policy_rejected": -1.8501157760620117, + "logits/reference_chosen": -2.9830873012542725, + "logits/reference_rejected": -2.656494140625, + "logps/policy_chosen": -87.76195526123047, + "logps/policy_rejected": -603.6038208007812, + "logps/reference_chosen": -330.90252685546875, + "logps/reference_rejected": -258.64581298828125, + "loss": 0.1816, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.4314053058624268, + "rewards/margins": 5.880985260009766, + "rewards/rejected": -3.449579954147339, + "step": 373, + "tokens/chosen_response_len_mean": 536.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 377.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.3, + "grad_norm": 4.790162201367013, + "learning_rate": 4.4136064141530547e-07, + "logits/policy_chosen": -1.7553645372390747, + "logits/policy_rejected": -1.6190441846847534, + "logits/reference_chosen": -2.7890069484710693, + "logits/reference_rejected": -2.1159238815307617, + "logps/policy_chosen": -67.688232421875, + "logps/policy_rejected": -195.291748046875, + "logps/reference_chosen": -179.9830322265625, + "logps/reference_rejected": -114.24205780029297, + "loss": 0.1921, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.1229479312896729, + "rewards/margins": 1.9334447383880615, + "rewards/rejected": -0.8104968070983887, + "step": 374, + "tokens/chosen_response_len_mean": 431.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 405.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.3, + "grad_norm": 1.2307538383465877, + "learning_rate": 4.409082283062523e-07, + "logits/policy_chosen": -1.5772703886032104, + "logits/policy_rejected": -1.6785141229629517, + "logits/reference_chosen": -3.0875000953674316, + "logits/reference_rejected": -2.7498016357421875, + "logps/policy_chosen": -74.79031372070312, + "logps/policy_rejected": -750.72998046875, + "logps/reference_chosen": -317.4873352050781, + "logps/reference_rejected": -347.156982421875, + "loss": 0.0851, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.4269702434539795, + "rewards/margins": 6.462699890136719, + "rewards/rejected": -4.03572940826416, + "step": 375, + "tokens/chosen_response_len_mean": 458.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 365.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.3, + "grad_norm": 3.5542378235491863, + "learning_rate": 4.4045431044864616e-07, + "logits/policy_chosen": -1.6069151163101196, + "logits/policy_rejected": -1.7103867530822754, + "logits/reference_chosen": -2.6566100120544434, + "logits/reference_rejected": -2.6409106254577637, + "logps/policy_chosen": -87.49906921386719, + "logps/policy_rejected": -665.1824951171875, + "logps/reference_chosen": -305.02557373046875, + "logps/reference_rejected": -320.46221923828125, + "loss": 0.2368, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.175264835357666, + "rewards/margins": 5.622467517852783, + "rewards/rejected": -3.447202682495117, + "step": 376, + "tokens/chosen_response_len_mean": 411.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 312.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.3, + "grad_norm": 5.281625587606173, + "learning_rate": 4.3999889142029126e-07, + "logits/policy_chosen": -2.1449053287506104, + "logits/policy_rejected": -2.194610118865967, + "logits/reference_chosen": -2.8323793411254883, + "logits/reference_rejected": -2.804175853729248, + "logps/policy_chosen": -30.32268524169922, + "logps/policy_rejected": -170.59368896484375, + "logps/reference_chosen": -67.28926086425781, + "logps/reference_rejected": -86.84202575683594, + "loss": 0.3036, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.3696657121181488, + "rewards/margins": 1.20718252658844, + "rewards/rejected": -0.8375166654586792, + "step": 377, + "tokens/chosen_response_len_mean": 183.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 105.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.3, + "grad_norm": 1.9215327012324088, + "learning_rate": 4.395419748108241e-07, + "logits/policy_chosen": -1.883148431777954, + "logits/policy_rejected": -1.9562857151031494, + "logits/reference_chosen": -3.1288890838623047, + "logits/reference_rejected": -2.9048759937286377, + "logps/policy_chosen": -83.87744903564453, + "logps/policy_rejected": -814.286865234375, + "logps/reference_chosen": -300.5325622558594, + "logps/reference_rejected": -364.4478759765625, + "loss": 0.1434, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.166551113128662, + "rewards/margins": 6.66494083404541, + "rewards/rejected": -4.498389720916748, + "step": 378, + "tokens/chosen_response_len_mean": 353.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 284.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.3, + "grad_norm": 3.0494184783882075, + "learning_rate": 4.390835642216848e-07, + "logits/policy_chosen": -1.1829478740692139, + "logits/policy_rejected": -1.6148120164871216, + "logits/reference_chosen": -2.8968608379364014, + "logits/reference_rejected": -2.4937872886657715, + "logps/policy_chosen": -101.84024047851562, + "logps/policy_rejected": -1095.6416015625, + "logps/reference_chosen": -356.5980224609375, + "logps/reference_rejected": -464.4998474121094, + "loss": 0.0851, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.5475780963897705, + "rewards/margins": 8.85899543762207, + "rewards/rejected": -6.311417579650879, + "step": 379, + "tokens/chosen_response_len_mean": 750.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 420.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.31, + "grad_norm": 1.9839846610083902, + "learning_rate": 4.386236632660897e-07, + "logits/policy_chosen": -1.9111047983169556, + "logits/policy_rejected": -1.440907597541809, + "logits/reference_chosen": -2.8704569339752197, + "logits/reference_rejected": -2.470503091812134, + "logps/policy_chosen": -63.218116760253906, + "logps/policy_rejected": -509.4794616699219, + "logps/reference_chosen": -221.18771362304688, + "logps/reference_rejected": -222.36109924316406, + "loss": 0.0839, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.5796959400177002, + "rewards/margins": 4.4508795738220215, + "rewards/rejected": -2.871183395385742, + "step": 380, + "tokens/chosen_response_len_mean": 366.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 483.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.31, + "grad_norm": 3.547953034306201, + "learning_rate": 4.3816227556900174e-07, + "logits/policy_chosen": -2.1110708713531494, + "logits/policy_rejected": -1.5525974035263062, + "logits/reference_chosen": -3.074847459793091, + "logits/reference_rejected": -2.299776315689087, + "logps/policy_chosen": -53.559783935546875, + "logps/policy_rejected": -340.3197937011719, + "logps/reference_chosen": -210.47630310058594, + "logps/reference_rejected": -170.03268432617188, + "loss": 0.2292, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.5691652297973633, + "rewards/margins": 3.27203631401062, + "rewards/rejected": -1.7028712034225464, + "step": 381, + "tokens/chosen_response_len_mean": 434.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 429.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.31, + "grad_norm": 4.4598115746078175, + "learning_rate": 4.376994047671028e-07, + "logits/policy_chosen": -1.8473024368286133, + "logits/policy_rejected": -1.9738845825195312, + "logits/reference_chosen": -2.9645516872406006, + "logits/reference_rejected": -2.810880184173584, + "logps/policy_chosen": -37.34379196166992, + "logps/policy_rejected": -328.9185485839844, + "logps/reference_chosen": -105.24017333984375, + "logps/reference_rejected": -159.31472778320312, + "loss": 0.1772, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.6789637804031372, + "rewards/margins": 2.375002145767212, + "rewards/rejected": -1.6960381269454956, + "step": 382, + "tokens/chosen_response_len_mean": 204.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 216.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.31, + "grad_norm": 2.895844402337817, + "learning_rate": 4.3723505450876454e-07, + "logits/policy_chosen": -1.5201809406280518, + "logits/policy_rejected": -1.8310809135437012, + "logits/reference_chosen": -2.652768135070801, + "logits/reference_rejected": -2.6164894104003906, + "logps/policy_chosen": -89.18895721435547, + "logps/policy_rejected": -672.33349609375, + "logps/reference_chosen": -299.8074951171875, + "logps/reference_rejected": -316.7373962402344, + "loss": 0.1478, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.1061851978302, + "rewards/margins": 5.66214656829834, + "rewards/rejected": -3.5559608936309814, + "step": 383, + "tokens/chosen_response_len_mean": 622.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 480.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.31, + "grad_norm": 3.2525327057187328, + "learning_rate": 4.3676922845401963e-07, + "logits/policy_chosen": -1.4135372638702393, + "logits/policy_rejected": -1.6460597515106201, + "logits/reference_chosen": -2.6158249378204346, + "logits/reference_rejected": -2.269045352935791, + "logps/policy_chosen": -36.46479797363281, + "logps/policy_rejected": -372.2850341796875, + "logps/reference_chosen": -101.00570678710938, + "logps/reference_rejected": -160.78067016601562, + "loss": 0.1283, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.645409107208252, + "rewards/margins": 2.76045298576355, + "rewards/rejected": -2.115043878555298, + "step": 384, + "tokens/chosen_response_len_mean": 362.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 215.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.31, + "grad_norm": 2.683005455867468, + "learning_rate": 4.363019302745334e-07, + "logits/policy_chosen": -2.2419838905334473, + "logits/policy_rejected": -2.421705961227417, + "logits/reference_chosen": -3.0781402587890625, + "logits/reference_rejected": -2.9199631214141846, + "logps/policy_chosen": -49.374847412109375, + "logps/policy_rejected": -359.7245178222656, + "logps/reference_chosen": -158.96978759765625, + "logps/reference_rejected": -127.78620910644531, + "loss": 0.2265, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.0959495306015015, + "rewards/margins": 3.415332317352295, + "rewards/rejected": -2.319382667541504, + "step": 385, + "tokens/chosen_response_len_mean": 254.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 102.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.31, + "grad_norm": 4.3953584290948156, + "learning_rate": 4.3583316365357416e-07, + "logits/policy_chosen": -1.6831576824188232, + "logits/policy_rejected": -1.8844032287597656, + "logits/reference_chosen": -2.9197726249694824, + "logits/reference_rejected": -2.744558811187744, + "logps/policy_chosen": -82.87364959716797, + "logps/policy_rejected": -661.0844116210938, + "logps/reference_chosen": -322.45697021484375, + "logps/reference_rejected": -285.1172180175781, + "loss": 0.1246, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.3958332538604736, + "rewards/margins": 6.155505657196045, + "rewards/rejected": -3.7596724033355713, + "step": 386, + "tokens/chosen_response_len_mean": 471.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 260.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.31, + "grad_norm": 4.3894384552460215, + "learning_rate": 4.3536293228598474e-07, + "logits/policy_chosen": -1.8510702848434448, + "logits/policy_rejected": -1.8795132637023926, + "logits/reference_chosen": -2.6799912452697754, + "logits/reference_rejected": -2.4659016132354736, + "logps/policy_chosen": -32.05808639526367, + "logps/policy_rejected": -213.7239990234375, + "logps/reference_chosen": -59.347633361816406, + "logps/reference_rejected": -68.51786804199219, + "loss": 0.2542, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.2728954553604126, + "rewards/margins": 1.7249568700790405, + "rewards/rejected": -1.4520612955093384, + "step": 387, + "tokens/chosen_response_len_mean": 63.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 50.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.31, + "grad_norm": 9.267866458567886, + "learning_rate": 4.3489123987815325e-07, + "logits/policy_chosen": -1.4151192903518677, + "logits/policy_rejected": -1.5208919048309326, + "logits/reference_chosen": -2.8274762630462646, + "logits/reference_rejected": -2.905880928039551, + "logps/policy_chosen": -118.6706771850586, + "logps/policy_rejected": -557.0550537109375, + "logps/reference_chosen": -374.5296936035156, + "logps/reference_rejected": -424.9022521972656, + "loss": 0.1827, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.5585901737213135, + "rewards/margins": 3.8801183700561523, + "rewards/rejected": -1.321528434753418, + "step": 388, + "tokens/chosen_response_len_mean": 562.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 549.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.31, + "grad_norm": 4.103907207414104, + "learning_rate": 4.344180901479836e-07, + "logits/policy_chosen": -1.689210295677185, + "logits/policy_rejected": -1.1707533597946167, + "logits/reference_chosen": -2.7496488094329834, + "logits/reference_rejected": -2.5293631553649902, + "logps/policy_chosen": -95.53358459472656, + "logps/policy_rejected": -641.15380859375, + "logps/reference_chosen": -288.0838623046875, + "logps/reference_rejected": -270.31884765625, + "loss": 0.1837, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.925502896308899, + "rewards/margins": 5.633852958679199, + "rewards/rejected": -3.7083497047424316, + "step": 389, + "tokens/chosen_response_len_mean": 366.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 445.75, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.31, + "grad_norm": 4.588056708293884, + "learning_rate": 4.339434868248665e-07, + "logits/policy_chosen": -1.6808419227600098, + "logits/policy_rejected": -1.7554115056991577, + "logits/reference_chosen": -2.79362154006958, + "logits/reference_rejected": -2.6228065490722656, + "logps/policy_chosen": -59.64699172973633, + "logps/policy_rejected": -678.405029296875, + "logps/reference_chosen": -213.72731018066406, + "logps/reference_rejected": -335.5184326171875, + "loss": 0.2293, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.54080331325531, + "rewards/margins": 4.969668865203857, + "rewards/rejected": -3.428865909576416, + "step": 390, + "tokens/chosen_response_len_mean": 348.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 301.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.31, + "grad_norm": 3.0510324137868525, + "learning_rate": 4.3346743364965e-07, + "logits/policy_chosen": -1.049216628074646, + "logits/policy_rejected": -1.8208162784576416, + "logits/reference_chosen": -2.591167449951172, + "logits/reference_rejected": -2.324847459793091, + "logps/policy_chosen": -121.5611572265625, + "logps/policy_rejected": -607.802001953125, + "logps/reference_chosen": -316.2970275878906, + "logps/reference_rejected": -231.97964477539062, + "loss": 0.1775, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.9473586082458496, + "rewards/margins": 5.705582141876221, + "rewards/rejected": -3.758223056793213, + "step": 391, + "tokens/chosen_response_len_mean": 472.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 194.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.32, + "grad_norm": 4.799092230789488, + "learning_rate": 4.3298993437460984e-07, + "logits/policy_chosen": -1.3143819570541382, + "logits/policy_rejected": -1.4681634902954102, + "logits/reference_chosen": -2.5000357627868652, + "logits/reference_rejected": -2.514500379562378, + "logps/policy_chosen": -81.29645538330078, + "logps/policy_rejected": -878.3502197265625, + "logps/reference_chosen": -289.78057861328125, + "logps/reference_rejected": -434.30841064453125, + "loss": 0.2684, + "rewards/accuracies": 0.75, + "rewards/chosen": 2.084841012954712, + "rewards/margins": 6.525258541107178, + "rewards/rejected": -4.440417766571045, + "step": 392, + "tokens/chosen_response_len_mean": 493.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 346.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.32, + "grad_norm": 5.0096860526087985, + "learning_rate": 4.325109927634202e-07, + "logits/policy_chosen": -1.745060682296753, + "logits/policy_rejected": -1.8800654411315918, + "logits/reference_chosen": -2.9817285537719727, + "logits/reference_rejected": -2.4398183822631836, + "logps/policy_chosen": -67.2919921875, + "logps/policy_rejected": -429.408203125, + "logps/reference_chosen": -218.21548461914062, + "logps/reference_rejected": -260.8848571777344, + "loss": 0.239, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.50923490524292, + "rewards/margins": 3.1944684982299805, + "rewards/rejected": -1.6852335929870605, + "step": 393, + "tokens/chosen_response_len_mean": 263.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 170.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.32, + "grad_norm": 3.990192788421323, + "learning_rate": 4.320306125911236e-07, + "logits/policy_chosen": -2.1053833961486816, + "logits/policy_rejected": -2.0984866619110107, + "logits/reference_chosen": -2.8531177043914795, + "logits/reference_rejected": -2.862253189086914, + "logps/policy_chosen": -26.376094818115234, + "logps/policy_rejected": -305.2973327636719, + "logps/reference_chosen": -47.91350555419922, + "logps/reference_rejected": -131.7420196533203, + "loss": 0.2484, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.21537409722805023, + "rewards/margins": 1.9509271383285522, + "rewards/rejected": -1.7355530261993408, + "step": 394, + "tokens/chosen_response_len_mean": 129.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 149.0, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.32, + "grad_norm": 6.374622408586315, + "learning_rate": 4.315487976441013e-07, + "logits/policy_chosen": -1.2467350959777832, + "logits/policy_rejected": -1.5044665336608887, + "logits/reference_chosen": -2.920576572418213, + "logits/reference_rejected": -2.6558072566986084, + "logps/policy_chosen": -138.9432373046875, + "logps/policy_rejected": -726.0556030273438, + "logps/reference_chosen": -493.89862060546875, + "logps/reference_rejected": -350.41497802734375, + "loss": 0.0826, + "rewards/accuracies": 1.0, + "rewards/chosen": 3.549553394317627, + "rewards/margins": 7.305959224700928, + "rewards/rejected": -3.756405830383301, + "step": 395, + "tokens/chosen_response_len_mean": 683.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 443.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.32, + "grad_norm": 2.7801583537276717, + "learning_rate": 4.31065551720044e-07, + "logits/policy_chosen": -1.164729118347168, + "logits/policy_rejected": -1.1291731595993042, + "logits/reference_chosen": -2.5515594482421875, + "logits/reference_rejected": -2.443310022354126, + "logps/policy_chosen": -66.39266967773438, + "logps/policy_rejected": -558.2879638671875, + "logps/reference_chosen": -186.88536071777344, + "logps/reference_rejected": -314.25531005859375, + "loss": 0.19, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.2049269676208496, + "rewards/margins": 3.6452534198760986, + "rewards/rejected": -2.440326452255249, + "step": 396, + "tokens/chosen_response_len_mean": 448.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 443.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.32, + "grad_norm": 2.9706769612280617, + "learning_rate": 4.3058087862792097e-07, + "logits/policy_chosen": -1.182519555091858, + "logits/policy_rejected": -1.190351128578186, + "logits/reference_chosen": -2.546760082244873, + "logits/reference_rejected": -2.415588617324829, + "logps/policy_chosen": -64.53900146484375, + "logps/policy_rejected": -359.20123291015625, + "logps/reference_chosen": -171.31234741210938, + "logps/reference_rejected": -217.0635986328125, + "loss": 0.1348, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.0677335262298584, + "rewards/margins": 2.489109992980957, + "rewards/rejected": -1.4213764667510986, + "step": 397, + "tokens/chosen_response_len_mean": 421.0, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 385.25, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.32, + "grad_norm": 4.30537555824225, + "learning_rate": 4.300947821879506e-07, + "logits/policy_chosen": -1.8363888263702393, + "logits/policy_rejected": -2.059960126876831, + "logits/reference_chosen": -2.9730725288391113, + "logits/reference_rejected": -2.7597203254699707, + "logps/policy_chosen": -53.318687438964844, + "logps/policy_rejected": -473.90850830078125, + "logps/reference_chosen": -151.44744873046875, + "logps/reference_rejected": -189.46267700195312, + "loss": 0.261, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.9812875390052795, + "rewards/margins": 3.8257455825805664, + "rewards/rejected": -2.8444581031799316, + "step": 398, + "tokens/chosen_response_len_mean": 290.75, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 171.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.32, + "grad_norm": 1.3903903548910281, + "learning_rate": 4.296072662315704e-07, + "logits/policy_chosen": -1.257875919342041, + "logits/policy_rejected": -1.1928701400756836, + "logits/reference_chosen": -2.869974374771118, + "logits/reference_rejected": -2.889599323272705, + "logps/policy_chosen": -72.68022155761719, + "logps/policy_rejected": -879.3326416015625, + "logps/reference_chosen": -297.6036071777344, + "logps/reference_rejected": -490.29583740234375, + "loss": 0.1009, + "rewards/accuracies": 1.0, + "rewards/chosen": 2.2492337226867676, + "rewards/margins": 6.139601707458496, + "rewards/rejected": -3.8903677463531494, + "step": 399, + "tokens/chosen_response_len_mean": 611.25, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 622.5, + "tokens/rejected_response_len_std": 0.0 + }, + { + "epoch": 0.32, + "grad_norm": 2.979855295546658, + "learning_rate": 4.291183346014062e-07, + "logits/policy_chosen": -2.134671926498413, + "logits/policy_rejected": -2.2338736057281494, + "logits/reference_chosen": -3.2482404708862305, + "logits/reference_rejected": -3.0586061477661133, + "logps/policy_chosen": -53.64082336425781, + "logps/policy_rejected": -510.7403869628906, + "logps/reference_chosen": -155.8409423828125, + "logps/reference_rejected": -154.61331176757812, + "loss": 0.2301, + "rewards/accuracies": 1.0, + "rewards/chosen": 1.0220012664794922, + "rewards/margins": 4.5832719802856445, + "rewards/rejected": -3.5612707138061523, + "step": 400, + "tokens/chosen_response_len_mean": 202.5, + "tokens/chosen_response_len_std": 0.0, + "tokens/rejected_response_len_mean": 134.25, + "tokens/rejected_response_len_std": 0.0 + } + ], + "logging_steps": 1, + "max_steps": 1244, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 100, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +}