| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 2.9765925925925925, | |
| "eval_steps": 500, | |
| "global_step": 315, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.047407407407407405, | |
| "grad_norm": 1064.504150390625, | |
| "learning_rate": 7.8125e-06, | |
| "log_odds_chosen": -1.0592396259307861, | |
| "log_odds_ratio": -11.0020170211792, | |
| "logps/chosen": -22.70431900024414, | |
| "logps/rejected": -21.645305633544922, | |
| "loss": 450.5827, | |
| "nll_loss": 9.83934497833252, | |
| "po_loss": 4.241364479064941, | |
| "rewards/accuracies": 0.4781250059604645, | |
| "rewards/chosen": -11.35215950012207, | |
| "rewards/margins": -0.5295066237449646, | |
| "rewards/rejected": -10.822652816772461, | |
| "step": 5 | |
| }, | |
| { | |
| "epoch": 0.09481481481481481, | |
| "grad_norm": 610.734619140625, | |
| "learning_rate": 1.5625e-05, | |
| "log_odds_chosen": -3.1254055500030518, | |
| "log_odds_ratio": -11.053281784057617, | |
| "logps/chosen": -22.00815200805664, | |
| "logps/rejected": -18.883081436157227, | |
| "loss": 422.0994, | |
| "nll_loss": 8.754120826721191, | |
| "po_loss": 4.43648624420166, | |
| "rewards/accuracies": 0.44999998807907104, | |
| "rewards/chosen": -11.00407600402832, | |
| "rewards/margins": -1.5625368356704712, | |
| "rewards/rejected": -9.441540718078613, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.14222222222222222, | |
| "grad_norm": 600.33056640625, | |
| "learning_rate": 2.34375e-05, | |
| "log_odds_chosen": -1.30233633518219, | |
| "log_odds_ratio": -11.293824195861816, | |
| "logps/chosen": -21.929264068603516, | |
| "logps/rejected": -20.62543487548828, | |
| "loss": 392.0485, | |
| "nll_loss": 7.978579044342041, | |
| "po_loss": 4.2729363441467285, | |
| "rewards/accuracies": 0.515625, | |
| "rewards/chosen": -10.964632034301758, | |
| "rewards/margins": -0.6519135236740112, | |
| "rewards/rejected": -10.31271743774414, | |
| "step": 15 | |
| }, | |
| { | |
| "epoch": 0.18962962962962962, | |
| "grad_norm": 3273.544677734375, | |
| "learning_rate": 3.125e-05, | |
| "log_odds_chosen": -3.9110000133514404, | |
| "log_odds_ratio": -8.825319290161133, | |
| "logps/chosen": -16.157787322998047, | |
| "logps/rejected": -12.249563217163086, | |
| "loss": 296.8894, | |
| "nll_loss": 5.963077545166016, | |
| "po_loss": 3.314715623855591, | |
| "rewards/accuracies": 0.44062501192092896, | |
| "rewards/chosen": -8.078893661499023, | |
| "rewards/margins": -1.954110860824585, | |
| "rewards/rejected": -6.124781608581543, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.23703703703703705, | |
| "grad_norm": 309.38031005859375, | |
| "learning_rate": 3.90625e-05, | |
| "log_odds_chosen": -0.052033402025699615, | |
| "log_odds_ratio": -1.5327208042144775, | |
| "logps/chosen": -3.302628755569458, | |
| "logps/rejected": -3.230381488800049, | |
| "loss": 83.2066, | |
| "nll_loss": 2.531467914581299, | |
| "po_loss": 0.06873763352632523, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -1.651314377784729, | |
| "rewards/margins": -0.036123599857091904, | |
| "rewards/rejected": -1.6151907444000244, | |
| "step": 25 | |
| }, | |
| { | |
| "epoch": 0.28444444444444444, | |
| "grad_norm": 1114.235107421875, | |
| "learning_rate": 4.6875e-05, | |
| "log_odds_chosen": 0.07359764724969864, | |
| "log_odds_ratio": -0.942546546459198, | |
| "logps/chosen": -1.9597572088241577, | |
| "logps/rejected": -2.0194180011749268, | |
| "loss": 64.8739, | |
| "nll_loss": 2.057140827178955, | |
| "po_loss": -0.029830310493707657, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.9798786044120789, | |
| "rewards/margins": 0.029830310493707657, | |
| "rewards/rejected": -1.0097090005874634, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.33185185185185184, | |
| "grad_norm": 388.6631774902344, | |
| "learning_rate": 4.998613757348784e-05, | |
| "log_odds_chosen": 0.3581487536430359, | |
| "log_odds_ratio": -0.841637909412384, | |
| "logps/chosen": -1.9078718423843384, | |
| "logps/rejected": -2.2348716259002686, | |
| "loss": 58.6657, | |
| "nll_loss": 1.9908554553985596, | |
| "po_loss": -0.15755358338356018, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.9539359211921692, | |
| "rewards/margins": 0.1634998619556427, | |
| "rewards/rejected": -1.1174358129501343, | |
| "step": 35 | |
| }, | |
| { | |
| "epoch": 0.37925925925925924, | |
| "grad_norm": 215.6595458984375, | |
| "learning_rate": 4.990147841143462e-05, | |
| "log_odds_chosen": 0.49525055289268494, | |
| "log_odds_ratio": -0.6980496644973755, | |
| "logps/chosen": -1.7104499340057373, | |
| "logps/rejected": -2.14109206199646, | |
| "loss": 55.0966, | |
| "nll_loss": 1.9370899200439453, | |
| "po_loss": -0.21532109379768372, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": -0.8552249670028687, | |
| "rewards/margins": 0.21532109379768372, | |
| "rewards/rejected": -1.07054603099823, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.4266666666666667, | |
| "grad_norm": 490.9734191894531, | |
| "learning_rate": 4.97401218720448e-05, | |
| "log_odds_chosen": 1.0681084394454956, | |
| "log_odds_ratio": -0.748261034488678, | |
| "logps/chosen": -1.9336668252944946, | |
| "logps/rejected": -2.933429718017578, | |
| "loss": 45.8169, | |
| "nll_loss": 1.9130370616912842, | |
| "po_loss": -0.4812597632408142, | |
| "rewards/accuracies": 0.671875, | |
| "rewards/chosen": -0.9668334126472473, | |
| "rewards/margins": 0.4998815655708313, | |
| "rewards/rejected": -1.466714859008789, | |
| "step": 45 | |
| }, | |
| { | |
| "epoch": 0.4740740740740741, | |
| "grad_norm": 687.996826171875, | |
| "learning_rate": 4.9502564938797946e-05, | |
| "log_odds_chosen": 1.5803465843200684, | |
| "log_odds_ratio": -0.8952202796936035, | |
| "logps/chosen": -2.2172279357910156, | |
| "logps/rejected": -3.716336488723755, | |
| "loss": 51.598, | |
| "nll_loss": 2.2828214168548584, | |
| "po_loss": -0.6703845262527466, | |
| "rewards/accuracies": 0.6937500238418579, | |
| "rewards/chosen": -1.1086139678955078, | |
| "rewards/margins": 0.7495540976524353, | |
| "rewards/rejected": -1.8581682443618774, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.5214814814814814, | |
| "grad_norm": 4070.76123046875, | |
| "learning_rate": 4.918953929490768e-05, | |
| "log_odds_chosen": 2.9766428470611572, | |
| "log_odds_ratio": -0.919791042804718, | |
| "logps/chosen": -2.4137585163116455, | |
| "logps/rejected": -5.309506416320801, | |
| "loss": 37.3237, | |
| "nll_loss": 2.3748691082000732, | |
| "po_loss": -1.2085039615631104, | |
| "rewards/accuracies": 0.6656249761581421, | |
| "rewards/chosen": -1.2068792581558228, | |
| "rewards/margins": 1.4478741884231567, | |
| "rewards/rejected": -2.6547532081604004, | |
| "step": 55 | |
| }, | |
| { | |
| "epoch": 0.5688888888888889, | |
| "grad_norm": 569.17724609375, | |
| "learning_rate": 4.88020090697132e-05, | |
| "log_odds_chosen": 5.00119686126709, | |
| "log_odds_ratio": -1.5456047058105469, | |
| "logps/chosen": -6.171454906463623, | |
| "logps/rejected": -11.093535423278809, | |
| "loss": 103.9391, | |
| "nll_loss": 4.901269912719727, | |
| "po_loss": -1.653172492980957, | |
| "rewards/accuracies": 0.721875011920929, | |
| "rewards/chosen": -3.0857274532318115, | |
| "rewards/margins": 2.461040496826172, | |
| "rewards/rejected": -5.546767711639404, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.6162962962962963, | |
| "grad_norm": 614.133544921875, | |
| "learning_rate": 4.834116786912897e-05, | |
| "log_odds_chosen": 1.351292371749878, | |
| "log_odds_ratio": -1.3231507539749146, | |
| "logps/chosen": -2.553697109222412, | |
| "logps/rejected": -3.8612685203552246, | |
| "loss": 63.6228, | |
| "nll_loss": 2.556739091873169, | |
| "po_loss": -0.5685266256332397, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -1.276848554611206, | |
| "rewards/margins": 0.6537858843803406, | |
| "rewards/rejected": -1.9306342601776123, | |
| "step": 65 | |
| }, | |
| { | |
| "epoch": 0.6637037037037037, | |
| "grad_norm": 386.0463562011719, | |
| "learning_rate": 4.7808435099299045e-05, | |
| "log_odds_chosen": 2.089937210083008, | |
| "log_odds_ratio": -0.670052170753479, | |
| "logps/chosen": -1.7493152618408203, | |
| "logps/rejected": -3.7693347930908203, | |
| "loss": 34.9169, | |
| "nll_loss": 1.8565772771835327, | |
| "po_loss": -0.7654256820678711, | |
| "rewards/accuracies": 0.6937500238418579, | |
| "rewards/chosen": -0.8746576309204102, | |
| "rewards/margins": 1.010009765625, | |
| "rewards/rejected": -1.8846673965454102, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.7111111111111111, | |
| "grad_norm": 3206.72705078125, | |
| "learning_rate": 4.720545159477922e-05, | |
| "log_odds_chosen": 2.741664409637451, | |
| "log_odds_ratio": -1.1608259677886963, | |
| "logps/chosen": -3.1741726398468018, | |
| "logps/rejected": -5.838489532470703, | |
| "loss": 55.9823, | |
| "nll_loss": 2.62483811378479, | |
| "po_loss": -0.8753900527954102, | |
| "rewards/accuracies": 0.6968749761581421, | |
| "rewards/chosen": -1.5870863199234009, | |
| "rewards/margins": 1.3321585655212402, | |
| "rewards/rejected": -2.9192447662353516, | |
| "step": 75 | |
| }, | |
| { | |
| "epoch": 0.7585185185185185, | |
| "grad_norm": 412.6952209472656, | |
| "learning_rate": 4.653407456471222e-05, | |
| "log_odds_chosen": 1.9728152751922607, | |
| "log_odds_ratio": -0.9150087237358093, | |
| "logps/chosen": -2.0698251724243164, | |
| "logps/rejected": -3.9561798572540283, | |
| "loss": 41.4899, | |
| "nll_loss": 2.0737736225128174, | |
| "po_loss": -0.7772130370140076, | |
| "rewards/accuracies": 0.653124988079071, | |
| "rewards/chosen": -1.0349125862121582, | |
| "rewards/margins": 0.9431774020195007, | |
| "rewards/rejected": -1.9780899286270142, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.8059259259259259, | |
| "grad_norm": 1100.900634765625, | |
| "learning_rate": 4.579637187256222e-05, | |
| "log_odds_chosen": 2.417811870574951, | |
| "log_odds_ratio": -0.9469805955886841, | |
| "logps/chosen": -2.1366405487060547, | |
| "logps/rejected": -4.464547157287598, | |
| "loss": 36.3301, | |
| "nll_loss": 2.163440704345703, | |
| "po_loss": -1.0281257629394531, | |
| "rewards/accuracies": 0.6812499761581421, | |
| "rewards/chosen": -1.0683202743530273, | |
| "rewards/margins": 1.163953185081482, | |
| "rewards/rejected": -2.232273578643799, | |
| "step": 85 | |
| }, | |
| { | |
| "epoch": 0.8533333333333334, | |
| "grad_norm": 785.2890625, | |
| "learning_rate": 4.499461566702685e-05, | |
| "log_odds_chosen": 3.7706730365753174, | |
| "log_odds_ratio": -1.2985801696777344, | |
| "logps/chosen": -3.598789691925049, | |
| "logps/rejected": -7.259984016418457, | |
| "loss": 42.6548, | |
| "nll_loss": 2.9308180809020996, | |
| "po_loss": -1.597855567932129, | |
| "rewards/accuracies": 0.690625011920929, | |
| "rewards/chosen": -1.7993948459625244, | |
| "rewards/margins": 1.830596923828125, | |
| "rewards/rejected": -3.6299920082092285, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.9007407407407407, | |
| "grad_norm": 3162.0009765625, | |
| "learning_rate": 4.413127538374411e-05, | |
| "log_odds_chosen": 2.615981340408325, | |
| "log_odds_ratio": -1.0526683330535889, | |
| "logps/chosen": -2.3276286125183105, | |
| "logps/rejected": -4.851325988769531, | |
| "loss": 31.8051, | |
| "nll_loss": 2.035627841949463, | |
| "po_loss": -1.0417195558547974, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -1.1638143062591553, | |
| "rewards/margins": 1.2618486881256104, | |
| "rewards/rejected": -2.4256629943847656, | |
| "step": 95 | |
| }, | |
| { | |
| "epoch": 0.9481481481481482, | |
| "grad_norm": 571.5673217773438, | |
| "learning_rate": 4.320901013934887e-05, | |
| "log_odds_chosen": 4.09200382232666, | |
| "log_odds_ratio": -1.3848493099212646, | |
| "logps/chosen": -3.318943500518799, | |
| "logps/rejected": -7.310236930847168, | |
| "loss": 37.0821, | |
| "nll_loss": 2.7883472442626953, | |
| "po_loss": -1.629533052444458, | |
| "rewards/accuracies": 0.6812499761581421, | |
| "rewards/chosen": -1.6594717502593994, | |
| "rewards/margins": 1.9956468343734741, | |
| "rewards/rejected": -3.655118465423584, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.9955555555555555, | |
| "grad_norm": 251.86697387695312, | |
| "learning_rate": 4.223066054130568e-05, | |
| "log_odds_chosen": 3.148193359375, | |
| "log_odds_ratio": -0.6689270734786987, | |
| "logps/chosen": -1.6926807165145874, | |
| "logps/rejected": -4.753729820251465, | |
| "loss": 24.9079, | |
| "nll_loss": 1.8303325176239014, | |
| "po_loss": -1.05196213722229, | |
| "rewards/accuracies": 0.6781250238418579, | |
| "rewards/chosen": -0.8463403582572937, | |
| "rewards/margins": 1.5305248498916626, | |
| "rewards/rejected": -2.3768649101257324, | |
| "step": 105 | |
| }, | |
| { | |
| "epoch": 1.037925925925926, | |
| "grad_norm": 235.2350616455078, | |
| "learning_rate": 4.1199239938743797e-05, | |
| "log_odds_chosen": 3.255824565887451, | |
| "log_odds_ratio": -0.6982910633087158, | |
| "logps/chosen": -1.715062141418457, | |
| "logps/rejected": -4.823176383972168, | |
| "loss": 23.0323, | |
| "nll_loss": 1.716023325920105, | |
| "po_loss": -0.9106981754302979, | |
| "rewards/accuracies": 0.7307692170143127, | |
| "rewards/chosen": -0.8575310707092285, | |
| "rewards/margins": 1.5540571212768555, | |
| "rewards/rejected": -2.411588191986084, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 1.0853333333333333, | |
| "grad_norm": 457.39080810546875, | |
| "learning_rate": 4.0117925141242174e-05, | |
| "log_odds_chosen": 5.342764854431152, | |
| "log_odds_ratio": -0.9483366012573242, | |
| "logps/chosen": -2.369607448577881, | |
| "logps/rejected": -7.539316654205322, | |
| "loss": 17.0777, | |
| "nll_loss": 2.0790820121765137, | |
| "po_loss": -1.5454037189483643, | |
| "rewards/accuracies": 0.7593749761581421, | |
| "rewards/chosen": -1.1848037242889404, | |
| "rewards/margins": 2.5848546028137207, | |
| "rewards/rejected": -3.769658327102661, | |
| "step": 115 | |
| }, | |
| { | |
| "epoch": 1.1327407407407408, | |
| "grad_norm": 478.9156799316406, | |
| "learning_rate": 3.899004663415084e-05, | |
| "log_odds_chosen": 4.5327959060668945, | |
| "log_odds_ratio": -0.7982605695724487, | |
| "logps/chosen": -2.2072598934173584, | |
| "logps/rejected": -6.586657524108887, | |
| "loss": 12.6686, | |
| "nll_loss": 1.9718843698501587, | |
| "po_loss": -1.5759899616241455, | |
| "rewards/accuracies": 0.7406250238418579, | |
| "rewards/chosen": -1.1036299467086792, | |
| "rewards/margins": 2.1896984577178955, | |
| "rewards/rejected": -3.2933287620544434, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 1.1801481481481482, | |
| "grad_norm": 760.638916015625, | |
| "learning_rate": 3.781907832058587e-05, | |
| "log_odds_chosen": 4.349984645843506, | |
| "log_odds_ratio": -1.0327547788619995, | |
| "logps/chosen": -2.662911891937256, | |
| "logps/rejected": -6.870619297027588, | |
| "loss": 32.4605, | |
| "nll_loss": 2.5304787158966064, | |
| "po_loss": -1.5160895586013794, | |
| "rewards/accuracies": 0.7406250238418579, | |
| "rewards/chosen": -1.331455945968628, | |
| "rewards/margins": 2.103853702545166, | |
| "rewards/rejected": -3.435309648513794, | |
| "step": 125 | |
| }, | |
| { | |
| "epoch": 1.2275555555555555, | |
| "grad_norm": 1151.731201171875, | |
| "learning_rate": 3.660862682169282e-05, | |
| "log_odds_chosen": 6.421099662780762, | |
| "log_odds_ratio": -1.0163640975952148, | |
| "logps/chosen": -2.875323534011841, | |
| "logps/rejected": -9.088385581970215, | |
| "loss": -4.7801, | |
| "nll_loss": 2.3773765563964844, | |
| "po_loss": -2.5267534255981445, | |
| "rewards/accuracies": 0.762499988079071, | |
| "rewards/chosen": -1.4376617670059204, | |
| "rewards/margins": 3.1065304279327393, | |
| "rewards/rejected": -4.544192790985107, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 1.274962962962963, | |
| "grad_norm": 535.7034912109375, | |
| "learning_rate": 3.5362420368134356e-05, | |
| "log_odds_chosen": 3.287085771560669, | |
| "log_odds_ratio": -0.6112745404243469, | |
| "logps/chosen": -1.6216014623641968, | |
| "logps/rejected": -4.728818893432617, | |
| "loss": 9.9669, | |
| "nll_loss": 1.7199676036834717, | |
| "po_loss": -1.4085030555725098, | |
| "rewards/accuracies": 0.7437499761581421, | |
| "rewards/chosen": -0.8108007311820984, | |
| "rewards/margins": 1.553608775138855, | |
| "rewards/rejected": -2.3644094467163086, | |
| "step": 135 | |
| }, | |
| { | |
| "epoch": 1.3223703703703704, | |
| "grad_norm": 1074.8087158203125, | |
| "learning_rate": 3.408429731701635e-05, | |
| "log_odds_chosen": 3.7467429637908936, | |
| "log_odds_ratio": -0.7782642841339111, | |
| "logps/chosen": -2.253432512283325, | |
| "logps/rejected": -5.834208011627197, | |
| "loss": 18.4068, | |
| "nll_loss": 2.049558401107788, | |
| "po_loss": -1.4743449687957764, | |
| "rewards/accuracies": 0.734375, | |
| "rewards/chosen": -1.1267162561416626, | |
| "rewards/margins": 1.790387749671936, | |
| "rewards/rejected": -2.9171040058135986, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 1.3697777777777778, | |
| "grad_norm": 2433.367919921875, | |
| "learning_rate": 3.2778194329621104e-05, | |
| "log_odds_chosen": 6.9977288246154785, | |
| "log_odds_ratio": -0.8477832078933716, | |
| "logps/chosen": -2.8123180866241455, | |
| "logps/rejected": -9.62822151184082, | |
| "loss": -7.1341, | |
| "nll_loss": 2.3402633666992188, | |
| "po_loss": -2.5632026195526123, | |
| "rewards/accuracies": 0.7718750238418579, | |
| "rewards/chosen": -1.4061590433120728, | |
| "rewards/margins": 3.407951831817627, | |
| "rewards/rejected": -4.81411075592041, | |
| "step": 145 | |
| }, | |
| { | |
| "epoch": 1.417185185185185, | |
| "grad_norm": 1035.2568359375, | |
| "learning_rate": 3.144813424636031e-05, | |
| "log_odds_chosen": 7.531359672546387, | |
| "log_odds_ratio": -0.9453743696212769, | |
| "logps/chosen": -2.4776880741119385, | |
| "logps/rejected": -9.81294059753418, | |
| "loss": -8.8755, | |
| "nll_loss": 2.127579689025879, | |
| "po_loss": -2.404938220977783, | |
| "rewards/accuracies": 0.762499988079071, | |
| "rewards/chosen": -1.2388440370559692, | |
| "rewards/margins": 3.667625904083252, | |
| "rewards/rejected": -4.90647029876709, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 1.4645925925925927, | |
| "grad_norm": 1741.77978515625, | |
| "learning_rate": 3.0098213696293542e-05, | |
| "log_odds_chosen": 4.925074577331543, | |
| "log_odds_ratio": -1.4115755558013916, | |
| "logps/chosen": -2.6048836708068848, | |
| "logps/rejected": -7.376564025878906, | |
| "loss": 23.5026, | |
| "nll_loss": 2.325968027114868, | |
| "po_loss": -1.5915124416351318, | |
| "rewards/accuracies": 0.6968749761581421, | |
| "rewards/chosen": -1.3024418354034424, | |
| "rewards/margins": 2.3858399391174316, | |
| "rewards/rejected": -3.688282012939453, | |
| "step": 155 | |
| }, | |
| { | |
| "epoch": 1.512, | |
| "grad_norm": 1471.9173583984375, | |
| "learning_rate": 2.8732590479375165e-05, | |
| "log_odds_chosen": 6.550325870513916, | |
| "log_odds_ratio": -0.9376071691513062, | |
| "logps/chosen": -2.9662528038024902, | |
| "logps/rejected": -9.300813674926758, | |
| "loss": -3.3757, | |
| "nll_loss": 2.572293758392334, | |
| "po_loss": -2.677783250808716, | |
| "rewards/accuracies": 0.762499988079071, | |
| "rewards/chosen": -1.4831264019012451, | |
| "rewards/margins": 3.1672801971435547, | |
| "rewards/rejected": -4.650406837463379, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 1.5594074074074074, | |
| "grad_norm": 562.9378051757812, | |
| "learning_rate": 2.7355470760292956e-05, | |
| "log_odds_chosen": 5.505670547485352, | |
| "log_odds_ratio": -0.8015021085739136, | |
| "logps/chosen": -2.046816349029541, | |
| "logps/rejected": -7.327232360839844, | |
| "loss": -4.0531, | |
| "nll_loss": 2.0913338661193848, | |
| "po_loss": -2.217992067337036, | |
| "rewards/accuracies": 0.768750011920929, | |
| "rewards/chosen": -1.0234081745147705, | |
| "rewards/margins": 2.6402080059051514, | |
| "rewards/rejected": -3.663616180419922, | |
| "step": 165 | |
| }, | |
| { | |
| "epoch": 1.6068148148148147, | |
| "grad_norm": 1370.6552734375, | |
| "learning_rate": 2.597109611334169e-05, | |
| "log_odds_chosen": 9.098596572875977, | |
| "log_odds_ratio": -1.227461576461792, | |
| "logps/chosen": -3.2510433197021484, | |
| "logps/rejected": -12.151667594909668, | |
| "loss": -13.6098, | |
| "nll_loss": 2.593132495880127, | |
| "po_loss": -3.018439769744873, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -1.6255216598510742, | |
| "rewards/margins": 4.450311660766602, | |
| "rewards/rejected": -6.075833797454834, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 1.6542222222222223, | |
| "grad_norm": 1004.2781982421875, | |
| "learning_rate": 2.458373045823404e-05, | |
| "log_odds_chosen": 6.2596330642700195, | |
| "log_odds_ratio": -0.9497385025024414, | |
| "logps/chosen": -2.4776344299316406, | |
| "logps/rejected": -8.515233039855957, | |
| "loss": -1.5757, | |
| "nll_loss": 2.2205429077148438, | |
| "po_loss": -2.2697837352752686, | |
| "rewards/accuracies": 0.7718750238418579, | |
| "rewards/chosen": -1.2388172149658203, | |
| "rewards/margins": 3.018799066543579, | |
| "rewards/rejected": -4.2576165199279785, | |
| "step": 175 | |
| }, | |
| { | |
| "epoch": 1.7016296296296296, | |
| "grad_norm": 3024.766845703125, | |
| "learning_rate": 2.3197646927086697e-05, | |
| "log_odds_chosen": 7.638102054595947, | |
| "log_odds_ratio": -1.0293067693710327, | |
| "logps/chosen": -3.2467892169952393, | |
| "logps/rejected": -10.673727989196777, | |
| "loss": -11.2032, | |
| "nll_loss": 2.4954748153686523, | |
| "po_loss": -2.845574378967285, | |
| "rewards/accuracies": 0.768750011920929, | |
| "rewards/chosen": -1.6233946084976196, | |
| "rewards/margins": 3.7134690284729004, | |
| "rewards/rejected": -5.336863994598389, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 1.749037037037037, | |
| "grad_norm": 976.9481201171875, | |
| "learning_rate": 2.1817114703032176e-05, | |
| "log_odds_chosen": 7.674588203430176, | |
| "log_odds_ratio": -1.1676676273345947, | |
| "logps/chosen": -3.3499820232391357, | |
| "logps/rejected": -10.849446296691895, | |
| "loss": -16.751, | |
| "nll_loss": 2.3611953258514404, | |
| "po_loss": -2.8846640586853027, | |
| "rewards/accuracies": 0.7593749761581421, | |
| "rewards/chosen": -1.6749910116195679, | |
| "rewards/margins": 3.749732255935669, | |
| "rewards/rejected": -5.424723148345947, | |
| "step": 185 | |
| }, | |
| { | |
| "epoch": 1.7964444444444445, | |
| "grad_norm": 1403.0980224609375, | |
| "learning_rate": 2.0446385870993467e-05, | |
| "log_odds_chosen": 8.678237915039062, | |
| "log_odds_ratio": -0.6416040062904358, | |
| "logps/chosen": -2.349724054336548, | |
| "logps/rejected": -10.788393020629883, | |
| "loss": -39.35, | |
| "nll_loss": 1.945783257484436, | |
| "po_loss": -3.175469398498535, | |
| "rewards/accuracies": 0.815625011920929, | |
| "rewards/chosen": -1.174862027168274, | |
| "rewards/margins": 4.219334602355957, | |
| "rewards/rejected": -5.394196510314941, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 1.8438518518518519, | |
| "grad_norm": 2194.176025390625, | |
| "learning_rate": 1.9089682321121834e-05, | |
| "log_odds_chosen": 11.073338508605957, | |
| "log_odds_ratio": -1.7082958221435547, | |
| "logps/chosen": -4.391510486602783, | |
| "logps/rejected": -15.270744323730469, | |
| "loss": -2.4223, | |
| "nll_loss": 3.056588649749756, | |
| "po_loss": -3.132284641265869, | |
| "rewards/accuracies": 0.7718750238418579, | |
| "rewards/chosen": -2.1957552433013916, | |
| "rewards/margins": 5.439617156982422, | |
| "rewards/rejected": -7.635372161865234, | |
| "step": 195 | |
| }, | |
| { | |
| "epoch": 1.8912592592592592, | |
| "grad_norm": 1372.590576171875, | |
| "learning_rate": 1.775118274523545e-05, | |
| "log_odds_chosen": 7.143240928649902, | |
| "log_odds_ratio": -1.3253802061080933, | |
| "logps/chosen": -3.159841537475586, | |
| "logps/rejected": -10.11519718170166, | |
| "loss": -3.8252, | |
| "nll_loss": 2.4054999351501465, | |
| "po_loss": -2.525035858154297, | |
| "rewards/accuracies": 0.715624988079071, | |
| "rewards/chosen": -1.579920768737793, | |
| "rewards/margins": 3.477677583694458, | |
| "rewards/rejected": -5.05759859085083, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 1.9386666666666668, | |
| "grad_norm": 1031.638916015625, | |
| "learning_rate": 1.643500976631037e-05, | |
| "log_odds_chosen": 5.937264442443848, | |
| "log_odds_ratio": -0.8166869878768921, | |
| "logps/chosen": -2.044642210006714, | |
| "logps/rejected": -7.761441230773926, | |
| "loss": -10.1847, | |
| "nll_loss": 1.9633842706680298, | |
| "po_loss": -2.281655788421631, | |
| "rewards/accuracies": 0.7906249761581421, | |
| "rewards/chosen": -1.022321105003357, | |
| "rewards/margins": 2.8583993911743164, | |
| "rewards/rejected": -3.880720615386963, | |
| "step": 205 | |
| }, | |
| { | |
| "epoch": 1.986074074074074, | |
| "grad_norm": 1609.1983642578125, | |
| "learning_rate": 1.514521724066537e-05, | |
| "log_odds_chosen": 7.844486236572266, | |
| "log_odds_ratio": -1.5023125410079956, | |
| "logps/chosen": -3.9588305950164795, | |
| "logps/rejected": -11.614171981811523, | |
| "loss": -3.4843, | |
| "nll_loss": 2.696570873260498, | |
| "po_loss": -2.8054568767547607, | |
| "rewards/accuracies": 0.746874988079071, | |
| "rewards/chosen": -1.9794152975082397, | |
| "rewards/margins": 3.827669858932495, | |
| "rewards/rejected": -5.807085990905762, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 2.0284444444444443, | |
| "grad_norm": 1158.2188720703125, | |
| "learning_rate": 1.3885777771950348e-05, | |
| "log_odds_chosen": 10.403746604919434, | |
| "log_odds_ratio": -1.2075212001800537, | |
| "logps/chosen": -3.1166441440582275, | |
| "logps/rejected": -13.249801635742188, | |
| "loss": -41.5962, | |
| "nll_loss": 2.3673253059387207, | |
| "po_loss": -3.821737766265869, | |
| "rewards/accuracies": 0.7867133021354675, | |
| "rewards/chosen": -1.5583220720291138, | |
| "rewards/margins": 5.066579341888428, | |
| "rewards/rejected": -6.624900817871094, | |
| "step": 215 | |
| }, | |
| { | |
| "epoch": 2.075851851851852, | |
| "grad_norm": 1111.02734375, | |
| "learning_rate": 1.2660570475395683e-05, | |
| "log_odds_chosen": 8.10044002532959, | |
| "log_odds_ratio": -1.2627264261245728, | |
| "logps/chosen": -3.747706890106201, | |
| "logps/rejected": -11.619832992553711, | |
| "loss": -19.3237, | |
| "nll_loss": 2.5686912536621094, | |
| "po_loss": -3.172555923461914, | |
| "rewards/accuracies": 0.765625, | |
| "rewards/chosen": -1.8738534450531006, | |
| "rewards/margins": 3.936063289642334, | |
| "rewards/rejected": -5.8099164962768555, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 2.1232592592592594, | |
| "grad_norm": 2720.02490234375, | |
| "learning_rate": 1.1473369030008974e-05, | |
| "log_odds_chosen": 11.025764465332031, | |
| "log_odds_ratio": -0.7701746821403503, | |
| "logps/chosen": -3.573103666305542, | |
| "logps/rejected": -14.34417724609375, | |
| "loss": -58.0462, | |
| "nll_loss": 2.4860126972198486, | |
| "po_loss": -4.299956798553467, | |
| "rewards/accuracies": 0.8374999761581421, | |
| "rewards/chosen": -1.786551833152771, | |
| "rewards/margins": 5.385537147521973, | |
| "rewards/rejected": -7.172088623046875, | |
| "step": 225 | |
| }, | |
| { | |
| "epoch": 2.1706666666666665, | |
| "grad_norm": 1029.630126953125, | |
| "learning_rate": 1.0327830055518842e-05, | |
| "log_odds_chosen": 10.32060432434082, | |
| "log_odds_ratio": -0.8015488386154175, | |
| "logps/chosen": -3.26597261428833, | |
| "logps/rejected": -13.332249641418457, | |
| "loss": -49.913, | |
| "nll_loss": 2.330512762069702, | |
| "po_loss": -3.890293836593628, | |
| "rewards/accuracies": 0.8062499761581421, | |
| "rewards/chosen": -1.632986307144165, | |
| "rewards/margins": 5.033139228820801, | |
| "rewards/rejected": -6.6661248207092285, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 2.218074074074074, | |
| "grad_norm": 1995.1678466796875, | |
| "learning_rate": 9.227481849865235e-06, | |
| "log_odds_chosen": 11.123466491699219, | |
| "log_odds_ratio": -0.6873899102210999, | |
| "logps/chosen": -2.032088041305542, | |
| "logps/rejected": -12.853292465209961, | |
| "loss": -69.5192, | |
| "nll_loss": 1.9619925022125244, | |
| "po_loss": -4.1344685554504395, | |
| "rewards/accuracies": 0.8187500238418579, | |
| "rewards/chosen": -1.016044020652771, | |
| "rewards/margins": 5.410602569580078, | |
| "rewards/rejected": -6.4266462326049805, | |
| "step": 235 | |
| }, | |
| { | |
| "epoch": 2.2654814814814817, | |
| "grad_norm": 2002.389892578125, | |
| "learning_rate": 8.175713521924978e-06, | |
| "log_odds_chosen": 12.202537536621094, | |
| "log_odds_ratio": -0.6559484004974365, | |
| "logps/chosen": -2.459198236465454, | |
| "logps/rejected": -14.35517692565918, | |
| "loss": -66.0372, | |
| "nll_loss": 2.1631743907928467, | |
| "po_loss": -4.226839065551758, | |
| "rewards/accuracies": 0.859375, | |
| "rewards/chosen": -1.229599118232727, | |
| "rewards/margins": 5.947988986968994, | |
| "rewards/rejected": -7.17758846282959, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 2.3128888888888888, | |
| "grad_norm": 1811.5279541015625, | |
| "learning_rate": 7.1757645529443665e-06, | |
| "log_odds_chosen": 12.961212158203125, | |
| "log_odds_ratio": -0.48759812116622925, | |
| "logps/chosen": -2.229421615600586, | |
| "logps/rejected": -14.900075912475586, | |
| "loss": -85.5446, | |
| "nll_loss": 1.9955413341522217, | |
| "po_loss": -4.668810844421387, | |
| "rewards/accuracies": 0.8531249761581421, | |
| "rewards/chosen": -1.114710807800293, | |
| "rewards/margins": 6.335326671600342, | |
| "rewards/rejected": -7.450037956237793, | |
| "step": 245 | |
| }, | |
| { | |
| "epoch": 2.3602962962962963, | |
| "grad_norm": 1374.3148193359375, | |
| "learning_rate": 6.230714818829733e-06, | |
| "log_odds_chosen": 13.162447929382324, | |
| "log_odds_ratio": -0.579684853553772, | |
| "logps/chosen": -2.2635672092437744, | |
| "logps/rejected": -15.118783950805664, | |
| "loss": -78.0242, | |
| "nll_loss": 1.9523729085922241, | |
| "po_loss": -4.39063024520874, | |
| "rewards/accuracies": 0.824999988079071, | |
| "rewards/chosen": -1.1317836046218872, | |
| "rewards/margins": 6.427608489990234, | |
| "rewards/rejected": -7.559391975402832, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 2.407703703703704, | |
| "grad_norm": 1782.409423828125, | |
| "learning_rate": 5.343475104027743e-06, | |
| "log_odds_chosen": 12.180627822875977, | |
| "log_odds_ratio": -0.7269414663314819, | |
| "logps/chosen": -2.7276577949523926, | |
| "logps/rejected": -14.642044067382812, | |
| "loss": -61.2515, | |
| "nll_loss": 2.155933380126953, | |
| "po_loss": -4.07004451751709, | |
| "rewards/accuracies": 0.831250011920929, | |
| "rewards/chosen": -1.3638288974761963, | |
| "rewards/margins": 5.957192897796631, | |
| "rewards/rejected": -7.321022033691406, | |
| "step": 255 | |
| }, | |
| { | |
| "epoch": 2.455111111111111, | |
| "grad_norm": 1750.225341796875, | |
| "learning_rate": 4.516778136213037e-06, | |
| "log_odds_chosen": 12.096226692199707, | |
| "log_odds_ratio": -0.6728618741035461, | |
| "logps/chosen": -2.2444777488708496, | |
| "logps/rejected": -14.059532165527344, | |
| "loss": -72.1379, | |
| "nll_loss": 1.928434133529663, | |
| "po_loss": -4.182744026184082, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -1.1222388744354248, | |
| "rewards/margins": 5.907527923583984, | |
| "rewards/rejected": -7.029766082763672, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 2.5025185185185186, | |
| "grad_norm": 1919.8836669921875, | |
| "learning_rate": 3.7531701693965554e-06, | |
| "log_odds_chosen": 11.939556121826172, | |
| "log_odds_ratio": -0.5605658888816833, | |
| "logps/chosen": -2.554964780807495, | |
| "logps/rejected": -14.228480339050293, | |
| "loss": -72.1107, | |
| "nll_loss": 2.096379518508911, | |
| "po_loss": -4.3498382568359375, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -1.2774823904037476, | |
| "rewards/margins": 5.836758136749268, | |
| "rewards/rejected": -7.1142401695251465, | |
| "step": 265 | |
| }, | |
| { | |
| "epoch": 2.549925925925926, | |
| "grad_norm": 3865.446044921875, | |
| "learning_rate": 3.055003141378948e-06, | |
| "log_odds_chosen": 13.130399703979492, | |
| "log_odds_ratio": -0.7432569265365601, | |
| "logps/chosen": -2.868731737136841, | |
| "logps/rejected": -15.705291748046875, | |
| "loss": -71.6455, | |
| "nll_loss": 2.3375840187072754, | |
| "po_loss": -4.576504230499268, | |
| "rewards/accuracies": 0.8374999761581421, | |
| "rewards/chosen": -1.4343658685684204, | |
| "rewards/margins": 6.418280124664307, | |
| "rewards/rejected": -7.8526458740234375, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 2.5973333333333333, | |
| "grad_norm": 1720.9822998046875, | |
| "learning_rate": 2.424427429704365e-06, | |
| "log_odds_chosen": 12.388298034667969, | |
| "log_odds_ratio": -0.7917383313179016, | |
| "logps/chosen": -2.539780378341675, | |
| "logps/rejected": -14.631436347961426, | |
| "loss": -74.1033, | |
| "nll_loss": 2.096407651901245, | |
| "po_loss": -4.412135124206543, | |
| "rewards/accuracies": 0.862500011920929, | |
| "rewards/chosen": -1.2698901891708374, | |
| "rewards/margins": 6.045827865600586, | |
| "rewards/rejected": -7.315718173980713, | |
| "step": 275 | |
| }, | |
| { | |
| "epoch": 2.644740740740741, | |
| "grad_norm": 1882.84033203125, | |
| "learning_rate": 1.8633852284264508e-06, | |
| "log_odds_chosen": 13.360041618347168, | |
| "log_odds_ratio": -0.7875075936317444, | |
| "logps/chosen": -2.570291757583618, | |
| "logps/rejected": -15.660143852233887, | |
| "loss": -80.8755, | |
| "nll_loss": 2.1218926906585693, | |
| "po_loss": -4.649253845214844, | |
| "rewards/accuracies": 0.840624988079071, | |
| "rewards/chosen": -1.285145878791809, | |
| "rewards/margins": 6.544926643371582, | |
| "rewards/rejected": -7.830071926116943, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 2.6921481481481484, | |
| "grad_norm": 2601.312255859375, | |
| "learning_rate": 1.3736045660864034e-06, | |
| "log_odds_chosen": 12.91968059539795, | |
| "log_odds_ratio": -0.935935378074646, | |
| "logps/chosen": -3.055326461791992, | |
| "logps/rejected": -15.716508865356445, | |
| "loss": -68.9803, | |
| "nll_loss": 2.237934112548828, | |
| "po_loss": -4.393568515777588, | |
| "rewards/accuracies": 0.846875011920929, | |
| "rewards/chosen": -1.527663230895996, | |
| "rewards/margins": 6.33059024810791, | |
| "rewards/rejected": -7.858254432678223, | |
| "step": 285 | |
| }, | |
| { | |
| "epoch": 2.7395555555555555, | |
| "grad_norm": 2271.935302734375, | |
| "learning_rate": 9.565939833279192e-07, | |
| "log_odds_chosen": 12.334630966186523, | |
| "log_odds_ratio": -0.5848469734191895, | |
| "logps/chosen": -3.0595030784606934, | |
| "logps/rejected": -15.129420280456543, | |
| "loss": -64.562, | |
| "nll_loss": 2.4212985038757324, | |
| "po_loss": -4.438861846923828, | |
| "rewards/accuracies": 0.8374999761581421, | |
| "rewards/chosen": -1.5297515392303467, | |
| "rewards/margins": 6.034958362579346, | |
| "rewards/rejected": -7.5647101402282715, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 2.786962962962963, | |
| "grad_norm": 1566.753173828125, | |
| "learning_rate": 6.136378865420872e-07, | |
| "log_odds_chosen": 11.438375473022461, | |
| "log_odds_ratio": -0.9271749258041382, | |
| "logps/chosen": -2.976278781890869, | |
| "logps/rejected": -14.156593322753906, | |
| "loss": -57.0123, | |
| "nll_loss": 2.327953577041626, | |
| "po_loss": -4.109589576721191, | |
| "rewards/accuracies": 0.8187500238418579, | |
| "rewards/chosen": -1.4881393909454346, | |
| "rewards/margins": 5.5901570320129395, | |
| "rewards/rejected": -7.078296661376953, | |
| "step": 295 | |
| }, | |
| { | |
| "epoch": 2.83437037037037, | |
| "grad_norm": 2821.74609375, | |
| "learning_rate": 3.45792591853214e-07, | |
| "log_odds_chosen": 11.852496147155762, | |
| "log_odds_ratio": -0.5676802396774292, | |
| "logps/chosen": -2.4296622276306152, | |
| "logps/rejected": -14.001643180847168, | |
| "loss": -73.6596, | |
| "nll_loss": 1.9556903839111328, | |
| "po_loss": -4.2575531005859375, | |
| "rewards/accuracies": 0.815625011920929, | |
| "rewards/chosen": -1.2148311138153076, | |
| "rewards/margins": 5.785989761352539, | |
| "rewards/rejected": -7.000821590423584, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 2.8817777777777778, | |
| "grad_norm": 2135.990966796875, | |
| "learning_rate": 1.538830716302092e-07, | |
| "log_odds_chosen": 12.745733261108398, | |
| "log_odds_ratio": -0.7597203850746155, | |
| "logps/chosen": -2.1967830657958984, | |
| "logps/rejected": -14.625683784484863, | |
| "loss": -78.7585, | |
| "nll_loss": 1.9788535833358765, | |
| "po_loss": -4.440057277679443, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -1.0983915328979492, | |
| "rewards/margins": 6.214449882507324, | |
| "rewards/rejected": -7.312841892242432, | |
| "step": 305 | |
| }, | |
| { | |
| "epoch": 2.9291851851851853, | |
| "grad_norm": 1398.578369140625, | |
| "learning_rate": 3.8500413544415025e-08, | |
| "log_odds_chosen": 13.022871017456055, | |
| "log_odds_ratio": -0.4309860169887543, | |
| "logps/chosen": -2.221106767654419, | |
| "logps/rejected": -14.942235946655273, | |
| "loss": -86.5137, | |
| "nll_loss": 1.9662892818450928, | |
| "po_loss": -4.66984224319458, | |
| "rewards/accuracies": 0.871874988079071, | |
| "rewards/chosen": -1.1105533838272095, | |
| "rewards/margins": 6.360564708709717, | |
| "rewards/rejected": -7.471117973327637, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 2.9765925925925925, | |
| "grad_norm": 1498.622314453125, | |
| "learning_rate": 0.0, | |
| "log_odds_chosen": 11.79759407043457, | |
| "log_odds_ratio": -0.305752694606781, | |
| "logps/chosen": -1.7462921142578125, | |
| "logps/rejected": -13.216029167175293, | |
| "loss": -82.7571, | |
| "nll_loss": 1.7008321285247803, | |
| "po_loss": -4.286990642547607, | |
| "rewards/accuracies": 0.859375, | |
| "rewards/chosen": -0.8731460571289062, | |
| "rewards/margins": 5.734868049621582, | |
| "rewards/rejected": -6.6080145835876465, | |
| "step": 315 | |
| }, | |
| { | |
| "epoch": 2.9765925925925925, | |
| "step": 315, | |
| "total_flos": 0.0, | |
| "train_loss": 16.270644454350546, | |
| "train_runtime": 9547.5166, | |
| "train_samples_per_second": 2.121, | |
| "train_steps_per_second": 0.033 | |
| } | |
| ], | |
| "logging_steps": 5, | |
| "max_steps": 315, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 100000, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 1, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |