| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 2.9765925925925925, | |
| "eval_steps": 500, | |
| "global_step": 315, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.047407407407407405, | |
| "grad_norm": 1199.2445068359375, | |
| "learning_rate": 7.8125e-06, | |
| "log_odds_chosen": -1.0599775314331055, | |
| "log_odds_ratio": -11.002888679504395, | |
| "logps/chosen": -22.7018985748291, | |
| "logps/rejected": -21.64215660095215, | |
| "loss": 482.2551, | |
| "nll_loss": 9.837621688842773, | |
| "po_loss": 5.232847690582275, | |
| "rewards/accuracies": 0.4781250059604645, | |
| "rewards/chosen": -11.35094928741455, | |
| "rewards/margins": -0.5298711061477661, | |
| "rewards/rejected": -10.821078300476074, | |
| "step": 5 | |
| }, | |
| { | |
| "epoch": 0.09481481481481481, | |
| "grad_norm": 597.969970703125, | |
| "learning_rate": 1.5625e-05, | |
| "log_odds_chosen": -3.1238608360290527, | |
| "log_odds_ratio": -11.037403106689453, | |
| "logps/chosen": -21.978214263916016, | |
| "logps/rejected": -18.854707717895508, | |
| "loss": 448.5505, | |
| "nll_loss": 8.742585182189941, | |
| "po_loss": 5.274618148803711, | |
| "rewards/accuracies": 0.4468750059604645, | |
| "rewards/chosen": -10.989107131958008, | |
| "rewards/margins": -1.5617557764053345, | |
| "rewards/rejected": -9.427353858947754, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.14222222222222222, | |
| "grad_norm": 562.1139526367188, | |
| "learning_rate": 2.34375e-05, | |
| "log_odds_chosen": -1.3050577640533447, | |
| "log_odds_ratio": -11.273977279663086, | |
| "logps/chosen": -21.88943862915039, | |
| "logps/rejected": -20.582889556884766, | |
| "loss": 423.8629, | |
| "nll_loss": 7.963566780090332, | |
| "po_loss": 5.2821502685546875, | |
| "rewards/accuracies": 0.515625, | |
| "rewards/chosen": -10.944719314575195, | |
| "rewards/margins": -0.6532740592956543, | |
| "rewards/rejected": -10.291444778442383, | |
| "step": 15 | |
| }, | |
| { | |
| "epoch": 0.18962962962962962, | |
| "grad_norm": 3476.161865234375, | |
| "learning_rate": 3.125e-05, | |
| "log_odds_chosen": -3.9486403465270996, | |
| "log_odds_ratio": -8.879361152648926, | |
| "logps/chosen": -16.218509674072266, | |
| "logps/rejected": -12.272605895996094, | |
| "loss": 321.7442, | |
| "nll_loss": 5.964278221130371, | |
| "po_loss": 4.09022855758667, | |
| "rewards/accuracies": 0.4312500059604645, | |
| "rewards/chosen": -8.109254837036133, | |
| "rewards/margins": -1.972952127456665, | |
| "rewards/rejected": -6.136302947998047, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.23703703703703705, | |
| "grad_norm": 253.1682586669922, | |
| "learning_rate": 3.90625e-05, | |
| "log_odds_chosen": -0.06654863059520721, | |
| "log_odds_ratio": -1.5640277862548828, | |
| "logps/chosen": -3.363724946975708, | |
| "logps/rejected": -3.2796008586883545, | |
| "loss": 89.1255, | |
| "nll_loss": 2.5614538192749023, | |
| "po_loss": 0.223719522356987, | |
| "rewards/accuracies": 0.5531250238418579, | |
| "rewards/chosen": -1.681862473487854, | |
| "rewards/margins": -0.042061954736709595, | |
| "rewards/rejected": -1.6398004293441772, | |
| "step": 25 | |
| }, | |
| { | |
| "epoch": 0.28444444444444444, | |
| "grad_norm": 201.95008850097656, | |
| "learning_rate": 4.6875e-05, | |
| "log_odds_chosen": 0.09966959804296494, | |
| "log_odds_ratio": -0.8472908735275269, | |
| "logps/chosen": -1.7985613346099854, | |
| "logps/rejected": -1.8834549188613892, | |
| "loss": 62.7189, | |
| "nll_loss": 2.001840591430664, | |
| "po_loss": -0.04187442362308502, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.8992806673049927, | |
| "rewards/margins": 0.04244700074195862, | |
| "rewards/rejected": -0.9417274594306946, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.33185185185185184, | |
| "grad_norm": 226.0881805419922, | |
| "learning_rate": 4.998613757348784e-05, | |
| "log_odds_chosen": 0.3407457172870636, | |
| "log_odds_ratio": -0.7940031886100769, | |
| "logps/chosen": -1.766862154006958, | |
| "logps/rejected": -2.0710058212280273, | |
| "loss": 56.0033, | |
| "nll_loss": 1.9001868963241577, | |
| "po_loss": -0.15008527040481567, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.883431077003479, | |
| "rewards/margins": 0.15207180380821228, | |
| "rewards/rejected": -1.0355029106140137, | |
| "step": 35 | |
| }, | |
| { | |
| "epoch": 0.37925925925925924, | |
| "grad_norm": 266.1627197265625, | |
| "learning_rate": 4.990147841143462e-05, | |
| "log_odds_chosen": 0.5402604341506958, | |
| "log_odds_ratio": -0.7131898999214172, | |
| "logps/chosen": -1.7148605585098267, | |
| "logps/rejected": -2.1804628372192383, | |
| "loss": 53.4684, | |
| "nll_loss": 1.883787751197815, | |
| "po_loss": -0.21290016174316406, | |
| "rewards/accuracies": 0.6625000238418579, | |
| "rewards/chosen": -0.8574302792549133, | |
| "rewards/margins": 0.2328011691570282, | |
| "rewards/rejected": -1.0902314186096191, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.4266666666666667, | |
| "grad_norm": 335.4718322753906, | |
| "learning_rate": 4.97401218720448e-05, | |
| "log_odds_chosen": 1.1245437860488892, | |
| "log_odds_ratio": -0.6130434274673462, | |
| "logps/chosen": -1.6581999063491821, | |
| "logps/rejected": -2.69628643989563, | |
| "loss": 45.3042, | |
| "nll_loss": 1.8416335582733154, | |
| "po_loss": -0.4258776307106018, | |
| "rewards/accuracies": 0.715624988079071, | |
| "rewards/chosen": -0.8290999531745911, | |
| "rewards/margins": 0.5190432667732239, | |
| "rewards/rejected": -1.348143219947815, | |
| "step": 45 | |
| }, | |
| { | |
| "epoch": 0.4740740740740741, | |
| "grad_norm": 170.5546875, | |
| "learning_rate": 4.9502564938797946e-05, | |
| "log_odds_chosen": 0.9557003974914551, | |
| "log_odds_ratio": -0.6759911775588989, | |
| "logps/chosen": -1.6203289031982422, | |
| "logps/rejected": -2.4938411712646484, | |
| "loss": 47.41, | |
| "nll_loss": 1.8312504291534424, | |
| "po_loss": -0.34968656301498413, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": -0.8101644515991211, | |
| "rewards/margins": 0.4367561340332031, | |
| "rewards/rejected": -1.2469205856323242, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.5214814814814814, | |
| "grad_norm": 225.52430725097656, | |
| "learning_rate": 4.918953929490768e-05, | |
| "log_odds_chosen": 1.1236488819122314, | |
| "log_odds_ratio": -0.6741882562637329, | |
| "logps/chosen": -1.6877021789550781, | |
| "logps/rejected": -2.727043390274048, | |
| "loss": 45.2223, | |
| "nll_loss": 1.8263689279556274, | |
| "po_loss": -0.41317272186279297, | |
| "rewards/accuracies": 0.6781250238418579, | |
| "rewards/chosen": -0.8438510894775391, | |
| "rewards/margins": 0.5196704864501953, | |
| "rewards/rejected": -1.363521695137024, | |
| "step": 55 | |
| }, | |
| { | |
| "epoch": 0.5688888888888889, | |
| "grad_norm": 355.39068603515625, | |
| "learning_rate": 4.88020090697132e-05, | |
| "log_odds_chosen": 1.1692771911621094, | |
| "log_odds_ratio": -0.6426931619644165, | |
| "logps/chosen": -1.5684707164764404, | |
| "logps/rejected": -2.65775728225708, | |
| "loss": 44.6871, | |
| "nll_loss": 1.8004539012908936, | |
| "po_loss": -0.40398064255714417, | |
| "rewards/accuracies": 0.653124988079071, | |
| "rewards/chosen": -0.7842353582382202, | |
| "rewards/margins": 0.544643223285675, | |
| "rewards/rejected": -1.32887864112854, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.6162962962962963, | |
| "grad_norm": 162.62863159179688, | |
| "learning_rate": 4.834116786912897e-05, | |
| "log_odds_chosen": 1.382725715637207, | |
| "log_odds_ratio": -0.705934464931488, | |
| "logps/chosen": -1.7009910345077515, | |
| "logps/rejected": -2.989063024520874, | |
| "loss": 43.7142, | |
| "nll_loss": 1.8565829992294312, | |
| "po_loss": -0.49051398038864136, | |
| "rewards/accuracies": 0.6937500238418579, | |
| "rewards/chosen": -0.8504955172538757, | |
| "rewards/margins": 0.6440361142158508, | |
| "rewards/rejected": -1.494531512260437, | |
| "step": 65 | |
| }, | |
| { | |
| "epoch": 0.6637037037037037, | |
| "grad_norm": 889.6281127929688, | |
| "learning_rate": 4.7808435099299045e-05, | |
| "log_odds_chosen": 3.606097459793091, | |
| "log_odds_ratio": -0.7058836221694946, | |
| "logps/chosen": -2.0993778705596924, | |
| "logps/rejected": -5.576085090637207, | |
| "loss": 33.5164, | |
| "nll_loss": 2.1465699672698975, | |
| "po_loss": -1.0991829633712769, | |
| "rewards/accuracies": 0.7593749761581421, | |
| "rewards/chosen": -1.0496889352798462, | |
| "rewards/margins": 1.7383534908294678, | |
| "rewards/rejected": -2.7880425453186035, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.7111111111111111, | |
| "grad_norm": 218.24794006347656, | |
| "learning_rate": 4.720545159477922e-05, | |
| "log_odds_chosen": 2.234952926635742, | |
| "log_odds_ratio": -0.8444352149963379, | |
| "logps/chosen": -1.784437894821167, | |
| "logps/rejected": -3.926750659942627, | |
| "loss": 35.6981, | |
| "nll_loss": 1.8649476766586304, | |
| "po_loss": -0.7493817806243896, | |
| "rewards/accuracies": 0.690625011920929, | |
| "rewards/chosen": -0.8922189474105835, | |
| "rewards/margins": 1.071156620979309, | |
| "rewards/rejected": -1.9633753299713135, | |
| "step": 75 | |
| }, | |
| { | |
| "epoch": 0.7585185185185185, | |
| "grad_norm": 1788.55126953125, | |
| "learning_rate": 4.653407456471222e-05, | |
| "log_odds_chosen": 1.6183449029922485, | |
| "log_odds_ratio": -0.846682071685791, | |
| "logps/chosen": -1.728458046913147, | |
| "logps/rejected": -3.26391339302063, | |
| "loss": 42.0319, | |
| "nll_loss": 1.8288921117782593, | |
| "po_loss": -0.5153962969779968, | |
| "rewards/accuracies": 0.621874988079071, | |
| "rewards/chosen": -0.8642290234565735, | |
| "rewards/margins": 0.7677275538444519, | |
| "rewards/rejected": -1.631956696510315, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.8059259259259259, | |
| "grad_norm": 608.4232788085938, | |
| "learning_rate": 4.579637187256222e-05, | |
| "log_odds_chosen": 2.1726527214050293, | |
| "log_odds_ratio": -0.8868170976638794, | |
| "logps/chosen": -2.386735200881958, | |
| "logps/rejected": -4.454265594482422, | |
| "loss": 40.3963, | |
| "nll_loss": 2.1039323806762695, | |
| "po_loss": -0.8415490388870239, | |
| "rewards/accuracies": 0.671875, | |
| "rewards/chosen": -1.193367600440979, | |
| "rewards/margins": 1.0337653160095215, | |
| "rewards/rejected": -2.227132797241211, | |
| "step": 85 | |
| }, | |
| { | |
| "epoch": 0.8533333333333334, | |
| "grad_norm": 2436.2255859375, | |
| "learning_rate": 4.499461566702685e-05, | |
| "log_odds_chosen": 3.146548271179199, | |
| "log_odds_ratio": -0.9941431283950806, | |
| "logps/chosen": -2.020655393600464, | |
| "logps/rejected": -5.0438432693481445, | |
| "loss": 36.2214, | |
| "nll_loss": 2.022157669067383, | |
| "po_loss": -0.890238881111145, | |
| "rewards/accuracies": 0.715624988079071, | |
| "rewards/chosen": -1.010327696800232, | |
| "rewards/margins": 1.5115936994552612, | |
| "rewards/rejected": -2.5219216346740723, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.9007407407407407, | |
| "grad_norm": 1364.458251953125, | |
| "learning_rate": 4.413127538374411e-05, | |
| "log_odds_chosen": 2.372877597808838, | |
| "log_odds_ratio": -0.8819985389709473, | |
| "logps/chosen": -1.8147878646850586, | |
| "logps/rejected": -4.103789806365967, | |
| "loss": 35.7507, | |
| "nll_loss": 1.7868406772613525, | |
| "po_loss": -0.6696311235427856, | |
| "rewards/accuracies": 0.640625, | |
| "rewards/chosen": -0.9073939323425293, | |
| "rewards/margins": 1.144500970840454, | |
| "rewards/rejected": -2.0518949031829834, | |
| "step": 95 | |
| }, | |
| { | |
| "epoch": 0.9481481481481482, | |
| "grad_norm": 1274.0474853515625, | |
| "learning_rate": 4.320901013934887e-05, | |
| "log_odds_chosen": 2.14143705368042, | |
| "log_odds_ratio": -0.786421000957489, | |
| "logps/chosen": -1.7358782291412354, | |
| "logps/rejected": -3.7556350231170654, | |
| "loss": 33.5542, | |
| "nll_loss": 1.854790449142456, | |
| "po_loss": -0.8062206506729126, | |
| "rewards/accuracies": 0.6968749761581421, | |
| "rewards/chosen": -0.8679391145706177, | |
| "rewards/margins": 1.0098785161972046, | |
| "rewards/rejected": -1.8778175115585327, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.9955555555555555, | |
| "grad_norm": 568.3045043945312, | |
| "learning_rate": 4.223066054130568e-05, | |
| "log_odds_chosen": 2.152684450149536, | |
| "log_odds_ratio": -0.7120019197463989, | |
| "logps/chosen": -1.5902575254440308, | |
| "logps/rejected": -3.6419010162353516, | |
| "loss": 31.711, | |
| "nll_loss": 1.7339718341827393, | |
| "po_loss": -0.7430030107498169, | |
| "rewards/accuracies": 0.6812499761581421, | |
| "rewards/chosen": -0.7951287627220154, | |
| "rewards/margins": 1.0258219242095947, | |
| "rewards/rejected": -1.8209505081176758, | |
| "step": 105 | |
| }, | |
| { | |
| "epoch": 1.037925925925926, | |
| "grad_norm": 628.1900634765625, | |
| "learning_rate": 4.1199239938743797e-05, | |
| "log_odds_chosen": 3.4075300693511963, | |
| "log_odds_ratio": -0.5813586115837097, | |
| "logps/chosen": -1.8704923391342163, | |
| "logps/rejected": -5.093435287475586, | |
| "loss": 16.708, | |
| "nll_loss": 1.6286418437957764, | |
| "po_loss": -1.0444445610046387, | |
| "rewards/accuracies": 0.7552447319030762, | |
| "rewards/chosen": -0.9352461695671082, | |
| "rewards/margins": 1.6114716529846191, | |
| "rewards/rejected": -2.546717643737793, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 1.0853333333333333, | |
| "grad_norm": 1226.1982421875, | |
| "learning_rate": 4.0117925141242174e-05, | |
| "log_odds_chosen": 3.840623378753662, | |
| "log_odds_ratio": -0.8993012309074402, | |
| "logps/chosen": -1.9903583526611328, | |
| "logps/rejected": -5.609218597412109, | |
| "loss": 14.4574, | |
| "nll_loss": 1.720370888710022, | |
| "po_loss": -1.2685784101486206, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.9951791763305664, | |
| "rewards/margins": 1.8094301223754883, | |
| "rewards/rejected": -2.8046092987060547, | |
| "step": 115 | |
| }, | |
| { | |
| "epoch": 1.1327407407407408, | |
| "grad_norm": 1661.664794921875, | |
| "learning_rate": 3.899004663415084e-05, | |
| "log_odds_chosen": 3.6831917762756348, | |
| "log_odds_ratio": -0.5217957496643066, | |
| "logps/chosen": -1.4346517324447632, | |
| "logps/rejected": -4.917321681976318, | |
| "loss": 14.9163, | |
| "nll_loss": 1.5880098342895508, | |
| "po_loss": -1.1218748092651367, | |
| "rewards/accuracies": 0.7593749761581421, | |
| "rewards/chosen": -0.7173258662223816, | |
| "rewards/margins": 1.7413349151611328, | |
| "rewards/rejected": -2.458660840988159, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 1.1801481481481482, | |
| "grad_norm": 571.31982421875, | |
| "learning_rate": 3.781907832058587e-05, | |
| "log_odds_chosen": 6.352164268493652, | |
| "log_odds_ratio": -0.6368833780288696, | |
| "logps/chosen": -1.6029341220855713, | |
| "logps/rejected": -7.754278659820557, | |
| "loss": 11.4892, | |
| "nll_loss": 1.7506815195083618, | |
| "po_loss": -1.3916442394256592, | |
| "rewards/accuracies": 0.765625, | |
| "rewards/chosen": -0.8014670610427856, | |
| "rewards/margins": 3.075672149658203, | |
| "rewards/rejected": -3.8771393299102783, | |
| "step": 125 | |
| }, | |
| { | |
| "epoch": 1.2275555555555555, | |
| "grad_norm": 831.4838256835938, | |
| "learning_rate": 3.660862682169282e-05, | |
| "log_odds_chosen": 5.919846534729004, | |
| "log_odds_ratio": -0.5129478573799133, | |
| "logps/chosen": -1.4314855337142944, | |
| "logps/rejected": -7.115793704986572, | |
| "loss": 9.806, | |
| "nll_loss": 1.6523067951202393, | |
| "po_loss": -1.3458693027496338, | |
| "rewards/accuracies": 0.7875000238418579, | |
| "rewards/chosen": -0.7157427668571472, | |
| "rewards/margins": 2.842153549194336, | |
| "rewards/rejected": -3.557896852493286, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 1.274962962962963, | |
| "grad_norm": 586.3059692382812, | |
| "learning_rate": 3.5362420368134356e-05, | |
| "log_odds_chosen": 5.599277019500732, | |
| "log_odds_ratio": -0.624258279800415, | |
| "logps/chosen": -1.7905899286270142, | |
| "logps/rejected": -7.161185264587402, | |
| "loss": 12.6168, | |
| "nll_loss": 1.760801076889038, | |
| "po_loss": -1.3665275573730469, | |
| "rewards/accuracies": 0.778124988079071, | |
| "rewards/chosen": -0.8952949643135071, | |
| "rewards/margins": 2.6852974891662598, | |
| "rewards/rejected": -3.580592632293701, | |
| "step": 135 | |
| }, | |
| { | |
| "epoch": 1.3223703703703704, | |
| "grad_norm": 880.5230102539062, | |
| "learning_rate": 3.408429731701635e-05, | |
| "log_odds_chosen": 3.740973711013794, | |
| "log_odds_ratio": -0.5780373215675354, | |
| "logps/chosen": -1.4464441537857056, | |
| "logps/rejected": -4.967782020568848, | |
| "loss": 15.2271, | |
| "nll_loss": 1.6292192935943604, | |
| "po_loss": -1.1533715724945068, | |
| "rewards/accuracies": 0.762499988079071, | |
| "rewards/chosen": -0.7232220768928528, | |
| "rewards/margins": 1.7606691122055054, | |
| "rewards/rejected": -2.483891010284424, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 1.3697777777777778, | |
| "grad_norm": 663.084228515625, | |
| "learning_rate": 3.2778194329621104e-05, | |
| "log_odds_chosen": 6.911275386810303, | |
| "log_odds_ratio": -0.8403680920600891, | |
| "logps/chosen": -1.9615440368652344, | |
| "logps/rejected": -8.633264541625977, | |
| "loss": 8.7443, | |
| "nll_loss": 1.8974125385284424, | |
| "po_loss": -1.624152421951294, | |
| "rewards/accuracies": 0.796875, | |
| "rewards/chosen": -0.9807720184326172, | |
| "rewards/margins": 3.335860013961792, | |
| "rewards/rejected": -4.316632270812988, | |
| "step": 145 | |
| }, | |
| { | |
| "epoch": 1.417185185185185, | |
| "grad_norm": 902.1053466796875, | |
| "learning_rate": 3.144813424636031e-05, | |
| "log_odds_chosen": 7.530759334564209, | |
| "log_odds_ratio": -0.5000367164611816, | |
| "logps/chosen": -1.5284488201141357, | |
| "logps/rejected": -8.809328079223633, | |
| "loss": -5.3961, | |
| "nll_loss": 1.633239984512329, | |
| "po_loss": -1.8018696308135986, | |
| "rewards/accuracies": 0.809374988079071, | |
| "rewards/chosen": -0.7642244100570679, | |
| "rewards/margins": 3.640439510345459, | |
| "rewards/rejected": -4.404664039611816, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 1.4645925925925927, | |
| "grad_norm": 641.8519897460938, | |
| "learning_rate": 3.0098213696293542e-05, | |
| "log_odds_chosen": 6.644770622253418, | |
| "log_odds_ratio": -1.1336581707000732, | |
| "logps/chosen": -2.2592227458953857, | |
| "logps/rejected": -8.666966438293457, | |
| "loss": 19.4596, | |
| "nll_loss": 2.1993162631988525, | |
| "po_loss": -1.591204047203064, | |
| "rewards/accuracies": 0.765625, | |
| "rewards/chosen": -1.1296113729476929, | |
| "rewards/margins": 3.203871488571167, | |
| "rewards/rejected": -4.3334832191467285, | |
| "step": 155 | |
| }, | |
| { | |
| "epoch": 1.512, | |
| "grad_norm": 991.56591796875, | |
| "learning_rate": 2.8732590479375165e-05, | |
| "log_odds_chosen": 7.27960205078125, | |
| "log_odds_ratio": -0.6811318397521973, | |
| "logps/chosen": -1.9771201610565186, | |
| "logps/rejected": -8.984837532043457, | |
| "loss": -4.91, | |
| "nll_loss": 1.8900007009506226, | |
| "po_loss": -2.043437957763672, | |
| "rewards/accuracies": 0.768750011920929, | |
| "rewards/chosen": -0.9885600805282593, | |
| "rewards/margins": 3.5038580894470215, | |
| "rewards/rejected": -4.4924187660217285, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 1.5594074074074074, | |
| "grad_norm": 593.0498657226562, | |
| "learning_rate": 2.7355470760292956e-05, | |
| "log_odds_chosen": 6.110047340393066, | |
| "log_odds_ratio": -0.8749097585678101, | |
| "logps/chosen": -2.053755283355713, | |
| "logps/rejected": -7.891977787017822, | |
| "loss": 7.4888, | |
| "nll_loss": 2.0199038982391357, | |
| "po_loss": -1.7858794927597046, | |
| "rewards/accuracies": 0.8062499761581421, | |
| "rewards/chosen": -1.0268776416778564, | |
| "rewards/margins": 2.9191110134124756, | |
| "rewards/rejected": -3.945988893508911, | |
| "step": 165 | |
| }, | |
| { | |
| "epoch": 1.6068148148148147, | |
| "grad_norm": 883.1884765625, | |
| "learning_rate": 2.597109611334169e-05, | |
| "log_odds_chosen": 7.358190059661865, | |
| "log_odds_ratio": -0.6941929459571838, | |
| "logps/chosen": -1.9749581813812256, | |
| "logps/rejected": -9.061105728149414, | |
| "loss": -1.458, | |
| "nll_loss": 1.907533049583435, | |
| "po_loss": -1.9530951976776123, | |
| "rewards/accuracies": 0.8125, | |
| "rewards/chosen": -0.9874790906906128, | |
| "rewards/margins": 3.5430731773376465, | |
| "rewards/rejected": -4.530552864074707, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 1.6542222222222223, | |
| "grad_norm": 776.2947387695312, | |
| "learning_rate": 2.458373045823404e-05, | |
| "log_odds_chosen": 7.158226013183594, | |
| "log_odds_ratio": -0.9282143712043762, | |
| "logps/chosen": -2.6594738960266113, | |
| "logps/rejected": -9.560735702514648, | |
| "loss": 7.5769, | |
| "nll_loss": 2.254758596420288, | |
| "po_loss": -2.0179810523986816, | |
| "rewards/accuracies": 0.784375011920929, | |
| "rewards/chosen": -1.3297369480133057, | |
| "rewards/margins": 3.4506313800811768, | |
| "rewards/rejected": -4.780367851257324, | |
| "step": 175 | |
| }, | |
| { | |
| "epoch": 1.7016296296296296, | |
| "grad_norm": 2007.3345947265625, | |
| "learning_rate": 2.3197646927086697e-05, | |
| "log_odds_chosen": 7.059978485107422, | |
| "log_odds_ratio": -0.91358882188797, | |
| "logps/chosen": -2.309866428375244, | |
| "logps/rejected": -9.11004638671875, | |
| "loss": -8.0333, | |
| "nll_loss": 1.9352056980133057, | |
| "po_loss": -2.1862475872039795, | |
| "rewards/accuracies": 0.7718750238418579, | |
| "rewards/chosen": -1.154933214187622, | |
| "rewards/margins": 3.400089979171753, | |
| "rewards/rejected": -4.555023193359375, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 1.749037037037037, | |
| "grad_norm": 846.7615356445312, | |
| "learning_rate": 2.1817114703032176e-05, | |
| "log_odds_chosen": 7.948319435119629, | |
| "log_odds_ratio": -1.0959177017211914, | |
| "logps/chosen": -2.901278495788574, | |
| "logps/rejected": -10.634871482849121, | |
| "loss": -11.2961, | |
| "nll_loss": 2.0984649658203125, | |
| "po_loss": -2.451467514038086, | |
| "rewards/accuracies": 0.7718750238418579, | |
| "rewards/chosen": -1.450639247894287, | |
| "rewards/margins": 3.8667964935302734, | |
| "rewards/rejected": -5.3174357414245605, | |
| "step": 185 | |
| }, | |
| { | |
| "epoch": 1.7964444444444445, | |
| "grad_norm": 817.7583618164062, | |
| "learning_rate": 2.0446385870993467e-05, | |
| "log_odds_chosen": 6.724404335021973, | |
| "log_odds_ratio": -0.5374380350112915, | |
| "logps/chosen": -1.7088407278060913, | |
| "logps/rejected": -8.153546333312988, | |
| "loss": -27.5871, | |
| "nll_loss": 1.6332111358642578, | |
| "po_loss": -2.495307207107544, | |
| "rewards/accuracies": 0.796875, | |
| "rewards/chosen": -0.8544203639030457, | |
| "rewards/margins": 3.2223522663116455, | |
| "rewards/rejected": -4.076773166656494, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 1.8438518518518519, | |
| "grad_norm": 2679.26513671875, | |
| "learning_rate": 1.9089682321121834e-05, | |
| "log_odds_chosen": 9.504817008972168, | |
| "log_odds_ratio": -1.4054466485977173, | |
| "logps/chosen": -4.4236674308776855, | |
| "logps/rejected": -13.693811416625977, | |
| "loss": 20.7869, | |
| "nll_loss": 3.1105668544769287, | |
| "po_loss": -2.460975170135498, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": -2.2118337154388428, | |
| "rewards/margins": 4.635071754455566, | |
| "rewards/rejected": -6.846905708312988, | |
| "step": 195 | |
| }, | |
| { | |
| "epoch": 1.8912592592592592, | |
| "grad_norm": 896.3246459960938, | |
| "learning_rate": 1.775118274523545e-05, | |
| "log_odds_chosen": 6.952999114990234, | |
| "log_odds_ratio": -1.0420355796813965, | |
| "logps/chosen": -2.362539768218994, | |
| "logps/rejected": -9.074200630187988, | |
| "loss": -3.5533, | |
| "nll_loss": 2.0847651958465576, | |
| "po_loss": -2.1958060264587402, | |
| "rewards/accuracies": 0.753125011920929, | |
| "rewards/chosen": -1.181269884109497, | |
| "rewards/margins": 3.355830430984497, | |
| "rewards/rejected": -4.537100315093994, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 1.9386666666666668, | |
| "grad_norm": 602.3760986328125, | |
| "learning_rate": 1.643500976631037e-05, | |
| "log_odds_chosen": 4.505127429962158, | |
| "log_odds_ratio": -0.7301878333091736, | |
| "logps/chosen": -1.551640272140503, | |
| "logps/rejected": -5.79341983795166, | |
| "loss": 2.1605, | |
| "nll_loss": 1.6449111700057983, | |
| "po_loss": -1.5773966312408447, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": -0.7758201360702515, | |
| "rewards/margins": 2.120889663696289, | |
| "rewards/rejected": -2.89670991897583, | |
| "step": 205 | |
| }, | |
| { | |
| "epoch": 1.986074074074074, | |
| "grad_norm": 831.9571533203125, | |
| "learning_rate": 1.514521724066537e-05, | |
| "log_odds_chosen": 4.619518280029297, | |
| "log_odds_ratio": -0.8430620431900024, | |
| "logps/chosen": -1.8811943531036377, | |
| "logps/rejected": -6.274863243103027, | |
| "loss": 7.1653, | |
| "nll_loss": 1.774757742881775, | |
| "po_loss": -1.5508434772491455, | |
| "rewards/accuracies": 0.746874988079071, | |
| "rewards/chosen": -0.9405971765518188, | |
| "rewards/margins": 2.1968345642089844, | |
| "rewards/rejected": -3.1374316215515137, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 2.0284444444444443, | |
| "grad_norm": 1437.5135498046875, | |
| "learning_rate": 1.3885777771950348e-05, | |
| "log_odds_chosen": 6.645725727081299, | |
| "log_odds_ratio": -0.6343217492103577, | |
| "logps/chosen": -1.605512022972107, | |
| "logps/rejected": -7.886458396911621, | |
| "loss": -23.0718, | |
| "nll_loss": 1.6514700651168823, | |
| "po_loss": -2.4581761360168457, | |
| "rewards/accuracies": 0.8216783404350281, | |
| "rewards/chosen": -0.8027560114860535, | |
| "rewards/margins": 3.1404736042022705, | |
| "rewards/rejected": -3.9432291984558105, | |
| "step": 215 | |
| }, | |
| { | |
| "epoch": 2.075851851851852, | |
| "grad_norm": 878.4225463867188, | |
| "learning_rate": 1.2660570475395683e-05, | |
| "log_odds_chosen": 7.114099025726318, | |
| "log_odds_ratio": -0.7919400930404663, | |
| "logps/chosen": -1.9719266891479492, | |
| "logps/rejected": -8.785611152648926, | |
| "loss": -21.0157, | |
| "nll_loss": 1.8605375289916992, | |
| "po_loss": -2.517279624938965, | |
| "rewards/accuracies": 0.8031250238418579, | |
| "rewards/chosen": -0.9859633445739746, | |
| "rewards/margins": 3.4068424701690674, | |
| "rewards/rejected": -4.392805576324463, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 2.1232592592592594, | |
| "grad_norm": 1384.3018798828125, | |
| "learning_rate": 1.1473369030008974e-05, | |
| "log_odds_chosen": 9.657078742980957, | |
| "log_odds_ratio": -0.717284083366394, | |
| "logps/chosen": -2.1501874923706055, | |
| "logps/rejected": -11.44202995300293, | |
| "loss": -49.1628, | |
| "nll_loss": 1.9243361949920654, | |
| "po_loss": -3.460672378540039, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -1.0750937461853027, | |
| "rewards/margins": 4.645921230316162, | |
| "rewards/rejected": -5.721014976501465, | |
| "step": 225 | |
| }, | |
| { | |
| "epoch": 2.1706666666666665, | |
| "grad_norm": 1059.046142578125, | |
| "learning_rate": 1.0327830055518842e-05, | |
| "log_odds_chosen": 9.652680397033691, | |
| "log_odds_ratio": -0.5430900454521179, | |
| "logps/chosen": -1.8017524480819702, | |
| "logps/rejected": -11.105250358581543, | |
| "loss": -54.9112, | |
| "nll_loss": 1.7806711196899414, | |
| "po_loss": -3.496647596359253, | |
| "rewards/accuracies": 0.8343750238418579, | |
| "rewards/chosen": -0.9008762240409851, | |
| "rewards/margins": 4.651749610900879, | |
| "rewards/rejected": -5.5526251792907715, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 2.218074074074074, | |
| "grad_norm": 934.3763427734375, | |
| "learning_rate": 9.227481849865235e-06, | |
| "log_odds_chosen": 8.37989616394043, | |
| "log_odds_ratio": -0.548344075679779, | |
| "logps/chosen": -1.6562755107879639, | |
| "logps/rejected": -9.679137229919434, | |
| "loss": -44.0365, | |
| "nll_loss": 1.69879949092865, | |
| "po_loss": -3.074941635131836, | |
| "rewards/accuracies": 0.831250011920929, | |
| "rewards/chosen": -0.8281377553939819, | |
| "rewards/margins": 4.011431694030762, | |
| "rewards/rejected": -4.839568614959717, | |
| "step": 235 | |
| }, | |
| { | |
| "epoch": 2.2654814814814817, | |
| "grad_norm": 1470.9927978515625, | |
| "learning_rate": 8.175713521924978e-06, | |
| "log_odds_chosen": 9.9420166015625, | |
| "log_odds_ratio": -0.5666526556015015, | |
| "logps/chosen": -1.586470365524292, | |
| "logps/rejected": -11.144021987915039, | |
| "loss": -54.6849, | |
| "nll_loss": 1.6270078420639038, | |
| "po_loss": -3.335911989212036, | |
| "rewards/accuracies": 0.8812500238418579, | |
| "rewards/chosen": -0.793235182762146, | |
| "rewards/margins": 4.778776168823242, | |
| "rewards/rejected": -5.5720109939575195, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 2.3128888888888888, | |
| "grad_norm": 2696.906982421875, | |
| "learning_rate": 7.1757645529443665e-06, | |
| "log_odds_chosen": 11.02066421508789, | |
| "log_odds_ratio": -0.523024320602417, | |
| "logps/chosen": -1.5870704650878906, | |
| "logps/rejected": -12.236452102661133, | |
| "loss": -57.4956, | |
| "nll_loss": 1.6896440982818604, | |
| "po_loss": -3.4863803386688232, | |
| "rewards/accuracies": 0.862500011920929, | |
| "rewards/chosen": -0.7935352325439453, | |
| "rewards/margins": 5.324690341949463, | |
| "rewards/rejected": -6.118226051330566, | |
| "step": 245 | |
| }, | |
| { | |
| "epoch": 2.3602962962962963, | |
| "grad_norm": 2218.82373046875, | |
| "learning_rate": 6.230714818829733e-06, | |
| "log_odds_chosen": 12.59239673614502, | |
| "log_odds_ratio": -0.5458945035934448, | |
| "logps/chosen": -2.0517947673797607, | |
| "logps/rejected": -14.282119750976562, | |
| "loss": -61.3005, | |
| "nll_loss": 1.7380138635635376, | |
| "po_loss": -3.6536529064178467, | |
| "rewards/accuracies": 0.824999988079071, | |
| "rewards/chosen": -1.0258973836898804, | |
| "rewards/margins": 6.115162372589111, | |
| "rewards/rejected": -7.141059875488281, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 2.407703703703704, | |
| "grad_norm": 3879.0703125, | |
| "learning_rate": 5.343475104027743e-06, | |
| "log_odds_chosen": 13.796175956726074, | |
| "log_odds_ratio": -0.7049432992935181, | |
| "logps/chosen": -2.48822283744812, | |
| "logps/rejected": -15.938923835754395, | |
| "loss": -54.8681, | |
| "nll_loss": 2.094301700592041, | |
| "po_loss": -3.8089308738708496, | |
| "rewards/accuracies": 0.878125011920929, | |
| "rewards/chosen": -1.24411141872406, | |
| "rewards/margins": 6.725350379943848, | |
| "rewards/rejected": -7.969461917877197, | |
| "step": 255 | |
| }, | |
| { | |
| "epoch": 2.455111111111111, | |
| "grad_norm": 2860.3720703125, | |
| "learning_rate": 4.516778136213037e-06, | |
| "log_odds_chosen": 12.339117050170898, | |
| "log_odds_ratio": -0.8312928080558777, | |
| "logps/chosen": -2.22261381149292, | |
| "logps/rejected": -14.213083267211914, | |
| "loss": -51.4931, | |
| "nll_loss": 2.0115973949432373, | |
| "po_loss": -3.6207566261291504, | |
| "rewards/accuracies": 0.859375, | |
| "rewards/chosen": -1.11130690574646, | |
| "rewards/margins": 5.995234489440918, | |
| "rewards/rejected": -7.106541633605957, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 2.5025185185185186, | |
| "grad_norm": 1744.2803955078125, | |
| "learning_rate": 3.7531701693965554e-06, | |
| "log_odds_chosen": 10.812795639038086, | |
| "log_odds_ratio": -0.5689637064933777, | |
| "logps/chosen": -1.7130670547485352, | |
| "logps/rejected": -12.172760009765625, | |
| "loss": -62.0607, | |
| "nll_loss": 1.6393451690673828, | |
| "po_loss": -3.5787429809570312, | |
| "rewards/accuracies": 0.840624988079071, | |
| "rewards/chosen": -0.8565335273742676, | |
| "rewards/margins": 5.229846000671387, | |
| "rewards/rejected": -6.0863800048828125, | |
| "step": 265 | |
| }, | |
| { | |
| "epoch": 2.549925925925926, | |
| "grad_norm": 1753.0069580078125, | |
| "learning_rate": 3.055003141378948e-06, | |
| "log_odds_chosen": 11.131937026977539, | |
| "log_odds_ratio": -0.6443347930908203, | |
| "logps/chosen": -1.8285239934921265, | |
| "logps/rejected": -12.587448120117188, | |
| "loss": -64.4668, | |
| "nll_loss": 1.797912836074829, | |
| "po_loss": -3.8124988079071045, | |
| "rewards/accuracies": 0.8531249761581421, | |
| "rewards/chosen": -0.9142619967460632, | |
| "rewards/margins": 5.379462242126465, | |
| "rewards/rejected": -6.293724060058594, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 2.5973333333333333, | |
| "grad_norm": 2948.94775390625, | |
| "learning_rate": 2.424427429704365e-06, | |
| "log_odds_chosen": 10.50131893157959, | |
| "log_odds_ratio": -0.5303918123245239, | |
| "logps/chosen": -1.9819530248641968, | |
| "logps/rejected": -12.131158828735352, | |
| "loss": -54.0842, | |
| "nll_loss": 1.8271331787109375, | |
| "po_loss": -3.5172641277313232, | |
| "rewards/accuracies": 0.878125011920929, | |
| "rewards/chosen": -0.9909765124320984, | |
| "rewards/margins": 5.0746026039123535, | |
| "rewards/rejected": -6.065579414367676, | |
| "step": 275 | |
| }, | |
| { | |
| "epoch": 2.644740740740741, | |
| "grad_norm": 2592.339111328125, | |
| "learning_rate": 1.8633852284264508e-06, | |
| "log_odds_chosen": 12.00327205657959, | |
| "log_odds_ratio": -0.676723837852478, | |
| "logps/chosen": -1.9051532745361328, | |
| "logps/rejected": -13.546533584594727, | |
| "loss": -61.6846, | |
| "nll_loss": 1.8986514806747437, | |
| "po_loss": -3.826294422149658, | |
| "rewards/accuracies": 0.824999988079071, | |
| "rewards/chosen": -0.9525766372680664, | |
| "rewards/margins": 5.820690155029297, | |
| "rewards/rejected": -6.773266792297363, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 2.6921481481481484, | |
| "grad_norm": 2699.58544921875, | |
| "learning_rate": 1.3736045660864034e-06, | |
| "log_odds_chosen": 11.029436111450195, | |
| "log_odds_ratio": -0.5692117810249329, | |
| "logps/chosen": -1.7028005123138428, | |
| "logps/rejected": -12.390884399414062, | |
| "loss": -66.3219, | |
| "nll_loss": 1.6979774236679077, | |
| "po_loss": -3.770538330078125, | |
| "rewards/accuracies": 0.859375, | |
| "rewards/chosen": -0.8514002561569214, | |
| "rewards/margins": 5.344042778015137, | |
| "rewards/rejected": -6.195442199707031, | |
| "step": 285 | |
| }, | |
| { | |
| "epoch": 2.7395555555555555, | |
| "grad_norm": 2166.333984375, | |
| "learning_rate": 9.565939833279192e-07, | |
| "log_odds_chosen": 10.880716323852539, | |
| "log_odds_ratio": -0.7237260341644287, | |
| "logps/chosen": -2.316985607147217, | |
| "logps/rejected": -12.853070259094238, | |
| "loss": -52.5932, | |
| "nll_loss": 1.9568628072738647, | |
| "po_loss": -3.60040020942688, | |
| "rewards/accuracies": 0.8374999761581421, | |
| "rewards/chosen": -1.1584928035736084, | |
| "rewards/margins": 5.26804256439209, | |
| "rewards/rejected": -6.426535129547119, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 2.786962962962963, | |
| "grad_norm": 2485.855712890625, | |
| "learning_rate": 6.136378865420872e-07, | |
| "log_odds_chosen": 10.605603218078613, | |
| "log_odds_ratio": -0.8477842211723328, | |
| "logps/chosen": -2.2415826320648193, | |
| "logps/rejected": -12.530155181884766, | |
| "loss": -47.1903, | |
| "nll_loss": 1.9998095035552979, | |
| "po_loss": -3.474505662918091, | |
| "rewards/accuracies": 0.7906249761581421, | |
| "rewards/chosen": -1.1207913160324097, | |
| "rewards/margins": 5.144285678863525, | |
| "rewards/rejected": -6.265077590942383, | |
| "step": 295 | |
| }, | |
| { | |
| "epoch": 2.83437037037037, | |
| "grad_norm": 2430.19970703125, | |
| "learning_rate": 3.45792591853214e-07, | |
| "log_odds_chosen": 11.173027992248535, | |
| "log_odds_ratio": -0.5032259821891785, | |
| "logps/chosen": -1.4856630563735962, | |
| "logps/rejected": -12.28561019897461, | |
| "loss": -71.7924, | |
| "nll_loss": 1.597579002380371, | |
| "po_loss": -3.8410916328430176, | |
| "rewards/accuracies": 0.828125, | |
| "rewards/chosen": -0.7428315281867981, | |
| "rewards/margins": 5.399973392486572, | |
| "rewards/rejected": -6.142805099487305, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 2.8817777777777778, | |
| "grad_norm": 1150.9241943359375, | |
| "learning_rate": 1.538830716302092e-07, | |
| "log_odds_chosen": 11.992634773254395, | |
| "log_odds_ratio": -0.6470831632614136, | |
| "logps/chosen": -1.8577916622161865, | |
| "logps/rejected": -13.447807312011719, | |
| "loss": -69.0062, | |
| "nll_loss": 1.8080154657363892, | |
| "po_loss": -3.9644579887390137, | |
| "rewards/accuracies": 0.828125, | |
| "rewards/chosen": -0.9288958311080933, | |
| "rewards/margins": 5.795008182525635, | |
| "rewards/rejected": -6.723903656005859, | |
| "step": 305 | |
| }, | |
| { | |
| "epoch": 2.9291851851851853, | |
| "grad_norm": 1115.533447265625, | |
| "learning_rate": 3.8500413544415025e-08, | |
| "log_odds_chosen": 11.34982967376709, | |
| "log_odds_ratio": -0.5919870734214783, | |
| "logps/chosen": -1.6738865375518799, | |
| "logps/rejected": -12.641853332519531, | |
| "loss": -65.5454, | |
| "nll_loss": 1.8010927438735962, | |
| "po_loss": -3.849386692047119, | |
| "rewards/accuracies": 0.8656250238418579, | |
| "rewards/chosen": -0.8369432687759399, | |
| "rewards/margins": 5.483983993530273, | |
| "rewards/rejected": -6.320926666259766, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 2.9765925925925925, | |
| "grad_norm": 2356.891845703125, | |
| "learning_rate": 0.0, | |
| "log_odds_chosen": 11.529512405395508, | |
| "log_odds_ratio": -0.520513653755188, | |
| "logps/chosen": -1.613694429397583, | |
| "logps/rejected": -12.730021476745605, | |
| "loss": -73.7153, | |
| "nll_loss": 1.6499922275543213, | |
| "po_loss": -3.9535961151123047, | |
| "rewards/accuracies": 0.840624988079071, | |
| "rewards/chosen": -0.8068472146987915, | |
| "rewards/margins": 5.558164119720459, | |
| "rewards/rejected": -6.365010738372803, | |
| "step": 315 | |
| }, | |
| { | |
| "epoch": 2.9765925925925925, | |
| "step": 315, | |
| "total_flos": 0.0, | |
| "train_loss": 22.20340297638424, | |
| "train_runtime": 9567.4219, | |
| "train_samples_per_second": 2.117, | |
| "train_steps_per_second": 0.033 | |
| } | |
| ], | |
| "logging_steps": 5, | |
| "max_steps": 315, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 100000, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 1, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |