| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 2.9765925925925925, | |
| "eval_steps": 500, | |
| "global_step": 315, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.047407407407407405, | |
| "grad_norm": 1138.854736328125, | |
| "learning_rate": 7.8125e-06, | |
| "log_odds_chosen": -1.0697920322418213, | |
| "log_odds_ratio": -11.011500358581543, | |
| "logps/chosen": -22.719226837158203, | |
| "logps/rejected": -21.649660110473633, | |
| "loss": 450.84, | |
| "nll_loss": 9.846151351928711, | |
| "po_loss": 4.242598533630371, | |
| "rewards/accuracies": 0.48124998807907104, | |
| "rewards/chosen": -11.359613418579102, | |
| "rewards/margins": -0.5347837209701538, | |
| "rewards/rejected": -10.824830055236816, | |
| "step": 5 | |
| }, | |
| { | |
| "epoch": 0.09481481481481481, | |
| "grad_norm": 597.58203125, | |
| "learning_rate": 1.5625e-05, | |
| "log_odds_chosen": -3.116546392440796, | |
| "log_odds_ratio": -11.056388854980469, | |
| "logps/chosen": -22.023601531982422, | |
| "logps/rejected": -18.90738868713379, | |
| "loss": 421.2396, | |
| "nll_loss": 8.764582633972168, | |
| "po_loss": 4.39915132522583, | |
| "rewards/accuracies": 0.44999998807907104, | |
| "rewards/chosen": -11.011800765991211, | |
| "rewards/margins": -1.5581079721450806, | |
| "rewards/rejected": -9.453694343566895, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.14222222222222222, | |
| "grad_norm": 462.572265625, | |
| "learning_rate": 2.34375e-05, | |
| "log_odds_chosen": -1.2942144870758057, | |
| "log_odds_ratio": -11.329824447631836, | |
| "logps/chosen": -22.003665924072266, | |
| "logps/rejected": -20.707988739013672, | |
| "loss": 393.115, | |
| "nll_loss": 8.009772300720215, | |
| "po_loss": 4.2750701904296875, | |
| "rewards/accuracies": 0.515625, | |
| "rewards/chosen": -11.001832962036133, | |
| "rewards/margins": -0.6478384137153625, | |
| "rewards/rejected": -10.353994369506836, | |
| "step": 15 | |
| }, | |
| { | |
| "epoch": 0.18962962962962962, | |
| "grad_norm": 3452.147705078125, | |
| "learning_rate": 3.125e-05, | |
| "log_odds_chosen": -4.19545316696167, | |
| "log_odds_ratio": -9.310150146484375, | |
| "logps/chosen": -16.894468307495117, | |
| "logps/rejected": -12.701272964477539, | |
| "loss": 308.1191, | |
| "nll_loss": 6.174555778503418, | |
| "po_loss": 3.454167127609253, | |
| "rewards/accuracies": 0.4281249940395355, | |
| "rewards/chosen": -8.447234153747559, | |
| "rewards/margins": -2.0965964794158936, | |
| "rewards/rejected": -6.3506364822387695, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.23703703703703705, | |
| "grad_norm": 254.29014587402344, | |
| "learning_rate": 3.90625e-05, | |
| "log_odds_chosen": -0.08858927339315414, | |
| "log_odds_ratio": -1.799390435218811, | |
| "logps/chosen": -3.9111340045928955, | |
| "logps/rejected": -3.804535388946533, | |
| "loss": 90.7617, | |
| "nll_loss": 2.6883437633514404, | |
| "po_loss": 0.14795991778373718, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -1.9555670022964478, | |
| "rewards/margins": -0.05329931899905205, | |
| "rewards/rejected": -1.9022676944732666, | |
| "step": 25 | |
| }, | |
| { | |
| "epoch": 0.28444444444444444, | |
| "grad_norm": 269.8670349121094, | |
| "learning_rate": 4.6875e-05, | |
| "log_odds_chosen": 0.05695251747965813, | |
| "log_odds_ratio": -0.9221351742744446, | |
| "logps/chosen": -1.9418389797210693, | |
| "logps/rejected": -1.9890735149383545, | |
| "loss": 65.438, | |
| "nll_loss": 2.0685558319091797, | |
| "po_loss": -0.023617293685674667, | |
| "rewards/accuracies": 0.5406249761581421, | |
| "rewards/chosen": -0.9709194898605347, | |
| "rewards/margins": 0.023617293685674667, | |
| "rewards/rejected": -0.9945367574691772, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.33185185185185184, | |
| "grad_norm": 265.45452880859375, | |
| "learning_rate": 4.998613757348784e-05, | |
| "log_odds_chosen": 0.3491572439670563, | |
| "log_odds_ratio": -0.8370898962020874, | |
| "logps/chosen": -1.8574031591415405, | |
| "logps/rejected": -2.1753742694854736, | |
| "loss": 57.9215, | |
| "nll_loss": 1.967643141746521, | |
| "po_loss": -0.1575944572687149, | |
| "rewards/accuracies": 0.5406249761581421, | |
| "rewards/chosen": -0.9287015795707703, | |
| "rewards/margins": 0.15898552536964417, | |
| "rewards/rejected": -1.0876871347427368, | |
| "step": 35 | |
| }, | |
| { | |
| "epoch": 0.37925925925925924, | |
| "grad_norm": 194.72329711914062, | |
| "learning_rate": 4.990147841143462e-05, | |
| "log_odds_chosen": 0.4601282477378845, | |
| "log_odds_ratio": -0.6781095266342163, | |
| "logps/chosen": -1.6383177042007446, | |
| "logps/rejected": -2.0322861671447754, | |
| "loss": 54.2986, | |
| "nll_loss": 1.8938146829605103, | |
| "po_loss": -0.1969841569662094, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -0.8191588521003723, | |
| "rewards/margins": 0.1969841569662094, | |
| "rewards/rejected": -1.0161430835723877, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.4266666666666667, | |
| "grad_norm": 421.30731201171875, | |
| "learning_rate": 4.97401218720448e-05, | |
| "log_odds_chosen": 1.2004860639572144, | |
| "log_odds_ratio": -0.6465462446212769, | |
| "logps/chosen": -1.7445472478866577, | |
| "logps/rejected": -2.871616840362549, | |
| "loss": 42.8079, | |
| "nll_loss": 1.8902437686920166, | |
| "po_loss": -0.5524969100952148, | |
| "rewards/accuracies": 0.690625011920929, | |
| "rewards/chosen": -0.8722736239433289, | |
| "rewards/margins": 0.5635346174240112, | |
| "rewards/rejected": -1.4358084201812744, | |
| "step": 45 | |
| }, | |
| { | |
| "epoch": 0.4740740740740741, | |
| "grad_norm": 850.62548828125, | |
| "learning_rate": 4.9502564938797946e-05, | |
| "log_odds_chosen": 2.4571447372436523, | |
| "log_odds_ratio": -1.3941503763198853, | |
| "logps/chosen": -3.2160325050354004, | |
| "logps/rejected": -5.586895942687988, | |
| "loss": 68.2657, | |
| "nll_loss": 3.1136634349823, | |
| "po_loss": -0.9803593754768372, | |
| "rewards/accuracies": 0.706250011920929, | |
| "rewards/chosen": -1.6080162525177002, | |
| "rewards/margins": 1.1854315996170044, | |
| "rewards/rejected": -2.793447971343994, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.5214814814814814, | |
| "grad_norm": 3458.772216796875, | |
| "learning_rate": 4.918953929490768e-05, | |
| "log_odds_chosen": 3.8432812690734863, | |
| "log_odds_ratio": -0.9803560972213745, | |
| "logps/chosen": -2.737734317779541, | |
| "logps/rejected": -6.494544982910156, | |
| "loss": 47.9796, | |
| "nll_loss": 2.612356662750244, | |
| "po_loss": -1.1129950284957886, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": -1.3688671588897705, | |
| "rewards/margins": 1.8784055709838867, | |
| "rewards/rejected": -3.247272491455078, | |
| "step": 55 | |
| }, | |
| { | |
| "epoch": 0.5688888888888889, | |
| "grad_norm": 689.9036254882812, | |
| "learning_rate": 4.88020090697132e-05, | |
| "log_odds_chosen": 4.405730724334717, | |
| "log_odds_ratio": -0.9279032945632935, | |
| "logps/chosen": -2.4398763179779053, | |
| "logps/rejected": -6.7704877853393555, | |
| "loss": 34.5555, | |
| "nll_loss": 2.4739956855773926, | |
| "po_loss": -1.3941353559494019, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -1.2199381589889526, | |
| "rewards/margins": 2.1653053760528564, | |
| "rewards/rejected": -3.3852438926696777, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.6162962962962963, | |
| "grad_norm": 627.5538330078125, | |
| "learning_rate": 4.834116786912897e-05, | |
| "log_odds_chosen": 3.3358917236328125, | |
| "log_odds_ratio": -1.2476164102554321, | |
| "logps/chosen": -3.2438666820526123, | |
| "logps/rejected": -6.503542423248291, | |
| "loss": 59.5781, | |
| "nll_loss": 3.0055854320526123, | |
| "po_loss": -1.143770456314087, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": -1.6219333410263062, | |
| "rewards/margins": 1.6298377513885498, | |
| "rewards/rejected": -3.2517712116241455, | |
| "step": 65 | |
| }, | |
| { | |
| "epoch": 0.6637037037037037, | |
| "grad_norm": 1038.5926513671875, | |
| "learning_rate": 4.7808435099299045e-05, | |
| "log_odds_chosen": 5.5194196701049805, | |
| "log_odds_ratio": -1.1829580068588257, | |
| "logps/chosen": -3.315161943435669, | |
| "logps/rejected": -8.737940788269043, | |
| "loss": 18.1436, | |
| "nll_loss": 2.795347213745117, | |
| "po_loss": -2.228358030319214, | |
| "rewards/accuracies": 0.7437499761581421, | |
| "rewards/chosen": -1.6575809717178345, | |
| "rewards/margins": 2.7113900184631348, | |
| "rewards/rejected": -4.3689703941345215, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.7111111111111111, | |
| "grad_norm": 957.5834350585938, | |
| "learning_rate": 4.720545159477922e-05, | |
| "log_odds_chosen": 3.6360058784484863, | |
| "log_odds_ratio": -1.2177813053131104, | |
| "logps/chosen": -2.6733596324920654, | |
| "logps/rejected": -6.228394031524658, | |
| "loss": 39.0704, | |
| "nll_loss": 2.3507373332977295, | |
| "po_loss": -1.1297887563705444, | |
| "rewards/accuracies": 0.6812499761581421, | |
| "rewards/chosen": -1.3366798162460327, | |
| "rewards/margins": 1.7775170803070068, | |
| "rewards/rejected": -3.114197015762329, | |
| "step": 75 | |
| }, | |
| { | |
| "epoch": 0.7585185185185185, | |
| "grad_norm": 391.593505859375, | |
| "learning_rate": 4.653407456471222e-05, | |
| "log_odds_chosen": 4.495142459869385, | |
| "log_odds_ratio": -1.3136035203933716, | |
| "logps/chosen": -2.7727952003479004, | |
| "logps/rejected": -7.165794372558594, | |
| "loss": 31.409, | |
| "nll_loss": 2.5395021438598633, | |
| "po_loss": -1.5579702854156494, | |
| "rewards/accuracies": 0.628125011920929, | |
| "rewards/chosen": -1.3863976001739502, | |
| "rewards/margins": 2.196500062942505, | |
| "rewards/rejected": -3.582897186279297, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.8059259259259259, | |
| "grad_norm": 2327.972900390625, | |
| "learning_rate": 4.579637187256222e-05, | |
| "log_odds_chosen": 1.418734073638916, | |
| "log_odds_ratio": -0.8357670903205872, | |
| "logps/chosen": -1.7513229846954346, | |
| "logps/rejected": -3.0980544090270996, | |
| "loss": 41.072, | |
| "nll_loss": 1.9310455322265625, | |
| "po_loss": -0.6475440859794617, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.8756614923477173, | |
| "rewards/margins": 0.6733657121658325, | |
| "rewards/rejected": -1.5490272045135498, | |
| "step": 85 | |
| }, | |
| { | |
| "epoch": 0.8533333333333334, | |
| "grad_norm": 778.7096557617188, | |
| "learning_rate": 4.499461566702685e-05, | |
| "log_odds_chosen": 4.9186859130859375, | |
| "log_odds_ratio": -0.8927431106567383, | |
| "logps/chosen": -3.104053497314453, | |
| "logps/rejected": -7.928048610687256, | |
| "loss": 26.255, | |
| "nll_loss": 2.6208713054656982, | |
| "po_loss": -1.8004029989242554, | |
| "rewards/accuracies": 0.706250011920929, | |
| "rewards/chosen": -1.5520267486572266, | |
| "rewards/margins": 2.4119973182678223, | |
| "rewards/rejected": -3.964024305343628, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.9007407407407407, | |
| "grad_norm": 1113.090576171875, | |
| "learning_rate": 4.413127538374411e-05, | |
| "log_odds_chosen": 4.415733337402344, | |
| "log_odds_ratio": -1.0066527128219604, | |
| "logps/chosen": -2.505484104156494, | |
| "logps/rejected": -6.809667110443115, | |
| "loss": 15.2194, | |
| "nll_loss": 2.1575310230255127, | |
| "po_loss": -1.6819244623184204, | |
| "rewards/accuracies": 0.6937500238418579, | |
| "rewards/chosen": -1.252742052078247, | |
| "rewards/margins": 2.1520915031433105, | |
| "rewards/rejected": -3.4048335552215576, | |
| "step": 95 | |
| }, | |
| { | |
| "epoch": 0.9481481481481482, | |
| "grad_norm": 5262.9931640625, | |
| "learning_rate": 4.320901013934887e-05, | |
| "log_odds_chosen": 4.908814430236816, | |
| "log_odds_ratio": -1.2427480220794678, | |
| "logps/chosen": -2.8461787700653076, | |
| "logps/rejected": -7.642964839935303, | |
| "loss": 17.6349, | |
| "nll_loss": 2.5891518592834473, | |
| "po_loss": -2.0380616188049316, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -1.4230893850326538, | |
| "rewards/margins": 2.398393154144287, | |
| "rewards/rejected": -3.8214824199676514, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.9955555555555555, | |
| "grad_norm": 465.037841796875, | |
| "learning_rate": 4.223066054130568e-05, | |
| "log_odds_chosen": 2.4661428928375244, | |
| "log_odds_ratio": -0.8429304957389832, | |
| "logps/chosen": -1.9539477825164795, | |
| "logps/rejected": -4.3218865394592285, | |
| "loss": 27.8368, | |
| "nll_loss": 2.0108437538146973, | |
| "po_loss": -1.1409438848495483, | |
| "rewards/accuracies": 0.690625011920929, | |
| "rewards/chosen": -0.9769738912582397, | |
| "rewards/margins": 1.1839690208435059, | |
| "rewards/rejected": -2.1609432697296143, | |
| "step": 105 | |
| }, | |
| { | |
| "epoch": 1.037925925925926, | |
| "grad_norm": 1398.3836669921875, | |
| "learning_rate": 4.1199239938743797e-05, | |
| "log_odds_chosen": 6.5568623542785645, | |
| "log_odds_ratio": -1.0280228853225708, | |
| "logps/chosen": -3.167757511138916, | |
| "logps/rejected": -9.576342582702637, | |
| "loss": 6.365, | |
| "nll_loss": 2.601748466491699, | |
| "po_loss": -2.3791961669921875, | |
| "rewards/accuracies": 0.748251736164093, | |
| "rewards/chosen": -1.583878755569458, | |
| "rewards/margins": 3.2042922973632812, | |
| "rewards/rejected": -4.788171291351318, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 1.0853333333333333, | |
| "grad_norm": 468.6627502441406, | |
| "learning_rate": 4.0117925141242174e-05, | |
| "log_odds_chosen": 7.842375755310059, | |
| "log_odds_ratio": -0.8314769864082336, | |
| "logps/chosen": -2.4292473793029785, | |
| "logps/rejected": -10.092866897583008, | |
| "loss": -10.5443, | |
| "nll_loss": 2.201972007751465, | |
| "po_loss": -2.5314807891845703, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -1.2146236896514893, | |
| "rewards/margins": 3.8318095207214355, | |
| "rewards/rejected": -5.046433448791504, | |
| "step": 115 | |
| }, | |
| { | |
| "epoch": 1.1327407407407408, | |
| "grad_norm": 1342.4088134765625, | |
| "learning_rate": 3.899004663415084e-05, | |
| "log_odds_chosen": 6.2360992431640625, | |
| "log_odds_ratio": -0.7881489992141724, | |
| "logps/chosen": -2.0156655311584473, | |
| "logps/rejected": -8.078754425048828, | |
| "loss": -3.799, | |
| "nll_loss": 2.015702247619629, | |
| "po_loss": -2.1344211101531982, | |
| "rewards/accuracies": 0.7562500238418579, | |
| "rewards/chosen": -1.0078327655792236, | |
| "rewards/margins": 3.0315444469451904, | |
| "rewards/rejected": -4.039377212524414, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 1.1801481481481482, | |
| "grad_norm": 780.4815673828125, | |
| "learning_rate": 3.781907832058587e-05, | |
| "log_odds_chosen": 6.6184797286987305, | |
| "log_odds_ratio": -1.2134829759597778, | |
| "logps/chosen": -2.7371468544006348, | |
| "logps/rejected": -9.186931610107422, | |
| "loss": 11.0535, | |
| "nll_loss": 2.5944888591766357, | |
| "po_loss": -2.249067783355713, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -1.3685734272003174, | |
| "rewards/margins": 3.2248923778533936, | |
| "rewards/rejected": -4.593465805053711, | |
| "step": 125 | |
| }, | |
| { | |
| "epoch": 1.2275555555555555, | |
| "grad_norm": 827.787353515625, | |
| "learning_rate": 3.660862682169282e-05, | |
| "log_odds_chosen": 8.269857406616211, | |
| "log_odds_ratio": -1.018838882446289, | |
| "logps/chosen": -2.7996513843536377, | |
| "logps/rejected": -10.883502960205078, | |
| "loss": -11.0712, | |
| "nll_loss": 2.5594162940979004, | |
| "po_loss": -2.9053916931152344, | |
| "rewards/accuracies": 0.753125011920929, | |
| "rewards/chosen": -1.3998256921768188, | |
| "rewards/margins": 4.041925430297852, | |
| "rewards/rejected": -5.441751480102539, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 1.274962962962963, | |
| "grad_norm": 602.3118896484375, | |
| "learning_rate": 3.5362420368134356e-05, | |
| "log_odds_chosen": 5.271297454833984, | |
| "log_odds_ratio": -0.828192412853241, | |
| "logps/chosen": -1.9964498281478882, | |
| "logps/rejected": -7.077614784240723, | |
| "loss": 1.6283, | |
| "nll_loss": 2.024073362350464, | |
| "po_loss": -1.973189353942871, | |
| "rewards/accuracies": 0.7437499761581421, | |
| "rewards/chosen": -0.9982249140739441, | |
| "rewards/margins": 2.5405819416046143, | |
| "rewards/rejected": -3.5388073921203613, | |
| "step": 135 | |
| }, | |
| { | |
| "epoch": 1.3223703703703704, | |
| "grad_norm": 926.779052734375, | |
| "learning_rate": 3.408429731701635e-05, | |
| "log_odds_chosen": 5.869942665100098, | |
| "log_odds_ratio": -0.8702658414840698, | |
| "logps/chosen": -2.663573980331421, | |
| "logps/rejected": -8.38515853881836, | |
| "loss": 0.999, | |
| "nll_loss": 2.1175601482391357, | |
| "po_loss": -2.0863418579101562, | |
| "rewards/accuracies": 0.7437499761581421, | |
| "rewards/chosen": -1.3317869901657104, | |
| "rewards/margins": 2.8607921600341797, | |
| "rewards/rejected": -4.19257926940918, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 1.3697777777777778, | |
| "grad_norm": 1278.476806640625, | |
| "learning_rate": 3.2778194329621104e-05, | |
| "log_odds_chosen": 9.726934432983398, | |
| "log_odds_ratio": -1.0185357332229614, | |
| "logps/chosen": -3.2876574993133545, | |
| "logps/rejected": -12.812705993652344, | |
| "loss": -19.5231, | |
| "nll_loss": 2.5718603134155273, | |
| "po_loss": -3.181957483291626, | |
| "rewards/accuracies": 0.793749988079071, | |
| "rewards/chosen": -1.6438287496566772, | |
| "rewards/margins": 4.762524604797363, | |
| "rewards/rejected": -6.406352996826172, | |
| "step": 145 | |
| }, | |
| { | |
| "epoch": 1.417185185185185, | |
| "grad_norm": 1008.686279296875, | |
| "learning_rate": 3.144813424636031e-05, | |
| "log_odds_chosen": 7.952484130859375, | |
| "log_odds_ratio": -0.8371334075927734, | |
| "logps/chosen": -2.3361740112304688, | |
| "logps/rejected": -10.076719284057617, | |
| "loss": -22.1519, | |
| "nll_loss": 1.983311653137207, | |
| "po_loss": -2.6755595207214355, | |
| "rewards/accuracies": 0.7875000238418579, | |
| "rewards/chosen": -1.1680870056152344, | |
| "rewards/margins": 3.8702728748321533, | |
| "rewards/rejected": -5.038359642028809, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 1.4645925925925927, | |
| "grad_norm": 789.317138671875, | |
| "learning_rate": 3.0098213696293542e-05, | |
| "log_odds_chosen": 8.040031433105469, | |
| "log_odds_ratio": -1.153850793838501, | |
| "logps/chosen": -2.9109256267547607, | |
| "logps/rejected": -10.789738655090332, | |
| "loss": 0.2885, | |
| "nll_loss": 2.3872437477111816, | |
| "po_loss": -2.378229856491089, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": -1.4554628133773804, | |
| "rewards/margins": 3.939406156539917, | |
| "rewards/rejected": -5.394869327545166, | |
| "step": 155 | |
| }, | |
| { | |
| "epoch": 1.512, | |
| "grad_norm": 951.3047485351562, | |
| "learning_rate": 2.8732590479375165e-05, | |
| "log_odds_chosen": 7.912862300872803, | |
| "log_odds_ratio": -0.7059909105300903, | |
| "logps/chosen": -2.1726765632629395, | |
| "logps/rejected": -9.871898651123047, | |
| "loss": -31.1045, | |
| "nll_loss": 2.01442289352417, | |
| "po_loss": -2.9864375591278076, | |
| "rewards/accuracies": 0.768750011920929, | |
| "rewards/chosen": -1.0863382816314697, | |
| "rewards/margins": 3.8496108055114746, | |
| "rewards/rejected": -4.935949325561523, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 1.5594074074074074, | |
| "grad_norm": 2454.81591796875, | |
| "learning_rate": 2.7355470760292956e-05, | |
| "log_odds_chosen": 9.038829803466797, | |
| "log_odds_ratio": -1.0712902545928955, | |
| "logps/chosen": -3.265695095062256, | |
| "logps/rejected": -12.09699821472168, | |
| "loss": -28.5012, | |
| "nll_loss": 2.4790894985198975, | |
| "po_loss": -3.3697502613067627, | |
| "rewards/accuracies": 0.778124988079071, | |
| "rewards/chosen": -1.632847547531128, | |
| "rewards/margins": 4.415651798248291, | |
| "rewards/rejected": -6.04849910736084, | |
| "step": 165 | |
| }, | |
| { | |
| "epoch": 1.6068148148148147, | |
| "grad_norm": 1764.2366943359375, | |
| "learning_rate": 2.597109611334169e-05, | |
| "log_odds_chosen": 9.091514587402344, | |
| "log_odds_ratio": -1.6616445779800415, | |
| "logps/chosen": -4.427676200866699, | |
| "logps/rejected": -13.332109451293945, | |
| "loss": -3.2606, | |
| "nll_loss": 3.0899510383605957, | |
| "po_loss": -3.191845417022705, | |
| "rewards/accuracies": 0.7718750238418579, | |
| "rewards/chosen": -2.2138381004333496, | |
| "rewards/margins": 4.452216625213623, | |
| "rewards/rejected": -6.666054725646973, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 1.6542222222222223, | |
| "grad_norm": 1194.388916015625, | |
| "learning_rate": 2.458373045823404e-05, | |
| "log_odds_chosen": 7.1326398849487305, | |
| "log_odds_ratio": -0.9163221120834351, | |
| "logps/chosen": -2.580127716064453, | |
| "logps/rejected": -9.50422477722168, | |
| "loss": -18.0293, | |
| "nll_loss": 2.2237651348114014, | |
| "po_loss": -2.787180185317993, | |
| "rewards/accuracies": 0.762499988079071, | |
| "rewards/chosen": -1.2900638580322266, | |
| "rewards/margins": 3.462048292160034, | |
| "rewards/rejected": -4.75211238861084, | |
| "step": 175 | |
| }, | |
| { | |
| "epoch": 1.7016296296296296, | |
| "grad_norm": 2008.02197265625, | |
| "learning_rate": 2.3197646927086697e-05, | |
| "log_odds_chosen": 7.623831748962402, | |
| "log_odds_ratio": -0.8227715492248535, | |
| "logps/chosen": -2.170476198196411, | |
| "logps/rejected": -9.566916465759277, | |
| "loss": -24.3356, | |
| "nll_loss": 2.127624988555908, | |
| "po_loss": -2.888112783432007, | |
| "rewards/accuracies": 0.768750011920929, | |
| "rewards/chosen": -1.0852380990982056, | |
| "rewards/margins": 3.698220729827881, | |
| "rewards/rejected": -4.783458232879639, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 1.749037037037037, | |
| "grad_norm": 2212.9755859375, | |
| "learning_rate": 2.1817114703032176e-05, | |
| "log_odds_chosen": 11.049670219421387, | |
| "log_odds_ratio": -1.2048530578613281, | |
| "logps/chosen": -3.1207997798919678, | |
| "logps/rejected": -13.965868949890137, | |
| "loss": -42.3188, | |
| "nll_loss": 2.493201732635498, | |
| "po_loss": -3.8156638145446777, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": -1.5603998899459839, | |
| "rewards/margins": 5.422535419464111, | |
| "rewards/rejected": -6.982934474945068, | |
| "step": 185 | |
| }, | |
| { | |
| "epoch": 1.7964444444444445, | |
| "grad_norm": 1651.638671875, | |
| "learning_rate": 2.0446385870993467e-05, | |
| "log_odds_chosen": 9.86381721496582, | |
| "log_odds_ratio": -0.9923731088638306, | |
| "logps/chosen": -2.9157769680023193, | |
| "logps/rejected": -12.55068588256836, | |
| "loss": -36.9979, | |
| "nll_loss": 2.172974109649658, | |
| "po_loss": -3.329157590866089, | |
| "rewards/accuracies": 0.796875, | |
| "rewards/chosen": -1.4578884840011597, | |
| "rewards/margins": 4.8174543380737305, | |
| "rewards/rejected": -6.27534294128418, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 1.8438518518518519, | |
| "grad_norm": 5688.65869140625, | |
| "learning_rate": 1.9089682321121834e-05, | |
| "log_odds_chosen": 8.133454322814941, | |
| "log_odds_ratio": -1.3896712064743042, | |
| "logps/chosen": -3.334374189376831, | |
| "logps/rejected": -11.257940292358398, | |
| "loss": -14.291, | |
| "nll_loss": 2.494044303894043, | |
| "po_loss": -2.9406371116638184, | |
| "rewards/accuracies": 0.7906249761581421, | |
| "rewards/chosen": -1.6671870946884155, | |
| "rewards/margins": 3.961782932281494, | |
| "rewards/rejected": -5.628970146179199, | |
| "step": 195 | |
| }, | |
| { | |
| "epoch": 1.8912592592592592, | |
| "grad_norm": 4854.55810546875, | |
| "learning_rate": 1.775118274523545e-05, | |
| "log_odds_chosen": 8.46328353881836, | |
| "log_odds_ratio": -2.0774197578430176, | |
| "logps/chosen": -4.338870525360107, | |
| "logps/rejected": -12.631368637084961, | |
| "loss": 0.4968, | |
| "nll_loss": 3.0006825923919678, | |
| "po_loss": -2.985157012939453, | |
| "rewards/accuracies": 0.737500011920929, | |
| "rewards/chosen": -2.1694352626800537, | |
| "rewards/margins": 4.1462483406066895, | |
| "rewards/rejected": -6.3156843185424805, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 1.9386666666666668, | |
| "grad_norm": 1195.15283203125, | |
| "learning_rate": 1.643500976631037e-05, | |
| "log_odds_chosen": 8.545049667358398, | |
| "log_odds_ratio": -1.2959668636322021, | |
| "logps/chosen": -3.3477349281311035, | |
| "logps/rejected": -11.675881385803223, | |
| "loss": -24.4773, | |
| "nll_loss": 2.649782419204712, | |
| "po_loss": -3.414698362350464, | |
| "rewards/accuracies": 0.784375011920929, | |
| "rewards/chosen": -1.6738674640655518, | |
| "rewards/margins": 4.1640729904174805, | |
| "rewards/rejected": -5.837940692901611, | |
| "step": 205 | |
| }, | |
| { | |
| "epoch": 1.986074074074074, | |
| "grad_norm": 1435.34326171875, | |
| "learning_rate": 1.514521724066537e-05, | |
| "log_odds_chosen": 7.497690677642822, | |
| "log_odds_ratio": -1.6079471111297607, | |
| "logps/chosen": -3.8709499835968018, | |
| "logps/rejected": -11.176200866699219, | |
| "loss": -8.5711, | |
| "nll_loss": 2.686655282974243, | |
| "po_loss": -2.9545013904571533, | |
| "rewards/accuracies": 0.762499988079071, | |
| "rewards/chosen": -1.9354749917984009, | |
| "rewards/margins": 3.6526246070861816, | |
| "rewards/rejected": -5.588100433349609, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 2.0284444444444443, | |
| "grad_norm": 784.1499633789062, | |
| "learning_rate": 1.3885777771950348e-05, | |
| "log_odds_chosen": 8.465600967407227, | |
| "log_odds_ratio": -0.868893027305603, | |
| "logps/chosen": -2.0467655658721924, | |
| "logps/rejected": -10.241917610168457, | |
| "loss": -37.4481, | |
| "nll_loss": 1.8826793432235718, | |
| "po_loss": -3.1920523643493652, | |
| "rewards/accuracies": 0.7692307829856873, | |
| "rewards/chosen": -1.0233827829360962, | |
| "rewards/margins": 4.097576141357422, | |
| "rewards/rejected": -5.1209588050842285, | |
| "step": 215 | |
| }, | |
| { | |
| "epoch": 2.075851851851852, | |
| "grad_norm": 1654.237548828125, | |
| "learning_rate": 1.2660570475395683e-05, | |
| "log_odds_chosen": 8.654611587524414, | |
| "log_odds_ratio": -1.1223597526550293, | |
| "logps/chosen": -2.459672212600708, | |
| "logps/rejected": -10.875265121459961, | |
| "loss": -34.0465, | |
| "nll_loss": 2.0625672340393066, | |
| "po_loss": -3.126519203186035, | |
| "rewards/accuracies": 0.765625, | |
| "rewards/chosen": -1.229836106300354, | |
| "rewards/margins": 4.207796573638916, | |
| "rewards/rejected": -5.4376325607299805, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 2.1232592592592594, | |
| "grad_norm": 2589.6494140625, | |
| "learning_rate": 1.1473369030008974e-05, | |
| "log_odds_chosen": 15.148641586303711, | |
| "log_odds_ratio": -1.0744845867156982, | |
| "logps/chosen": -3.3443615436553955, | |
| "logps/rejected": -18.203109741210938, | |
| "loss": -73.6899, | |
| "nll_loss": 2.520362377166748, | |
| "po_loss": -4.8231706619262695, | |
| "rewards/accuracies": 0.840624988079071, | |
| "rewards/chosen": -1.6721807718276978, | |
| "rewards/margins": 7.429374694824219, | |
| "rewards/rejected": -9.101554870605469, | |
| "step": 225 | |
| }, | |
| { | |
| "epoch": 2.1706666666666665, | |
| "grad_norm": 1232.9119873046875, | |
| "learning_rate": 1.0327830055518842e-05, | |
| "log_odds_chosen": 14.175605773925781, | |
| "log_odds_ratio": -0.9750823974609375, | |
| "logps/chosen": -3.1031196117401123, | |
| "logps/rejected": -17.016685485839844, | |
| "loss": -66.4024, | |
| "nll_loss": 2.539062976837158, | |
| "po_loss": -4.614138603210449, | |
| "rewards/accuracies": 0.840624988079071, | |
| "rewards/chosen": -1.5515598058700562, | |
| "rewards/margins": 6.956783294677734, | |
| "rewards/rejected": -8.508342742919922, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 2.218074074074074, | |
| "grad_norm": 2281.852294921875, | |
| "learning_rate": 9.227481849865235e-06, | |
| "log_odds_chosen": 13.702494621276855, | |
| "log_odds_ratio": -0.6508721113204956, | |
| "logps/chosen": -2.776512861251831, | |
| "logps/rejected": -16.18694496154785, | |
| "loss": -76.9486, | |
| "nll_loss": 2.2473607063293457, | |
| "po_loss": -4.652005672454834, | |
| "rewards/accuracies": 0.831250011920929, | |
| "rewards/chosen": -1.3882564306259155, | |
| "rewards/margins": 6.705216407775879, | |
| "rewards/rejected": -8.093472480773926, | |
| "step": 235 | |
| }, | |
| { | |
| "epoch": 2.2654814814814817, | |
| "grad_norm": 1797.7962646484375, | |
| "learning_rate": 8.175713521924978e-06, | |
| "log_odds_chosen": 14.10627555847168, | |
| "log_odds_ratio": -0.9793645739555359, | |
| "logps/chosen": -3.2738196849823, | |
| "logps/rejected": -17.115571975708008, | |
| "loss": -71.8213, | |
| "nll_loss": 2.362848997116089, | |
| "po_loss": -4.607265472412109, | |
| "rewards/accuracies": 0.859375, | |
| "rewards/chosen": -1.63690984249115, | |
| "rewards/margins": 6.92087459564209, | |
| "rewards/rejected": -8.557785987854004, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 2.3128888888888888, | |
| "grad_norm": 2065.531005859375, | |
| "learning_rate": 7.1757645529443665e-06, | |
| "log_odds_chosen": 14.409799575805664, | |
| "log_odds_ratio": -0.6093363165855408, | |
| "logps/chosen": -2.691865921020508, | |
| "logps/rejected": -16.832874298095703, | |
| "loss": -86.8692, | |
| "nll_loss": 2.2543296813964844, | |
| "po_loss": -4.968992710113525, | |
| "rewards/accuracies": 0.8374999761581421, | |
| "rewards/chosen": -1.345932960510254, | |
| "rewards/margins": 7.070504188537598, | |
| "rewards/rejected": -8.416437149047852, | |
| "step": 245 | |
| }, | |
| { | |
| "epoch": 2.3602962962962963, | |
| "grad_norm": 1674.9385986328125, | |
| "learning_rate": 6.230714818829733e-06, | |
| "log_odds_chosen": 14.006881713867188, | |
| "log_odds_ratio": -0.6611822247505188, | |
| "logps/chosen": -2.298327684402466, | |
| "logps/rejected": -16.025455474853516, | |
| "loss": -73.411, | |
| "nll_loss": 2.0404696464538574, | |
| "po_loss": -4.3345627784729, | |
| "rewards/accuracies": 0.784375011920929, | |
| "rewards/chosen": -1.149163842201233, | |
| "rewards/margins": 6.863563537597656, | |
| "rewards/rejected": -8.012727737426758, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 2.407703703703704, | |
| "grad_norm": 1458.6246337890625, | |
| "learning_rate": 5.343475104027743e-06, | |
| "log_odds_chosen": 14.005816459655762, | |
| "log_odds_ratio": -0.6005424857139587, | |
| "logps/chosen": -2.2070460319519043, | |
| "logps/rejected": -15.93396282196045, | |
| "loss": -76.6993, | |
| "nll_loss": 1.9984962940216064, | |
| "po_loss": -4.395350456237793, | |
| "rewards/accuracies": 0.856249988079071, | |
| "rewards/chosen": -1.1035230159759521, | |
| "rewards/margins": 6.863458156585693, | |
| "rewards/rejected": -7.966981410980225, | |
| "step": 255 | |
| }, | |
| { | |
| "epoch": 2.455111111111111, | |
| "grad_norm": 1062.482666015625, | |
| "learning_rate": 4.516778136213037e-06, | |
| "log_odds_chosen": 13.041534423828125, | |
| "log_odds_ratio": -0.6385210752487183, | |
| "logps/chosen": -2.050607204437256, | |
| "logps/rejected": -14.810098648071289, | |
| "loss": -73.7325, | |
| "nll_loss": 1.8901704549789429, | |
| "po_loss": -4.19431209564209, | |
| "rewards/accuracies": 0.862500011920929, | |
| "rewards/chosen": -1.025303602218628, | |
| "rewards/margins": 6.379745006561279, | |
| "rewards/rejected": -7.4050493240356445, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 2.5025185185185186, | |
| "grad_norm": 2426.10986328125, | |
| "learning_rate": 3.7531701693965554e-06, | |
| "log_odds_chosen": 16.418107986450195, | |
| "log_odds_ratio": -0.8002341985702515, | |
| "logps/chosen": -2.38873028755188, | |
| "logps/rejected": -18.518217086791992, | |
| "loss": -90.9132, | |
| "nll_loss": 2.055996894836426, | |
| "po_loss": -4.897036075592041, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -1.19436514377594, | |
| "rewards/margins": 8.064743995666504, | |
| "rewards/rejected": -9.259108543395996, | |
| "step": 265 | |
| }, | |
| { | |
| "epoch": 2.549925925925926, | |
| "grad_norm": 2285.034423828125, | |
| "learning_rate": 3.055003141378948e-06, | |
| "log_odds_chosen": 18.129314422607422, | |
| "log_odds_ratio": -1.1312395334243774, | |
| "logps/chosen": -3.370629072189331, | |
| "logps/rejected": -21.226877212524414, | |
| "loss": -80.7968, | |
| "nll_loss": 2.6373114585876465, | |
| "po_loss": -5.162210941314697, | |
| "rewards/accuracies": 0.828125, | |
| "rewards/chosen": -1.6853145360946655, | |
| "rewards/margins": 8.928122520446777, | |
| "rewards/rejected": -10.613438606262207, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 2.5973333333333333, | |
| "grad_norm": 2841.40380859375, | |
| "learning_rate": 2.424427429704365e-06, | |
| "log_odds_chosen": 16.357511520385742, | |
| "log_odds_ratio": -1.1468102931976318, | |
| "logps/chosen": -3.206864595413208, | |
| "logps/rejected": -19.272470474243164, | |
| "loss": -74.3951, | |
| "nll_loss": 2.3466782569885254, | |
| "po_loss": -4.671523571014404, | |
| "rewards/accuracies": 0.856249988079071, | |
| "rewards/chosen": -1.603432297706604, | |
| "rewards/margins": 8.032803535461426, | |
| "rewards/rejected": -9.636235237121582, | |
| "step": 275 | |
| }, | |
| { | |
| "epoch": 2.644740740740741, | |
| "grad_norm": 1928.48876953125, | |
| "learning_rate": 1.8633852284264508e-06, | |
| "log_odds_chosen": 16.074771881103516, | |
| "log_odds_ratio": -0.9569836854934692, | |
| "logps/chosen": -2.4576973915100098, | |
| "logps/rejected": -18.23931884765625, | |
| "loss": -88.0982, | |
| "nll_loss": 2.244922161102295, | |
| "po_loss": -4.997992515563965, | |
| "rewards/accuracies": 0.828125, | |
| "rewards/chosen": -1.2288486957550049, | |
| "rewards/margins": 7.890810489654541, | |
| "rewards/rejected": -9.119659423828125, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 2.6921481481481484, | |
| "grad_norm": 1673.14990234375, | |
| "learning_rate": 1.3736045660864034e-06, | |
| "log_odds_chosen": 15.582837104797363, | |
| "log_odds_ratio": -0.819076657295227, | |
| "logps/chosen": -2.702998638153076, | |
| "logps/rejected": -18.022306442260742, | |
| "loss": -88.1281, | |
| "nll_loss": 2.126950740814209, | |
| "po_loss": -4.880953788757324, | |
| "rewards/accuracies": 0.8374999761581421, | |
| "rewards/chosen": -1.351499319076538, | |
| "rewards/margins": 7.659654140472412, | |
| "rewards/rejected": -9.011153221130371, | |
| "step": 285 | |
| }, | |
| { | |
| "epoch": 2.7395555555555555, | |
| "grad_norm": 1918.1820068359375, | |
| "learning_rate": 9.565939833279192e-07, | |
| "log_odds_chosen": 14.058061599731445, | |
| "log_odds_ratio": -1.0425951480865479, | |
| "logps/chosen": -3.089294910430908, | |
| "logps/rejected": -16.88263511657715, | |
| "loss": -59.2928, | |
| "nll_loss": 2.3471436500549316, | |
| "po_loss": -4.200043678283691, | |
| "rewards/accuracies": 0.8343750238418579, | |
| "rewards/chosen": -1.544647455215454, | |
| "rewards/margins": 6.896668434143066, | |
| "rewards/rejected": -8.441317558288574, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 2.786962962962963, | |
| "grad_norm": 1549.3990478515625, | |
| "learning_rate": 6.136378865420872e-07, | |
| "log_odds_chosen": 14.449419975280762, | |
| "log_odds_ratio": -0.8233845829963684, | |
| "logps/chosen": -2.452683448791504, | |
| "logps/rejected": -16.653139114379883, | |
| "loss": -82.7587, | |
| "nll_loss": 1.9743688106536865, | |
| "po_loss": -4.560579299926758, | |
| "rewards/accuracies": 0.8031250238418579, | |
| "rewards/chosen": -1.226341724395752, | |
| "rewards/margins": 7.100228309631348, | |
| "rewards/rejected": -8.326569557189941, | |
| "step": 295 | |
| }, | |
| { | |
| "epoch": 2.83437037037037, | |
| "grad_norm": 1157.6502685546875, | |
| "learning_rate": 3.45792591853214e-07, | |
| "log_odds_chosen": 15.727914810180664, | |
| "log_odds_ratio": -0.5168970227241516, | |
| "logps/chosen": -2.1314849853515625, | |
| "logps/rejected": -17.58350944519043, | |
| "loss": -96.6356, | |
| "nll_loss": 1.8993088006973267, | |
| "po_loss": -4.9191718101501465, | |
| "rewards/accuracies": 0.8374999761581421, | |
| "rewards/chosen": -1.0657424926757812, | |
| "rewards/margins": 7.72601318359375, | |
| "rewards/rejected": -8.791754722595215, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 2.8817777777777778, | |
| "grad_norm": 3823.998291015625, | |
| "learning_rate": 1.538830716302092e-07, | |
| "log_odds_chosen": 16.228670120239258, | |
| "log_odds_ratio": -0.9718038439750671, | |
| "logps/chosen": -2.4307305812835693, | |
| "logps/rejected": -18.350786209106445, | |
| "loss": -87.9496, | |
| "nll_loss": 2.063523769378662, | |
| "po_loss": -4.811949253082275, | |
| "rewards/accuracies": 0.828125, | |
| "rewards/chosen": -1.2153652906417847, | |
| "rewards/margins": 7.960028171539307, | |
| "rewards/rejected": -9.175393104553223, | |
| "step": 305 | |
| }, | |
| { | |
| "epoch": 2.9291851851851853, | |
| "grad_norm": 2266.873779296875, | |
| "learning_rate": 3.8500413544415025e-08, | |
| "log_odds_chosen": 16.433456420898438, | |
| "log_odds_ratio": -0.6722828149795532, | |
| "logps/chosen": -2.7718887329101562, | |
| "logps/rejected": -18.93569564819336, | |
| "loss": -83.3821, | |
| "nll_loss": 2.255375623703003, | |
| "po_loss": -4.861066818237305, | |
| "rewards/accuracies": 0.828125, | |
| "rewards/chosen": -1.3859443664550781, | |
| "rewards/margins": 8.081903457641602, | |
| "rewards/rejected": -9.46784782409668, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 2.9765925925925925, | |
| "grad_norm": 1334.365234375, | |
| "learning_rate": 0.0, | |
| "log_odds_chosen": 15.443585395812988, | |
| "log_odds_ratio": -0.6007210612297058, | |
| "logps/chosen": -1.9952449798583984, | |
| "logps/rejected": -17.120744705200195, | |
| "loss": -98.7664, | |
| "nll_loss": 1.7973524332046509, | |
| "po_loss": -4.8838019371032715, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": -0.9976224899291992, | |
| "rewards/margins": 7.562750339508057, | |
| "rewards/rejected": -8.560372352600098, | |
| "step": 315 | |
| }, | |
| { | |
| "epoch": 2.9765925925925925, | |
| "step": 315, | |
| "total_flos": 0.0, | |
| "train_loss": 6.844926292177231, | |
| "train_runtime": 9518.1794, | |
| "train_samples_per_second": 2.128, | |
| "train_steps_per_second": 0.033 | |
| } | |
| ], | |
| "logging_steps": 5, | |
| "max_steps": 315, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 100000, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 1, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |