| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 2.994495412844037, | |
| "eval_steps": 500, | |
| "global_step": 408, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.014678899082568808, | |
| "grad_norm": 11.81737232208252, | |
| "learning_rate": 2.439024390243903e-07, | |
| "logits/chosen": -0.9879676103591919, | |
| "logits/rejected": -1.9993298053741455, | |
| "logps/chosen": -269.27239990234375, | |
| "logps/rejected": -186.47621154785156, | |
| "loss": 0.6931, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 2 | |
| }, | |
| { | |
| "epoch": 0.029357798165137616, | |
| "grad_norm": 11.950206756591797, | |
| "learning_rate": 4.878048780487805e-07, | |
| "logits/chosen": -1.0342975854873657, | |
| "logits/rejected": -1.9880424737930298, | |
| "logps/chosen": -290.81072998046875, | |
| "logps/rejected": -204.50514221191406, | |
| "loss": 0.712, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.009406615048646927, | |
| "rewards/margins": 0.020395996049046516, | |
| "rewards/rejected": -0.029802614822983742, | |
| "step": 4 | |
| }, | |
| { | |
| "epoch": 0.044036697247706424, | |
| "grad_norm": 11.719453811645508, | |
| "learning_rate": 7.317073170731707e-07, | |
| "logits/chosen": -1.1187832355499268, | |
| "logits/rejected": -2.125272750854492, | |
| "logps/chosen": -295.85894775390625, | |
| "logps/rejected": -203.1645050048828, | |
| "loss": 0.6642, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": 0.07618961483240128, | |
| "rewards/margins": 0.09595101326704025, | |
| "rewards/rejected": -0.01976138912141323, | |
| "step": 6 | |
| }, | |
| { | |
| "epoch": 0.05871559633027523, | |
| "grad_norm": 12.171574592590332, | |
| "learning_rate": 9.75609756097561e-07, | |
| "logits/chosen": -1.250899076461792, | |
| "logits/rejected": -2.1083037853240967, | |
| "logps/chosen": -252.51145935058594, | |
| "logps/rejected": -164.40138244628906, | |
| "loss": 0.7179, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": 0.044141992926597595, | |
| "rewards/margins": 0.003628704696893692, | |
| "rewards/rejected": 0.040513284504413605, | |
| "step": 8 | |
| }, | |
| { | |
| "epoch": 0.07339449541284404, | |
| "grad_norm": 13.125951766967773, | |
| "learning_rate": 1.2195121951219514e-06, | |
| "logits/chosen": -1.0734999179840088, | |
| "logits/rejected": -2.204047679901123, | |
| "logps/chosen": -306.6387939453125, | |
| "logps/rejected": -158.70912170410156, | |
| "loss": 0.7397, | |
| "rewards/accuracies": 0.515625, | |
| "rewards/chosen": -0.0127907395362854, | |
| "rewards/margins": -0.031406134366989136, | |
| "rewards/rejected": 0.01861538738012314, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.08807339449541285, | |
| "grad_norm": 15.099513053894043, | |
| "learning_rate": 1.4634146341463414e-06, | |
| "logits/chosen": -1.0108157396316528, | |
| "logits/rejected": -1.977769374847412, | |
| "logps/chosen": -344.31402587890625, | |
| "logps/rejected": -223.3643798828125, | |
| "loss": 0.7584, | |
| "rewards/accuracies": 0.40625, | |
| "rewards/chosen": -0.06078364700078964, | |
| "rewards/margins": -0.0867958813905716, | |
| "rewards/rejected": 0.026012245565652847, | |
| "step": 12 | |
| }, | |
| { | |
| "epoch": 0.10275229357798166, | |
| "grad_norm": 13.785890579223633, | |
| "learning_rate": 1.707317073170732e-06, | |
| "logits/chosen": -0.9762290120124817, | |
| "logits/rejected": -1.9721505641937256, | |
| "logps/chosen": -259.58258056640625, | |
| "logps/rejected": -167.8755645751953, | |
| "loss": 0.7235, | |
| "rewards/accuracies": 0.46875, | |
| "rewards/chosen": 0.023126909509301186, | |
| "rewards/margins": -0.02050386182963848, | |
| "rewards/rejected": 0.043630778789520264, | |
| "step": 14 | |
| }, | |
| { | |
| "epoch": 0.11743119266055047, | |
| "grad_norm": 13.014513969421387, | |
| "learning_rate": 1.951219512195122e-06, | |
| "logits/chosen": -1.1472342014312744, | |
| "logits/rejected": -2.0296616554260254, | |
| "logps/chosen": -269.7952575683594, | |
| "logps/rejected": -186.65452575683594, | |
| "loss": 0.7405, | |
| "rewards/accuracies": 0.453125, | |
| "rewards/chosen": 0.004093457013368607, | |
| "rewards/margins": -0.044701721519231796, | |
| "rewards/rejected": 0.0487951785326004, | |
| "step": 16 | |
| }, | |
| { | |
| "epoch": 0.13211009174311927, | |
| "grad_norm": 12.20093059539795, | |
| "learning_rate": 2.1951219512195125e-06, | |
| "logits/chosen": -1.0266412496566772, | |
| "logits/rejected": -2.0891737937927246, | |
| "logps/chosen": -313.8085021972656, | |
| "logps/rejected": -197.85943603515625, | |
| "loss": 0.6731, | |
| "rewards/accuracies": 0.609375, | |
| "rewards/chosen": 0.09984610974788666, | |
| "rewards/margins": 0.0959281176328659, | |
| "rewards/rejected": 0.003917992115020752, | |
| "step": 18 | |
| }, | |
| { | |
| "epoch": 0.14678899082568808, | |
| "grad_norm": 12.344905853271484, | |
| "learning_rate": 2.4390243902439027e-06, | |
| "logits/chosen": -1.0662198066711426, | |
| "logits/rejected": -2.0889832973480225, | |
| "logps/chosen": -308.8189697265625, | |
| "logps/rejected": -156.6934814453125, | |
| "loss": 0.6784, | |
| "rewards/accuracies": 0.546875, | |
| "rewards/chosen": 0.08540838956832886, | |
| "rewards/margins": 0.07292439043521881, | |
| "rewards/rejected": 0.012484000064432621, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.1614678899082569, | |
| "grad_norm": 11.898660659790039, | |
| "learning_rate": 2.682926829268293e-06, | |
| "logits/chosen": -1.2143007516860962, | |
| "logits/rejected": -2.262324571609497, | |
| "logps/chosen": -298.7814636230469, | |
| "logps/rejected": -186.76119995117188, | |
| "loss": 0.6781, | |
| "rewards/accuracies": 0.515625, | |
| "rewards/chosen": 0.08213196694850922, | |
| "rewards/margins": 0.08361663669347763, | |
| "rewards/rejected": -0.0014846734702587128, | |
| "step": 22 | |
| }, | |
| { | |
| "epoch": 0.1761467889908257, | |
| "grad_norm": 13.68064022064209, | |
| "learning_rate": 2.926829268292683e-06, | |
| "logits/chosen": -1.0233314037322998, | |
| "logits/rejected": -2.1899986267089844, | |
| "logps/chosen": -370.8209228515625, | |
| "logps/rejected": -156.96270751953125, | |
| "loss": 0.7306, | |
| "rewards/accuracies": 0.515625, | |
| "rewards/chosen": -0.007536953315138817, | |
| "rewards/margins": -0.009052609093487263, | |
| "rewards/rejected": 0.0015156615991145372, | |
| "step": 24 | |
| }, | |
| { | |
| "epoch": 0.1908256880733945, | |
| "grad_norm": 12.661199569702148, | |
| "learning_rate": 3.1707317073170736e-06, | |
| "logits/chosen": -1.2463735342025757, | |
| "logits/rejected": -2.1673622131347656, | |
| "logps/chosen": -326.9246520996094, | |
| "logps/rejected": -182.17701721191406, | |
| "loss": 0.7175, | |
| "rewards/accuracies": 0.546875, | |
| "rewards/chosen": -0.010405808687210083, | |
| "rewards/margins": 0.007835682481527328, | |
| "rewards/rejected": -0.01824149303138256, | |
| "step": 26 | |
| }, | |
| { | |
| "epoch": 0.20550458715596331, | |
| "grad_norm": 11.61974811553955, | |
| "learning_rate": 3.414634146341464e-06, | |
| "logits/chosen": -1.1716669797897339, | |
| "logits/rejected": -2.2106716632843018, | |
| "logps/chosen": -284.443603515625, | |
| "logps/rejected": -165.102783203125, | |
| "loss": 0.7409, | |
| "rewards/accuracies": 0.546875, | |
| "rewards/chosen": 0.007994448766112328, | |
| "rewards/margins": -0.035433441400527954, | |
| "rewards/rejected": 0.04342789575457573, | |
| "step": 28 | |
| }, | |
| { | |
| "epoch": 0.22018348623853212, | |
| "grad_norm": 10.777989387512207, | |
| "learning_rate": 3.6585365853658537e-06, | |
| "logits/chosen": -1.0662914514541626, | |
| "logits/rejected": -2.1156551837921143, | |
| "logps/chosen": -289.4057922363281, | |
| "logps/rejected": -197.46649169921875, | |
| "loss": 0.6371, | |
| "rewards/accuracies": 0.609375, | |
| "rewards/chosen": 0.1247626319527626, | |
| "rewards/margins": 0.15936096012592316, | |
| "rewards/rejected": -0.03459831699728966, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.23486238532110093, | |
| "grad_norm": 12.190910339355469, | |
| "learning_rate": 3.902439024390244e-06, | |
| "logits/chosen": -1.1755316257476807, | |
| "logits/rejected": -2.1449058055877686, | |
| "logps/chosen": -288.5774841308594, | |
| "logps/rejected": -163.59588623046875, | |
| "loss": 0.6733, | |
| "rewards/accuracies": 0.546875, | |
| "rewards/chosen": 0.09892146289348602, | |
| "rewards/margins": 0.08179756253957748, | |
| "rewards/rejected": 0.01712390035390854, | |
| "step": 32 | |
| }, | |
| { | |
| "epoch": 0.24954128440366974, | |
| "grad_norm": 13.154803276062012, | |
| "learning_rate": 4.146341463414634e-06, | |
| "logits/chosen": -1.1496777534484863, | |
| "logits/rejected": -2.2045750617980957, | |
| "logps/chosen": -324.6558837890625, | |
| "logps/rejected": -164.45327758789062, | |
| "loss": 0.6478, | |
| "rewards/accuracies": 0.640625, | |
| "rewards/chosen": 0.08885271847248077, | |
| "rewards/margins": 0.1558375358581543, | |
| "rewards/rejected": -0.06698483228683472, | |
| "step": 34 | |
| }, | |
| { | |
| "epoch": 0.26422018348623855, | |
| "grad_norm": 11.07314682006836, | |
| "learning_rate": 4.390243902439025e-06, | |
| "logits/chosen": -1.1677134037017822, | |
| "logits/rejected": -2.0850350856781006, | |
| "logps/chosen": -311.2884216308594, | |
| "logps/rejected": -204.43142700195312, | |
| "loss": 0.6193, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": 0.20451843738555908, | |
| "rewards/margins": 0.22681473195552826, | |
| "rewards/rejected": -0.02229629084467888, | |
| "step": 36 | |
| }, | |
| { | |
| "epoch": 0.27889908256880735, | |
| "grad_norm": 12.431696891784668, | |
| "learning_rate": 4.634146341463416e-06, | |
| "logits/chosen": -1.195428729057312, | |
| "logits/rejected": -2.197521686553955, | |
| "logps/chosen": -294.04962158203125, | |
| "logps/rejected": -200.2810516357422, | |
| "loss": 0.6598, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": 0.15507417917251587, | |
| "rewards/margins": 0.11412172019481659, | |
| "rewards/rejected": 0.04095245152711868, | |
| "step": 38 | |
| }, | |
| { | |
| "epoch": 0.29357798165137616, | |
| "grad_norm": 11.575589179992676, | |
| "learning_rate": 4.8780487804878055e-06, | |
| "logits/chosen": -1.0411652326583862, | |
| "logits/rejected": -2.03951096534729, | |
| "logps/chosen": -345.9762268066406, | |
| "logps/rejected": -181.34144592285156, | |
| "loss": 0.6186, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": 0.14570173621177673, | |
| "rewards/margins": 0.20660607516765594, | |
| "rewards/rejected": -0.06090431660413742, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.30825688073394497, | |
| "grad_norm": 12.716486930847168, | |
| "learning_rate": 4.999908404322799e-06, | |
| "logits/chosen": -1.0371800661087036, | |
| "logits/rejected": -2.2317895889282227, | |
| "logps/chosen": -319.42755126953125, | |
| "logps/rejected": -172.60479736328125, | |
| "loss": 0.6248, | |
| "rewards/accuracies": 0.640625, | |
| "rewards/chosen": 0.13961191475391388, | |
| "rewards/margins": 0.19473902881145477, | |
| "rewards/rejected": -0.055127132683992386, | |
| "step": 42 | |
| }, | |
| { | |
| "epoch": 0.3229357798165138, | |
| "grad_norm": 10.400399208068848, | |
| "learning_rate": 4.999175679175577e-06, | |
| "logits/chosen": -1.1097325086593628, | |
| "logits/rejected": -2.1328647136688232, | |
| "logps/chosen": -251.92745971679688, | |
| "logps/rejected": -161.21292114257812, | |
| "loss": 0.5849, | |
| "rewards/accuracies": 0.765625, | |
| "rewards/chosen": 0.25156256556510925, | |
| "rewards/margins": 0.2756442427635193, | |
| "rewards/rejected": -0.02408166043460369, | |
| "step": 44 | |
| }, | |
| { | |
| "epoch": 0.3376146788990826, | |
| "grad_norm": 11.084893226623535, | |
| "learning_rate": 4.997710443643461e-06, | |
| "logits/chosen": -1.1712064743041992, | |
| "logits/rejected": -2.0722293853759766, | |
| "logps/chosen": -259.9323425292969, | |
| "logps/rejected": -206.37510681152344, | |
| "loss": 0.6109, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": 0.22726279497146606, | |
| "rewards/margins": 0.24400296807289124, | |
| "rewards/rejected": -0.016740169376134872, | |
| "step": 46 | |
| }, | |
| { | |
| "epoch": 0.3522935779816514, | |
| "grad_norm": 13.230236053466797, | |
| "learning_rate": 4.995513127188151e-06, | |
| "logits/chosen": -1.0816175937652588, | |
| "logits/rejected": -2.215028762817383, | |
| "logps/chosen": -365.7675476074219, | |
| "logps/rejected": -183.13980102539062, | |
| "loss": 0.5456, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.34264349937438965, | |
| "rewards/margins": 0.38213008642196655, | |
| "rewards/rejected": -0.03948655351996422, | |
| "step": 48 | |
| }, | |
| { | |
| "epoch": 0.3669724770642202, | |
| "grad_norm": 11.37851333618164, | |
| "learning_rate": 4.992584373844853e-06, | |
| "logits/chosen": -1.2096611261367798, | |
| "logits/rejected": -2.082951784133911, | |
| "logps/chosen": -345.7232971191406, | |
| "logps/rejected": -184.25949096679688, | |
| "loss": 0.5091, | |
| "rewards/accuracies": 0.8125, | |
| "rewards/chosen": 0.413723886013031, | |
| "rewards/margins": 0.4999043643474579, | |
| "rewards/rejected": -0.08618048578500748, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.381651376146789, | |
| "grad_norm": 9.676469802856445, | |
| "learning_rate": 4.98892504203351e-06, | |
| "logits/chosen": -1.2248896360397339, | |
| "logits/rejected": -2.1341745853424072, | |
| "logps/chosen": -282.0457763671875, | |
| "logps/rejected": -158.89736938476562, | |
| "loss": 0.501, | |
| "rewards/accuracies": 0.921875, | |
| "rewards/chosen": 0.42736518383026123, | |
| "rewards/margins": 0.5060732960700989, | |
| "rewards/rejected": -0.07870808988809586, | |
| "step": 52 | |
| }, | |
| { | |
| "epoch": 0.3963302752293578, | |
| "grad_norm": 9.402766227722168, | |
| "learning_rate": 4.9845362043071925e-06, | |
| "logits/chosen": -1.0192848443984985, | |
| "logits/rejected": -2.0682382583618164, | |
| "logps/chosen": -290.6011962890625, | |
| "logps/rejected": -163.6627197265625, | |
| "loss": 0.4541, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": 0.5030941963195801, | |
| "rewards/margins": 0.6462306380271912, | |
| "rewards/rejected": -0.14313644170761108, | |
| "step": 54 | |
| }, | |
| { | |
| "epoch": 0.41100917431192663, | |
| "grad_norm": 10.944356918334961, | |
| "learning_rate": 4.97941914703774e-06, | |
| "logits/chosen": -1.1482800245285034, | |
| "logits/rejected": -2.151231050491333, | |
| "logps/chosen": -287.7913513183594, | |
| "logps/rejected": -201.2919464111328, | |
| "loss": 0.4487, | |
| "rewards/accuracies": 0.875, | |
| "rewards/chosen": 0.6401927471160889, | |
| "rewards/margins": 0.7009615898132324, | |
| "rewards/rejected": -0.06076894700527191, | |
| "step": 56 | |
| }, | |
| { | |
| "epoch": 0.42568807339449544, | |
| "grad_norm": 8.618446350097656, | |
| "learning_rate": 4.973575370038718e-06, | |
| "logits/chosen": -1.0707895755767822, | |
| "logits/rejected": -2.049323558807373, | |
| "logps/chosen": -305.2084045410156, | |
| "logps/rejected": -193.321533203125, | |
| "loss": 0.3851, | |
| "rewards/accuracies": 0.90625, | |
| "rewards/chosen": 0.8177109956741333, | |
| "rewards/margins": 0.9303702116012573, | |
| "rewards/rejected": -0.11265924572944641, | |
| "step": 58 | |
| }, | |
| { | |
| "epoch": 0.44036697247706424, | |
| "grad_norm": 7.712850093841553, | |
| "learning_rate": 4.967006586125827e-06, | |
| "logits/chosen": -1.240044355392456, | |
| "logits/rejected": -2.0774481296539307, | |
| "logps/chosen": -301.3046569824219, | |
| "logps/rejected": -186.58460998535156, | |
| "loss": 0.35, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": 0.9353222846984863, | |
| "rewards/margins": 1.0043295621871948, | |
| "rewards/rejected": -0.06900733709335327, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.45504587155963305, | |
| "grad_norm": 8.133475303649902, | |
| "learning_rate": 4.959714720614871e-06, | |
| "logits/chosen": -1.1756389141082764, | |
| "logits/rejected": -2.2198028564453125, | |
| "logps/chosen": -319.236083984375, | |
| "logps/rejected": -184.04647827148438, | |
| "loss": 0.3239, | |
| "rewards/accuracies": 0.953125, | |
| "rewards/chosen": 0.9475828409194946, | |
| "rewards/margins": 1.177114725112915, | |
| "rewards/rejected": -0.22953176498413086, | |
| "step": 62 | |
| }, | |
| { | |
| "epoch": 0.46972477064220186, | |
| "grad_norm": 6.613894462585449, | |
| "learning_rate": 4.951701910757446e-06, | |
| "logits/chosen": -1.1599823236465454, | |
| "logits/rejected": -2.064751148223877, | |
| "logps/chosen": -253.94537353515625, | |
| "logps/rejected": -188.87652587890625, | |
| "loss": 0.3088, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 1.0339241027832031, | |
| "rewards/margins": 1.2678444385528564, | |
| "rewards/rejected": -0.2339203655719757, | |
| "step": 64 | |
| }, | |
| { | |
| "epoch": 0.48440366972477067, | |
| "grad_norm": 8.49493408203125, | |
| "learning_rate": 4.942970505114514e-06, | |
| "logits/chosen": -1.0440397262573242, | |
| "logits/rejected": -2.1136162281036377, | |
| "logps/chosen": -308.4583435058594, | |
| "logps/rejected": -176.34474182128906, | |
| "loss": 0.268, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.1736990213394165, | |
| "rewards/margins": 1.4280885457992554, | |
| "rewards/rejected": -0.25438952445983887, | |
| "step": 66 | |
| }, | |
| { | |
| "epoch": 0.4990825688073395, | |
| "grad_norm": 6.022420883178711, | |
| "learning_rate": 4.933523062868033e-06, | |
| "logits/chosen": -1.0774444341659546, | |
| "logits/rejected": -2.1658172607421875, | |
| "logps/chosen": -269.4661560058594, | |
| "logps/rejected": -164.7786865234375, | |
| "loss": 0.2372, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 1.3157860040664673, | |
| "rewards/margins": 1.5686390399932861, | |
| "rewards/rejected": -0.25285303592681885, | |
| "step": 68 | |
| }, | |
| { | |
| "epoch": 0.5137614678899083, | |
| "grad_norm": 4.839372634887695, | |
| "learning_rate": 4.923362353070859e-06, | |
| "logits/chosen": -0.8954001665115356, | |
| "logits/rejected": -2.1572980880737305, | |
| "logps/chosen": -287.38250732421875, | |
| "logps/rejected": -159.82025146484375, | |
| "loss": 0.2079, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.4153721332550049, | |
| "rewards/margins": 1.8501354455947876, | |
| "rewards/rejected": -0.43476346135139465, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.5284403669724771, | |
| "grad_norm": 5.355666160583496, | |
| "learning_rate": 4.912491353835138e-06, | |
| "logits/chosen": -1.1590656042099, | |
| "logits/rejected": -2.088367462158203, | |
| "logps/chosen": -260.02386474609375, | |
| "logps/rejected": -185.47396850585938, | |
| "loss": 0.2185, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 1.4196313619613647, | |
| "rewards/margins": 1.858705997467041, | |
| "rewards/rejected": -0.43907448649406433, | |
| "step": 72 | |
| }, | |
| { | |
| "epoch": 0.5431192660550459, | |
| "grad_norm": 4.641209602355957, | |
| "learning_rate": 4.900913251459418e-06, | |
| "logits/chosen": -1.0761524438858032, | |
| "logits/rejected": -2.0451908111572266, | |
| "logps/chosen": -264.9051513671875, | |
| "logps/rejected": -173.16702270507812, | |
| "loss": 0.1769, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 1.5420759916305542, | |
| "rewards/margins": 2.071654796600342, | |
| "rewards/rejected": -0.5295785069465637, | |
| "step": 74 | |
| }, | |
| { | |
| "epoch": 0.5577981651376147, | |
| "grad_norm": 4.564330101013184, | |
| "learning_rate": 4.8886314394947396e-06, | |
| "logits/chosen": -0.9936952590942383, | |
| "logits/rejected": -2.070539951324463, | |
| "logps/chosen": -278.8867492675781, | |
| "logps/rejected": -185.91055297851562, | |
| "loss": 0.1608, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 1.9348686933517456, | |
| "rewards/margins": 2.52958083152771, | |
| "rewards/rejected": -0.5947118997573853, | |
| "step": 76 | |
| }, | |
| { | |
| "epoch": 0.5724770642201835, | |
| "grad_norm": 5.782593250274658, | |
| "learning_rate": 4.875649517749985e-06, | |
| "logits/chosen": -1.0427924394607544, | |
| "logits/rejected": -2.180347442626953, | |
| "logps/chosen": -282.06732177734375, | |
| "logps/rejected": -191.30137634277344, | |
| "loss": 0.1548, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 1.9499953985214233, | |
| "rewards/margins": 2.6421873569488525, | |
| "rewards/rejected": -0.6921918392181396, | |
| "step": 78 | |
| }, | |
| { | |
| "epoch": 0.5871559633027523, | |
| "grad_norm": 4.356126308441162, | |
| "learning_rate": 4.861971291236772e-06, | |
| "logits/chosen": -1.134873390197754, | |
| "logits/rejected": -2.047222852706909, | |
| "logps/chosen": -328.65509033203125, | |
| "logps/rejected": -191.76483154296875, | |
| "loss": 0.1841, | |
| "rewards/accuracies": 0.9375, | |
| "rewards/chosen": 2.2117769718170166, | |
| "rewards/margins": 2.5882744789123535, | |
| "rewards/rejected": -0.37649768590927124, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.6018348623853211, | |
| "grad_norm": 3.70808482170105, | |
| "learning_rate": 4.847600769054201e-06, | |
| "logits/chosen": -1.1773045063018799, | |
| "logits/rejected": -2.071323871612549, | |
| "logps/chosen": -365.7237243652344, | |
| "logps/rejected": -221.5764923095703, | |
| "loss": 0.1093, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 2.4926247596740723, | |
| "rewards/margins": 3.0623347759246826, | |
| "rewards/rejected": -0.5697098970413208, | |
| "step": 82 | |
| }, | |
| { | |
| "epoch": 0.6165137614678899, | |
| "grad_norm": 2.8207852840423584, | |
| "learning_rate": 4.832542163213787e-06, | |
| "logits/chosen": -1.0239057540893555, | |
| "logits/rejected": -2.1960628032684326, | |
| "logps/chosen": -261.3912658691406, | |
| "logps/rejected": -155.67286682128906, | |
| "loss": 0.1073, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 2.281486749649048, | |
| "rewards/margins": 3.123940944671631, | |
| "rewards/rejected": -0.8424541354179382, | |
| "step": 84 | |
| }, | |
| { | |
| "epoch": 0.6311926605504588, | |
| "grad_norm": 2.6905996799468994, | |
| "learning_rate": 4.816799887404911e-06, | |
| "logits/chosen": -1.2185587882995605, | |
| "logits/rejected": -2.146491289138794, | |
| "logps/chosen": -300.77069091796875, | |
| "logps/rejected": -185.7276153564453, | |
| "loss": 0.1277, | |
| "rewards/accuracies": 0.96875, | |
| "rewards/chosen": 2.3313047885894775, | |
| "rewards/margins": 3.0290822982788086, | |
| "rewards/rejected": -0.6977773904800415, | |
| "step": 86 | |
| }, | |
| { | |
| "epoch": 0.6458715596330276, | |
| "grad_norm": 1.891965389251709, | |
| "learning_rate": 4.800378555701168e-06, | |
| "logits/chosen": -1.056377649307251, | |
| "logits/rejected": -2.001763343811035, | |
| "logps/chosen": -354.14990234375, | |
| "logps/rejected": -186.62448120117188, | |
| "loss": 0.1089, | |
| "rewards/accuracies": 0.953125, | |
| "rewards/chosen": 2.5092077255249023, | |
| "rewards/margins": 3.3948686122894287, | |
| "rewards/rejected": -0.8856609463691711, | |
| "step": 88 | |
| }, | |
| { | |
| "epoch": 0.6605504587155964, | |
| "grad_norm": 3.7145261764526367, | |
| "learning_rate": 4.783282981207979e-06, | |
| "logits/chosen": -1.1021761894226074, | |
| "logits/rejected": -2.2725181579589844, | |
| "logps/chosen": -296.32763671875, | |
| "logps/rejected": -169.7439727783203, | |
| "loss": 0.0866, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 2.6951088905334473, | |
| "rewards/margins": 3.6553006172180176, | |
| "rewards/rejected": -0.9601919054985046, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.6752293577981652, | |
| "grad_norm": 2.50156307220459, | |
| "learning_rate": 4.765518174651864e-06, | |
| "logits/chosen": -1.1074126958847046, | |
| "logits/rejected": -2.051131248474121, | |
| "logps/chosen": -285.9756164550781, | |
| "logps/rejected": -190.58448791503906, | |
| "loss": 0.0852, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 2.7018895149230957, | |
| "rewards/margins": 3.8313865661621094, | |
| "rewards/rejected": -1.1294972896575928, | |
| "step": 92 | |
| }, | |
| { | |
| "epoch": 0.689908256880734, | |
| "grad_norm": 2.272671699523926, | |
| "learning_rate": 4.747089342911793e-06, | |
| "logits/chosen": -0.9693321585655212, | |
| "logits/rejected": -2.168473720550537, | |
| "logps/chosen": -291.7270812988281, | |
| "logps/rejected": -175.2049560546875, | |
| "loss": 0.0446, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 2.954824209213257, | |
| "rewards/margins": 4.128055095672607, | |
| "rewards/rejected": -1.1732308864593506, | |
| "step": 94 | |
| }, | |
| { | |
| "epoch": 0.7045871559633028, | |
| "grad_norm": 2.2310574054718018, | |
| "learning_rate": 4.728001887493048e-06, | |
| "logits/chosen": -0.9781808853149414, | |
| "logits/rejected": -2.155506134033203, | |
| "logps/chosen": -299.66314697265625, | |
| "logps/rejected": -194.56436157226562, | |
| "loss": 0.067, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 3.1443114280700684, | |
| "rewards/margins": 4.226352214813232, | |
| "rewards/rejected": -1.0820410251617432, | |
| "step": 96 | |
| }, | |
| { | |
| "epoch": 0.7192660550458716, | |
| "grad_norm": 1.7269368171691895, | |
| "learning_rate": 4.708261402944036e-06, | |
| "logits/chosen": -1.0619006156921387, | |
| "logits/rejected": -2.1256189346313477, | |
| "logps/chosen": -315.5987548828125, | |
| "logps/rejected": -188.52439880371094, | |
| "loss": 0.06, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 3.3185007572174072, | |
| "rewards/margins": 4.740314960479736, | |
| "rewards/rejected": -1.4218144416809082, | |
| "step": 98 | |
| }, | |
| { | |
| "epoch": 0.7339449541284404, | |
| "grad_norm": 3.1934289932250977, | |
| "learning_rate": 4.687873675216522e-06, | |
| "logits/chosen": -0.9534860253334045, | |
| "logits/rejected": -1.9718412160873413, | |
| "logps/chosen": -303.17181396484375, | |
| "logps/rejected": -199.40789794921875, | |
| "loss": 0.0892, | |
| "rewards/accuracies": 0.953125, | |
| "rewards/chosen": 3.4963011741638184, | |
| "rewards/margins": 4.650891304016113, | |
| "rewards/rejected": -1.1545898914337158, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.7486238532110092, | |
| "grad_norm": 1.0567034482955933, | |
| "learning_rate": 4.666844679969765e-06, | |
| "logits/chosen": -1.287552833557129, | |
| "logits/rejected": -2.272284507751465, | |
| "logps/chosen": -299.2529296875, | |
| "logps/rejected": -208.53785705566406, | |
| "loss": 0.0373, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 3.132131814956665, | |
| "rewards/margins": 4.880558967590332, | |
| "rewards/rejected": -1.748427152633667, | |
| "step": 102 | |
| }, | |
| { | |
| "epoch": 0.763302752293578, | |
| "grad_norm": 1.3455036878585815, | |
| "learning_rate": 4.6451805808190464e-06, | |
| "logits/chosen": -1.049391508102417, | |
| "logits/rejected": -2.1182594299316406, | |
| "logps/chosen": -284.2237548828125, | |
| "logps/rejected": -176.08627319335938, | |
| "loss": 0.0317, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 3.437601089477539, | |
| "rewards/margins": 5.258786678314209, | |
| "rewards/rejected": -1.821185827255249, | |
| "step": 104 | |
| }, | |
| { | |
| "epoch": 0.7779816513761468, | |
| "grad_norm": 1.6037604808807373, | |
| "learning_rate": 4.622887727529104e-06, | |
| "logits/chosen": -1.0589053630828857, | |
| "logits/rejected": -2.095472812652588, | |
| "logps/chosen": -257.8381042480469, | |
| "logps/rejected": -207.0792236328125, | |
| "loss": 0.029, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 3.2390363216400146, | |
| "rewards/margins": 5.350650310516357, | |
| "rewards/rejected": -2.1116137504577637, | |
| "step": 106 | |
| }, | |
| { | |
| "epoch": 0.7926605504587156, | |
| "grad_norm": 1.9005062580108643, | |
| "learning_rate": 4.599972654153018e-06, | |
| "logits/chosen": -0.9298142194747925, | |
| "logits/rejected": -2.0814666748046875, | |
| "logps/chosen": -301.68865966796875, | |
| "logps/rejected": -174.01010131835938, | |
| "loss": 0.0311, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 3.7089574337005615, | |
| "rewards/margins": 5.569860458374023, | |
| "rewards/rejected": -1.8609036207199097, | |
| "step": 108 | |
| }, | |
| { | |
| "epoch": 0.8073394495412844, | |
| "grad_norm": 0.712770402431488, | |
| "learning_rate": 4.5764420771170735e-06, | |
| "logits/chosen": -0.9678480625152588, | |
| "logits/rejected": -2.0447123050689697, | |
| "logps/chosen": -278.64398193359375, | |
| "logps/rejected": -192.5853729248047, | |
| "loss": 0.0215, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 3.513016939163208, | |
| "rewards/margins": 5.6180419921875, | |
| "rewards/rejected": -2.105024814605713, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.8220183486238533, | |
| "grad_norm": 1.3919163942337036, | |
| "learning_rate": 4.552302893252166e-06, | |
| "logits/chosen": -1.2199370861053467, | |
| "logits/rejected": -2.197056293487549, | |
| "logps/chosen": -306.26080322265625, | |
| "logps/rejected": -205.06845092773438, | |
| "loss": 0.0296, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 3.367818832397461, | |
| "rewards/margins": 5.316436290740967, | |
| "rewards/rejected": -1.948617696762085, | |
| "step": 112 | |
| }, | |
| { | |
| "epoch": 0.8366972477064221, | |
| "grad_norm": 3.037362575531006, | |
| "learning_rate": 4.52756217777234e-06, | |
| "logits/chosen": -1.2299991846084595, | |
| "logits/rejected": -2.1640126705169678, | |
| "logps/chosen": -311.70574951171875, | |
| "logps/rejected": -207.38746643066406, | |
| "loss": 0.0398, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 3.595170021057129, | |
| "rewards/margins": 5.463950157165527, | |
| "rewards/rejected": -1.8687800168991089, | |
| "step": 114 | |
| }, | |
| { | |
| "epoch": 0.8513761467889909, | |
| "grad_norm": 0.8069730401039124, | |
| "learning_rate": 4.502227182201035e-06, | |
| "logits/chosen": -0.9528835415840149, | |
| "logits/rejected": -1.977004051208496, | |
| "logps/chosen": -264.5509033203125, | |
| "logps/rejected": -174.93551635742188, | |
| "loss": 0.0227, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 3.867180109024048, | |
| "rewards/margins": 6.007584571838379, | |
| "rewards/rejected": -2.140403985977173, | |
| "step": 116 | |
| }, | |
| { | |
| "epoch": 0.8660550458715597, | |
| "grad_norm": 1.4102082252502441, | |
| "learning_rate": 4.476305332245662e-06, | |
| "logits/chosen": -1.0918750762939453, | |
| "logits/rejected": -2.3146743774414062, | |
| "logps/chosen": -314.5960998535156, | |
| "logps/rejected": -152.3535614013672, | |
| "loss": 0.028, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 3.734642744064331, | |
| "rewards/margins": 6.135974407196045, | |
| "rewards/rejected": -2.4013314247131348, | |
| "step": 118 | |
| }, | |
| { | |
| "epoch": 0.8807339449541285, | |
| "grad_norm": 2.8867928981781006, | |
| "learning_rate": 4.449804225621116e-06, | |
| "logits/chosen": -1.0288662910461426, | |
| "logits/rejected": -2.0701658725738525, | |
| "logps/chosen": -279.2713317871094, | |
| "logps/rejected": -180.374267578125, | |
| "loss": 0.0485, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 3.5982298851013184, | |
| "rewards/margins": 5.633719444274902, | |
| "rewards/rejected": -2.0354888439178467, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.8954128440366973, | |
| "grad_norm": 0.7778434753417969, | |
| "learning_rate": 4.422731629822887e-06, | |
| "logits/chosen": -0.9540915489196777, | |
| "logits/rejected": -1.9875534772872925, | |
| "logps/chosen": -314.85003662109375, | |
| "logps/rejected": -194.16896057128906, | |
| "loss": 0.0315, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 3.723828077316284, | |
| "rewards/margins": 6.088706016540527, | |
| "rewards/rejected": -2.364877462387085, | |
| "step": 122 | |
| }, | |
| { | |
| "epoch": 0.9100917431192661, | |
| "grad_norm": 1.9667764902114868, | |
| "learning_rate": 4.395095479850396e-06, | |
| "logits/chosen": -0.9676120281219482, | |
| "logits/rejected": -1.9072697162628174, | |
| "logps/chosen": -287.99981689453125, | |
| "logps/rejected": -186.82659912109375, | |
| "loss": 0.0548, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 3.7472550868988037, | |
| "rewards/margins": 6.025314807891846, | |
| "rewards/rejected": -2.278059482574463, | |
| "step": 124 | |
| }, | |
| { | |
| "epoch": 0.9247706422018349, | |
| "grad_norm": 0.4268924593925476, | |
| "learning_rate": 4.366903875881243e-06, | |
| "logits/chosen": -1.0968043804168701, | |
| "logits/rejected": -2.334925651550293, | |
| "logps/chosen": -275.3115234375, | |
| "logps/rejected": -164.202392578125, | |
| "loss": 0.0128, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 3.771523952484131, | |
| "rewards/margins": 6.7282609939575195, | |
| "rewards/rejected": -2.9567372798919678, | |
| "step": 126 | |
| }, | |
| { | |
| "epoch": 0.9394495412844037, | |
| "grad_norm": 1.4270014762878418, | |
| "learning_rate": 4.3381650808970365e-06, | |
| "logits/chosen": -1.0460113286972046, | |
| "logits/rejected": -1.9695379734039307, | |
| "logps/chosen": -254.8202667236328, | |
| "logps/rejected": -185.63243103027344, | |
| "loss": 0.0226, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 3.706533432006836, | |
| "rewards/margins": 6.099806785583496, | |
| "rewards/rejected": -2.39327335357666, | |
| "step": 128 | |
| }, | |
| { | |
| "epoch": 0.9541284403669725, | |
| "grad_norm": 0.6754117012023926, | |
| "learning_rate": 4.308887518261507e-06, | |
| "logits/chosen": -0.8909565210342407, | |
| "logits/rejected": -1.9432121515274048, | |
| "logps/chosen": -278.40216064453125, | |
| "logps/rejected": -195.16552734375, | |
| "loss": 0.0194, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.237884521484375, | |
| "rewards/margins": 6.639657974243164, | |
| "rewards/rejected": -2.4017739295959473, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.9688073394495413, | |
| "grad_norm": 0.7388483285903931, | |
| "learning_rate": 4.279079769251617e-06, | |
| "logits/chosen": -1.2244815826416016, | |
| "logits/rejected": -2.1885085105895996, | |
| "logps/chosen": -351.21783447265625, | |
| "logps/rejected": -210.98890686035156, | |
| "loss": 0.0184, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.041647911071777, | |
| "rewards/margins": 6.690797328948975, | |
| "rewards/rejected": -2.6491494178771973, | |
| "step": 132 | |
| }, | |
| { | |
| "epoch": 0.9834862385321101, | |
| "grad_norm": 0.7370263934135437, | |
| "learning_rate": 4.248750570542373e-06, | |
| "logits/chosen": -1.0081679821014404, | |
| "logits/rejected": -2.0711734294891357, | |
| "logps/chosen": -272.2639465332031, | |
| "logps/rejected": -179.82412719726562, | |
| "loss": 0.0231, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 3.885261058807373, | |
| "rewards/margins": 6.468730449676514, | |
| "rewards/rejected": -2.5834696292877197, | |
| "step": 134 | |
| }, | |
| { | |
| "epoch": 0.998165137614679, | |
| "grad_norm": 2.1839847564697266, | |
| "learning_rate": 4.21790881164611e-06, | |
| "logits/chosen": -0.9589763879776001, | |
| "logits/rejected": -2.103942394256592, | |
| "logps/chosen": -282.6980285644531, | |
| "logps/rejected": -193.8739776611328, | |
| "loss": 0.0379, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.211104869842529, | |
| "rewards/margins": 7.221211910247803, | |
| "rewards/rejected": -3.0101072788238525, | |
| "step": 136 | |
| }, | |
| { | |
| "epoch": 1.0128440366972478, | |
| "grad_norm": 2.379425525665283, | |
| "learning_rate": 4.186563532306957e-06, | |
| "logits/chosen": -0.9432098865509033, | |
| "logits/rejected": -2.0608460903167725, | |
| "logps/chosen": -288.9028625488281, | |
| "logps/rejected": -168.07359313964844, | |
| "loss": 0.028, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.061460494995117, | |
| "rewards/margins": 7.086147308349609, | |
| "rewards/rejected": -3.0246872901916504, | |
| "step": 138 | |
| }, | |
| { | |
| "epoch": 1.0275229357798166, | |
| "grad_norm": 2.2438290119171143, | |
| "learning_rate": 4.154723919851291e-06, | |
| "logits/chosen": -1.1197127103805542, | |
| "logits/rejected": -2.0973258018493652, | |
| "logps/chosen": -290.60296630859375, | |
| "logps/rejected": -173.36465454101562, | |
| "loss": 0.0308, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 3.5277624130249023, | |
| "rewards/margins": 6.31058406829834, | |
| "rewards/rejected": -2.7828218936920166, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 1.0422018348623854, | |
| "grad_norm": 0.38025742769241333, | |
| "learning_rate": 4.122399306494918e-06, | |
| "logits/chosen": -1.1321005821228027, | |
| "logits/rejected": -2.2533721923828125, | |
| "logps/chosen": -336.11224365234375, | |
| "logps/rejected": -198.53457641601562, | |
| "loss": 0.0211, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.0418477058410645, | |
| "rewards/margins": 7.016913890838623, | |
| "rewards/rejected": -2.975067138671875, | |
| "step": 142 | |
| }, | |
| { | |
| "epoch": 1.0568807339449542, | |
| "grad_norm": 1.0832823514938354, | |
| "learning_rate": 4.089599166607794e-06, | |
| "logits/chosen": -1.0980923175811768, | |
| "logits/rejected": -2.007105588912964, | |
| "logps/chosen": -292.0760803222656, | |
| "logps/rejected": -186.78787231445312, | |
| "loss": 0.0121, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 3.942317008972168, | |
| "rewards/margins": 7.760876178741455, | |
| "rewards/rejected": -3.818559169769287, | |
| "step": 144 | |
| }, | |
| { | |
| "epoch": 1.071559633027523, | |
| "grad_norm": 0.542005717754364, | |
| "learning_rate": 4.05633311393708e-06, | |
| "logits/chosen": -0.9787145853042603, | |
| "logits/rejected": -2.0150396823883057, | |
| "logps/chosen": -257.6767883300781, | |
| "logps/rejected": -172.47512817382812, | |
| "loss": 0.0268, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.031050682067871, | |
| "rewards/margins": 7.134464263916016, | |
| "rewards/rejected": -3.1034140586853027, | |
| "step": 146 | |
| }, | |
| { | |
| "epoch": 1.0862385321100918, | |
| "grad_norm": 1.513509750366211, | |
| "learning_rate": 4.022610898789349e-06, | |
| "logits/chosen": -1.008697509765625, | |
| "logits/rejected": -2.0967135429382324, | |
| "logps/chosen": -266.4443664550781, | |
| "logps/rejected": -186.60263061523438, | |
| "loss": 0.0169, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.120553970336914, | |
| "rewards/margins": 7.502930641174316, | |
| "rewards/rejected": -3.382376194000244, | |
| "step": 148 | |
| }, | |
| { | |
| "epoch": 1.1009174311926606, | |
| "grad_norm": 1.2189836502075195, | |
| "learning_rate": 3.988442405172755e-06, | |
| "logits/chosen": -0.8885701894760132, | |
| "logits/rejected": -2.0014257431030273, | |
| "logps/chosen": -281.70147705078125, | |
| "logps/rejected": -201.9718780517578, | |
| "loss": 0.0121, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.45000696182251, | |
| "rewards/margins": 7.705287933349609, | |
| "rewards/rejected": -3.2552807331085205, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 1.1155963302752294, | |
| "grad_norm": 0.2563473880290985, | |
| "learning_rate": 3.953837647900031e-06, | |
| "logits/chosen": -0.9757863283157349, | |
| "logits/rejected": -2.0974419116973877, | |
| "logps/chosen": -273.5846862792969, | |
| "logps/rejected": -195.75936889648438, | |
| "loss": 0.0107, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.658951759338379, | |
| "rewards/margins": 8.117878913879395, | |
| "rewards/rejected": -3.4589266777038574, | |
| "step": 152 | |
| }, | |
| { | |
| "epoch": 1.1302752293577982, | |
| "grad_norm": 2.6809535026550293, | |
| "learning_rate": 3.918806769653135e-06, | |
| "logits/chosen": -0.8756412863731384, | |
| "logits/rejected": -1.9975080490112305, | |
| "logps/chosen": -318.453857421875, | |
| "logps/rejected": -195.71372985839844, | |
| "loss": 0.0324, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.540386199951172, | |
| "rewards/margins": 7.758340358734131, | |
| "rewards/rejected": -3.217954158782959, | |
| "step": 154 | |
| }, | |
| { | |
| "epoch": 1.144954128440367, | |
| "grad_norm": 0.34194982051849365, | |
| "learning_rate": 3.88336003801042e-06, | |
| "logits/chosen": -0.9494649171829224, | |
| "logits/rejected": -2.052715301513672, | |
| "logps/chosen": -255.02169799804688, | |
| "logps/rejected": -178.14224243164062, | |
| "loss": 0.0114, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 3.8328986167907715, | |
| "rewards/margins": 7.075186729431152, | |
| "rewards/rejected": -3.2422876358032227, | |
| "step": 156 | |
| }, | |
| { | |
| "epoch": 1.1596330275229358, | |
| "grad_norm": 0.37112390995025635, | |
| "learning_rate": 3.847507842437205e-06, | |
| "logits/chosen": -0.8547274470329285, | |
| "logits/rejected": -2.1034629344940186, | |
| "logps/chosen": -296.8822021484375, | |
| "logps/rejected": -171.6925048828125, | |
| "loss": 0.007, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.401730060577393, | |
| "rewards/margins": 8.271763801574707, | |
| "rewards/rejected": -3.8700337409973145, | |
| "step": 158 | |
| }, | |
| { | |
| "epoch": 1.1743119266055047, | |
| "grad_norm": 0.6065702438354492, | |
| "learning_rate": 3.811260691240604e-06, | |
| "logits/chosen": -0.894873857498169, | |
| "logits/rejected": -2.086596965789795, | |
| "logps/chosen": -340.1643981933594, | |
| "logps/rejected": -188.5568389892578, | |
| "loss": 0.009, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.769918441772461, | |
| "rewards/margins": 8.063861846923828, | |
| "rewards/rejected": -3.293943166732788, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 1.1889908256880735, | |
| "grad_norm": 0.38738325238227844, | |
| "learning_rate": 3.774629208489547e-06, | |
| "logits/chosen": -0.9661360383033752, | |
| "logits/rejected": -2.0905256271362305, | |
| "logps/chosen": -241.7164764404297, | |
| "logps/rejected": -172.8728790283203, | |
| "loss": 0.0084, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 3.948549270629883, | |
| "rewards/margins": 7.093012809753418, | |
| "rewards/rejected": -3.1444640159606934, | |
| "step": 162 | |
| }, | |
| { | |
| "epoch": 1.2036697247706423, | |
| "grad_norm": 0.4064182639122009, | |
| "learning_rate": 3.7376241309008433e-06, | |
| "logits/chosen": -1.1252474784851074, | |
| "logits/rejected": -2.123969793319702, | |
| "logps/chosen": -326.73370361328125, | |
| "logps/rejected": -183.0895233154297, | |
| "loss": 0.0133, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.725476264953613, | |
| "rewards/margins": 7.916323661804199, | |
| "rewards/rejected": -3.190847635269165, | |
| "step": 164 | |
| }, | |
| { | |
| "epoch": 1.218348623853211, | |
| "grad_norm": 0.3772048056125641, | |
| "learning_rate": 3.7002563046922502e-06, | |
| "logits/chosen": -1.0913598537445068, | |
| "logits/rejected": -2.229214668273926, | |
| "logps/chosen": -326.4932861328125, | |
| "logps/rejected": -173.02989196777344, | |
| "loss": 0.0044, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.724410533905029, | |
| "rewards/margins": 8.843596458435059, | |
| "rewards/rejected": -4.119184970855713, | |
| "step": 166 | |
| }, | |
| { | |
| "epoch": 1.2330275229357799, | |
| "grad_norm": 0.48601555824279785, | |
| "learning_rate": 3.6625366824034337e-06, | |
| "logits/chosen": -0.8681567907333374, | |
| "logits/rejected": -2.067228317260742, | |
| "logps/chosen": -279.7916259765625, | |
| "logps/rejected": -206.9775390625, | |
| "loss": 0.0139, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.7722625732421875, | |
| "rewards/margins": 9.153127670288086, | |
| "rewards/rejected": -4.380865097045898, | |
| "step": 168 | |
| }, | |
| { | |
| "epoch": 1.2477064220183487, | |
| "grad_norm": 1.1044621467590332, | |
| "learning_rate": 3.6244763196857714e-06, | |
| "logits/chosen": -0.9898172616958618, | |
| "logits/rejected": -2.130460262298584, | |
| "logps/chosen": -296.6734619140625, | |
| "logps/rejected": -181.456298828125, | |
| "loss": 0.013, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.674115180969238, | |
| "rewards/margins": 8.792963981628418, | |
| "rewards/rejected": -4.1188483238220215, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 1.2623853211009175, | |
| "grad_norm": 1.3868632316589355, | |
| "learning_rate": 3.5860863720619333e-06, | |
| "logits/chosen": -1.0125945806503296, | |
| "logits/rejected": -2.080739736557007, | |
| "logps/chosen": -289.3682861328125, | |
| "logps/rejected": -184.58253479003906, | |
| "loss": 0.0137, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.030690670013428, | |
| "rewards/margins": 8.200502395629883, | |
| "rewards/rejected": -3.1698126792907715, | |
| "step": 172 | |
| }, | |
| { | |
| "epoch": 1.2770642201834863, | |
| "grad_norm": 0.4792233407497406, | |
| "learning_rate": 3.547378091656186e-06, | |
| "logits/chosen": -0.9022351503372192, | |
| "logits/rejected": -2.0722413063049316, | |
| "logps/chosen": -293.7245178222656, | |
| "logps/rejected": -173.53054809570312, | |
| "loss": 0.0092, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.439302444458008, | |
| "rewards/margins": 8.599308013916016, | |
| "rewards/rejected": -4.160006046295166, | |
| "step": 174 | |
| }, | |
| { | |
| "epoch": 1.2917431192660551, | |
| "grad_norm": 0.7374489903450012, | |
| "learning_rate": 3.5083628238963913e-06, | |
| "logits/chosen": -1.085463047027588, | |
| "logits/rejected": -1.969193935394287, | |
| "logps/chosen": -234.489013671875, | |
| "logps/rejected": -175.44613647460938, | |
| "loss": 0.0147, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.414360523223877, | |
| "rewards/margins": 7.9778642654418945, | |
| "rewards/rejected": -3.5635030269622803, | |
| "step": 176 | |
| }, | |
| { | |
| "epoch": 1.306422018348624, | |
| "grad_norm": 1.7487801313400269, | |
| "learning_rate": 3.4690520041886473e-06, | |
| "logits/chosen": -0.9150568246841431, | |
| "logits/rejected": -2.0502333641052246, | |
| "logps/chosen": -275.4502258300781, | |
| "logps/rejected": -212.3257598876953, | |
| "loss": 0.0157, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.58690071105957, | |
| "rewards/margins": 8.234541893005371, | |
| "rewards/rejected": -3.647641181945801, | |
| "step": 178 | |
| }, | |
| { | |
| "epoch": 1.3211009174311927, | |
| "grad_norm": 0.12792479991912842, | |
| "learning_rate": 3.4294571545655653e-06, | |
| "logits/chosen": -0.91706383228302, | |
| "logits/rejected": -2.196730613708496, | |
| "logps/chosen": -293.5966796875, | |
| "logps/rejected": -180.54701232910156, | |
| "loss": 0.0033, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.876993656158447, | |
| "rewards/margins": 9.287820816040039, | |
| "rewards/rejected": -4.41082763671875, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 1.3357798165137615, | |
| "grad_norm": 1.574942708015442, | |
| "learning_rate": 3.38958988030915e-06, | |
| "logits/chosen": -1.1890692710876465, | |
| "logits/rejected": -2.066960334777832, | |
| "logps/chosen": -274.7825622558594, | |
| "logps/rejected": -224.47837829589844, | |
| "loss": 0.0592, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 4.447505950927734, | |
| "rewards/margins": 8.213159561157227, | |
| "rewards/rejected": -3.7656538486480713, | |
| "step": 182 | |
| }, | |
| { | |
| "epoch": 1.3504587155963304, | |
| "grad_norm": 1.0345042943954468, | |
| "learning_rate": 3.3494618665492833e-06, | |
| "logits/chosen": -1.1099860668182373, | |
| "logits/rejected": -2.0204684734344482, | |
| "logps/chosen": -255.40478515625, | |
| "logps/rejected": -192.52752685546875, | |
| "loss": 0.0152, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.036734104156494, | |
| "rewards/margins": 7.486913681030273, | |
| "rewards/rejected": -3.4501795768737793, | |
| "step": 184 | |
| }, | |
| { | |
| "epoch": 1.3651376146788992, | |
| "grad_norm": 0.1829257309436798, | |
| "learning_rate": 3.3090848748388042e-06, | |
| "logits/chosen": -1.0115846395492554, | |
| "logits/rejected": -2.1213629245758057, | |
| "logps/chosen": -353.5410461425781, | |
| "logps/rejected": -192.9250030517578, | |
| "loss": 0.0061, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.421483993530273, | |
| "rewards/margins": 8.999296188354492, | |
| "rewards/rejected": -4.577812194824219, | |
| "step": 186 | |
| }, | |
| { | |
| "epoch": 1.379816513761468, | |
| "grad_norm": 0.3030329942703247, | |
| "learning_rate": 3.2684707397061887e-06, | |
| "logits/chosen": -1.0969910621643066, | |
| "logits/rejected": -2.0923759937286377, | |
| "logps/chosen": -293.1423645019531, | |
| "logps/rejected": -173.88784790039062, | |
| "loss": 0.0124, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.567940711975098, | |
| "rewards/margins": 8.394798278808594, | |
| "rewards/rejected": -3.8268580436706543, | |
| "step": 188 | |
| }, | |
| { | |
| "epoch": 1.3944954128440368, | |
| "grad_norm": 0.8538657426834106, | |
| "learning_rate": 3.2276313651868364e-06, | |
| "logits/chosen": -0.9523632526397705, | |
| "logits/rejected": -2.0854203701019287, | |
| "logps/chosen": -297.4543762207031, | |
| "logps/rejected": -162.13568115234375, | |
| "loss": 0.0139, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.548934459686279, | |
| "rewards/margins": 8.458111763000488, | |
| "rewards/rejected": -3.909177541732788, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 1.4091743119266056, | |
| "grad_norm": 0.4353146553039551, | |
| "learning_rate": 3.1865787213339926e-06, | |
| "logits/chosen": -0.9564714431762695, | |
| "logits/rejected": -2.0908193588256836, | |
| "logps/chosen": -281.3487243652344, | |
| "logps/rejected": -186.54757690429688, | |
| "loss": 0.0115, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.7654829025268555, | |
| "rewards/margins": 9.230217933654785, | |
| "rewards/rejected": -4.464734077453613, | |
| "step": 192 | |
| }, | |
| { | |
| "epoch": 1.4238532110091744, | |
| "grad_norm": 0.40312162041664124, | |
| "learning_rate": 3.1453248407103156e-06, | |
| "logits/chosen": -0.9966449737548828, | |
| "logits/rejected": -2.1248295307159424, | |
| "logps/chosen": -287.3999328613281, | |
| "logps/rejected": -169.44496154785156, | |
| "loss": 0.008, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.19376802444458, | |
| "rewards/margins": 8.428169250488281, | |
| "rewards/rejected": -4.234401702880859, | |
| "step": 194 | |
| }, | |
| { | |
| "epoch": 1.4385321100917432, | |
| "grad_norm": 1.217081904411316, | |
| "learning_rate": 3.1038818148611178e-06, | |
| "logits/chosen": -1.022183895111084, | |
| "logits/rejected": -2.0069739818573, | |
| "logps/chosen": -312.16973876953125, | |
| "logps/rejected": -181.82955932617188, | |
| "loss": 0.0132, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.685446262359619, | |
| "rewards/margins": 8.785126686096191, | |
| "rewards/rejected": -4.099679946899414, | |
| "step": 196 | |
| }, | |
| { | |
| "epoch": 1.453211009174312, | |
| "grad_norm": 0.6292124390602112, | |
| "learning_rate": 3.062261790770331e-06, | |
| "logits/chosen": -0.8997288942337036, | |
| "logits/rejected": -1.9895069599151611, | |
| "logps/chosen": -259.7031555175781, | |
| "logps/rejected": -182.4678497314453, | |
| "loss": 0.0253, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.453324794769287, | |
| "rewards/margins": 7.995000839233398, | |
| "rewards/rejected": -3.541675567626953, | |
| "step": 198 | |
| }, | |
| { | |
| "epoch": 1.4678899082568808, | |
| "grad_norm": 0.46019911766052246, | |
| "learning_rate": 3.0204769673002123e-06, | |
| "logits/chosen": -0.981975793838501, | |
| "logits/rejected": -2.123629331588745, | |
| "logps/chosen": -333.59722900390625, | |
| "logps/rejected": -198.2655487060547, | |
| "loss": 0.0062, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.988855838775635, | |
| "rewards/margins": 8.937817573547363, | |
| "rewards/rejected": -3.948960781097412, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 1.4825688073394496, | |
| "grad_norm": 1.5788525342941284, | |
| "learning_rate": 2.978539591615848e-06, | |
| "logits/chosen": -1.0232621431350708, | |
| "logits/rejected": -1.9014160633087158, | |
| "logps/chosen": -299.21649169921875, | |
| "logps/rejected": -196.33389282226562, | |
| "loss": 0.0167, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 4.068594932556152, | |
| "rewards/margins": 8.521183013916016, | |
| "rewards/rejected": -4.452587604522705, | |
| "step": 202 | |
| }, | |
| { | |
| "epoch": 1.4972477064220184, | |
| "grad_norm": 0.18567878007888794, | |
| "learning_rate": 2.936461955595501e-06, | |
| "logits/chosen": -1.0283303260803223, | |
| "logits/rejected": -2.1100425720214844, | |
| "logps/chosen": -298.8528137207031, | |
| "logps/rejected": -191.35086059570312, | |
| "loss": 0.0068, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.9132208824157715, | |
| "rewards/margins": 8.617488861083984, | |
| "rewards/rejected": -3.704267978668213, | |
| "step": 204 | |
| }, | |
| { | |
| "epoch": 1.5119266055045872, | |
| "grad_norm": 0.13169872760772705, | |
| "learning_rate": 2.8942563922278487e-06, | |
| "logits/chosen": -1.0413228273391724, | |
| "logits/rejected": -2.1321609020233154, | |
| "logps/chosen": -286.2435607910156, | |
| "logps/rejected": -196.66256713867188, | |
| "loss": 0.011, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.733799457550049, | |
| "rewards/margins": 9.53592586517334, | |
| "rewards/rejected": -4.802126407623291, | |
| "step": 206 | |
| }, | |
| { | |
| "epoch": 1.526605504587156, | |
| "grad_norm": 2.0917320251464844, | |
| "learning_rate": 2.8519352719971783e-06, | |
| "logits/chosen": -1.097141981124878, | |
| "logits/rejected": -2.0799503326416016, | |
| "logps/chosen": -316.93597412109375, | |
| "logps/rejected": -201.67100524902344, | |
| "loss": 0.0309, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 4.9373931884765625, | |
| "rewards/margins": 8.860273361206055, | |
| "rewards/rejected": -3.9228808879852295, | |
| "step": 208 | |
| }, | |
| { | |
| "epoch": 1.5412844036697249, | |
| "grad_norm": 0.6296855807304382, | |
| "learning_rate": 2.8095109992575824e-06, | |
| "logits/chosen": -0.9797852039337158, | |
| "logits/rejected": -2.088029146194458, | |
| "logps/chosen": -328.76251220703125, | |
| "logps/rejected": -201.71078491210938, | |
| "loss": 0.0049, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.399056911468506, | |
| "rewards/margins": 9.34478759765625, | |
| "rewards/rejected": -3.945730209350586, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 1.5559633027522937, | |
| "grad_norm": 0.3409838080406189, | |
| "learning_rate": 2.7669960085972407e-06, | |
| "logits/chosen": -0.9346829652786255, | |
| "logits/rejected": -2.2055399417877197, | |
| "logps/chosen": -351.57489013671875, | |
| "logps/rejected": -219.8714141845703, | |
| "loss": 0.0047, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.243146896362305, | |
| "rewards/margins": 9.388729095458984, | |
| "rewards/rejected": -4.145582675933838, | |
| "step": 212 | |
| }, | |
| { | |
| "epoch": 1.5706422018348625, | |
| "grad_norm": 0.791716456413269, | |
| "learning_rate": 2.7244027611938247e-06, | |
| "logits/chosen": -0.8380637764930725, | |
| "logits/rejected": -1.925654649734497, | |
| "logps/chosen": -251.362548828125, | |
| "logps/rejected": -220.16436767578125, | |
| "loss": 0.0215, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.47086763381958, | |
| "rewards/margins": 8.536866188049316, | |
| "rewards/rejected": -4.065998554229736, | |
| "step": 214 | |
| }, | |
| { | |
| "epoch": 1.5853211009174313, | |
| "grad_norm": 0.2506906986236572, | |
| "learning_rate": 2.6817437411621194e-06, | |
| "logits/chosen": -0.9830411076545715, | |
| "logits/rejected": -2.0578300952911377, | |
| "logps/chosen": -342.7967529296875, | |
| "logps/rejected": -237.17025756835938, | |
| "loss": 0.007, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.036384582519531, | |
| "rewards/margins": 9.013383865356445, | |
| "rewards/rejected": -3.976999521255493, | |
| "step": 216 | |
| }, | |
| { | |
| "epoch": 1.6, | |
| "grad_norm": 0.22265683114528656, | |
| "learning_rate": 2.639031451894923e-06, | |
| "logits/chosen": -1.028990387916565, | |
| "logits/rejected": -1.9095451831817627, | |
| "logps/chosen": -330.1585998535156, | |
| "logps/rejected": -222.2620086669922, | |
| "loss": 0.0033, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.05686616897583, | |
| "rewards/margins": 9.352256774902344, | |
| "rewards/rejected": -4.2953901290893555, | |
| "step": 218 | |
| }, | |
| { | |
| "epoch": 1.614678899082569, | |
| "grad_norm": 0.857473611831665, | |
| "learning_rate": 2.5962784123982843e-06, | |
| "logits/chosen": -1.049895167350769, | |
| "logits/rejected": -2.1700665950775146, | |
| "logps/chosen": -305.7288513183594, | |
| "logps/rejected": -198.574462890625, | |
| "loss": 0.0122, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.7103071212768555, | |
| "rewards/margins": 9.439103126525879, | |
| "rewards/rejected": -4.728795528411865, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 1.6293577981651377, | |
| "grad_norm": 0.2698463797569275, | |
| "learning_rate": 2.5534971536221804e-06, | |
| "logits/chosen": -0.861595630645752, | |
| "logits/rejected": -1.9534931182861328, | |
| "logps/chosen": -268.7453918457031, | |
| "logps/rejected": -191.55238342285156, | |
| "loss": 0.024, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 4.435842514038086, | |
| "rewards/margins": 9.0311918258667, | |
| "rewards/rejected": -4.595349311828613, | |
| "step": 222 | |
| }, | |
| { | |
| "epoch": 1.6440366972477065, | |
| "grad_norm": 1.8740975856781006, | |
| "learning_rate": 2.5107002147876814e-06, | |
| "logits/chosen": -1.010701298713684, | |
| "logits/rejected": -1.9186618328094482, | |
| "logps/chosen": -263.1980895996094, | |
| "logps/rejected": -206.22360229492188, | |
| "loss": 0.0135, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.652899742126465, | |
| "rewards/margins": 9.141688346862793, | |
| "rewards/rejected": -4.488787651062012, | |
| "step": 224 | |
| }, | |
| { | |
| "epoch": 1.6587155963302753, | |
| "grad_norm": 2.0281364917755127, | |
| "learning_rate": 2.467900139711693e-06, | |
| "logits/chosen": -1.0440551042556763, | |
| "logits/rejected": -1.971301555633545, | |
| "logps/chosen": -272.8301696777344, | |
| "logps/rejected": -197.07269287109375, | |
| "loss": 0.0238, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.243446350097656, | |
| "rewards/margins": 8.737079620361328, | |
| "rewards/rejected": -4.493633270263672, | |
| "step": 226 | |
| }, | |
| { | |
| "epoch": 1.6733944954128441, | |
| "grad_norm": 0.23005536198616028, | |
| "learning_rate": 2.4251094731303586e-06, | |
| "logits/chosen": -0.9269182085990906, | |
| "logits/rejected": -2.089838981628418, | |
| "logps/chosen": -291.0529479980469, | |
| "logps/rejected": -179.94895935058594, | |
| "loss": 0.0057, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.278434753417969, | |
| "rewards/margins": 9.40982723236084, | |
| "rewards/rejected": -4.131391525268555, | |
| "step": 228 | |
| }, | |
| { | |
| "epoch": 1.688073394495413, | |
| "grad_norm": 0.0681939348578453, | |
| "learning_rate": 2.3823407570221812e-06, | |
| "logits/chosen": -0.8353657126426697, | |
| "logits/rejected": -2.02689266204834, | |
| "logps/chosen": -300.3406982421875, | |
| "logps/rejected": -175.13296508789062, | |
| "loss": 0.006, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.921877384185791, | |
| "rewards/margins": 9.119339942932129, | |
| "rewards/rejected": -4.197463035583496, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 1.7027522935779817, | |
| "grad_norm": 0.24333705008029938, | |
| "learning_rate": 2.3396065269319655e-06, | |
| "logits/chosen": -1.0092397928237915, | |
| "logits/rejected": -2.1053268909454346, | |
| "logps/chosen": -300.02294921875, | |
| "logps/rejected": -172.78187561035156, | |
| "loss": 0.0091, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.987635612487793, | |
| "rewards/margins": 9.38530445098877, | |
| "rewards/rejected": -4.397668361663818, | |
| "step": 232 | |
| }, | |
| { | |
| "epoch": 1.7174311926605506, | |
| "grad_norm": 0.6797487139701843, | |
| "learning_rate": 2.2969193082966353e-06, | |
| "logits/chosen": -0.8851895332336426, | |
| "logits/rejected": -2.036161422729492, | |
| "logps/chosen": -285.2466735839844, | |
| "logps/rejected": -189.85882568359375, | |
| "loss": 0.0061, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.067909240722656, | |
| "rewards/margins": 9.919548988342285, | |
| "rewards/rejected": -4.851640701293945, | |
| "step": 234 | |
| }, | |
| { | |
| "epoch": 1.7321100917431194, | |
| "grad_norm": 0.6367282271385193, | |
| "learning_rate": 2.2542916127740194e-06, | |
| "logits/chosen": -0.8543779253959656, | |
| "logits/rejected": -1.752845048904419, | |
| "logps/chosen": -312.6046142578125, | |
| "logps/rejected": -234.28988647460938, | |
| "loss": 0.0121, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.974181175231934, | |
| "rewards/margins": 9.56247329711914, | |
| "rewards/rejected": -4.588292121887207, | |
| "step": 236 | |
| }, | |
| { | |
| "epoch": 1.7467889908256882, | |
| "grad_norm": 0.2897071838378906, | |
| "learning_rate": 2.211735934575674e-06, | |
| "logits/chosen": -0.9410618543624878, | |
| "logits/rejected": -2.174349308013916, | |
| "logps/chosen": -281.28863525390625, | |
| "logps/rejected": -165.9616241455078, | |
| "loss": 0.0067, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.256411552429199, | |
| "rewards/margins": 9.012039184570312, | |
| "rewards/rejected": -4.755627632141113, | |
| "step": 238 | |
| }, | |
| { | |
| "epoch": 1.761467889908257, | |
| "grad_norm": 0.41199827194213867, | |
| "learning_rate": 2.1692647468048235e-06, | |
| "logits/chosen": -1.0583674907684326, | |
| "logits/rejected": -2.0003695487976074, | |
| "logps/chosen": -307.04766845703125, | |
| "logps/rejected": -206.0718231201172, | |
| "loss": 0.0062, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.970597743988037, | |
| "rewards/margins": 10.884113311767578, | |
| "rewards/rejected": -5.913515567779541, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 1.7761467889908258, | |
| "grad_norm": 0.5175734162330627, | |
| "learning_rate": 2.126890497800477e-06, | |
| "logits/chosen": -1.0432560443878174, | |
| "logits/rejected": -1.912244439125061, | |
| "logps/chosen": -297.5209655761719, | |
| "logps/rejected": -201.7635498046875, | |
| "loss": 0.0184, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.594603538513184, | |
| "rewards/margins": 8.852926254272461, | |
| "rewards/rejected": -4.258323669433594, | |
| "step": 242 | |
| }, | |
| { | |
| "epoch": 1.7908256880733946, | |
| "grad_norm": 0.6547983288764954, | |
| "learning_rate": 2.084625607488816e-06, | |
| "logits/chosen": -0.9311404228210449, | |
| "logits/rejected": -2.1106457710266113, | |
| "logps/chosen": -275.57183837890625, | |
| "logps/rejected": -188.14370727539062, | |
| "loss": 0.0105, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.0517778396606445, | |
| "rewards/margins": 9.953323364257812, | |
| "rewards/rejected": -4.901544094085693, | |
| "step": 244 | |
| }, | |
| { | |
| "epoch": 1.8055045871559634, | |
| "grad_norm": 0.33203306794166565, | |
| "learning_rate": 2.0424824637428995e-06, | |
| "logits/chosen": -0.9116280674934387, | |
| "logits/rejected": -2.247035026550293, | |
| "logps/chosen": -267.2120361328125, | |
| "logps/rejected": -171.6895751953125, | |
| "loss": 0.0084, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.797155857086182, | |
| "rewards/margins": 9.531312942504883, | |
| "rewards/rejected": -4.734157562255859, | |
| "step": 246 | |
| }, | |
| { | |
| "epoch": 1.8201834862385322, | |
| "grad_norm": 0.5430265069007874, | |
| "learning_rate": 2.0004734187517744e-06, | |
| "logits/chosen": -1.082189917564392, | |
| "logits/rejected": -1.9552661180496216, | |
| "logps/chosen": -318.3630676269531, | |
| "logps/rejected": -176.5325469970703, | |
| "loss": 0.0103, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.9373297691345215, | |
| "rewards/margins": 9.501139640808105, | |
| "rewards/rejected": -4.563809871673584, | |
| "step": 248 | |
| }, | |
| { | |
| "epoch": 1.834862385321101, | |
| "grad_norm": 0.19558808207511902, | |
| "learning_rate": 1.9586107854000327e-06, | |
| "logits/chosen": -1.1152639389038086, | |
| "logits/rejected": -2.129647731781006, | |
| "logps/chosen": -296.6053466796875, | |
| "logps/rejected": -169.00213623046875, | |
| "loss": 0.0058, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.840038299560547, | |
| "rewards/margins": 9.542232513427734, | |
| "rewards/rejected": -4.702193737030029, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 1.8495412844036698, | |
| "grad_norm": 0.45886340737342834, | |
| "learning_rate": 1.916906833658899e-06, | |
| "logits/chosen": -0.8982828855514526, | |
| "logits/rejected": -2.0570406913757324, | |
| "logps/chosen": -324.3260803222656, | |
| "logps/rejected": -216.9913330078125, | |
| "loss": 0.0041, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.856814861297607, | |
| "rewards/margins": 9.941521644592285, | |
| "rewards/rejected": -5.084706783294678, | |
| "step": 252 | |
| }, | |
| { | |
| "epoch": 1.8642201834862386, | |
| "grad_norm": 0.9119444489479065, | |
| "learning_rate": 1.8753737869898921e-06, | |
| "logits/chosen": -0.972162663936615, | |
| "logits/rejected": -2.016150951385498, | |
| "logps/chosen": -248.53463745117188, | |
| "logps/rejected": -184.32382202148438, | |
| "loss": 0.007, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.359373092651367, | |
| "rewards/margins": 10.132734298706055, | |
| "rewards/rejected": -5.7733612060546875, | |
| "step": 254 | |
| }, | |
| { | |
| "epoch": 1.8788990825688074, | |
| "grad_norm": 0.12387188524007797, | |
| "learning_rate": 1.8340238187621185e-06, | |
| "logits/chosen": -0.8442805409431458, | |
| "logits/rejected": -1.9759818315505981, | |
| "logps/chosen": -262.56671142578125, | |
| "logps/rejected": -175.5653839111328, | |
| "loss": 0.0327, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 4.67296028137207, | |
| "rewards/margins": 9.085709571838379, | |
| "rewards/rejected": -4.412749290466309, | |
| "step": 256 | |
| }, | |
| { | |
| "epoch": 1.8935779816513763, | |
| "grad_norm": 0.5057358145713806, | |
| "learning_rate": 1.7928690486842438e-06, | |
| "logits/chosen": -1.015974760055542, | |
| "logits/rejected": -2.1081368923187256, | |
| "logps/chosen": -253.27394104003906, | |
| "logps/rejected": -160.09469604492188, | |
| "loss": 0.0064, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.807024002075195, | |
| "rewards/margins": 9.237527847290039, | |
| "rewards/rejected": -4.4305033683776855, | |
| "step": 258 | |
| }, | |
| { | |
| "epoch": 1.908256880733945, | |
| "grad_norm": 0.6048524379730225, | |
| "learning_rate": 1.7519215392522026e-06, | |
| "logits/chosen": -0.9711456298828125, | |
| "logits/rejected": -2.1203389167785645, | |
| "logps/chosen": -282.3438720703125, | |
| "logps/rejected": -166.2510528564453, | |
| "loss": 0.0064, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.09495735168457, | |
| "rewards/margins": 9.728025436401367, | |
| "rewards/rejected": -4.633067607879639, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 1.9229357798165139, | |
| "grad_norm": 0.5542910695075989, | |
| "learning_rate": 1.7111932922136715e-06, | |
| "logits/chosen": -0.9748891592025757, | |
| "logits/rejected": -1.8318710327148438, | |
| "logps/chosen": -253.21209716796875, | |
| "logps/rejected": -202.5255889892578, | |
| "loss": 0.019, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 4.323376178741455, | |
| "rewards/margins": 9.282448768615723, | |
| "rewards/rejected": -4.959072589874268, | |
| "step": 262 | |
| }, | |
| { | |
| "epoch": 1.9376146788990827, | |
| "grad_norm": 0.24626314640045166, | |
| "learning_rate": 1.6706962450503408e-06, | |
| "logits/chosen": -0.8283478617668152, | |
| "logits/rejected": -2.0624401569366455, | |
| "logps/chosen": -282.2995300292969, | |
| "logps/rejected": -189.75595092773438, | |
| "loss": 0.0059, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.900084495544434, | |
| "rewards/margins": 10.33100414276123, | |
| "rewards/rejected": -5.4309186935424805, | |
| "step": 264 | |
| }, | |
| { | |
| "epoch": 1.9522935779816515, | |
| "grad_norm": 1.167913794517517, | |
| "learning_rate": 1.630442267479034e-06, | |
| "logits/chosen": -0.789318323135376, | |
| "logits/rejected": -1.9187240600585938, | |
| "logps/chosen": -266.4274597167969, | |
| "logps/rejected": -198.47540283203125, | |
| "loss": 0.0113, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.9975690841674805, | |
| "rewards/margins": 9.762743949890137, | |
| "rewards/rejected": -4.765174865722656, | |
| "step": 266 | |
| }, | |
| { | |
| "epoch": 1.9669724770642203, | |
| "grad_norm": 0.05298791825771332, | |
| "learning_rate": 1.5904431579726837e-06, | |
| "logits/chosen": -0.9226531982421875, | |
| "logits/rejected": -2.0884995460510254, | |
| "logps/chosen": -295.6236267089844, | |
| "logps/rejected": -165.60801696777344, | |
| "loss": 0.0066, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.444765567779541, | |
| "rewards/margins": 9.557327270507812, | |
| "rewards/rejected": -5.1125617027282715, | |
| "step": 268 | |
| }, | |
| { | |
| "epoch": 1.981651376146789, | |
| "grad_norm": 0.11875250190496445, | |
| "learning_rate": 1.5507106403021897e-06, | |
| "logits/chosen": -0.8945147395133972, | |
| "logits/rejected": -2.1213436126708984, | |
| "logps/chosen": -329.32354736328125, | |
| "logps/rejected": -205.64938354492188, | |
| "loss": 0.0028, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.863536834716797, | |
| "rewards/margins": 10.598997116088867, | |
| "rewards/rejected": -4.735459804534912, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 1.996330275229358, | |
| "grad_norm": 0.16463226079940796, | |
| "learning_rate": 1.511256360100171e-06, | |
| "logits/chosen": -0.8653547167778015, | |
| "logits/rejected": -2.120985746383667, | |
| "logps/chosen": -294.7039489746094, | |
| "logps/rejected": -191.1700897216797, | |
| "loss": 0.0052, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.6077494621276855, | |
| "rewards/margins": 9.97114372253418, | |
| "rewards/rejected": -5.363394737243652, | |
| "step": 272 | |
| }, | |
| { | |
| "epoch": 2.0110091743119267, | |
| "grad_norm": 0.5620644092559814, | |
| "learning_rate": 1.4720918814476234e-06, | |
| "logits/chosen": -1.0870428085327148, | |
| "logits/rejected": -2.203629493713379, | |
| "logps/chosen": -255.451171875, | |
| "logps/rejected": -179.9131317138672, | |
| "loss": 0.0107, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.301146984100342, | |
| "rewards/margins": 10.202719688415527, | |
| "rewards/rejected": -5.9015727043151855, | |
| "step": 274 | |
| }, | |
| { | |
| "epoch": 2.0256880733944955, | |
| "grad_norm": 0.22174260020256042, | |
| "learning_rate": 1.4332286834844792e-06, | |
| "logits/chosen": -1.1182466745376587, | |
| "logits/rejected": -2.1164536476135254, | |
| "logps/chosen": -286.516357421875, | |
| "logps/rejected": -188.3638458251953, | |
| "loss": 0.0033, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.640047550201416, | |
| "rewards/margins": 9.481383323669434, | |
| "rewards/rejected": -4.841336250305176, | |
| "step": 276 | |
| }, | |
| { | |
| "epoch": 2.0403669724770643, | |
| "grad_norm": 0.33157217502593994, | |
| "learning_rate": 1.3946781570450563e-06, | |
| "logits/chosen": -0.9743894338607788, | |
| "logits/rejected": -2.0844216346740723, | |
| "logps/chosen": -303.1180419921875, | |
| "logps/rejected": -197.849853515625, | |
| "loss": 0.0026, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.488700866699219, | |
| "rewards/margins": 10.087553024291992, | |
| "rewards/rejected": -4.598852157592773, | |
| "step": 278 | |
| }, | |
| { | |
| "epoch": 2.055045871559633, | |
| "grad_norm": 0.2071988433599472, | |
| "learning_rate": 1.3564516013194023e-06, | |
| "logits/chosen": -0.7817774415016174, | |
| "logits/rejected": -1.967786431312561, | |
| "logps/chosen": -266.5663757324219, | |
| "logps/rejected": -185.63877868652344, | |
| "loss": 0.004, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.6336894035339355, | |
| "rewards/margins": 10.230566024780273, | |
| "rewards/rejected": -5.596876621246338, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 2.069724770642202, | |
| "grad_norm": 0.35437583923339844, | |
| "learning_rate": 1.3185602205414894e-06, | |
| "logits/chosen": -0.9503396153450012, | |
| "logits/rejected": -2.0260818004608154, | |
| "logps/chosen": -269.90093994140625, | |
| "logps/rejected": -172.7965850830078, | |
| "loss": 0.0059, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.775392532348633, | |
| "rewards/margins": 9.074084281921387, | |
| "rewards/rejected": -4.298691272735596, | |
| "step": 282 | |
| }, | |
| { | |
| "epoch": 2.0844036697247708, | |
| "grad_norm": 0.09949786216020584, | |
| "learning_rate": 1.2810151207052465e-06, | |
| "logits/chosen": -1.025212049484253, | |
| "logits/rejected": -2.090640068054199, | |
| "logps/chosen": -335.35882568359375, | |
| "logps/rejected": -221.98355102539062, | |
| "loss": 0.0042, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.815802097320557, | |
| "rewards/margins": 9.805765151977539, | |
| "rewards/rejected": -4.989964008331299, | |
| "step": 284 | |
| }, | |
| { | |
| "epoch": 2.0990825688073396, | |
| "grad_norm": 0.1902090311050415, | |
| "learning_rate": 1.2438273063093811e-06, | |
| "logits/chosen": -0.8500208854675293, | |
| "logits/rejected": -1.9380009174346924, | |
| "logps/chosen": -277.2483215332031, | |
| "logps/rejected": -168.717529296875, | |
| "loss": 0.0127, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.573535442352295, | |
| "rewards/margins": 9.026323318481445, | |
| "rewards/rejected": -4.452788352966309, | |
| "step": 286 | |
| }, | |
| { | |
| "epoch": 2.1137614678899084, | |
| "grad_norm": 0.3208858072757721, | |
| "learning_rate": 1.2070076771319536e-06, | |
| "logits/chosen": -1.082637906074524, | |
| "logits/rejected": -1.9498220682144165, | |
| "logps/chosen": -353.7499084472656, | |
| "logps/rejected": -200.4058074951172, | |
| "loss": 0.0101, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.898225784301758, | |
| "rewards/margins": 8.967299461364746, | |
| "rewards/rejected": -4.0690741539001465, | |
| "step": 288 | |
| }, | |
| { | |
| "epoch": 2.128440366972477, | |
| "grad_norm": 1.1437596082687378, | |
| "learning_rate": 1.1705670250356417e-06, | |
| "logits/chosen": -0.8648325800895691, | |
| "logits/rejected": -2.037424087524414, | |
| "logps/chosen": -311.7782287597656, | |
| "logps/rejected": -195.8933563232422, | |
| "loss": 0.0121, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.304541110992432, | |
| "rewards/margins": 10.098506927490234, | |
| "rewards/rejected": -4.7939653396606445, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 2.143119266055046, | |
| "grad_norm": 0.16270968317985535, | |
| "learning_rate": 1.1345160308046413e-06, | |
| "logits/chosen": -0.9791809916496277, | |
| "logits/rejected": -2.24078369140625, | |
| "logps/chosen": -382.8855895996094, | |
| "logps/rejected": -205.8779754638672, | |
| "loss": 0.0044, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.505633354187012, | |
| "rewards/margins": 11.220186233520508, | |
| "rewards/rejected": -5.714553356170654, | |
| "step": 292 | |
| }, | |
| { | |
| "epoch": 2.157798165137615, | |
| "grad_norm": 2.2618370056152344, | |
| "learning_rate": 1.0988652610141154e-06, | |
| "logits/chosen": -0.9164503216743469, | |
| "logits/rejected": -1.9510498046875, | |
| "logps/chosen": -276.1203918457031, | |
| "logps/rejected": -212.5254669189453, | |
| "loss": 0.0214, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 4.720728874206543, | |
| "rewards/margins": 9.504093170166016, | |
| "rewards/rejected": -4.783364295959473, | |
| "step": 294 | |
| }, | |
| { | |
| "epoch": 2.1724770642201836, | |
| "grad_norm": 0.08572974801063538, | |
| "learning_rate": 1.063625164933124e-06, | |
| "logits/chosen": -0.8781817555427551, | |
| "logits/rejected": -2.0793867111206055, | |
| "logps/chosen": -330.6020202636719, | |
| "logps/rejected": -208.72425842285156, | |
| "loss": 0.0081, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.462750434875488, | |
| "rewards/margins": 11.203109741210938, | |
| "rewards/rejected": -5.740358352661133, | |
| "step": 296 | |
| }, | |
| { | |
| "epoch": 2.1871559633027524, | |
| "grad_norm": 0.5944895148277283, | |
| "learning_rate": 1.0288060714619359e-06, | |
| "logits/chosen": -1.1157301664352417, | |
| "logits/rejected": -2.214977502822876, | |
| "logps/chosen": -316.9060363769531, | |
| "logps/rejected": -167.6466522216797, | |
| "loss": 0.0098, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.41005277633667, | |
| "rewards/margins": 10.054335594177246, | |
| "rewards/rejected": -4.644283294677734, | |
| "step": 298 | |
| }, | |
| { | |
| "epoch": 2.2018348623853212, | |
| "grad_norm": 0.5239315629005432, | |
| "learning_rate": 9.944181861046188e-07, | |
| "logits/chosen": -0.8929880857467651, | |
| "logits/rejected": -1.9771008491516113, | |
| "logps/chosen": -334.0789489746094, | |
| "logps/rejected": -201.40476989746094, | |
| "loss": 0.0112, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.018026351928711, | |
| "rewards/margins": 10.597818374633789, | |
| "rewards/rejected": -5.579792499542236, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 2.21651376146789, | |
| "grad_norm": 0.17582310736179352, | |
| "learning_rate": 9.604715879777986e-07, | |
| "logits/chosen": -0.9466437101364136, | |
| "logits/rejected": -2.1750948429107666, | |
| "logps/chosen": -279.5908203125, | |
| "logps/rejected": -154.12644958496094, | |
| "loss": 0.0055, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.104902744293213, | |
| "rewards/margins": 10.134696006774902, | |
| "rewards/rejected": -5.029792785644531, | |
| "step": 302 | |
| }, | |
| { | |
| "epoch": 2.231192660550459, | |
| "grad_norm": 0.44052350521087646, | |
| "learning_rate": 9.269762268564616e-07, | |
| "logits/chosen": -1.0591435432434082, | |
| "logits/rejected": -2.134446382522583, | |
| "logps/chosen": -255.2496337890625, | |
| "logps/rejected": -161.16136169433594, | |
| "loss": 0.012, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.842598915100098, | |
| "rewards/margins": 9.835264205932617, | |
| "rewards/rejected": -4.992665767669678, | |
| "step": 304 | |
| }, | |
| { | |
| "epoch": 2.2458715596330276, | |
| "grad_norm": 0.702462375164032, | |
| "learning_rate": 8.939419202576694e-07, | |
| "logits/chosen": -0.768172025680542, | |
| "logits/rejected": -1.7977386713027954, | |
| "logps/chosen": -258.4624938964844, | |
| "logps/rejected": -183.80621337890625, | |
| "loss": 0.0133, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.580810070037842, | |
| "rewards/margins": 7.86204719543457, | |
| "rewards/rejected": -3.2812376022338867, | |
| "step": 306 | |
| }, | |
| { | |
| "epoch": 2.2605504587155965, | |
| "grad_norm": 0.4431416690349579, | |
| "learning_rate": 8.61378350563033e-07, | |
| "logits/chosen": -0.9345456957817078, | |
| "logits/rejected": -1.9868954420089722, | |
| "logps/chosen": -250.33721923828125, | |
| "logps/rejected": -193.64549255371094, | |
| "loss": 0.0059, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.593289375305176, | |
| "rewards/margins": 9.057455062866211, | |
| "rewards/rejected": -4.464165210723877, | |
| "step": 308 | |
| }, | |
| { | |
| "epoch": 2.2752293577981653, | |
| "grad_norm": 0.30388739705085754, | |
| "learning_rate": 8.292950621808022e-07, | |
| "logits/chosen": -0.9780189990997314, | |
| "logits/rejected": -2.0176703929901123, | |
| "logps/chosen": -285.4472961425781, | |
| "logps/rejected": -191.96495056152344, | |
| "loss": 0.005, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.00777006149292, | |
| "rewards/margins": 10.013311386108398, | |
| "rewards/rejected": -5.005540370941162, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 2.289908256880734, | |
| "grad_norm": 0.3784541189670563, | |
| "learning_rate": 7.977014587483925e-07, | |
| "logits/chosen": -1.0011767148971558, | |
| "logits/rejected": -2.0550498962402344, | |
| "logps/chosen": -273.92138671875, | |
| "logps/rejected": -228.16741943359375, | |
| "loss": 0.011, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.854176998138428, | |
| "rewards/margins": 9.765246391296387, | |
| "rewards/rejected": -4.911068916320801, | |
| "step": 312 | |
| }, | |
| { | |
| "epoch": 2.304587155963303, | |
| "grad_norm": 0.2992611527442932, | |
| "learning_rate": 7.666068003761684e-07, | |
| "logits/chosen": -0.9273378849029541, | |
| "logits/rejected": -2.042013645172119, | |
| "logps/chosen": -296.50616455078125, | |
| "logps/rejected": -169.5068817138672, | |
| "loss": 0.0027, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.002749443054199, | |
| "rewards/margins": 10.678738594055176, | |
| "rewards/rejected": -5.675989151000977, | |
| "step": 314 | |
| }, | |
| { | |
| "epoch": 2.3192660550458717, | |
| "grad_norm": 0.23903429508209229, | |
| "learning_rate": 7.360202009332993e-07, | |
| "logits/chosen": -1.0399566888809204, | |
| "logits/rejected": -2.143623113632202, | |
| "logps/chosen": -296.7044677734375, | |
| "logps/rejected": -185.7502899169922, | |
| "loss": 0.0048, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.675933361053467, | |
| "rewards/margins": 10.062166213989258, | |
| "rewards/rejected": -5.386232376098633, | |
| "step": 316 | |
| }, | |
| { | |
| "epoch": 2.3339449541284405, | |
| "grad_norm": 0.23702357709407806, | |
| "learning_rate": 7.059506253764773e-07, | |
| "logits/chosen": -0.9900916814804077, | |
| "logits/rejected": -2.093594789505005, | |
| "logps/chosen": -314.4263000488281, | |
| "logps/rejected": -194.73269653320312, | |
| "loss": 0.0066, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.672959327697754, | |
| "rewards/margins": 10.092663764953613, | |
| "rewards/rejected": -5.419704437255859, | |
| "step": 318 | |
| }, | |
| { | |
| "epoch": 2.3486238532110093, | |
| "grad_norm": 0.40923863649368286, | |
| "learning_rate": 6.764068871222825e-07, | |
| "logits/chosen": -0.7488622069358826, | |
| "logits/rejected": -1.9413087368011475, | |
| "logps/chosen": -287.322021484375, | |
| "logps/rejected": -188.12283325195312, | |
| "loss": 0.0055, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.020465850830078, | |
| "rewards/margins": 9.374917984008789, | |
| "rewards/rejected": -4.354452133178711, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 2.363302752293578, | |
| "grad_norm": 0.19543257355690002, | |
| "learning_rate": 6.473976454639608e-07, | |
| "logits/chosen": -0.9299582839012146, | |
| "logits/rejected": -2.107851266860962, | |
| "logps/chosen": -293.9842529296875, | |
| "logps/rejected": -168.69400024414062, | |
| "loss": 0.0029, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.329434871673584, | |
| "rewards/margins": 10.282726287841797, | |
| "rewards/rejected": -4.9532904624938965, | |
| "step": 322 | |
| }, | |
| { | |
| "epoch": 2.377981651376147, | |
| "grad_norm": 1.0940320491790771, | |
| "learning_rate": 6.189314030333796e-07, | |
| "logits/chosen": -0.8577584624290466, | |
| "logits/rejected": -1.934208631515503, | |
| "logps/chosen": -280.2929382324219, | |
| "logps/rejected": -220.2890167236328, | |
| "loss": 0.0154, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.774515628814697, | |
| "rewards/margins": 10.356229782104492, | |
| "rewards/rejected": -5.581714153289795, | |
| "step": 324 | |
| }, | |
| { | |
| "epoch": 2.3926605504587157, | |
| "grad_norm": 0.12193372845649719, | |
| "learning_rate": 5.910165033089e-07, | |
| "logits/chosen": -0.8733283281326294, | |
| "logits/rejected": -2.079462766647339, | |
| "logps/chosen": -316.6996765136719, | |
| "logps/rejected": -201.25564575195312, | |
| "loss": 0.0022, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.122347831726074, | |
| "rewards/margins": 9.807957649230957, | |
| "rewards/rejected": -4.685609817504883, | |
| "step": 326 | |
| }, | |
| { | |
| "epoch": 2.4073394495412845, | |
| "grad_norm": 0.43534737825393677, | |
| "learning_rate": 5.636611281698956e-07, | |
| "logits/chosen": -0.8986641764640808, | |
| "logits/rejected": -1.9822278022766113, | |
| "logps/chosen": -262.6693420410156, | |
| "logps/rejected": -185.32846069335938, | |
| "loss": 0.0067, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.460582733154297, | |
| "rewards/margins": 9.22741413116455, | |
| "rewards/rejected": -4.766830921173096, | |
| "step": 328 | |
| }, | |
| { | |
| "epoch": 2.4220183486238533, | |
| "grad_norm": 0.17293158173561096, | |
| "learning_rate": 5.368732954986389e-07, | |
| "logits/chosen": -1.0250214338302612, | |
| "logits/rejected": -2.0870189666748047, | |
| "logps/chosen": -279.42999267578125, | |
| "logps/rejected": -196.77059936523438, | |
| "loss": 0.0069, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.643675327301025, | |
| "rewards/margins": 9.602448463439941, | |
| "rewards/rejected": -4.958773136138916, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 2.436697247706422, | |
| "grad_norm": 0.20987118780612946, | |
| "learning_rate": 5.106608568302504e-07, | |
| "logits/chosen": -1.066097617149353, | |
| "logits/rejected": -2.057497978210449, | |
| "logps/chosen": -257.912109375, | |
| "logps/rejected": -195.58677673339844, | |
| "loss": 0.0204, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 4.727499961853027, | |
| "rewards/margins": 9.989591598510742, | |
| "rewards/rejected": -5.262092590332031, | |
| "step": 332 | |
| }, | |
| { | |
| "epoch": 2.451376146788991, | |
| "grad_norm": 1.3423670530319214, | |
| "learning_rate": 4.850314950514124e-07, | |
| "logits/chosen": -0.8067299127578735, | |
| "logits/rejected": -1.9319019317626953, | |
| "logps/chosen": -281.423583984375, | |
| "logps/rejected": -192.34666442871094, | |
| "loss": 0.0104, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.9796462059021, | |
| "rewards/margins": 9.925731658935547, | |
| "rewards/rejected": -4.9460859298706055, | |
| "step": 334 | |
| }, | |
| { | |
| "epoch": 2.4660550458715598, | |
| "grad_norm": 0.2133161723613739, | |
| "learning_rate": 4.599927221485034e-07, | |
| "logits/chosen": -0.9198440909385681, | |
| "logits/rejected": -2.121577024459839, | |
| "logps/chosen": -277.7024230957031, | |
| "logps/rejected": -171.4612579345703, | |
| "loss": 0.0033, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.626633167266846, | |
| "rewards/margins": 9.858685493469238, | |
| "rewards/rejected": -5.232051849365234, | |
| "step": 336 | |
| }, | |
| { | |
| "epoch": 2.4807339449541286, | |
| "grad_norm": 0.16850100457668304, | |
| "learning_rate": 4.3555187700583175e-07, | |
| "logits/chosen": -0.8522999882698059, | |
| "logits/rejected": -2.053220748901367, | |
| "logps/chosen": -265.3820495605469, | |
| "logps/rejected": -188.9971160888672, | |
| "loss": 0.0036, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.703529357910156, | |
| "rewards/margins": 10.37534236907959, | |
| "rewards/rejected": -5.671813011169434, | |
| "step": 338 | |
| }, | |
| { | |
| "epoch": 2.4954128440366974, | |
| "grad_norm": 0.15111279487609863, | |
| "learning_rate": 4.1171612325460244e-07, | |
| "logits/chosen": -0.9065884351730347, | |
| "logits/rejected": -1.9212383031845093, | |
| "logps/chosen": -279.0539245605469, | |
| "logps/rejected": -185.0900115966797, | |
| "loss": 0.0045, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.483066082000732, | |
| "rewards/margins": 9.553812026977539, | |
| "rewards/rejected": -5.070746421813965, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 2.510091743119266, | |
| "grad_norm": 0.06084302440285683, | |
| "learning_rate": 3.8849244717325206e-07, | |
| "logits/chosen": -0.9317240715026855, | |
| "logits/rejected": -1.988271713256836, | |
| "logps/chosen": -268.8980407714844, | |
| "logps/rejected": -202.74929809570312, | |
| "loss": 0.0096, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.079599380493164, | |
| "rewards/margins": 10.533794403076172, | |
| "rewards/rejected": -5.454195022583008, | |
| "step": 342 | |
| }, | |
| { | |
| "epoch": 2.524770642201835, | |
| "grad_norm": 0.8741805553436279, | |
| "learning_rate": 3.658876556397628e-07, | |
| "logits/chosen": -1.1219009160995483, | |
| "logits/rejected": -2.1234138011932373, | |
| "logps/chosen": -255.28311157226562, | |
| "logps/rejected": -171.67091369628906, | |
| "loss": 0.0099, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.061553478240967, | |
| "rewards/margins": 10.136541366577148, | |
| "rewards/rejected": -5.074987411499023, | |
| "step": 344 | |
| }, | |
| { | |
| "epoch": 2.539449541284404, | |
| "grad_norm": 0.4372842013835907, | |
| "learning_rate": 3.4390837413656256e-07, | |
| "logits/chosen": -0.9813422560691833, | |
| "logits/rejected": -2.116903781890869, | |
| "logps/chosen": -278.31292724609375, | |
| "logps/rejected": -204.9643096923828, | |
| "loss": 0.0083, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.085036277770996, | |
| "rewards/margins": 10.61630630493164, | |
| "rewards/rejected": -5.531269073486328, | |
| "step": 346 | |
| }, | |
| { | |
| "epoch": 2.5541284403669726, | |
| "grad_norm": 1.2550814151763916, | |
| "learning_rate": 3.225610448085903e-07, | |
| "logits/chosen": -0.9581831693649292, | |
| "logits/rejected": -2.0414552688598633, | |
| "logps/chosen": -270.668701171875, | |
| "logps/rejected": -183.82034301757812, | |
| "loss": 0.0113, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.899204730987549, | |
| "rewards/margins": 10.034035682678223, | |
| "rewards/rejected": -5.134830474853516, | |
| "step": 348 | |
| }, | |
| { | |
| "epoch": 2.5688073394495414, | |
| "grad_norm": 0.03559936583042145, | |
| "learning_rate": 3.018519245750989e-07, | |
| "logits/chosen": -0.9744287729263306, | |
| "logits/rejected": -1.9595189094543457, | |
| "logps/chosen": -321.4473876953125, | |
| "logps/rejected": -223.63467407226562, | |
| "loss": 0.0091, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.675147533416748, | |
| "rewards/margins": 10.108884811401367, | |
| "rewards/rejected": -5.433738708496094, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 2.5834862385321102, | |
| "grad_norm": 0.2746826708316803, | |
| "learning_rate": 2.817870832957459e-07, | |
| "logits/chosen": -0.8869858980178833, | |
| "logits/rejected": -2.016246795654297, | |
| "logps/chosen": -259.2815856933594, | |
| "logps/rejected": -180.26258850097656, | |
| "loss": 0.0056, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.067638874053955, | |
| "rewards/margins": 10.355432510375977, | |
| "rewards/rejected": -5.287793159484863, | |
| "step": 352 | |
| }, | |
| { | |
| "epoch": 2.598165137614679, | |
| "grad_norm": 0.17304402589797974, | |
| "learning_rate": 2.6237240199151386e-07, | |
| "logits/chosen": -1.0045228004455566, | |
| "logits/rejected": -2.091106414794922, | |
| "logps/chosen": -264.62774658203125, | |
| "logps/rejected": -172.3504638671875, | |
| "loss": 0.0051, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.949882507324219, | |
| "rewards/margins": 9.181565284729004, | |
| "rewards/rejected": -4.231683254241943, | |
| "step": 354 | |
| }, | |
| { | |
| "epoch": 2.612844036697248, | |
| "grad_norm": 0.9380022883415222, | |
| "learning_rate": 2.436135711209786e-07, | |
| "logits/chosen": -1.1858479976654053, | |
| "logits/rejected": -2.1570074558258057, | |
| "logps/chosen": -279.8266296386719, | |
| "logps/rejected": -165.30809020996094, | |
| "loss": 0.009, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.504940032958984, | |
| "rewards/margins": 9.417243957519531, | |
| "rewards/rejected": -4.912304401397705, | |
| "step": 356 | |
| }, | |
| { | |
| "epoch": 2.6275229357798167, | |
| "grad_norm": 0.6058441400527954, | |
| "learning_rate": 2.2551608891243026e-07, | |
| "logits/chosen": -1.1764850616455078, | |
| "logits/rejected": -2.1525368690490723, | |
| "logps/chosen": -352.7016296386719, | |
| "logps/rejected": -213.2824249267578, | |
| "loss": 0.0043, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.686002731323242, | |
| "rewards/margins": 9.250012397766113, | |
| "rewards/rejected": -4.564009666442871, | |
| "step": 358 | |
| }, | |
| { | |
| "epoch": 2.6422018348623855, | |
| "grad_norm": 0.14885057508945465, | |
| "learning_rate": 2.0808525975233807e-07, | |
| "logits/chosen": -0.8036705255508423, | |
| "logits/rejected": -2.0143167972564697, | |
| "logps/chosen": -282.6025085449219, | |
| "logps/rejected": -200.5447540283203, | |
| "loss": 0.015, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.5807576179504395, | |
| "rewards/margins": 9.530784606933594, | |
| "rewards/rejected": -4.950027942657471, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 2.6568807339449543, | |
| "grad_norm": 0.40349748730659485, | |
| "learning_rate": 1.9132619263063144e-07, | |
| "logits/chosen": -0.8986431360244751, | |
| "logits/rejected": -2.059335231781006, | |
| "logps/chosen": -346.6067810058594, | |
| "logps/rejected": -212.26348876953125, | |
| "loss": 0.0096, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.356535911560059, | |
| "rewards/margins": 10.730274200439453, | |
| "rewards/rejected": -5.373737812042236, | |
| "step": 362 | |
| }, | |
| { | |
| "epoch": 2.671559633027523, | |
| "grad_norm": 0.02223406359553337, | |
| "learning_rate": 1.7524379964325155e-07, | |
| "logits/chosen": -0.9592161774635315, | |
| "logits/rejected": -2.094557523727417, | |
| "logps/chosen": -327.5130310058594, | |
| "logps/rejected": -203.91741943359375, | |
| "loss": 0.0038, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.003190517425537, | |
| "rewards/margins": 10.261466979980469, | |
| "rewards/rejected": -5.258275985717773, | |
| "step": 364 | |
| }, | |
| { | |
| "epoch": 2.686238532110092, | |
| "grad_norm": 0.21345356106758118, | |
| "learning_rate": 1.5984279455240975e-07, | |
| "logits/chosen": -0.9917050004005432, | |
| "logits/rejected": -2.0196518898010254, | |
| "logps/chosen": -282.2841796875, | |
| "logps/rejected": -191.79910278320312, | |
| "loss": 0.0036, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.118269443511963, | |
| "rewards/margins": 10.18971061706543, | |
| "rewards/rejected": -5.071441173553467, | |
| "step": 366 | |
| }, | |
| { | |
| "epoch": 2.7009174311926607, | |
| "grad_norm": 0.0874081626534462, | |
| "learning_rate": 1.451276914049818e-07, | |
| "logits/chosen": -0.9789815545082092, | |
| "logits/rejected": -2.004281997680664, | |
| "logps/chosen": -256.3831787109375, | |
| "logps/rejected": -178.12074279785156, | |
| "loss": 0.0037, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.682791233062744, | |
| "rewards/margins": 10.118135452270508, | |
| "rewards/rejected": -5.435344219207764, | |
| "step": 368 | |
| }, | |
| { | |
| "epoch": 2.7155963302752295, | |
| "grad_norm": 0.15522974729537964, | |
| "learning_rate": 1.3110280320943692e-07, | |
| "logits/chosen": -0.89200758934021, | |
| "logits/rejected": -2.112806797027588, | |
| "logps/chosen": -271.0398254394531, | |
| "logps/rejected": -171.77919006347656, | |
| "loss": 0.0016, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.064676284790039, | |
| "rewards/margins": 10.34743881225586, | |
| "rewards/rejected": -5.28276252746582, | |
| "step": 370 | |
| }, | |
| { | |
| "epoch": 2.7302752293577983, | |
| "grad_norm": 0.5010592341423035, | |
| "learning_rate": 1.1777224067169218e-07, | |
| "logits/chosen": -0.8353609442710876, | |
| "logits/rejected": -1.9892935752868652, | |
| "logps/chosen": -279.2965087890625, | |
| "logps/rejected": -191.65855407714844, | |
| "loss": 0.0049, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.053516864776611, | |
| "rewards/margins": 10.337114334106445, | |
| "rewards/rejected": -5.283597469329834, | |
| "step": 372 | |
| }, | |
| { | |
| "epoch": 2.744954128440367, | |
| "grad_norm": 0.48516571521759033, | |
| "learning_rate": 1.0513991099025872e-07, | |
| "logits/chosen": -1.016608476638794, | |
| "logits/rejected": -2.1301956176757812, | |
| "logps/chosen": -323.5552673339844, | |
| "logps/rejected": -193.77996826171875, | |
| "loss": 0.0062, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.830180644989014, | |
| "rewards/margins": 9.39232349395752, | |
| "rewards/rejected": -4.562142372131348, | |
| "step": 374 | |
| }, | |
| { | |
| "epoch": 2.759633027522936, | |
| "grad_norm": 0.18496806919574738, | |
| "learning_rate": 9.320951671104194e-08, | |
| "logits/chosen": -0.9126584529876709, | |
| "logits/rejected": -2.1175155639648438, | |
| "logps/chosen": -314.1302795410156, | |
| "logps/rejected": -191.1002960205078, | |
| "loss": 0.0048, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.795359134674072, | |
| "rewards/margins": 10.335535049438477, | |
| "rewards/rejected": -4.540175914764404, | |
| "step": 376 | |
| }, | |
| { | |
| "epoch": 2.7743119266055047, | |
| "grad_norm": 0.158527210354805, | |
| "learning_rate": 8.198455464212108e-08, | |
| "logits/chosen": -0.9621077179908752, | |
| "logits/rejected": -2.066542148590088, | |
| "logps/chosen": -293.69940185546875, | |
| "logps/rejected": -176.17442321777344, | |
| "loss": 0.0017, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.256305694580078, | |
| "rewards/margins": 10.846275329589844, | |
| "rewards/rejected": -5.589971542358398, | |
| "step": 378 | |
| }, | |
| { | |
| "epoch": 2.7889908256880735, | |
| "grad_norm": 0.11951223015785217, | |
| "learning_rate": 7.146831482883115e-08, | |
| "logits/chosen": -0.7449550628662109, | |
| "logits/rejected": -2.0898332595825195, | |
| "logps/chosen": -297.49365234375, | |
| "logps/rejected": -172.2970428466797, | |
| "loss": 0.0027, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.16118860244751, | |
| "rewards/margins": 11.0027437210083, | |
| "rewards/rejected": -5.841555595397949, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 2.8036697247706424, | |
| "grad_norm": 0.32903870940208435, | |
| "learning_rate": 6.16638795894492e-08, | |
| "logits/chosen": -0.9001256823539734, | |
| "logits/rejected": -1.9853109121322632, | |
| "logps/chosen": -261.6986389160156, | |
| "logps/rejected": -200.32876586914062, | |
| "loss": 0.0049, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.192695617675781, | |
| "rewards/margins": 10.185223579406738, | |
| "rewards/rejected": -4.992527961730957, | |
| "step": 382 | |
| }, | |
| { | |
| "epoch": 2.818348623853211, | |
| "grad_norm": 1.1356521844863892, | |
| "learning_rate": 5.257412261176375e-08, | |
| "logits/chosen": -1.0478947162628174, | |
| "logits/rejected": -2.031193971633911, | |
| "logps/chosen": -272.9176025390625, | |
| "logps/rejected": -191.03363037109375, | |
| "loss": 0.008, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.408777236938477, | |
| "rewards/margins": 10.281020164489746, | |
| "rewards/rejected": -4.8722429275512695, | |
| "step": 384 | |
| }, | |
| { | |
| "epoch": 2.83302752293578, | |
| "grad_norm": 0.36936327815055847, | |
| "learning_rate": 4.4201708110795384e-08, | |
| "logits/chosen": -0.9411278963088989, | |
| "logits/rejected": -1.9795866012573242, | |
| "logps/chosen": -292.65386962890625, | |
| "logps/rejected": -204.53778076171875, | |
| "loss": 0.0039, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.8684539794921875, | |
| "rewards/margins": 9.500937461853027, | |
| "rewards/rejected": -4.632482528686523, | |
| "step": 386 | |
| }, | |
| { | |
| "epoch": 2.847706422018349, | |
| "grad_norm": 0.16482090950012207, | |
| "learning_rate": 3.654909004791152e-08, | |
| "logits/chosen": -0.938539981842041, | |
| "logits/rejected": -2.1424248218536377, | |
| "logps/chosen": -293.4642333984375, | |
| "logps/rejected": -184.5415802001953, | |
| "loss": 0.0121, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.669407844543457, | |
| "rewards/margins": 10.090935707092285, | |
| "rewards/rejected": -5.421527862548828, | |
| "step": 388 | |
| }, | |
| { | |
| "epoch": 2.8623853211009176, | |
| "grad_norm": 0.13458868861198425, | |
| "learning_rate": 2.9618511411570462e-08, | |
| "logits/chosen": -1.0025708675384521, | |
| "logits/rejected": -2.083418607711792, | |
| "logps/chosen": -284.4067687988281, | |
| "logps/rejected": -172.37875366210938, | |
| "loss": 0.0069, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.38177490234375, | |
| "rewards/margins": 9.599076271057129, | |
| "rewards/rejected": -5.2173004150390625, | |
| "step": 390 | |
| }, | |
| { | |
| "epoch": 2.8770642201834864, | |
| "grad_norm": 1.0483838319778442, | |
| "learning_rate": 2.3412003559898088e-08, | |
| "logits/chosen": -0.8990004658699036, | |
| "logits/rejected": -1.8701345920562744, | |
| "logps/chosen": -271.26129150390625, | |
| "logps/rejected": -207.7753143310547, | |
| "loss": 0.0125, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.918183326721191, | |
| "rewards/margins": 9.391782760620117, | |
| "rewards/rejected": -4.473598957061768, | |
| "step": 392 | |
| }, | |
| { | |
| "epoch": 2.891743119266055, | |
| "grad_norm": 0.25216034054756165, | |
| "learning_rate": 1.793138562529634e-08, | |
| "logits/chosen": -0.971919059753418, | |
| "logits/rejected": -2.1569983959198, | |
| "logps/chosen": -346.71875, | |
| "logps/rejected": -184.22348022460938, | |
| "loss": 0.0027, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.533580780029297, | |
| "rewards/margins": 10.083108901977539, | |
| "rewards/rejected": -4.549527168273926, | |
| "step": 394 | |
| }, | |
| { | |
| "epoch": 2.906422018348624, | |
| "grad_norm": 0.8910009860992432, | |
| "learning_rate": 1.317826398125277e-08, | |
| "logits/chosen": -1.062324047088623, | |
| "logits/rejected": -2.1035232543945312, | |
| "logps/chosen": -293.03125, | |
| "logps/rejected": -204.35723876953125, | |
| "loss": 0.0135, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.314833641052246, | |
| "rewards/margins": 10.953380584716797, | |
| "rewards/rejected": -5.638547420501709, | |
| "step": 396 | |
| }, | |
| { | |
| "epoch": 2.921100917431193, | |
| "grad_norm": 0.3026532828807831, | |
| "learning_rate": 9.15403177151275e-09, | |
| "logits/chosen": -0.9711483716964722, | |
| "logits/rejected": -1.8983428478240967, | |
| "logps/chosen": -276.14398193359375, | |
| "logps/rejected": -217.97817993164062, | |
| "loss": 0.0064, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.347715377807617, | |
| "rewards/margins": 10.437253952026367, | |
| "rewards/rejected": -5.08953857421875, | |
| "step": 398 | |
| }, | |
| { | |
| "epoch": 2.9357798165137616, | |
| "grad_norm": 0.21673916280269623, | |
| "learning_rate": 5.85986850174608e-09, | |
| "logits/chosen": -0.8715996146202087, | |
| "logits/rejected": -2.193289279937744, | |
| "logps/chosen": -312.49847412109375, | |
| "logps/rejected": -185.29078674316406, | |
| "loss": 0.0027, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.40484619140625, | |
| "rewards/margins": 10.567187309265137, | |
| "rewards/rejected": -5.162341117858887, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 2.9504587155963304, | |
| "grad_norm": 0.18232221901416779, | |
| "learning_rate": 3.296739693834927e-09, | |
| "logits/chosen": -1.094886302947998, | |
| "logits/rejected": -1.9882696866989136, | |
| "logps/chosen": -305.0465087890625, | |
| "logps/rejected": -184.07928466796875, | |
| "loss": 0.0032, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.589092254638672, | |
| "rewards/margins": 9.142921447753906, | |
| "rewards/rejected": -4.553828239440918, | |
| "step": 402 | |
| }, | |
| { | |
| "epoch": 2.9651376146788992, | |
| "grad_norm": 0.28508853912353516, | |
| "learning_rate": 1.4653966028774225e-09, | |
| "logits/chosen": -0.9431482553482056, | |
| "logits/rejected": -1.953324556350708, | |
| "logps/chosen": -313.6567077636719, | |
| "logps/rejected": -213.5366973876953, | |
| "loss": 0.0034, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 4.910001277923584, | |
| "rewards/margins": 10.507518768310547, | |
| "rewards/rejected": -5.597517490386963, | |
| "step": 404 | |
| }, | |
| { | |
| "epoch": 2.979816513761468, | |
| "grad_norm": 0.14594300091266632, | |
| "learning_rate": 3.6637599699351766e-10, | |
| "logits/chosen": -0.940761387348175, | |
| "logits/rejected": -2.1918911933898926, | |
| "logps/chosen": -289.67626953125, | |
| "logps/rejected": -180.724365234375, | |
| "loss": 0.0028, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 5.103169918060303, | |
| "rewards/margins": 9.98257827758789, | |
| "rewards/rejected": -4.879408359527588, | |
| "step": 406 | |
| }, | |
| { | |
| "epoch": 2.994495412844037, | |
| "grad_norm": 0.16170361638069153, | |
| "learning_rate": 0.0, | |
| "logits/chosen": -1.0162718296051025, | |
| "logits/rejected": -1.9849637746810913, | |
| "logps/chosen": -319.7056579589844, | |
| "logps/rejected": -213.12435913085938, | |
| "loss": 0.0316, | |
| "rewards/accuracies": 0.984375, | |
| "rewards/chosen": 5.299257755279541, | |
| "rewards/margins": 10.258605003356934, | |
| "rewards/rejected": -4.959346771240234, | |
| "step": 408 | |
| }, | |
| { | |
| "epoch": 2.994495412844037, | |
| "step": 408, | |
| "total_flos": 7.837376281021809e+17, | |
| "train_loss": 0.11720214437923905, | |
| "train_runtime": 8069.9016, | |
| "train_samples_per_second": 1.62, | |
| "train_steps_per_second": 0.051 | |
| } | |
| ], | |
| "logging_steps": 2, | |
| "max_steps": 408, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": false, | |
| "should_training_stop": false | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 7.837376281021809e+17, | |
| "train_batch_size": 4, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |