| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.9282399143163156, | |
| "eval_steps": 100, | |
| "global_step": 2600, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.003570153516601214, | |
| "grad_norm": 7.480371952056885, | |
| "learning_rate": 4.9821492324169946e-05, | |
| "logits/chosen": 24.812057495117188, | |
| "logits/rejected": 24.878061294555664, | |
| "logps/chosen": -20.12216567993164, | |
| "logps/rejected": -25.59366798400879, | |
| "loss": 0.6569, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": 0.08925116807222366, | |
| "rewards/margins": 0.21577796339988708, | |
| "rewards/rejected": -0.12652680277824402, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.007140307033202428, | |
| "grad_norm": 6.517239093780518, | |
| "learning_rate": 4.964298464833988e-05, | |
| "logits/chosen": 24.764490127563477, | |
| "logits/rejected": 24.641494750976562, | |
| "logps/chosen": -11.448348999023438, | |
| "logps/rejected": -12.761981964111328, | |
| "loss": 0.6509, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": 0.14247111976146698, | |
| "rewards/margins": 0.24720080196857452, | |
| "rewards/rejected": -0.10472966730594635, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.010710460549803642, | |
| "grad_norm": 6.2558159828186035, | |
| "learning_rate": 4.946447697250982e-05, | |
| "logits/chosen": 26.845245361328125, | |
| "logits/rejected": 26.89157485961914, | |
| "logps/chosen": -32.96189880371094, | |
| "logps/rejected": -36.98007583618164, | |
| "loss": 0.7495, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.26471006870269775, | |
| "rewards/margins": 0.1922953575849533, | |
| "rewards/rejected": -0.45700541138648987, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.014280614066404856, | |
| "grad_norm": 7.402151584625244, | |
| "learning_rate": 4.928596929667976e-05, | |
| "logits/chosen": 27.013961791992188, | |
| "logits/rejected": 27.773513793945312, | |
| "logps/chosen": -33.79491424560547, | |
| "logps/rejected": -45.16600799560547, | |
| "loss": 0.607, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.004278683569282293, | |
| "rewards/margins": 0.5446106195449829, | |
| "rewards/rejected": -0.5488892793655396, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.01785076758300607, | |
| "grad_norm": 8.540044784545898, | |
| "learning_rate": 4.9107461620849696e-05, | |
| "logits/chosen": 24.013198852539062, | |
| "logits/rejected": 24.5129337310791, | |
| "logps/chosen": -45.71046829223633, | |
| "logps/rejected": -38.64923858642578, | |
| "loss": 0.5787, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.3722326159477234, | |
| "rewards/margins": 0.14807865023612976, | |
| "rewards/rejected": -0.5203112363815308, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.021420921099607283, | |
| "grad_norm": 4.945995330810547, | |
| "learning_rate": 4.892895394501964e-05, | |
| "logits/chosen": 27.04281997680664, | |
| "logits/rejected": 26.753490447998047, | |
| "logps/chosen": -58.52327346801758, | |
| "logps/rejected": -42.7177619934082, | |
| "loss": 0.6459, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.732448399066925, | |
| "rewards/margins": 0.44616565108299255, | |
| "rewards/rejected": -1.1786140203475952, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.024991074616208496, | |
| "grad_norm": 7.011103630065918, | |
| "learning_rate": 4.875044626918958e-05, | |
| "logits/chosen": 25.751834869384766, | |
| "logits/rejected": 26.487178802490234, | |
| "logps/chosen": -59.837867736816406, | |
| "logps/rejected": -62.329681396484375, | |
| "loss": 0.6127, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.34982752799987793, | |
| "rewards/margins": 0.5455743074417114, | |
| "rewards/rejected": -0.8954018354415894, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.028561228132809712, | |
| "grad_norm": 4.859188556671143, | |
| "learning_rate": 4.8571938593359514e-05, | |
| "logits/chosen": 23.81821060180664, | |
| "logits/rejected": 23.5118465423584, | |
| "logps/chosen": -42.71343231201172, | |
| "logps/rejected": -37.387977600097656, | |
| "loss": 0.5896, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.011353443376719952, | |
| "rewards/margins": 0.47227078676223755, | |
| "rewards/rejected": -0.4836241602897644, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.03213138164941092, | |
| "grad_norm": 11.05513858795166, | |
| "learning_rate": 4.839343091752946e-05, | |
| "logits/chosen": 27.86802101135254, | |
| "logits/rejected": 27.87044334411621, | |
| "logps/chosen": -56.7071533203125, | |
| "logps/rejected": -48.46919631958008, | |
| "loss": 0.6217, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.01455842237919569, | |
| "rewards/margins": 0.41877785325050354, | |
| "rewards/rejected": -0.4333363175392151, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.03570153516601214, | |
| "grad_norm": 5.356033802032471, | |
| "learning_rate": 4.8214923241699396e-05, | |
| "logits/chosen": 23.89133644104004, | |
| "logits/rejected": 23.665489196777344, | |
| "logps/chosen": -63.13393020629883, | |
| "logps/rejected": -43.369056701660156, | |
| "loss": 0.6764, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.114576056599617, | |
| "rewards/margins": 0.36550942063331604, | |
| "rewards/rejected": -0.48008546233177185, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.03570153516601214, | |
| "eval_logits/chosen": 25.279199600219727, | |
| "eval_logits/rejected": 25.355634689331055, | |
| "eval_logps/chosen": -42.37962341308594, | |
| "eval_logps/rejected": -37.090999603271484, | |
| "eval_loss": 0.5741721987724304, | |
| "eval_rewards/accuracies": 0.6777777671813965, | |
| "eval_rewards/chosen": -0.07397282123565674, | |
| "eval_rewards/margins": 0.43140149116516113, | |
| "eval_rewards/rejected": -0.5053743124008179, | |
| "eval_runtime": 109.7281, | |
| "eval_samples_per_second": 13.05, | |
| "eval_steps_per_second": 0.82, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.039271688682613354, | |
| "grad_norm": 5.330838680267334, | |
| "learning_rate": 4.803641556586933e-05, | |
| "logits/chosen": 26.816951751708984, | |
| "logits/rejected": 27.308795928955078, | |
| "logps/chosen": -37.895626068115234, | |
| "logps/rejected": -35.43812942504883, | |
| "loss": 0.5487, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.1233404278755188, | |
| "rewards/margins": 0.47418642044067383, | |
| "rewards/rejected": -0.5975269079208374, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.042841842199214566, | |
| "grad_norm": 10.010088920593262, | |
| "learning_rate": 4.785790789003927e-05, | |
| "logits/chosen": 27.359798431396484, | |
| "logits/rejected": 27.50588035583496, | |
| "logps/chosen": -52.02180862426758, | |
| "logps/rejected": -48.30536651611328, | |
| "loss": 0.5954, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": 0.14739781618118286, | |
| "rewards/margins": 0.22387762367725372, | |
| "rewards/rejected": -0.07647980749607086, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.04641199571581578, | |
| "grad_norm": 7.994888782501221, | |
| "learning_rate": 4.7679400214209214e-05, | |
| "logits/chosen": 27.39333152770996, | |
| "logits/rejected": 26.996658325195312, | |
| "logps/chosen": -18.88664436340332, | |
| "logps/rejected": -39.95366668701172, | |
| "loss": 0.6306, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.09302469342947006, | |
| "rewards/margins": 0.047691889107227325, | |
| "rewards/rejected": -0.14071659743785858, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.04998214923241699, | |
| "grad_norm": 3.463163375854492, | |
| "learning_rate": 4.750089253837915e-05, | |
| "logits/chosen": 22.470857620239258, | |
| "logits/rejected": 22.843807220458984, | |
| "logps/chosen": -22.85379981994629, | |
| "logps/rejected": -19.81589698791504, | |
| "loss": 0.5663, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": 0.08994263410568237, | |
| "rewards/margins": 0.6263381242752075, | |
| "rewards/rejected": -0.5363954305648804, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.053552302749018205, | |
| "grad_norm": 7.127109527587891, | |
| "learning_rate": 4.732238486254909e-05, | |
| "logits/chosen": 26.753835678100586, | |
| "logits/rejected": 26.78439712524414, | |
| "logps/chosen": -46.176551818847656, | |
| "logps/rejected": -50.23821258544922, | |
| "loss": 0.5586, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.20682087540626526, | |
| "rewards/margins": 1.0148881673812866, | |
| "rewards/rejected": -0.8080674409866333, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.057122456265619424, | |
| "grad_norm": 6.003675937652588, | |
| "learning_rate": 4.714387718671903e-05, | |
| "logits/chosen": 29.764068603515625, | |
| "logits/rejected": 29.705978393554688, | |
| "logps/chosen": -23.824987411499023, | |
| "logps/rejected": -42.840850830078125, | |
| "loss": 0.5256, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.22749033570289612, | |
| "rewards/margins": 0.35782289505004883, | |
| "rewards/rejected": -0.5853132009506226, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.06069260978222064, | |
| "grad_norm": 6.028194427490234, | |
| "learning_rate": 4.696536951088897e-05, | |
| "logits/chosen": 23.769296646118164, | |
| "logits/rejected": 24.033222198486328, | |
| "logps/chosen": -41.425865173339844, | |
| "logps/rejected": -41.90104293823242, | |
| "loss": 0.6533, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.4930092692375183, | |
| "rewards/margins": 0.38698917627334595, | |
| "rewards/rejected": -0.8799983859062195, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 0.06426276329882184, | |
| "grad_norm": 4.039268493652344, | |
| "learning_rate": 4.678686183505891e-05, | |
| "logits/chosen": 25.150955200195312, | |
| "logits/rejected": 25.3060302734375, | |
| "logps/chosen": -33.606788635253906, | |
| "logps/rejected": -22.799793243408203, | |
| "loss": 0.5998, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.12118367105722427, | |
| "rewards/margins": 0.29234248399734497, | |
| "rewards/rejected": -0.41352614760398865, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.06783291681542307, | |
| "grad_norm": 8.811986923217773, | |
| "learning_rate": 4.660835415922885e-05, | |
| "logits/chosen": 28.551767349243164, | |
| "logits/rejected": 28.756189346313477, | |
| "logps/chosen": -45.27893829345703, | |
| "logps/rejected": -28.83548355102539, | |
| "loss": 0.5604, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.4300183653831482, | |
| "rewards/margins": 0.17230427265167236, | |
| "rewards/rejected": -0.6023226976394653, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 0.07140307033202428, | |
| "grad_norm": 7.31522798538208, | |
| "learning_rate": 4.642984648339879e-05, | |
| "logits/chosen": 25.931976318359375, | |
| "logits/rejected": 25.951126098632812, | |
| "logps/chosen": -50.394126892089844, | |
| "logps/rejected": -38.347755432128906, | |
| "loss": 0.6238, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.2471456527709961, | |
| "rewards/margins": 0.6917206048965454, | |
| "rewards/rejected": -0.9388662576675415, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.07140307033202428, | |
| "eval_logits/chosen": 25.649152755737305, | |
| "eval_logits/rejected": 25.710378646850586, | |
| "eval_logps/chosen": -43.097450256347656, | |
| "eval_logps/rejected": -38.84811782836914, | |
| "eval_loss": 0.5490943193435669, | |
| "eval_rewards/accuracies": 0.6833333373069763, | |
| "eval_rewards/chosen": -0.14575596153736115, | |
| "eval_rewards/margins": 0.5353304743766785, | |
| "eval_rewards/rejected": -0.6810863614082336, | |
| "eval_runtime": 97.7698, | |
| "eval_samples_per_second": 14.647, | |
| "eval_steps_per_second": 0.921, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.0749732238486255, | |
| "grad_norm": 5.679516792297363, | |
| "learning_rate": 4.6251338807568726e-05, | |
| "logits/chosen": 26.078624725341797, | |
| "logits/rejected": 25.89596939086914, | |
| "logps/chosen": -28.943302154541016, | |
| "logps/rejected": -45.90317916870117, | |
| "loss": 0.6111, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.14115412533283234, | |
| "rewards/margins": 0.2540377974510193, | |
| "rewards/rejected": -0.3951919376850128, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 0.07854337736522671, | |
| "grad_norm": 13.459559440612793, | |
| "learning_rate": 4.607283113173867e-05, | |
| "logits/chosen": 22.200332641601562, | |
| "logits/rejected": 22.71112060546875, | |
| "logps/chosen": -36.98976135253906, | |
| "logps/rejected": -30.201828002929688, | |
| "loss": 0.5806, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.00881283264607191, | |
| "rewards/margins": 0.8385305404663086, | |
| "rewards/rejected": -0.8473433256149292, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 0.08211353088182792, | |
| "grad_norm": 9.37973403930664, | |
| "learning_rate": 4.589432345590861e-05, | |
| "logits/chosen": 28.823495864868164, | |
| "logits/rejected": 28.9989013671875, | |
| "logps/chosen": -52.96837615966797, | |
| "logps/rejected": -48.45201110839844, | |
| "loss": 0.5957, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.45236507058143616, | |
| "rewards/margins": 0.9013611674308777, | |
| "rewards/rejected": -1.3537262678146362, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 0.08568368439842913, | |
| "grad_norm": 6.611434459686279, | |
| "learning_rate": 4.5715815780078545e-05, | |
| "logits/chosen": 19.64382553100586, | |
| "logits/rejected": 19.509660720825195, | |
| "logps/chosen": -48.31082534790039, | |
| "logps/rejected": -50.42847442626953, | |
| "loss": 0.6004, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.15059544146060944, | |
| "rewards/margins": 0.887191653251648, | |
| "rewards/rejected": -1.0377871990203857, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 0.08925383791503035, | |
| "grad_norm": 6.911257266998291, | |
| "learning_rate": 4.553730810424848e-05, | |
| "logits/chosen": 17.858783721923828, | |
| "logits/rejected": 18.209197998046875, | |
| "logps/chosen": -28.553043365478516, | |
| "logps/rejected": -39.740516662597656, | |
| "loss": 0.5548, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.13874831795692444, | |
| "rewards/margins": 0.48837804794311523, | |
| "rewards/rejected": -0.6271263360977173, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.09282399143163156, | |
| "grad_norm": 8.965351104736328, | |
| "learning_rate": 4.5358800428418426e-05, | |
| "logits/chosen": 25.02800178527832, | |
| "logits/rejected": 24.903820037841797, | |
| "logps/chosen": -41.33483123779297, | |
| "logps/rejected": -38.99028396606445, | |
| "loss": 0.5356, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": 0.14857253432273865, | |
| "rewards/margins": 0.6738235354423523, | |
| "rewards/rejected": -0.525251030921936, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 0.09639414494823277, | |
| "grad_norm": 6.815878868103027, | |
| "learning_rate": 4.5180292752588364e-05, | |
| "logits/chosen": 26.964553833007812, | |
| "logits/rejected": 26.749637603759766, | |
| "logps/chosen": -36.553131103515625, | |
| "logps/rejected": -19.335412979125977, | |
| "loss": 0.5599, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.019418681040406227, | |
| "rewards/margins": 0.6367809772491455, | |
| "rewards/rejected": -0.6561996936798096, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 0.09996429846483398, | |
| "grad_norm": 9.245991706848145, | |
| "learning_rate": 4.50017850767583e-05, | |
| "logits/chosen": 25.099380493164062, | |
| "logits/rejected": 25.418010711669922, | |
| "logps/chosen": -30.81514549255371, | |
| "logps/rejected": -27.115314483642578, | |
| "loss": 0.6413, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.4499152600765228, | |
| "rewards/margins": 0.7015028595924377, | |
| "rewards/rejected": -1.1514180898666382, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 0.1035344519814352, | |
| "grad_norm": 4.70521879196167, | |
| "learning_rate": 4.4823277400928245e-05, | |
| "logits/chosen": 26.4927978515625, | |
| "logits/rejected": 26.206491470336914, | |
| "logps/chosen": -50.66899871826172, | |
| "logps/rejected": -31.35797691345215, | |
| "loss": 0.5381, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.7465568780899048, | |
| "rewards/margins": 0.4888392388820648, | |
| "rewards/rejected": -1.235396146774292, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 0.10710460549803641, | |
| "grad_norm": 12.274664878845215, | |
| "learning_rate": 4.464476972509818e-05, | |
| "logits/chosen": 24.71265983581543, | |
| "logits/rejected": 24.88272476196289, | |
| "logps/chosen": -37.392120361328125, | |
| "logps/rejected": -31.838241577148438, | |
| "loss": 0.6747, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.5502563714981079, | |
| "rewards/margins": 0.21214011311531067, | |
| "rewards/rejected": -0.762396514415741, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.10710460549803641, | |
| "eval_logits/chosen": 24.666624069213867, | |
| "eval_logits/rejected": 24.732454299926758, | |
| "eval_logps/chosen": -43.291751861572266, | |
| "eval_logps/rejected": -40.54462432861328, | |
| "eval_loss": 0.5352813601493835, | |
| "eval_rewards/accuracies": 0.7222222089767456, | |
| "eval_rewards/chosen": -0.16518594324588776, | |
| "eval_rewards/margins": 0.6855509877204895, | |
| "eval_rewards/rejected": -0.8507369160652161, | |
| "eval_runtime": 97.8407, | |
| "eval_samples_per_second": 14.636, | |
| "eval_steps_per_second": 0.92, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.11067475901463764, | |
| "grad_norm": 7.402091979980469, | |
| "learning_rate": 4.446626204926812e-05, | |
| "logits/chosen": 25.197057723999023, | |
| "logits/rejected": 25.214290618896484, | |
| "logps/chosen": -53.834320068359375, | |
| "logps/rejected": -47.46216583251953, | |
| "loss": 0.5246, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.2350492775440216, | |
| "rewards/margins": 0.7798230051994324, | |
| "rewards/rejected": -1.0148723125457764, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 0.11424491253123885, | |
| "grad_norm": 7.803309917449951, | |
| "learning_rate": 4.4287754373438064e-05, | |
| "logits/chosen": 25.673919677734375, | |
| "logits/rejected": 26.028600692749023, | |
| "logps/chosen": -47.897422790527344, | |
| "logps/rejected": -32.04759216308594, | |
| "loss": 0.5493, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": 0.12382154166698456, | |
| "rewards/margins": 1.0582133531570435, | |
| "rewards/rejected": -0.934391975402832, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 0.11781506604784006, | |
| "grad_norm": 12.149727821350098, | |
| "learning_rate": 4.4109246697608e-05, | |
| "logits/chosen": 26.565237045288086, | |
| "logits/rejected": 26.2646541595459, | |
| "logps/chosen": -45.204307556152344, | |
| "logps/rejected": -54.38944625854492, | |
| "loss": 0.5822, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.24066975712776184, | |
| "rewards/margins": 0.3767842650413513, | |
| "rewards/rejected": -0.6174539923667908, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 0.12138521956444127, | |
| "grad_norm": 11.431114196777344, | |
| "learning_rate": 4.393073902177794e-05, | |
| "logits/chosen": 24.764408111572266, | |
| "logits/rejected": 23.79238510131836, | |
| "logps/chosen": -22.893648147583008, | |
| "logps/rejected": -48.49061584472656, | |
| "loss": 0.6424, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.6392322778701782, | |
| "rewards/margins": 0.1874510645866394, | |
| "rewards/rejected": -0.8266833424568176, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 0.12495537308104249, | |
| "grad_norm": 13.246600151062012, | |
| "learning_rate": 4.375223134594788e-05, | |
| "logits/chosen": 23.024621963500977, | |
| "logits/rejected": 22.838958740234375, | |
| "logps/chosen": -46.08310317993164, | |
| "logps/rejected": -63.932945251464844, | |
| "loss": 0.5871, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.7620424032211304, | |
| "rewards/margins": 0.4331347346305847, | |
| "rewards/rejected": -1.1951771974563599, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 0.12852552659764369, | |
| "grad_norm": 6.875857830047607, | |
| "learning_rate": 4.357372367011782e-05, | |
| "logits/chosen": 24.33723258972168, | |
| "logits/rejected": 24.633174896240234, | |
| "logps/chosen": -36.276363372802734, | |
| "logps/rejected": -41.762107849121094, | |
| "loss": 0.538, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.5144018530845642, | |
| "rewards/margins": 0.7962431907653809, | |
| "rewards/rejected": -1.3106449842453003, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 0.1320956801142449, | |
| "grad_norm": 8.577788352966309, | |
| "learning_rate": 4.339521599428776e-05, | |
| "logits/chosen": 18.497318267822266, | |
| "logits/rejected": 18.542171478271484, | |
| "logps/chosen": -54.802284240722656, | |
| "logps/rejected": -40.361900329589844, | |
| "loss": 0.5472, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.43400055170059204, | |
| "rewards/margins": 0.37733301520347595, | |
| "rewards/rejected": -0.8113336563110352, | |
| "step": 370 | |
| }, | |
| { | |
| "epoch": 0.13566583363084614, | |
| "grad_norm": 7.8961052894592285, | |
| "learning_rate": 4.3216708318457694e-05, | |
| "logits/chosen": 20.79564094543457, | |
| "logits/rejected": 21.202953338623047, | |
| "logps/chosen": -46.6911735534668, | |
| "logps/rejected": -25.177892684936523, | |
| "loss": 0.5404, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.20933791995048523, | |
| "rewards/margins": 0.9722681045532227, | |
| "rewards/rejected": -1.1816058158874512, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 0.13923598714744734, | |
| "grad_norm": 6.78292989730835, | |
| "learning_rate": 4.303820064262764e-05, | |
| "logits/chosen": 24.46206283569336, | |
| "logits/rejected": 24.468271255493164, | |
| "logps/chosen": -17.142438888549805, | |
| "logps/rejected": -8.94920539855957, | |
| "loss": 0.5916, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.5404640436172485, | |
| "rewards/margins": -0.012588715180754662, | |
| "rewards/rejected": -0.5278753042221069, | |
| "step": 390 | |
| }, | |
| { | |
| "epoch": 0.14280614066404856, | |
| "grad_norm": 6.525446891784668, | |
| "learning_rate": 4.2859692966797575e-05, | |
| "logits/chosen": 27.94113540649414, | |
| "logits/rejected": 27.818674087524414, | |
| "logps/chosen": -28.574228286743164, | |
| "logps/rejected": -41.67361831665039, | |
| "loss": 0.5795, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.17134228348731995, | |
| "rewards/margins": 0.6803145408630371, | |
| "rewards/rejected": -0.8516567945480347, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 0.14280614066404856, | |
| "eval_logits/chosen": 23.894601821899414, | |
| "eval_logits/rejected": 23.941455841064453, | |
| "eval_logps/chosen": -42.414432525634766, | |
| "eval_logps/rejected": -40.24538040161133, | |
| "eval_loss": 0.5327069759368896, | |
| "eval_rewards/accuracies": 0.7944444417953491, | |
| "eval_rewards/chosen": -0.07745402306318283, | |
| "eval_rewards/margins": 0.7433586716651917, | |
| "eval_rewards/rejected": -0.8208127617835999, | |
| "eval_runtime": 97.7382, | |
| "eval_samples_per_second": 14.651, | |
| "eval_steps_per_second": 0.921, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 0.14637629418064976, | |
| "grad_norm": 7.846954345703125, | |
| "learning_rate": 4.268118529096751e-05, | |
| "logits/chosen": 25.257925033569336, | |
| "logits/rejected": 25.105932235717773, | |
| "logps/chosen": -45.8572883605957, | |
| "logps/rejected": -26.36654281616211, | |
| "loss": 0.4999, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.5164770483970642, | |
| "rewards/margins": 0.7112953662872314, | |
| "rewards/rejected": -1.2277723550796509, | |
| "step": 410 | |
| }, | |
| { | |
| "epoch": 0.149946447697251, | |
| "grad_norm": 7.722918510437012, | |
| "learning_rate": 4.250267761513746e-05, | |
| "logits/chosen": 21.332164764404297, | |
| "logits/rejected": 21.323444366455078, | |
| "logps/chosen": -37.462974548339844, | |
| "logps/rejected": -28.17437171936035, | |
| "loss": 0.6035, | |
| "rewards/accuracies": 0.4000000059604645, | |
| "rewards/chosen": -0.4826648235321045, | |
| "rewards/margins": 0.2844996452331543, | |
| "rewards/rejected": -0.7671645283699036, | |
| "step": 420 | |
| }, | |
| { | |
| "epoch": 0.1535166012138522, | |
| "grad_norm": 8.375965118408203, | |
| "learning_rate": 4.232416993930739e-05, | |
| "logits/chosen": 18.349498748779297, | |
| "logits/rejected": 18.25157356262207, | |
| "logps/chosen": -53.23881912231445, | |
| "logps/rejected": -62.28485107421875, | |
| "loss": 0.5301, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.14654412865638733, | |
| "rewards/margins": 0.404157817363739, | |
| "rewards/rejected": -0.550701916217804, | |
| "step": 430 | |
| }, | |
| { | |
| "epoch": 0.15708675473045342, | |
| "grad_norm": 6.275489807128906, | |
| "learning_rate": 4.214566226347733e-05, | |
| "logits/chosen": 22.851360321044922, | |
| "logits/rejected": 23.03590202331543, | |
| "logps/chosen": -32.48253631591797, | |
| "logps/rejected": -36.65414047241211, | |
| "loss": 0.4959, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.13281583786010742, | |
| "rewards/margins": 1.3949477672576904, | |
| "rewards/rejected": -1.262131929397583, | |
| "step": 440 | |
| }, | |
| { | |
| "epoch": 0.16065690824705461, | |
| "grad_norm": 9.541354179382324, | |
| "learning_rate": 4.1967154587647275e-05, | |
| "logits/chosen": 25.5571231842041, | |
| "logits/rejected": 25.35956573486328, | |
| "logps/chosen": -33.83552169799805, | |
| "logps/rejected": -34.80134963989258, | |
| "loss": 0.5615, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.1620870977640152, | |
| "rewards/margins": 1.014452576637268, | |
| "rewards/rejected": -1.176539659500122, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 0.16422706176365584, | |
| "grad_norm": 4.77647590637207, | |
| "learning_rate": 4.1788646911817206e-05, | |
| "logits/chosen": 25.47999382019043, | |
| "logits/rejected": 24.808551788330078, | |
| "logps/chosen": -39.025047302246094, | |
| "logps/rejected": -71.5890884399414, | |
| "loss": 0.4752, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.2537313401699066, | |
| "rewards/margins": 1.16329026222229, | |
| "rewards/rejected": -1.4170215129852295, | |
| "step": 460 | |
| }, | |
| { | |
| "epoch": 0.16779721528025704, | |
| "grad_norm": 4.253603458404541, | |
| "learning_rate": 4.161013923598715e-05, | |
| "logits/chosen": 20.190471649169922, | |
| "logits/rejected": 20.048721313476562, | |
| "logps/chosen": -68.62611389160156, | |
| "logps/rejected": -72.08489227294922, | |
| "loss": 0.5801, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.9637469053268433, | |
| "rewards/margins": 0.6993335485458374, | |
| "rewards/rejected": -1.6630805730819702, | |
| "step": 470 | |
| }, | |
| { | |
| "epoch": 0.17136736879685827, | |
| "grad_norm": 9.119401931762695, | |
| "learning_rate": 4.1431631560157094e-05, | |
| "logits/chosen": 19.260848999023438, | |
| "logits/rejected": 18.897022247314453, | |
| "logps/chosen": -39.502891540527344, | |
| "logps/rejected": -37.60371398925781, | |
| "loss": 0.6457, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.7391226291656494, | |
| "rewards/margins": 0.16527147591114044, | |
| "rewards/rejected": -0.9043940305709839, | |
| "step": 480 | |
| }, | |
| { | |
| "epoch": 0.1749375223134595, | |
| "grad_norm": 6.5403876304626465, | |
| "learning_rate": 4.1253123884327025e-05, | |
| "logits/chosen": 24.00229263305664, | |
| "logits/rejected": 24.137210845947266, | |
| "logps/chosen": -45.11564636230469, | |
| "logps/rejected": -63.3198356628418, | |
| "loss": 0.492, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.5857070684432983, | |
| "rewards/margins": 1.3458783626556396, | |
| "rewards/rejected": -1.9315853118896484, | |
| "step": 490 | |
| }, | |
| { | |
| "epoch": 0.1785076758300607, | |
| "grad_norm": 6.147622108459473, | |
| "learning_rate": 4.107461620849697e-05, | |
| "logits/chosen": 20.132909774780273, | |
| "logits/rejected": 20.044574737548828, | |
| "logps/chosen": -40.866424560546875, | |
| "logps/rejected": -50.83662414550781, | |
| "loss": 0.4606, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.40833067893981934, | |
| "rewards/margins": 0.7871369123458862, | |
| "rewards/rejected": -1.195467472076416, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 0.1785076758300607, | |
| "eval_logits/chosen": 22.78708839416504, | |
| "eval_logits/rejected": 22.809024810791016, | |
| "eval_logps/chosen": -47.07508850097656, | |
| "eval_logps/rejected": -45.6721076965332, | |
| "eval_loss": 0.5191536545753479, | |
| "eval_rewards/accuracies": 0.75, | |
| "eval_rewards/chosen": -0.5435196161270142, | |
| "eval_rewards/margins": 0.8199654221534729, | |
| "eval_rewards/rejected": -1.3634849786758423, | |
| "eval_runtime": 97.7087, | |
| "eval_samples_per_second": 14.656, | |
| "eval_steps_per_second": 0.921, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 0.18207782934666192, | |
| "grad_norm": 12.392146110534668, | |
| "learning_rate": 4.0896108532666906e-05, | |
| "logits/chosen": 19.699203491210938, | |
| "logits/rejected": 19.52513885498047, | |
| "logps/chosen": -57.56768798828125, | |
| "logps/rejected": -71.2248764038086, | |
| "loss": 0.632, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.3169898986816406, | |
| "rewards/margins": 1.7359142303466797, | |
| "rewards/rejected": -2.0529043674468994, | |
| "step": 510 | |
| }, | |
| { | |
| "epoch": 0.18564798286326312, | |
| "grad_norm": 5.329209804534912, | |
| "learning_rate": 4.071760085683684e-05, | |
| "logits/chosen": 20.416704177856445, | |
| "logits/rejected": 20.525035858154297, | |
| "logps/chosen": -38.03459930419922, | |
| "logps/rejected": -38.064720153808594, | |
| "loss": 0.5251, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.5582042932510376, | |
| "rewards/margins": 0.7457180023193359, | |
| "rewards/rejected": -1.3039222955703735, | |
| "step": 520 | |
| }, | |
| { | |
| "epoch": 0.18921813637986434, | |
| "grad_norm": 5.097146987915039, | |
| "learning_rate": 4.053909318100679e-05, | |
| "logits/chosen": 22.233572006225586, | |
| "logits/rejected": 22.370027542114258, | |
| "logps/chosen": -15.693742752075195, | |
| "logps/rejected": -16.01146125793457, | |
| "loss": 0.4981, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.28356021642684937, | |
| "rewards/margins": 0.759593665599823, | |
| "rewards/rejected": -1.0431538820266724, | |
| "step": 530 | |
| }, | |
| { | |
| "epoch": 0.19278828989646554, | |
| "grad_norm": 10.173523902893066, | |
| "learning_rate": 4.0360585505176725e-05, | |
| "logits/chosen": 22.36469268798828, | |
| "logits/rejected": 22.462005615234375, | |
| "logps/chosen": -22.046735763549805, | |
| "logps/rejected": -29.99558448791504, | |
| "loss": 0.5417, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.5353186726570129, | |
| "rewards/margins": 0.14299364387989044, | |
| "rewards/rejected": -0.6783123016357422, | |
| "step": 540 | |
| }, | |
| { | |
| "epoch": 0.19635844341306677, | |
| "grad_norm": 4.725411891937256, | |
| "learning_rate": 4.018207782934666e-05, | |
| "logits/chosen": 22.934490203857422, | |
| "logits/rejected": 22.83697509765625, | |
| "logps/chosen": -50.47562789916992, | |
| "logps/rejected": -39.98368453979492, | |
| "loss": 0.656, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.5939350724220276, | |
| "rewards/margins": 0.7087148427963257, | |
| "rewards/rejected": -1.3026498556137085, | |
| "step": 550 | |
| }, | |
| { | |
| "epoch": 0.19992859692966797, | |
| "grad_norm": 16.693065643310547, | |
| "learning_rate": 4.00035701535166e-05, | |
| "logits/chosen": 26.147781372070312, | |
| "logits/rejected": 25.938640594482422, | |
| "logps/chosen": -66.49282836914062, | |
| "logps/rejected": -62.023353576660156, | |
| "loss": 0.5596, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.1642974466085434, | |
| "rewards/margins": 1.2225383520126343, | |
| "rewards/rejected": -1.3868358135223389, | |
| "step": 560 | |
| }, | |
| { | |
| "epoch": 0.2034987504462692, | |
| "grad_norm": 4.374217987060547, | |
| "learning_rate": 3.982506247768654e-05, | |
| "logits/chosen": 23.22292709350586, | |
| "logits/rejected": 22.756343841552734, | |
| "logps/chosen": -47.733238220214844, | |
| "logps/rejected": -65.7103042602539, | |
| "loss": 0.5128, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.5647065043449402, | |
| "rewards/margins": 0.45924702286720276, | |
| "rewards/rejected": -1.0239535570144653, | |
| "step": 570 | |
| }, | |
| { | |
| "epoch": 0.2070689039628704, | |
| "grad_norm": 5.655895233154297, | |
| "learning_rate": 3.964655480185648e-05, | |
| "logits/chosen": 22.537179946899414, | |
| "logits/rejected": 22.3128662109375, | |
| "logps/chosen": -33.57523727416992, | |
| "logps/rejected": -46.982025146484375, | |
| "loss": 0.485, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.5356764793395996, | |
| "rewards/margins": 1.0464246273040771, | |
| "rewards/rejected": -1.5821009874343872, | |
| "step": 580 | |
| }, | |
| { | |
| "epoch": 0.21063905747947162, | |
| "grad_norm": 8.67690372467041, | |
| "learning_rate": 3.946804712602642e-05, | |
| "logits/chosen": 24.053617477416992, | |
| "logits/rejected": 24.086387634277344, | |
| "logps/chosen": -30.85910415649414, | |
| "logps/rejected": -17.82439422607422, | |
| "loss": 0.4658, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.6462145447731018, | |
| "rewards/margins": 0.6275963187217712, | |
| "rewards/rejected": -1.273810863494873, | |
| "step": 590 | |
| }, | |
| { | |
| "epoch": 0.21420921099607282, | |
| "grad_norm": 12.361294746398926, | |
| "learning_rate": 3.928953945019636e-05, | |
| "logits/chosen": 19.749366760253906, | |
| "logits/rejected": 20.843948364257812, | |
| "logps/chosen": -47.06502151489258, | |
| "logps/rejected": -38.376441955566406, | |
| "loss": 0.4708, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.37672120332717896, | |
| "rewards/margins": 1.4848692417144775, | |
| "rewards/rejected": -1.8615906238555908, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 0.21420921099607282, | |
| "eval_logits/chosen": 22.7338924407959, | |
| "eval_logits/rejected": 22.768844604492188, | |
| "eval_logps/chosen": -43.45345687866211, | |
| "eval_logps/rejected": -43.90707778930664, | |
| "eval_loss": 0.5074149966239929, | |
| "eval_rewards/accuracies": 0.7944444417953491, | |
| "eval_rewards/chosen": -0.18135611712932587, | |
| "eval_rewards/margins": 1.0056260824203491, | |
| "eval_rewards/rejected": -1.1869820356369019, | |
| "eval_runtime": 97.81, | |
| "eval_samples_per_second": 14.641, | |
| "eval_steps_per_second": 0.92, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 0.21777936451267405, | |
| "grad_norm": 11.45566177368164, | |
| "learning_rate": 3.91110317743663e-05, | |
| "logits/chosen": 17.696277618408203, | |
| "logits/rejected": 18.161190032958984, | |
| "logps/chosen": -61.38924026489258, | |
| "logps/rejected": -28.698049545288086, | |
| "loss": 0.6026, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.9514519572257996, | |
| "rewards/margins": -0.2889273166656494, | |
| "rewards/rejected": -0.6625245809555054, | |
| "step": 610 | |
| }, | |
| { | |
| "epoch": 0.22134951802927527, | |
| "grad_norm": 11.354263305664062, | |
| "learning_rate": 3.8932524098536237e-05, | |
| "logits/chosen": 23.74215316772461, | |
| "logits/rejected": 23.932151794433594, | |
| "logps/chosen": -39.733665466308594, | |
| "logps/rejected": -52.69596481323242, | |
| "loss": 0.5451, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.11807258427143097, | |
| "rewards/margins": 0.5899603962898254, | |
| "rewards/rejected": -0.7080329656600952, | |
| "step": 620 | |
| }, | |
| { | |
| "epoch": 0.22491967154587647, | |
| "grad_norm": 8.663636207580566, | |
| "learning_rate": 3.875401642270618e-05, | |
| "logits/chosen": 20.996337890625, | |
| "logits/rejected": 21.191204071044922, | |
| "logps/chosen": -42.43983840942383, | |
| "logps/rejected": -30.458621978759766, | |
| "loss": 0.5347, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.20720069110393524, | |
| "rewards/margins": 0.8300245404243469, | |
| "rewards/rejected": -1.0372252464294434, | |
| "step": 630 | |
| }, | |
| { | |
| "epoch": 0.2284898250624777, | |
| "grad_norm": 6.389614105224609, | |
| "learning_rate": 3.857550874687612e-05, | |
| "logits/chosen": 23.66379737854004, | |
| "logits/rejected": 24.068124771118164, | |
| "logps/chosen": -56.00236129760742, | |
| "logps/rejected": -52.503746032714844, | |
| "loss": 0.5791, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.09035615622997284, | |
| "rewards/margins": 1.4734671115875244, | |
| "rewards/rejected": -1.5638234615325928, | |
| "step": 640 | |
| }, | |
| { | |
| "epoch": 0.2320599785790789, | |
| "grad_norm": 5.053744792938232, | |
| "learning_rate": 3.8397001071046055e-05, | |
| "logits/chosen": 24.763042449951172, | |
| "logits/rejected": 24.731813430786133, | |
| "logps/chosen": -34.47309494018555, | |
| "logps/rejected": -24.708209991455078, | |
| "loss": 0.5188, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.23519682884216309, | |
| "rewards/margins": 0.7381345629692078, | |
| "rewards/rejected": -0.9733314514160156, | |
| "step": 650 | |
| }, | |
| { | |
| "epoch": 0.23563013209568012, | |
| "grad_norm": 14.828216552734375, | |
| "learning_rate": 3.8218493395216e-05, | |
| "logits/chosen": 23.33843421936035, | |
| "logits/rejected": 23.605775833129883, | |
| "logps/chosen": -38.09233856201172, | |
| "logps/rejected": -36.02016830444336, | |
| "loss": 0.6089, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.3239721357822418, | |
| "rewards/margins": 0.35063061118125916, | |
| "rewards/rejected": -0.6746028661727905, | |
| "step": 660 | |
| }, | |
| { | |
| "epoch": 0.23920028561228132, | |
| "grad_norm": 6.591442108154297, | |
| "learning_rate": 3.8039985719385937e-05, | |
| "logits/chosen": 22.064315795898438, | |
| "logits/rejected": 22.158384323120117, | |
| "logps/chosen": -28.91001319885254, | |
| "logps/rejected": -23.76300048828125, | |
| "loss": 0.5987, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.16806618869304657, | |
| "rewards/margins": 0.20326104760169983, | |
| "rewards/rejected": -0.3713272213935852, | |
| "step": 670 | |
| }, | |
| { | |
| "epoch": 0.24277043912888255, | |
| "grad_norm": 9.390283584594727, | |
| "learning_rate": 3.7861478043555874e-05, | |
| "logits/chosen": 23.296916961669922, | |
| "logits/rejected": 23.495471954345703, | |
| "logps/chosen": -51.231292724609375, | |
| "logps/rejected": -32.36138916015625, | |
| "loss": 0.4838, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.4357140064239502, | |
| "rewards/margins": 0.5686477422714233, | |
| "rewards/rejected": -1.0043617486953735, | |
| "step": 680 | |
| }, | |
| { | |
| "epoch": 0.24634059264548375, | |
| "grad_norm": 7.05921745300293, | |
| "learning_rate": 3.768297036772581e-05, | |
| "logits/chosen": 24.67749786376953, | |
| "logits/rejected": 25.084362030029297, | |
| "logps/chosen": -61.90864944458008, | |
| "logps/rejected": -47.69279861450195, | |
| "loss": 0.6434, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -1.0670510530471802, | |
| "rewards/margins": 0.16362956166267395, | |
| "rewards/rejected": -1.2306805849075317, | |
| "step": 690 | |
| }, | |
| { | |
| "epoch": 0.24991074616208497, | |
| "grad_norm": 6.927039623260498, | |
| "learning_rate": 3.7504462691895755e-05, | |
| "logits/chosen": 22.120304107666016, | |
| "logits/rejected": 22.317134857177734, | |
| "logps/chosen": -47.007747650146484, | |
| "logps/rejected": -29.523571014404297, | |
| "loss": 0.5162, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.47515907883644104, | |
| "rewards/margins": 0.9105289578437805, | |
| "rewards/rejected": -1.385688066482544, | |
| "step": 700 | |
| }, | |
| { | |
| "epoch": 0.24991074616208497, | |
| "eval_logits/chosen": 23.21824073791504, | |
| "eval_logits/rejected": 23.2376651763916, | |
| "eval_logps/chosen": -43.74300765991211, | |
| "eval_logps/rejected": -43.685081481933594, | |
| "eval_loss": 0.5032530426979065, | |
| "eval_rewards/accuracies": 0.7888888716697693, | |
| "eval_rewards/chosen": -0.21031144261360168, | |
| "eval_rewards/margins": 0.9544711709022522, | |
| "eval_rewards/rejected": -1.1647826433181763, | |
| "eval_runtime": 97.9268, | |
| "eval_samples_per_second": 14.623, | |
| "eval_steps_per_second": 0.919, | |
| "step": 700 | |
| }, | |
| { | |
| "epoch": 0.2534808996786862, | |
| "grad_norm": 6.4891767501831055, | |
| "learning_rate": 3.732595501606569e-05, | |
| "logits/chosen": 23.012346267700195, | |
| "logits/rejected": 23.133983612060547, | |
| "logps/chosen": -32.99153137207031, | |
| "logps/rejected": -30.8564453125, | |
| "loss": 0.5311, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.13405704498291016, | |
| "rewards/margins": 0.8106959462165833, | |
| "rewards/rejected": -0.9447528719902039, | |
| "step": 710 | |
| }, | |
| { | |
| "epoch": 0.25705105319528737, | |
| "grad_norm": 7.658117771148682, | |
| "learning_rate": 3.714744734023563e-05, | |
| "logits/chosen": 22.539342880249023, | |
| "logits/rejected": 22.32984161376953, | |
| "logps/chosen": -41.30803298950195, | |
| "logps/rejected": -37.77648162841797, | |
| "loss": 0.583, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.5999592542648315, | |
| "rewards/margins": 0.26343995332717896, | |
| "rewards/rejected": -0.8633992075920105, | |
| "step": 720 | |
| }, | |
| { | |
| "epoch": 0.2606212067118886, | |
| "grad_norm": 7.918398857116699, | |
| "learning_rate": 3.6968939664405574e-05, | |
| "logits/chosen": 22.939983367919922, | |
| "logits/rejected": 23.18207359313965, | |
| "logps/chosen": -46.46357345581055, | |
| "logps/rejected": -35.31732940673828, | |
| "loss": 0.5581, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.043735720217227936, | |
| "rewards/margins": 0.8847159147262573, | |
| "rewards/rejected": -0.928451657295227, | |
| "step": 730 | |
| }, | |
| { | |
| "epoch": 0.2641913602284898, | |
| "grad_norm": 6.20953893661499, | |
| "learning_rate": 3.6790431988575504e-05, | |
| "logits/chosen": 28.148590087890625, | |
| "logits/rejected": 27.938528060913086, | |
| "logps/chosen": -51.41779327392578, | |
| "logps/rejected": -52.49924850463867, | |
| "loss": 0.523, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -1.0679657459259033, | |
| "rewards/margins": 0.6626083254814148, | |
| "rewards/rejected": -1.7305742502212524, | |
| "step": 740 | |
| }, | |
| { | |
| "epoch": 0.26776151374509105, | |
| "grad_norm": 10.36694049835205, | |
| "learning_rate": 3.661192431274545e-05, | |
| "logits/chosen": 24.841873168945312, | |
| "logits/rejected": 25.12506675720215, | |
| "logps/chosen": -21.518558502197266, | |
| "logps/rejected": -28.731252670288086, | |
| "loss": 0.5559, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.3871726393699646, | |
| "rewards/margins": 0.5390633344650269, | |
| "rewards/rejected": -0.9262359738349915, | |
| "step": 750 | |
| }, | |
| { | |
| "epoch": 0.2713316672616923, | |
| "grad_norm": 6.616639137268066, | |
| "learning_rate": 3.643341663691539e-05, | |
| "logits/chosen": 23.483844757080078, | |
| "logits/rejected": 23.462881088256836, | |
| "logps/chosen": -20.33786964416504, | |
| "logps/rejected": -41.595367431640625, | |
| "loss": 0.5861, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.26747018098831177, | |
| "rewards/margins": 0.5755705833435059, | |
| "rewards/rejected": -0.8430407643318176, | |
| "step": 760 | |
| }, | |
| { | |
| "epoch": 0.27490182077829345, | |
| "grad_norm": 7.002477169036865, | |
| "learning_rate": 3.625490896108533e-05, | |
| "logits/chosen": 23.841524124145508, | |
| "logits/rejected": 23.905414581298828, | |
| "logps/chosen": -54.80754852294922, | |
| "logps/rejected": -60.40642166137695, | |
| "loss": 0.5478, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.7754853367805481, | |
| "rewards/margins": 0.7477506995201111, | |
| "rewards/rejected": -1.5232360363006592, | |
| "step": 770 | |
| }, | |
| { | |
| "epoch": 0.2784719742948947, | |
| "grad_norm": 4.716040134429932, | |
| "learning_rate": 3.607640128525527e-05, | |
| "logits/chosen": 21.66192054748535, | |
| "logits/rejected": 21.678348541259766, | |
| "logps/chosen": -59.828773498535156, | |
| "logps/rejected": -51.79111862182617, | |
| "loss": 0.3989, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.5894995927810669, | |
| "rewards/margins": 1.3722957372665405, | |
| "rewards/rejected": -1.9617952108383179, | |
| "step": 780 | |
| }, | |
| { | |
| "epoch": 0.2820421278114959, | |
| "grad_norm": 8.946788787841797, | |
| "learning_rate": 3.589789360942521e-05, | |
| "logits/chosen": 24.498876571655273, | |
| "logits/rejected": 24.739574432373047, | |
| "logps/chosen": -56.08790969848633, | |
| "logps/rejected": -50.47938919067383, | |
| "loss": 0.5822, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -1.2977168560028076, | |
| "rewards/margins": 0.1555618941783905, | |
| "rewards/rejected": -1.45327889919281, | |
| "step": 790 | |
| }, | |
| { | |
| "epoch": 0.28561228132809713, | |
| "grad_norm": 5.08765983581543, | |
| "learning_rate": 3.571938593359515e-05, | |
| "logits/chosen": 26.44881820678711, | |
| "logits/rejected": 26.592966079711914, | |
| "logps/chosen": -26.545644760131836, | |
| "logps/rejected": -46.947174072265625, | |
| "loss": 0.5121, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.5634913444519043, | |
| "rewards/margins": 0.8930961489677429, | |
| "rewards/rejected": -1.456587553024292, | |
| "step": 800 | |
| }, | |
| { | |
| "epoch": 0.28561228132809713, | |
| "eval_logits/chosen": 23.41709327697754, | |
| "eval_logits/rejected": 23.457468032836914, | |
| "eval_logps/chosen": -47.103492736816406, | |
| "eval_logps/rejected": -47.543006896972656, | |
| "eval_loss": 0.4901362657546997, | |
| "eval_rewards/accuracies": 0.7833333611488342, | |
| "eval_rewards/chosen": -0.5463601350784302, | |
| "eval_rewards/margins": 1.0042147636413574, | |
| "eval_rewards/rejected": -1.5505748987197876, | |
| "eval_runtime": 97.7883, | |
| "eval_samples_per_second": 14.644, | |
| "eval_steps_per_second": 0.92, | |
| "step": 800 | |
| }, | |
| { | |
| "epoch": 0.2891824348446983, | |
| "grad_norm": 5.668002605438232, | |
| "learning_rate": 3.5540878257765086e-05, | |
| "logits/chosen": 15.257682800292969, | |
| "logits/rejected": 15.15269947052002, | |
| "logps/chosen": -28.859485626220703, | |
| "logps/rejected": -39.70137023925781, | |
| "loss": 0.6225, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.8887822031974792, | |
| "rewards/margins": 0.40193691849708557, | |
| "rewards/rejected": -1.2907191514968872, | |
| "step": 810 | |
| }, | |
| { | |
| "epoch": 0.2927525883612995, | |
| "grad_norm": 6.119519233703613, | |
| "learning_rate": 3.536237058193502e-05, | |
| "logits/chosen": 20.487186431884766, | |
| "logits/rejected": 20.569347381591797, | |
| "logps/chosen": -26.315814971923828, | |
| "logps/rejected": -24.049549102783203, | |
| "loss": 0.6334, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.2173776626586914, | |
| "rewards/margins": 0.8700194358825684, | |
| "rewards/rejected": -1.0873970985412598, | |
| "step": 820 | |
| }, | |
| { | |
| "epoch": 0.29632274187790075, | |
| "grad_norm": 9.75696849822998, | |
| "learning_rate": 3.518386290610497e-05, | |
| "logits/chosen": 28.570262908935547, | |
| "logits/rejected": 29.040294647216797, | |
| "logps/chosen": -46.7459831237793, | |
| "logps/rejected": -25.238162994384766, | |
| "loss": 0.4854, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.20457518100738525, | |
| "rewards/margins": 1.082558274269104, | |
| "rewards/rejected": -1.2871334552764893, | |
| "step": 830 | |
| }, | |
| { | |
| "epoch": 0.299892895394502, | |
| "grad_norm": 4.077060699462891, | |
| "learning_rate": 3.5005355230274904e-05, | |
| "logits/chosen": 22.926097869873047, | |
| "logits/rejected": 23.03900718688965, | |
| "logps/chosen": -43.289085388183594, | |
| "logps/rejected": -39.03928756713867, | |
| "loss": 0.4416, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.38202500343322754, | |
| "rewards/margins": 0.7754106521606445, | |
| "rewards/rejected": -1.1574357748031616, | |
| "step": 840 | |
| }, | |
| { | |
| "epoch": 0.30346304891110315, | |
| "grad_norm": 6.117227554321289, | |
| "learning_rate": 3.482684755444484e-05, | |
| "logits/chosen": 26.401382446289062, | |
| "logits/rejected": 25.781219482421875, | |
| "logps/chosen": -30.348541259765625, | |
| "logps/rejected": -47.32704162597656, | |
| "loss": 0.4493, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.6118322610855103, | |
| "rewards/margins": 1.2445160150527954, | |
| "rewards/rejected": -1.8563482761383057, | |
| "step": 850 | |
| }, | |
| { | |
| "epoch": 0.3070332024277044, | |
| "grad_norm": 11.844883918762207, | |
| "learning_rate": 3.4648339878614786e-05, | |
| "logits/chosen": 23.075183868408203, | |
| "logits/rejected": 22.080493927001953, | |
| "logps/chosen": -49.947914123535156, | |
| "logps/rejected": -58.764617919921875, | |
| "loss": 0.5217, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -1.0053069591522217, | |
| "rewards/margins": 0.48744481801986694, | |
| "rewards/rejected": -1.4927518367767334, | |
| "step": 860 | |
| }, | |
| { | |
| "epoch": 0.3106033559443056, | |
| "grad_norm": 12.23678207397461, | |
| "learning_rate": 3.4469832202784716e-05, | |
| "logits/chosen": 26.47686767578125, | |
| "logits/rejected": 26.53334617614746, | |
| "logps/chosen": -27.09230613708496, | |
| "logps/rejected": -39.659324645996094, | |
| "loss": 0.5951, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.6957861185073853, | |
| "rewards/margins": 0.6590690016746521, | |
| "rewards/rejected": -1.3548550605773926, | |
| "step": 870 | |
| }, | |
| { | |
| "epoch": 0.31417350946090683, | |
| "grad_norm": 8.197884559631348, | |
| "learning_rate": 3.429132452695466e-05, | |
| "logits/chosen": 22.180370330810547, | |
| "logits/rejected": 22.334489822387695, | |
| "logps/chosen": -45.77663040161133, | |
| "logps/rejected": -41.45575714111328, | |
| "loss": 0.4294, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.40475016832351685, | |
| "rewards/margins": 1.7321338653564453, | |
| "rewards/rejected": -2.1368842124938965, | |
| "step": 880 | |
| }, | |
| { | |
| "epoch": 0.31774366297750806, | |
| "grad_norm": 6.394644260406494, | |
| "learning_rate": 3.4112816851124604e-05, | |
| "logits/chosen": 21.572330474853516, | |
| "logits/rejected": 21.563013076782227, | |
| "logps/chosen": -42.58979034423828, | |
| "logps/rejected": -32.326332092285156, | |
| "loss": 0.5278, | |
| "rewards/accuracies": 0.30000001192092896, | |
| "rewards/chosen": -1.3842201232910156, | |
| "rewards/margins": -0.4129442572593689, | |
| "rewards/rejected": -0.9712759256362915, | |
| "step": 890 | |
| }, | |
| { | |
| "epoch": 0.32131381649410923, | |
| "grad_norm": 11.288488388061523, | |
| "learning_rate": 3.3934309175294535e-05, | |
| "logits/chosen": 23.486909866333008, | |
| "logits/rejected": 23.229293823242188, | |
| "logps/chosen": -26.536273956298828, | |
| "logps/rejected": -35.171104431152344, | |
| "loss": 0.4865, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.39322757720947266, | |
| "rewards/margins": 0.7671092748641968, | |
| "rewards/rejected": -1.160336971282959, | |
| "step": 900 | |
| }, | |
| { | |
| "epoch": 0.32131381649410923, | |
| "eval_logits/chosen": 22.803043365478516, | |
| "eval_logits/rejected": 22.84783172607422, | |
| "eval_logps/chosen": -47.309364318847656, | |
| "eval_logps/rejected": -47.91334533691406, | |
| "eval_loss": 0.4862760305404663, | |
| "eval_rewards/accuracies": 0.7777777910232544, | |
| "eval_rewards/chosen": -0.5669471025466919, | |
| "eval_rewards/margins": 1.0206618309020996, | |
| "eval_rewards/rejected": -1.587609052658081, | |
| "eval_runtime": 97.8776, | |
| "eval_samples_per_second": 14.631, | |
| "eval_steps_per_second": 0.92, | |
| "step": 900 | |
| }, | |
| { | |
| "epoch": 0.32488397001071045, | |
| "grad_norm": 9.65831470489502, | |
| "learning_rate": 3.375580149946448e-05, | |
| "logits/chosen": 26.497304916381836, | |
| "logits/rejected": 26.356307983398438, | |
| "logps/chosen": -28.236690521240234, | |
| "logps/rejected": -34.156455993652344, | |
| "loss": 0.4985, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.3999934196472168, | |
| "rewards/margins": 0.484555184841156, | |
| "rewards/rejected": -0.884548544883728, | |
| "step": 910 | |
| }, | |
| { | |
| "epoch": 0.3284541235273117, | |
| "grad_norm": 6.470170021057129, | |
| "learning_rate": 3.357729382363442e-05, | |
| "logits/chosen": 23.454336166381836, | |
| "logits/rejected": 23.263614654541016, | |
| "logps/chosen": -65.40380859375, | |
| "logps/rejected": -39.344261169433594, | |
| "loss": 0.5108, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.46298104524612427, | |
| "rewards/margins": 1.2011882066726685, | |
| "rewards/rejected": -1.6641693115234375, | |
| "step": 920 | |
| }, | |
| { | |
| "epoch": 0.3320242770439129, | |
| "grad_norm": 10.672679901123047, | |
| "learning_rate": 3.3398786147804354e-05, | |
| "logits/chosen": 24.854284286499023, | |
| "logits/rejected": 24.732633590698242, | |
| "logps/chosen": -60.40422821044922, | |
| "logps/rejected": -35.107269287109375, | |
| "loss": 0.5479, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.48133277893066406, | |
| "rewards/margins": 0.2392297238111496, | |
| "rewards/rejected": -0.7205625176429749, | |
| "step": 930 | |
| }, | |
| { | |
| "epoch": 0.3355944305605141, | |
| "grad_norm": 11.122824668884277, | |
| "learning_rate": 3.32202784719743e-05, | |
| "logits/chosen": 25.70233726501465, | |
| "logits/rejected": 25.91379737854004, | |
| "logps/chosen": -45.25204086303711, | |
| "logps/rejected": -32.91322326660156, | |
| "loss": 0.4858, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.7527903318405151, | |
| "rewards/margins": 0.9621620178222656, | |
| "rewards/rejected": -1.7149524688720703, | |
| "step": 940 | |
| }, | |
| { | |
| "epoch": 0.3391645840771153, | |
| "grad_norm": 8.845693588256836, | |
| "learning_rate": 3.3041770796144235e-05, | |
| "logits/chosen": 23.679813385009766, | |
| "logits/rejected": 24.00821304321289, | |
| "logps/chosen": -53.314300537109375, | |
| "logps/rejected": -57.53212356567383, | |
| "loss": 0.4488, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.8183348774909973, | |
| "rewards/margins": 1.1887353658676147, | |
| "rewards/rejected": -2.007070302963257, | |
| "step": 950 | |
| }, | |
| { | |
| "epoch": 0.34273473759371653, | |
| "grad_norm": 11.808037757873535, | |
| "learning_rate": 3.286326312031417e-05, | |
| "logits/chosen": 21.081445693969727, | |
| "logits/rejected": 20.683502197265625, | |
| "logps/chosen": -59.89850997924805, | |
| "logps/rejected": -69.1084213256836, | |
| "loss": 0.5822, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.4903489649295807, | |
| "rewards/margins": 1.016610860824585, | |
| "rewards/rejected": -1.5069600343704224, | |
| "step": 960 | |
| }, | |
| { | |
| "epoch": 0.34630489111031776, | |
| "grad_norm": 4.3298845291137695, | |
| "learning_rate": 3.2684755444484116e-05, | |
| "logits/chosen": 23.46924591064453, | |
| "logits/rejected": 23.299514770507812, | |
| "logps/chosen": -45.10730743408203, | |
| "logps/rejected": -34.44489288330078, | |
| "loss": 0.5883, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.9016550779342651, | |
| "rewards/margins": 0.5788153409957886, | |
| "rewards/rejected": -1.4804704189300537, | |
| "step": 970 | |
| }, | |
| { | |
| "epoch": 0.349875044626919, | |
| "grad_norm": 5.846831798553467, | |
| "learning_rate": 3.2506247768654054e-05, | |
| "logits/chosen": 22.685070037841797, | |
| "logits/rejected": 23.413772583007812, | |
| "logps/chosen": -42.309078216552734, | |
| "logps/rejected": -63.73997116088867, | |
| "loss": 0.4721, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.5896610021591187, | |
| "rewards/margins": 1.2323236465454102, | |
| "rewards/rejected": -1.8219846487045288, | |
| "step": 980 | |
| }, | |
| { | |
| "epoch": 0.35344519814352016, | |
| "grad_norm": 8.441866874694824, | |
| "learning_rate": 3.232774009282399e-05, | |
| "logits/chosen": 22.13656997680664, | |
| "logits/rejected": 22.546173095703125, | |
| "logps/chosen": -39.778053283691406, | |
| "logps/rejected": -48.562049865722656, | |
| "loss": 0.5298, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.4805465340614319, | |
| "rewards/margins": 1.1920750141143799, | |
| "rewards/rejected": -1.6726213693618774, | |
| "step": 990 | |
| }, | |
| { | |
| "epoch": 0.3570153516601214, | |
| "grad_norm": 8.373982429504395, | |
| "learning_rate": 3.214923241699393e-05, | |
| "logits/chosen": 23.80282974243164, | |
| "logits/rejected": 23.995637893676758, | |
| "logps/chosen": -41.55686950683594, | |
| "logps/rejected": -39.182655334472656, | |
| "loss": 0.6191, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.36075398325920105, | |
| "rewards/margins": 1.393147349357605, | |
| "rewards/rejected": -1.7539011240005493, | |
| "step": 1000 | |
| }, | |
| { | |
| "epoch": 0.3570153516601214, | |
| "eval_logits/chosen": 21.82489013671875, | |
| "eval_logits/rejected": 21.88067054748535, | |
| "eval_logps/chosen": -47.92158126831055, | |
| "eval_logps/rejected": -49.27989196777344, | |
| "eval_loss": 0.474328875541687, | |
| "eval_rewards/accuracies": 0.7888888716697693, | |
| "eval_rewards/chosen": -0.6281694173812866, | |
| "eval_rewards/margins": 1.0960942506790161, | |
| "eval_rewards/rejected": -1.7242639064788818, | |
| "eval_runtime": 97.8094, | |
| "eval_samples_per_second": 14.641, | |
| "eval_steps_per_second": 0.92, | |
| "step": 1000 | |
| }, | |
| { | |
| "epoch": 0.3605855051767226, | |
| "grad_norm": 10.327641487121582, | |
| "learning_rate": 3.197072474116387e-05, | |
| "logits/chosen": 23.04856300354004, | |
| "logits/rejected": 23.33269691467285, | |
| "logps/chosen": -47.23090362548828, | |
| "logps/rejected": -35.351966857910156, | |
| "loss": 0.5248, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -1.2200241088867188, | |
| "rewards/margins": 0.10034622997045517, | |
| "rewards/rejected": -1.3203704357147217, | |
| "step": 1010 | |
| }, | |
| { | |
| "epoch": 0.36415565869332384, | |
| "grad_norm": 13.408589363098145, | |
| "learning_rate": 3.179221706533381e-05, | |
| "logits/chosen": 20.803436279296875, | |
| "logits/rejected": 20.647022247314453, | |
| "logps/chosen": -42.571128845214844, | |
| "logps/rejected": -34.00419235229492, | |
| "loss": 0.5064, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -1.104473352432251, | |
| "rewards/margins": 0.35474246740341187, | |
| "rewards/rejected": -1.4592156410217285, | |
| "step": 1020 | |
| }, | |
| { | |
| "epoch": 0.367725812209925, | |
| "grad_norm": 5.063046455383301, | |
| "learning_rate": 3.161370938950375e-05, | |
| "logits/chosen": 23.382953643798828, | |
| "logits/rejected": 23.356374740600586, | |
| "logps/chosen": -35.55125045776367, | |
| "logps/rejected": -40.64656066894531, | |
| "loss": 0.5387, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.9775670766830444, | |
| "rewards/margins": 0.6872422099113464, | |
| "rewards/rejected": -1.664809226989746, | |
| "step": 1030 | |
| }, | |
| { | |
| "epoch": 0.37129596572652623, | |
| "grad_norm": 7.303701877593994, | |
| "learning_rate": 3.143520171367369e-05, | |
| "logits/chosen": 24.133228302001953, | |
| "logits/rejected": 24.389375686645508, | |
| "logps/chosen": -50.63660430908203, | |
| "logps/rejected": -54.03656005859375, | |
| "loss": 0.4985, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.38200637698173523, | |
| "rewards/margins": 1.240886926651001, | |
| "rewards/rejected": -1.6228930950164795, | |
| "step": 1040 | |
| }, | |
| { | |
| "epoch": 0.37486611924312746, | |
| "grad_norm": 6.75851583480835, | |
| "learning_rate": 3.125669403784363e-05, | |
| "logits/chosen": 19.592905044555664, | |
| "logits/rejected": 19.829776763916016, | |
| "logps/chosen": -59.42816162109375, | |
| "logps/rejected": -58.9128303527832, | |
| "loss": 0.4953, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -1.082971215248108, | |
| "rewards/margins": 0.658551037311554, | |
| "rewards/rejected": -1.7415224313735962, | |
| "step": 1050 | |
| }, | |
| { | |
| "epoch": 0.3784362727597287, | |
| "grad_norm": 8.15499210357666, | |
| "learning_rate": 3.1078186362013566e-05, | |
| "logits/chosen": 25.108243942260742, | |
| "logits/rejected": 25.314802169799805, | |
| "logps/chosen": -44.924949645996094, | |
| "logps/rejected": -37.734554290771484, | |
| "loss": 0.4964, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.3476589620113373, | |
| "rewards/margins": 1.265342354774475, | |
| "rewards/rejected": -1.6130014657974243, | |
| "step": 1060 | |
| }, | |
| { | |
| "epoch": 0.38200642627632986, | |
| "grad_norm": 14.444954872131348, | |
| "learning_rate": 3.089967868618351e-05, | |
| "logits/chosen": 24.31263542175293, | |
| "logits/rejected": 24.08812141418457, | |
| "logps/chosen": -42.63981246948242, | |
| "logps/rejected": -52.27241897583008, | |
| "loss": 0.5519, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.5805258750915527, | |
| "rewards/margins": 0.8533035516738892, | |
| "rewards/rejected": -1.4338295459747314, | |
| "step": 1070 | |
| }, | |
| { | |
| "epoch": 0.3855765797929311, | |
| "grad_norm": 7.73518180847168, | |
| "learning_rate": 3.072117101035345e-05, | |
| "logits/chosen": 25.645715713500977, | |
| "logits/rejected": 25.8345947265625, | |
| "logps/chosen": -39.769630432128906, | |
| "logps/rejected": -35.580650329589844, | |
| "loss": 0.4351, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.4297657907009125, | |
| "rewards/margins": 1.0592445135116577, | |
| "rewards/rejected": -1.4890105724334717, | |
| "step": 1080 | |
| }, | |
| { | |
| "epoch": 0.3891467333095323, | |
| "grad_norm": 6.938913345336914, | |
| "learning_rate": 3.0542663334523384e-05, | |
| "logits/chosen": 21.569717407226562, | |
| "logits/rejected": 21.530803680419922, | |
| "logps/chosen": -42.12656021118164, | |
| "logps/rejected": -51.796173095703125, | |
| "loss": 0.5753, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.23102021217346191, | |
| "rewards/margins": 1.1891123056411743, | |
| "rewards/rejected": -1.4201325178146362, | |
| "step": 1090 | |
| }, | |
| { | |
| "epoch": 0.39271688682613354, | |
| "grad_norm": 14.953814506530762, | |
| "learning_rate": 3.0364155658693328e-05, | |
| "logits/chosen": 23.64889144897461, | |
| "logits/rejected": 24.02678680419922, | |
| "logps/chosen": -32.63819122314453, | |
| "logps/rejected": -34.16315841674805, | |
| "loss": 0.5176, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.6592009663581848, | |
| "rewards/margins": 1.2730789184570312, | |
| "rewards/rejected": -1.9322798252105713, | |
| "step": 1100 | |
| }, | |
| { | |
| "epoch": 0.39271688682613354, | |
| "eval_logits/chosen": 22.99826431274414, | |
| "eval_logits/rejected": 23.05336570739746, | |
| "eval_logps/chosen": -45.58966064453125, | |
| "eval_logps/rejected": -47.14959716796875, | |
| "eval_loss": 0.468368798494339, | |
| "eval_rewards/accuracies": 0.8055555820465088, | |
| "eval_rewards/chosen": -0.3949766457080841, | |
| "eval_rewards/margins": 1.116257905960083, | |
| "eval_rewards/rejected": -1.5112345218658447, | |
| "eval_runtime": 97.8365, | |
| "eval_samples_per_second": 14.637, | |
| "eval_steps_per_second": 0.92, | |
| "step": 1100 | |
| }, | |
| { | |
| "epoch": 0.39628704034273476, | |
| "grad_norm": 8.018348693847656, | |
| "learning_rate": 3.0185647982863262e-05, | |
| "logits/chosen": 21.868206024169922, | |
| "logits/rejected": 21.934436798095703, | |
| "logps/chosen": -12.12812614440918, | |
| "logps/rejected": -39.364479064941406, | |
| "loss": 0.6244, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.7536962032318115, | |
| "rewards/margins": 0.8047281503677368, | |
| "rewards/rejected": -1.5584242343902588, | |
| "step": 1110 | |
| }, | |
| { | |
| "epoch": 0.39985719385933594, | |
| "grad_norm": 14.252638816833496, | |
| "learning_rate": 3.0007140307033206e-05, | |
| "logits/chosen": 21.122318267822266, | |
| "logits/rejected": 21.303096771240234, | |
| "logps/chosen": -25.239931106567383, | |
| "logps/rejected": -44.85738754272461, | |
| "loss": 0.5106, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.9396845102310181, | |
| "rewards/margins": 0.4221257269382477, | |
| "rewards/rejected": -1.3618100881576538, | |
| "step": 1120 | |
| }, | |
| { | |
| "epoch": 0.40342734737593716, | |
| "grad_norm": 10.65608024597168, | |
| "learning_rate": 2.982863263120314e-05, | |
| "logits/chosen": 27.12192153930664, | |
| "logits/rejected": 27.36038589477539, | |
| "logps/chosen": -30.87465476989746, | |
| "logps/rejected": -33.55220031738281, | |
| "loss": 0.5345, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.76970374584198, | |
| "rewards/margins": 0.6138733625411987, | |
| "rewards/rejected": -1.3835771083831787, | |
| "step": 1130 | |
| }, | |
| { | |
| "epoch": 0.4069975008925384, | |
| "grad_norm": 8.322869300842285, | |
| "learning_rate": 2.965012495537308e-05, | |
| "logits/chosen": 21.752477645874023, | |
| "logits/rejected": 21.980819702148438, | |
| "logps/chosen": -45.920928955078125, | |
| "logps/rejected": -35.359962463378906, | |
| "loss": 0.4077, | |
| "rewards/accuracies": 0.4000000059604645, | |
| "rewards/chosen": -1.1174064874649048, | |
| "rewards/margins": 0.1437036395072937, | |
| "rewards/rejected": -1.2611101865768433, | |
| "step": 1140 | |
| }, | |
| { | |
| "epoch": 0.4105676544091396, | |
| "grad_norm": 7.159435272216797, | |
| "learning_rate": 2.9471617279543025e-05, | |
| "logits/chosen": 25.57335662841797, | |
| "logits/rejected": 25.599693298339844, | |
| "logps/chosen": -35.72122573852539, | |
| "logps/rejected": -55.0676383972168, | |
| "loss": 0.4664, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.37138763070106506, | |
| "rewards/margins": 1.5116512775421143, | |
| "rewards/rejected": -1.883039116859436, | |
| "step": 1150 | |
| }, | |
| { | |
| "epoch": 0.4141378079257408, | |
| "grad_norm": 6.634488582611084, | |
| "learning_rate": 2.929310960371296e-05, | |
| "logits/chosen": 20.41391944885254, | |
| "logits/rejected": 20.523845672607422, | |
| "logps/chosen": -42.240272521972656, | |
| "logps/rejected": -59.96808624267578, | |
| "loss": 0.3929, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.6876804828643799, | |
| "rewards/margins": 1.695648431777954, | |
| "rewards/rejected": -2.383328914642334, | |
| "step": 1160 | |
| }, | |
| { | |
| "epoch": 0.417707961442342, | |
| "grad_norm": 6.726844787597656, | |
| "learning_rate": 2.91146019278829e-05, | |
| "logits/chosen": 25.41510009765625, | |
| "logits/rejected": 25.722497940063477, | |
| "logps/chosen": -66.07826232910156, | |
| "logps/rejected": -49.482093811035156, | |
| "loss": 0.493, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.5786004066467285, | |
| "rewards/margins": 1.6916053295135498, | |
| "rewards/rejected": -2.2702059745788574, | |
| "step": 1170 | |
| }, | |
| { | |
| "epoch": 0.42127811495894324, | |
| "grad_norm": 5.256776332855225, | |
| "learning_rate": 2.8936094252052844e-05, | |
| "logits/chosen": 24.357757568359375, | |
| "logits/rejected": 24.397367477416992, | |
| "logps/chosen": -45.10001754760742, | |
| "logps/rejected": -79.18079376220703, | |
| "loss": 0.4627, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.1261012703180313, | |
| "rewards/margins": 1.3214435577392578, | |
| "rewards/rejected": -1.447544813156128, | |
| "step": 1180 | |
| }, | |
| { | |
| "epoch": 0.42484826847554447, | |
| "grad_norm": 23.727317810058594, | |
| "learning_rate": 2.8757586576222777e-05, | |
| "logits/chosen": 26.00099754333496, | |
| "logits/rejected": 25.485157012939453, | |
| "logps/chosen": -36.56196594238281, | |
| "logps/rejected": -52.7928581237793, | |
| "loss": 0.5167, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.9087120294570923, | |
| "rewards/margins": 1.3440439701080322, | |
| "rewards/rejected": -2.252755880355835, | |
| "step": 1190 | |
| }, | |
| { | |
| "epoch": 0.42841842199214564, | |
| "grad_norm": 7.284200191497803, | |
| "learning_rate": 2.8579078900392718e-05, | |
| "logits/chosen": 27.846277236938477, | |
| "logits/rejected": 28.00874900817871, | |
| "logps/chosen": -59.95824432373047, | |
| "logps/rejected": -73.82151794433594, | |
| "loss": 0.4865, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.411950021982193, | |
| "rewards/margins": 1.2028063535690308, | |
| "rewards/rejected": -1.6147563457489014, | |
| "step": 1200 | |
| }, | |
| { | |
| "epoch": 0.42841842199214564, | |
| "eval_logits/chosen": 22.84278106689453, | |
| "eval_logits/rejected": 22.89705467224121, | |
| "eval_logps/chosen": -46.42566680908203, | |
| "eval_logps/rejected": -48.32484817504883, | |
| "eval_loss": 0.46235060691833496, | |
| "eval_rewards/accuracies": 0.8222222328186035, | |
| "eval_rewards/chosen": -0.4785774350166321, | |
| "eval_rewards/margins": 1.1501818895339966, | |
| "eval_rewards/rejected": -1.6287592649459839, | |
| "eval_runtime": 97.7745, | |
| "eval_samples_per_second": 14.646, | |
| "eval_steps_per_second": 0.92, | |
| "step": 1200 | |
| }, | |
| { | |
| "epoch": 0.43198857550874686, | |
| "grad_norm": 10.374000549316406, | |
| "learning_rate": 2.8400571224562655e-05, | |
| "logits/chosen": 27.34882164001465, | |
| "logits/rejected": 26.882160186767578, | |
| "logps/chosen": -17.03693199157715, | |
| "logps/rejected": -37.147464752197266, | |
| "loss": 0.5116, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.975134015083313, | |
| "rewards/margins": 0.5972901582717896, | |
| "rewards/rejected": -1.5724241733551025, | |
| "step": 1210 | |
| }, | |
| { | |
| "epoch": 0.4355587290253481, | |
| "grad_norm": 8.28000259399414, | |
| "learning_rate": 2.8222063548732596e-05, | |
| "logits/chosen": 24.054485321044922, | |
| "logits/rejected": 24.115304946899414, | |
| "logps/chosen": -40.294517517089844, | |
| "logps/rejected": -29.917953491210938, | |
| "loss": 0.4612, | |
| "rewards/accuracies": 0.4000000059604645, | |
| "rewards/chosen": -0.7177508473396301, | |
| "rewards/margins": 0.1164645105600357, | |
| "rewards/rejected": -0.8342153429985046, | |
| "step": 1220 | |
| }, | |
| { | |
| "epoch": 0.4391288825419493, | |
| "grad_norm": 8.388548851013184, | |
| "learning_rate": 2.8043555872902537e-05, | |
| "logits/chosen": 25.40005111694336, | |
| "logits/rejected": 25.363679885864258, | |
| "logps/chosen": -33.07610321044922, | |
| "logps/rejected": -46.69367218017578, | |
| "loss": 0.5705, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.6370976567268372, | |
| "rewards/margins": 0.2812044620513916, | |
| "rewards/rejected": -0.9183019399642944, | |
| "step": 1230 | |
| }, | |
| { | |
| "epoch": 0.44269903605855054, | |
| "grad_norm": 10.984318733215332, | |
| "learning_rate": 2.7865048197072474e-05, | |
| "logits/chosen": 21.659027099609375, | |
| "logits/rejected": 21.85382843017578, | |
| "logps/chosen": -73.49362182617188, | |
| "logps/rejected": -69.6221694946289, | |
| "loss": 0.5531, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -1.2802386283874512, | |
| "rewards/margins": 0.23997211456298828, | |
| "rewards/rejected": -1.520210862159729, | |
| "step": 1240 | |
| }, | |
| { | |
| "epoch": 0.4462691895751517, | |
| "grad_norm": 6.512259006500244, | |
| "learning_rate": 2.7686540521242415e-05, | |
| "logits/chosen": 25.227018356323242, | |
| "logits/rejected": 24.743755340576172, | |
| "logps/chosen": -47.708919525146484, | |
| "logps/rejected": -72.66358947753906, | |
| "loss": 0.5644, | |
| "rewards/accuracies": 0.4000000059604645, | |
| "rewards/chosen": -1.2783235311508179, | |
| "rewards/margins": 0.3692890405654907, | |
| "rewards/rejected": -1.6476128101348877, | |
| "step": 1250 | |
| }, | |
| { | |
| "epoch": 0.44983934309175294, | |
| "grad_norm": 13.5468168258667, | |
| "learning_rate": 2.7508032845412352e-05, | |
| "logits/chosen": 22.638336181640625, | |
| "logits/rejected": 22.76083755493164, | |
| "logps/chosen": -33.97126007080078, | |
| "logps/rejected": -34.66718673706055, | |
| "loss": 0.486, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.6014578342437744, | |
| "rewards/margins": 1.0001842975616455, | |
| "rewards/rejected": -1.6016420125961304, | |
| "step": 1260 | |
| }, | |
| { | |
| "epoch": 0.45340949660835417, | |
| "grad_norm": 10.532015800476074, | |
| "learning_rate": 2.7329525169582293e-05, | |
| "logits/chosen": 20.29448699951172, | |
| "logits/rejected": 20.10554313659668, | |
| "logps/chosen": -36.69490051269531, | |
| "logps/rejected": -56.0795783996582, | |
| "loss": 0.5096, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.4149419665336609, | |
| "rewards/margins": 0.9424688220024109, | |
| "rewards/rejected": -1.3574106693267822, | |
| "step": 1270 | |
| }, | |
| { | |
| "epoch": 0.4569796501249554, | |
| "grad_norm": 5.038132190704346, | |
| "learning_rate": 2.7151017493752233e-05, | |
| "logits/chosen": 25.44969367980957, | |
| "logits/rejected": 25.12093734741211, | |
| "logps/chosen": -45.38970184326172, | |
| "logps/rejected": -48.31528091430664, | |
| "loss": 0.4794, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.48757249116897583, | |
| "rewards/margins": 0.5879091024398804, | |
| "rewards/rejected": -1.075481653213501, | |
| "step": 1280 | |
| }, | |
| { | |
| "epoch": 0.46054980364155657, | |
| "grad_norm": 14.402963638305664, | |
| "learning_rate": 2.697250981792217e-05, | |
| "logits/chosen": 24.284452438354492, | |
| "logits/rejected": 23.935733795166016, | |
| "logps/chosen": -44.071624755859375, | |
| "logps/rejected": -44.24872589111328, | |
| "loss": 0.5999, | |
| "rewards/accuracies": 0.4000000059604645, | |
| "rewards/chosen": -1.4019516706466675, | |
| "rewards/margins": 0.15506015717983246, | |
| "rewards/rejected": -1.5570119619369507, | |
| "step": 1290 | |
| }, | |
| { | |
| "epoch": 0.4641199571581578, | |
| "grad_norm": 7.078202247619629, | |
| "learning_rate": 2.679400214209211e-05, | |
| "logits/chosen": 20.088533401489258, | |
| "logits/rejected": 20.420536041259766, | |
| "logps/chosen": -50.00440216064453, | |
| "logps/rejected": -56.248382568359375, | |
| "loss": 0.4642, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.6917403936386108, | |
| "rewards/margins": 1.1690679788589478, | |
| "rewards/rejected": -1.8608081340789795, | |
| "step": 1300 | |
| }, | |
| { | |
| "epoch": 0.4641199571581578, | |
| "eval_logits/chosen": 21.782039642333984, | |
| "eval_logits/rejected": 21.82331085205078, | |
| "eval_logps/chosen": -46.87673568725586, | |
| "eval_logps/rejected": -49.23981475830078, | |
| "eval_loss": 0.4559466540813446, | |
| "eval_rewards/accuracies": 0.8388888835906982, | |
| "eval_rewards/chosen": -0.523684024810791, | |
| "eval_rewards/margins": 1.196572184562683, | |
| "eval_rewards/rejected": -1.7202562093734741, | |
| "eval_runtime": 97.7244, | |
| "eval_samples_per_second": 14.653, | |
| "eval_steps_per_second": 0.921, | |
| "step": 1300 | |
| }, | |
| { | |
| "epoch": 0.467690110674759, | |
| "grad_norm": 10.393937110900879, | |
| "learning_rate": 2.6615494466262052e-05, | |
| "logits/chosen": 20.803997039794922, | |
| "logits/rejected": 20.7797794342041, | |
| "logps/chosen": -44.06121826171875, | |
| "logps/rejected": -47.28874588012695, | |
| "loss": 0.5374, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.47202056646347046, | |
| "rewards/margins": 0.9966621398925781, | |
| "rewards/rejected": -1.4686826467514038, | |
| "step": 1310 | |
| }, | |
| { | |
| "epoch": 0.47126026419136025, | |
| "grad_norm": 6.863224983215332, | |
| "learning_rate": 2.643698679043199e-05, | |
| "logits/chosen": 19.93913459777832, | |
| "logits/rejected": 20.202621459960938, | |
| "logps/chosen": -37.7781867980957, | |
| "logps/rejected": -39.12306594848633, | |
| "loss": 0.5426, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -1.0681989192962646, | |
| "rewards/margins": 0.03398240730166435, | |
| "rewards/rejected": -1.1021811962127686, | |
| "step": 1320 | |
| }, | |
| { | |
| "epoch": 0.4748304177079614, | |
| "grad_norm": 9.727211952209473, | |
| "learning_rate": 2.625847911460193e-05, | |
| "logits/chosen": 19.134681701660156, | |
| "logits/rejected": 19.11941146850586, | |
| "logps/chosen": -57.157989501953125, | |
| "logps/rejected": -72.72858428955078, | |
| "loss": 0.507, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.9592474102973938, | |
| "rewards/margins": 1.3726739883422852, | |
| "rewards/rejected": -2.331921339035034, | |
| "step": 1330 | |
| }, | |
| { | |
| "epoch": 0.47840057122456264, | |
| "grad_norm": 9.726844787597656, | |
| "learning_rate": 2.6079971438771867e-05, | |
| "logits/chosen": 22.685604095458984, | |
| "logits/rejected": 23.21428108215332, | |
| "logps/chosen": -57.5455436706543, | |
| "logps/rejected": -83.27085876464844, | |
| "loss": 0.379, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -1.020816683769226, | |
| "rewards/margins": 1.3953444957733154, | |
| "rewards/rejected": -2.416161298751831, | |
| "step": 1340 | |
| }, | |
| { | |
| "epoch": 0.48197072474116387, | |
| "grad_norm": 9.248942375183105, | |
| "learning_rate": 2.5901463762941808e-05, | |
| "logits/chosen": 19.568103790283203, | |
| "logits/rejected": 19.459789276123047, | |
| "logps/chosen": -22.311328887939453, | |
| "logps/rejected": -22.42624282836914, | |
| "loss": 0.4095, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.12884020805358887, | |
| "rewards/margins": 1.4438714981079102, | |
| "rewards/rejected": -1.572711706161499, | |
| "step": 1350 | |
| }, | |
| { | |
| "epoch": 0.4855408782577651, | |
| "grad_norm": 9.093886375427246, | |
| "learning_rate": 2.572295608711175e-05, | |
| "logits/chosen": 19.389034271240234, | |
| "logits/rejected": 19.35251235961914, | |
| "logps/chosen": -33.7999382019043, | |
| "logps/rejected": -42.553993225097656, | |
| "loss": 0.4346, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.5412906408309937, | |
| "rewards/margins": 0.463665634393692, | |
| "rewards/rejected": -1.0049562454223633, | |
| "step": 1360 | |
| }, | |
| { | |
| "epoch": 0.4891110317743663, | |
| "grad_norm": 11.408600807189941, | |
| "learning_rate": 2.5544448411281686e-05, | |
| "logits/chosen": 19.231670379638672, | |
| "logits/rejected": 19.629993438720703, | |
| "logps/chosen": -31.9910831451416, | |
| "logps/rejected": -59.96710205078125, | |
| "loss": 0.6003, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.4423964023590088, | |
| "rewards/margins": 1.090103268623352, | |
| "rewards/rejected": -1.5324996709823608, | |
| "step": 1370 | |
| }, | |
| { | |
| "epoch": 0.4926811852909675, | |
| "grad_norm": 5.170421123504639, | |
| "learning_rate": 2.5365940735451627e-05, | |
| "logits/chosen": 27.88661766052246, | |
| "logits/rejected": 27.40877914428711, | |
| "logps/chosen": -45.87278366088867, | |
| "logps/rejected": -45.47796630859375, | |
| "loss": 0.4592, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.8744171261787415, | |
| "rewards/margins": 0.6523105502128601, | |
| "rewards/rejected": -1.5267279148101807, | |
| "step": 1380 | |
| }, | |
| { | |
| "epoch": 0.4962513388075687, | |
| "grad_norm": 5.33081579208374, | |
| "learning_rate": 2.5187433059621564e-05, | |
| "logits/chosen": 21.309518814086914, | |
| "logits/rejected": 21.40631866455078, | |
| "logps/chosen": -36.923614501953125, | |
| "logps/rejected": -56.65825271606445, | |
| "loss": 0.5338, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -1.4122052192687988, | |
| "rewards/margins": 0.27405181527137756, | |
| "rewards/rejected": -1.686257004737854, | |
| "step": 1390 | |
| }, | |
| { | |
| "epoch": 0.49982149232416995, | |
| "grad_norm": 10.743315696716309, | |
| "learning_rate": 2.5008925383791505e-05, | |
| "logits/chosen": 24.20417594909668, | |
| "logits/rejected": 24.082937240600586, | |
| "logps/chosen": -33.370670318603516, | |
| "logps/rejected": -40.95515441894531, | |
| "loss": 0.4764, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -1.0275386571884155, | |
| "rewards/margins": 0.8042508959770203, | |
| "rewards/rejected": -1.8317893743515015, | |
| "step": 1400 | |
| }, | |
| { | |
| "epoch": 0.49982149232416995, | |
| "eval_logits/chosen": 20.522747039794922, | |
| "eval_logits/rejected": 20.564208984375, | |
| "eval_logps/chosen": -48.346073150634766, | |
| "eval_logps/rejected": -51.10609817504883, | |
| "eval_loss": 0.4600922167301178, | |
| "eval_rewards/accuracies": 0.800000011920929, | |
| "eval_rewards/chosen": -0.6706183552742004, | |
| "eval_rewards/margins": 1.2362663745880127, | |
| "eval_rewards/rejected": -1.906884789466858, | |
| "eval_runtime": 99.1997, | |
| "eval_samples_per_second": 14.436, | |
| "eval_steps_per_second": 0.907, | |
| "step": 1400 | |
| }, | |
| { | |
| "epoch": 0.5033916458407711, | |
| "grad_norm": 7.706882476806641, | |
| "learning_rate": 2.4830417707961442e-05, | |
| "logits/chosen": 18.212251663208008, | |
| "logits/rejected": 17.935840606689453, | |
| "logps/chosen": -55.223960876464844, | |
| "logps/rejected": -50.36370086669922, | |
| "loss": 0.4297, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -1.1679266691207886, | |
| "rewards/margins": 1.6370041370391846, | |
| "rewards/rejected": -2.8049309253692627, | |
| "step": 1410 | |
| }, | |
| { | |
| "epoch": 0.5069617993573724, | |
| "grad_norm": 9.615565299987793, | |
| "learning_rate": 2.4651910032131383e-05, | |
| "logits/chosen": 22.241762161254883, | |
| "logits/rejected": 22.47495460510254, | |
| "logps/chosen": -67.49073791503906, | |
| "logps/rejected": -73.30606079101562, | |
| "loss": 0.4645, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -1.1100456714630127, | |
| "rewards/margins": 1.8767004013061523, | |
| "rewards/rejected": -2.986745834350586, | |
| "step": 1420 | |
| }, | |
| { | |
| "epoch": 0.5105319528739736, | |
| "grad_norm": 5.209810733795166, | |
| "learning_rate": 2.4473402356301323e-05, | |
| "logits/chosen": 22.897598266601562, | |
| "logits/rejected": 22.638469696044922, | |
| "logps/chosen": -53.96964645385742, | |
| "logps/rejected": -58.29240036010742, | |
| "loss": 0.4386, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.6232610940933228, | |
| "rewards/margins": 0.9309207201004028, | |
| "rewards/rejected": -1.554181694984436, | |
| "step": 1430 | |
| }, | |
| { | |
| "epoch": 0.5141021063905747, | |
| "grad_norm": 8.026928901672363, | |
| "learning_rate": 2.429489468047126e-05, | |
| "logits/chosen": 22.4190731048584, | |
| "logits/rejected": 22.359643936157227, | |
| "logps/chosen": -48.24509048461914, | |
| "logps/rejected": -47.77333450317383, | |
| "loss": 0.4485, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -1.1848137378692627, | |
| "rewards/margins": 0.29461732506752014, | |
| "rewards/rejected": -1.479430913925171, | |
| "step": 1440 | |
| }, | |
| { | |
| "epoch": 0.517672259907176, | |
| "grad_norm": 10.013033866882324, | |
| "learning_rate": 2.41163870046412e-05, | |
| "logits/chosen": 22.765010833740234, | |
| "logits/rejected": 22.88141441345215, | |
| "logps/chosen": -54.80376052856445, | |
| "logps/rejected": -59.78020477294922, | |
| "loss": 0.5059, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -1.1334106922149658, | |
| "rewards/margins": 0.6534554362297058, | |
| "rewards/rejected": -1.7868661880493164, | |
| "step": 1450 | |
| }, | |
| { | |
| "epoch": 0.5212424134237772, | |
| "grad_norm": 9.498845100402832, | |
| "learning_rate": 2.3937879328811142e-05, | |
| "logits/chosen": 22.87884521484375, | |
| "logits/rejected": 22.664630889892578, | |
| "logps/chosen": -38.91038131713867, | |
| "logps/rejected": -40.58343505859375, | |
| "loss": 0.4454, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.4328577518463135, | |
| "rewards/margins": 1.4672273397445679, | |
| "rewards/rejected": -1.9000848531723022, | |
| "step": 1460 | |
| }, | |
| { | |
| "epoch": 0.5248125669403785, | |
| "grad_norm": 9.389070510864258, | |
| "learning_rate": 2.375937165298108e-05, | |
| "logits/chosen": 23.82521629333496, | |
| "logits/rejected": 24.009113311767578, | |
| "logps/chosen": -60.59919357299805, | |
| "logps/rejected": -68.24311828613281, | |
| "loss": 0.4305, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -1.1264002323150635, | |
| "rewards/margins": 1.3599331378936768, | |
| "rewards/rejected": -2.4863333702087402, | |
| "step": 1470 | |
| }, | |
| { | |
| "epoch": 0.5283827204569796, | |
| "grad_norm": 11.059606552124023, | |
| "learning_rate": 2.358086397715102e-05, | |
| "logits/chosen": 25.180397033691406, | |
| "logits/rejected": 25.406417846679688, | |
| "logps/chosen": -53.621864318847656, | |
| "logps/rejected": -58.304664611816406, | |
| "loss": 0.471, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -1.3794161081314087, | |
| "rewards/margins": 0.9802428483963013, | |
| "rewards/rejected": -2.359658718109131, | |
| "step": 1480 | |
| }, | |
| { | |
| "epoch": 0.5319528739735808, | |
| "grad_norm": 19.903459548950195, | |
| "learning_rate": 2.3402356301320957e-05, | |
| "logits/chosen": 23.71100616455078, | |
| "logits/rejected": 23.361339569091797, | |
| "logps/chosen": -51.43365478515625, | |
| "logps/rejected": -51.136474609375, | |
| "loss": 0.4899, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.7299798727035522, | |
| "rewards/margins": 1.1167805194854736, | |
| "rewards/rejected": -1.8467603921890259, | |
| "step": 1490 | |
| }, | |
| { | |
| "epoch": 0.5355230274901821, | |
| "grad_norm": 9.738292694091797, | |
| "learning_rate": 2.3223848625490895e-05, | |
| "logits/chosen": 23.410078048706055, | |
| "logits/rejected": 23.206371307373047, | |
| "logps/chosen": -47.294952392578125, | |
| "logps/rejected": -54.903099060058594, | |
| "loss": 0.4734, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.9249691963195801, | |
| "rewards/margins": 0.9751961827278137, | |
| "rewards/rejected": -1.9001652002334595, | |
| "step": 1500 | |
| }, | |
| { | |
| "epoch": 0.5355230274901821, | |
| "eval_logits/chosen": 20.92539405822754, | |
| "eval_logits/rejected": 20.960508346557617, | |
| "eval_logps/chosen": -48.110416412353516, | |
| "eval_logps/rejected": -51.09627151489258, | |
| "eval_loss": 0.4577901065349579, | |
| "eval_rewards/accuracies": 0.800000011920929, | |
| "eval_rewards/chosen": -0.6470518112182617, | |
| "eval_rewards/margins": 1.2588502168655396, | |
| "eval_rewards/rejected": -1.9059022665023804, | |
| "eval_runtime": 97.9322, | |
| "eval_samples_per_second": 14.622, | |
| "eval_steps_per_second": 0.919, | |
| "step": 1500 | |
| }, | |
| { | |
| "epoch": 0.5390931810067833, | |
| "grad_norm": 8.651128768920898, | |
| "learning_rate": 2.304534094966084e-05, | |
| "logits/chosen": 22.864696502685547, | |
| "logits/rejected": 22.489166259765625, | |
| "logps/chosen": -24.272085189819336, | |
| "logps/rejected": -25.032752990722656, | |
| "loss": 0.5041, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.06202169135212898, | |
| "rewards/margins": 1.4595599174499512, | |
| "rewards/rejected": -1.5215816497802734, | |
| "step": 1510 | |
| }, | |
| { | |
| "epoch": 0.5426633345233846, | |
| "grad_norm": 13.196417808532715, | |
| "learning_rate": 2.2866833273830776e-05, | |
| "logits/chosen": 23.409244537353516, | |
| "logits/rejected": 23.42110824584961, | |
| "logps/chosen": -70.71308898925781, | |
| "logps/rejected": -65.17359161376953, | |
| "loss": 0.4541, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.825936496257782, | |
| "rewards/margins": 1.5356091260910034, | |
| "rewards/rejected": -2.3615458011627197, | |
| "step": 1520 | |
| }, | |
| { | |
| "epoch": 0.5462334880399857, | |
| "grad_norm": 13.838322639465332, | |
| "learning_rate": 2.2688325598000713e-05, | |
| "logits/chosen": 19.617794036865234, | |
| "logits/rejected": 19.378015518188477, | |
| "logps/chosen": -63.78108596801758, | |
| "logps/rejected": -57.444053649902344, | |
| "loss": 0.5129, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.7819554805755615, | |
| "rewards/margins": 1.4755408763885498, | |
| "rewards/rejected": -2.2574961185455322, | |
| "step": 1530 | |
| }, | |
| { | |
| "epoch": 0.5498036415565869, | |
| "grad_norm": 3.351306915283203, | |
| "learning_rate": 2.2509817922170654e-05, | |
| "logits/chosen": 18.362462997436523, | |
| "logits/rejected": 18.116527557373047, | |
| "logps/chosen": -61.28546905517578, | |
| "logps/rejected": -71.31098937988281, | |
| "loss": 0.667, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.7423151135444641, | |
| "rewards/margins": 1.1754957437515259, | |
| "rewards/rejected": -1.9178107976913452, | |
| "step": 1540 | |
| }, | |
| { | |
| "epoch": 0.5533737950731882, | |
| "grad_norm": 10.512808799743652, | |
| "learning_rate": 2.2331310246340595e-05, | |
| "logits/chosen": 23.116546630859375, | |
| "logits/rejected": 23.27178955078125, | |
| "logps/chosen": -71.55606079101562, | |
| "logps/rejected": -59.09543991088867, | |
| "loss": 0.5858, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -1.0912095308303833, | |
| "rewards/margins": 1.2087962627410889, | |
| "rewards/rejected": -2.3000059127807617, | |
| "step": 1550 | |
| }, | |
| { | |
| "epoch": 0.5569439485897894, | |
| "grad_norm": 6.164778709411621, | |
| "learning_rate": 2.2152802570510532e-05, | |
| "logits/chosen": 22.27012062072754, | |
| "logits/rejected": 22.54111671447754, | |
| "logps/chosen": -50.849666595458984, | |
| "logps/rejected": -51.00347900390625, | |
| "loss": 0.4023, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.8672236204147339, | |
| "rewards/margins": 0.6719032526016235, | |
| "rewards/rejected": -1.539126992225647, | |
| "step": 1560 | |
| }, | |
| { | |
| "epoch": 0.5605141021063905, | |
| "grad_norm": 8.524353981018066, | |
| "learning_rate": 2.1974294894680473e-05, | |
| "logits/chosen": 24.240970611572266, | |
| "logits/rejected": 24.447324752807617, | |
| "logps/chosen": -50.92961120605469, | |
| "logps/rejected": -25.607860565185547, | |
| "loss": 0.5566, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -1.0541558265686035, | |
| "rewards/margins": 0.1918071210384369, | |
| "rewards/rejected": -1.2459629774093628, | |
| "step": 1570 | |
| }, | |
| { | |
| "epoch": 0.5640842556229918, | |
| "grad_norm": 3.2097225189208984, | |
| "learning_rate": 2.179578721885041e-05, | |
| "logits/chosen": 21.733001708984375, | |
| "logits/rejected": 21.849323272705078, | |
| "logps/chosen": -59.861595153808594, | |
| "logps/rejected": -74.30784606933594, | |
| "loss": 0.4727, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -1.154857873916626, | |
| "rewards/margins": 1.554736852645874, | |
| "rewards/rejected": -2.709594249725342, | |
| "step": 1580 | |
| }, | |
| { | |
| "epoch": 0.567654409139593, | |
| "grad_norm": 10.112348556518555, | |
| "learning_rate": 2.1617279543020354e-05, | |
| "logits/chosen": 22.288734436035156, | |
| "logits/rejected": 22.532554626464844, | |
| "logps/chosen": -49.292564392089844, | |
| "logps/rejected": -46.22431182861328, | |
| "loss": 0.3915, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.7656620144844055, | |
| "rewards/margins": 0.9457171559333801, | |
| "rewards/rejected": -1.7113794088363647, | |
| "step": 1590 | |
| }, | |
| { | |
| "epoch": 0.5712245626561943, | |
| "grad_norm": 7.5602288246154785, | |
| "learning_rate": 2.143877186719029e-05, | |
| "logits/chosen": 16.03813934326172, | |
| "logits/rejected": 16.125709533691406, | |
| "logps/chosen": -48.10554504394531, | |
| "logps/rejected": -57.5333137512207, | |
| "loss": 0.4459, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.9440638422966003, | |
| "rewards/margins": 0.8066110610961914, | |
| "rewards/rejected": -1.750674843788147, | |
| "step": 1600 | |
| }, | |
| { | |
| "epoch": 0.5712245626561943, | |
| "eval_logits/chosen": 21.25445556640625, | |
| "eval_logits/rejected": 21.29209327697754, | |
| "eval_logps/chosen": -46.128841400146484, | |
| "eval_logps/rejected": -50.25452423095703, | |
| "eval_loss": 0.4501490294933319, | |
| "eval_rewards/accuracies": 0.8166666626930237, | |
| "eval_rewards/chosen": -0.4488947093486786, | |
| "eval_rewards/margins": 1.372832179069519, | |
| "eval_rewards/rejected": -1.8217267990112305, | |
| "eval_runtime": 97.5045, | |
| "eval_samples_per_second": 14.687, | |
| "eval_steps_per_second": 0.923, | |
| "step": 1600 | |
| }, | |
| { | |
| "epoch": 0.5747947161727954, | |
| "grad_norm": 21.815628051757812, | |
| "learning_rate": 2.126026419136023e-05, | |
| "logits/chosen": 22.183074951171875, | |
| "logits/rejected": 23.01669692993164, | |
| "logps/chosen": -25.743982315063477, | |
| "logps/rejected": -32.8138427734375, | |
| "loss": 0.4382, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.7791844606399536, | |
| "rewards/margins": 1.4727532863616943, | |
| "rewards/rejected": -2.2519376277923584, | |
| "step": 1610 | |
| }, | |
| { | |
| "epoch": 0.5783648696893966, | |
| "grad_norm": 9.609827041625977, | |
| "learning_rate": 2.108175651553017e-05, | |
| "logits/chosen": 21.618915557861328, | |
| "logits/rejected": 21.331867218017578, | |
| "logps/chosen": -36.85995864868164, | |
| "logps/rejected": -43.229026794433594, | |
| "loss": 0.7286, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.826258659362793, | |
| "rewards/margins": 1.1748714447021484, | |
| "rewards/rejected": -2.0011298656463623, | |
| "step": 1620 | |
| }, | |
| { | |
| "epoch": 0.5819350232059979, | |
| "grad_norm": 4.5630059242248535, | |
| "learning_rate": 2.0903248839700106e-05, | |
| "logits/chosen": 19.84273910522461, | |
| "logits/rejected": 19.884912490844727, | |
| "logps/chosen": -30.239971160888672, | |
| "logps/rejected": -48.68877029418945, | |
| "loss": 0.4667, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.15234649181365967, | |
| "rewards/margins": 1.829787015914917, | |
| "rewards/rejected": -1.9821338653564453, | |
| "step": 1630 | |
| }, | |
| { | |
| "epoch": 0.585505176722599, | |
| "grad_norm": 10.104663848876953, | |
| "learning_rate": 2.0724741163870047e-05, | |
| "logits/chosen": 22.992443084716797, | |
| "logits/rejected": 23.00868034362793, | |
| "logps/chosen": -63.122642517089844, | |
| "logps/rejected": -61.594627380371094, | |
| "loss": 0.4673, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.868377685546875, | |
| "rewards/margins": 1.27804696559906, | |
| "rewards/rejected": -2.1464245319366455, | |
| "step": 1640 | |
| }, | |
| { | |
| "epoch": 0.5890753302392003, | |
| "grad_norm": 15.089373588562012, | |
| "learning_rate": 2.0546233488039988e-05, | |
| "logits/chosen": 25.846508026123047, | |
| "logits/rejected": 25.832605361938477, | |
| "logps/chosen": -36.8404655456543, | |
| "logps/rejected": -45.35338592529297, | |
| "loss": 0.4712, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -1.2539387941360474, | |
| "rewards/margins": -0.1804993450641632, | |
| "rewards/rejected": -1.0734394788742065, | |
| "step": 1650 | |
| }, | |
| { | |
| "epoch": 0.5926454837558015, | |
| "grad_norm": 8.563141822814941, | |
| "learning_rate": 2.0367725812209925e-05, | |
| "logits/chosen": 25.35030746459961, | |
| "logits/rejected": 25.004871368408203, | |
| "logps/chosen": -14.423406600952148, | |
| "logps/rejected": -27.992664337158203, | |
| "loss": 0.538, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.5140976309776306, | |
| "rewards/margins": 0.7111214995384216, | |
| "rewards/rejected": -1.2252190113067627, | |
| "step": 1660 | |
| }, | |
| { | |
| "epoch": 0.5962156372724027, | |
| "grad_norm": 10.54287338256836, | |
| "learning_rate": 2.0189218136379866e-05, | |
| "logits/chosen": 23.3927001953125, | |
| "logits/rejected": 23.853105545043945, | |
| "logps/chosen": -48.61920928955078, | |
| "logps/rejected": -59.92339324951172, | |
| "loss": 0.6793, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.8948885202407837, | |
| "rewards/margins": 1.2263555526733398, | |
| "rewards/rejected": -2.121243953704834, | |
| "step": 1670 | |
| }, | |
| { | |
| "epoch": 0.599785790789004, | |
| "grad_norm": 10.524778366088867, | |
| "learning_rate": 2.0010710460549806e-05, | |
| "logits/chosen": 19.91085433959961, | |
| "logits/rejected": 19.805999755859375, | |
| "logps/chosen": -37.84980010986328, | |
| "logps/rejected": -42.93219757080078, | |
| "loss": 0.4365, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": 0.022962704300880432, | |
| "rewards/margins": 1.8917738199234009, | |
| "rewards/rejected": -1.868811011314392, | |
| "step": 1680 | |
| }, | |
| { | |
| "epoch": 0.6033559443056051, | |
| "grad_norm": 14.300918579101562, | |
| "learning_rate": 1.9832202784719744e-05, | |
| "logits/chosen": 18.957054138183594, | |
| "logits/rejected": 19.1351318359375, | |
| "logps/chosen": -44.51586151123047, | |
| "logps/rejected": -51.03856658935547, | |
| "loss": 0.6677, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.3876987397670746, | |
| "rewards/margins": 1.393635869026184, | |
| "rewards/rejected": -1.7813348770141602, | |
| "step": 1690 | |
| }, | |
| { | |
| "epoch": 0.6069260978222063, | |
| "grad_norm": 10.632288932800293, | |
| "learning_rate": 1.9653695108889684e-05, | |
| "logits/chosen": 23.2322998046875, | |
| "logits/rejected": 22.36001205444336, | |
| "logps/chosen": -33.04428482055664, | |
| "logps/rejected": -44.446144104003906, | |
| "loss": 0.5128, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.9377388954162598, | |
| "rewards/margins": 1.140350341796875, | |
| "rewards/rejected": -2.0780892372131348, | |
| "step": 1700 | |
| }, | |
| { | |
| "epoch": 0.6069260978222063, | |
| "eval_logits/chosen": 21.207813262939453, | |
| "eval_logits/rejected": 21.23162078857422, | |
| "eval_logps/chosen": -46.74406433105469, | |
| "eval_logps/rejected": -49.7892951965332, | |
| "eval_loss": 0.4637661278247833, | |
| "eval_rewards/accuracies": 0.8222222328186035, | |
| "eval_rewards/chosen": -0.510417103767395, | |
| "eval_rewards/margins": 1.2647870779037476, | |
| "eval_rewards/rejected": -1.7752044200897217, | |
| "eval_runtime": 97.7592, | |
| "eval_samples_per_second": 14.648, | |
| "eval_steps_per_second": 0.921, | |
| "step": 1700 | |
| }, | |
| { | |
| "epoch": 0.6104962513388076, | |
| "grad_norm": 9.3756742477417, | |
| "learning_rate": 1.9475187433059622e-05, | |
| "logits/chosen": 21.191104888916016, | |
| "logits/rejected": 21.159591674804688, | |
| "logps/chosen": -31.99550437927246, | |
| "logps/rejected": -44.552982330322266, | |
| "loss": 0.3974, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.8665735125541687, | |
| "rewards/margins": 1.130059838294983, | |
| "rewards/rejected": -1.9966332912445068, | |
| "step": 1710 | |
| }, | |
| { | |
| "epoch": 0.6140664048554088, | |
| "grad_norm": 21.877330780029297, | |
| "learning_rate": 1.929667975722956e-05, | |
| "logits/chosen": 23.222850799560547, | |
| "logits/rejected": 23.243497848510742, | |
| "logps/chosen": -36.72397232055664, | |
| "logps/rejected": -67.38380432128906, | |
| "loss": 0.5978, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.6531896591186523, | |
| "rewards/margins": 1.578351616859436, | |
| "rewards/rejected": -2.231541156768799, | |
| "step": 1720 | |
| }, | |
| { | |
| "epoch": 0.61763655837201, | |
| "grad_norm": 6.913092136383057, | |
| "learning_rate": 1.9118172081399503e-05, | |
| "logits/chosen": 22.709789276123047, | |
| "logits/rejected": 22.42465591430664, | |
| "logps/chosen": -80.70012664794922, | |
| "logps/rejected": -39.1956901550293, | |
| "loss": 0.5559, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -1.349876880645752, | |
| "rewards/margins": 0.47570380568504333, | |
| "rewards/rejected": -1.8255808353424072, | |
| "step": 1730 | |
| }, | |
| { | |
| "epoch": 0.6212067118886112, | |
| "grad_norm": 8.642501831054688, | |
| "learning_rate": 1.893966440556944e-05, | |
| "logits/chosen": 22.583799362182617, | |
| "logits/rejected": 22.996828079223633, | |
| "logps/chosen": -33.973365783691406, | |
| "logps/rejected": -40.40299987792969, | |
| "loss": 0.5488, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.8000438809394836, | |
| "rewards/margins": 1.3411608934402466, | |
| "rewards/rejected": -2.141204595565796, | |
| "step": 1740 | |
| }, | |
| { | |
| "epoch": 0.6247768654052124, | |
| "grad_norm": 7.104326248168945, | |
| "learning_rate": 1.8761156729739378e-05, | |
| "logits/chosen": 19.01543426513672, | |
| "logits/rejected": 19.197078704833984, | |
| "logps/chosen": -56.73799514770508, | |
| "logps/rejected": -57.29972457885742, | |
| "loss": 0.3864, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.8416491746902466, | |
| "rewards/margins": 1.005480170249939, | |
| "rewards/rejected": -1.847129464149475, | |
| "step": 1750 | |
| }, | |
| { | |
| "epoch": 0.6283470189218137, | |
| "grad_norm": 9.371530532836914, | |
| "learning_rate": 1.858264905390932e-05, | |
| "logits/chosen": 23.223146438598633, | |
| "logits/rejected": 23.122949600219727, | |
| "logps/chosen": -45.33961868286133, | |
| "logps/rejected": -50.840667724609375, | |
| "loss": 0.5181, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.8626285791397095, | |
| "rewards/margins": 0.9665843844413757, | |
| "rewards/rejected": -1.8292129039764404, | |
| "step": 1760 | |
| }, | |
| { | |
| "epoch": 0.6319171724384148, | |
| "grad_norm": 9.019303321838379, | |
| "learning_rate": 1.840414137807926e-05, | |
| "logits/chosen": 25.988510131835938, | |
| "logits/rejected": 25.829853057861328, | |
| "logps/chosen": -39.30087661743164, | |
| "logps/rejected": -33.74537658691406, | |
| "loss": 0.5334, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -1.3033127784729004, | |
| "rewards/margins": 0.49060168862342834, | |
| "rewards/rejected": -1.793914556503296, | |
| "step": 1770 | |
| }, | |
| { | |
| "epoch": 0.6354873259550161, | |
| "grad_norm": 23.491329193115234, | |
| "learning_rate": 1.8225633702249196e-05, | |
| "logits/chosen": 23.851768493652344, | |
| "logits/rejected": 24.093891143798828, | |
| "logps/chosen": -38.49626541137695, | |
| "logps/rejected": -29.503376007080078, | |
| "loss": 0.5504, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.6341218948364258, | |
| "rewards/margins": 0.8659729957580566, | |
| "rewards/rejected": -1.5000948905944824, | |
| "step": 1780 | |
| }, | |
| { | |
| "epoch": 0.6390574794716173, | |
| "grad_norm": 11.341009140014648, | |
| "learning_rate": 1.8047126026419137e-05, | |
| "logits/chosen": 20.8333683013916, | |
| "logits/rejected": 21.53514289855957, | |
| "logps/chosen": -30.507156372070312, | |
| "logps/rejected": -51.23711013793945, | |
| "loss": 0.5258, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.3501494526863098, | |
| "rewards/margins": 1.454388976097107, | |
| "rewards/rejected": -1.8045384883880615, | |
| "step": 1790 | |
| }, | |
| { | |
| "epoch": 0.6426276329882185, | |
| "grad_norm": 15.42497730255127, | |
| "learning_rate": 1.7868618350589074e-05, | |
| "logits/chosen": 23.256488800048828, | |
| "logits/rejected": 23.68168067932129, | |
| "logps/chosen": -23.0216007232666, | |
| "logps/rejected": -56.100563049316406, | |
| "loss": 0.4459, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -1.222777247428894, | |
| "rewards/margins": 1.2910280227661133, | |
| "rewards/rejected": -2.5138051509857178, | |
| "step": 1800 | |
| }, | |
| { | |
| "epoch": 0.6426276329882185, | |
| "eval_logits/chosen": 21.823591232299805, | |
| "eval_logits/rejected": 21.842632293701172, | |
| "eval_logps/chosen": -45.74694061279297, | |
| "eval_logps/rejected": -48.988624572753906, | |
| "eval_loss": 0.4546748399734497, | |
| "eval_rewards/accuracies": 0.8111110925674438, | |
| "eval_rewards/chosen": -0.4107048809528351, | |
| "eval_rewards/margins": 1.2844319343566895, | |
| "eval_rewards/rejected": -1.6951369047164917, | |
| "eval_runtime": 97.7489, | |
| "eval_samples_per_second": 14.65, | |
| "eval_steps_per_second": 0.921, | |
| "step": 1800 | |
| }, | |
| { | |
| "epoch": 0.6461977865048197, | |
| "grad_norm": 5.2203288078308105, | |
| "learning_rate": 1.769011067475902e-05, | |
| "logits/chosen": 24.605764389038086, | |
| "logits/rejected": 24.0284366607666, | |
| "logps/chosen": -44.47723388671875, | |
| "logps/rejected": -43.85586929321289, | |
| "loss": 0.3868, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.3751835823059082, | |
| "rewards/margins": 2.202699899673462, | |
| "rewards/rejected": -2.57788348197937, | |
| "step": 1810 | |
| }, | |
| { | |
| "epoch": 0.6497679400214209, | |
| "grad_norm": 15.71258544921875, | |
| "learning_rate": 1.7511602998928956e-05, | |
| "logits/chosen": 17.750822067260742, | |
| "logits/rejected": 18.020795822143555, | |
| "logps/chosen": -12.859057426452637, | |
| "logps/rejected": -30.338369369506836, | |
| "loss": 0.4199, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.33368760347366333, | |
| "rewards/margins": 1.292131781578064, | |
| "rewards/rejected": -1.625819444656372, | |
| "step": 1820 | |
| }, | |
| { | |
| "epoch": 0.6533380935380221, | |
| "grad_norm": 4.521580219268799, | |
| "learning_rate": 1.7333095323098893e-05, | |
| "logits/chosen": 23.34914207458496, | |
| "logits/rejected": 23.413654327392578, | |
| "logps/chosen": -37.8521614074707, | |
| "logps/rejected": -67.8910903930664, | |
| "loss": 0.4586, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.48519086837768555, | |
| "rewards/margins": 1.7367101907730103, | |
| "rewards/rejected": -2.2219011783599854, | |
| "step": 1830 | |
| }, | |
| { | |
| "epoch": 0.6569082470546234, | |
| "grad_norm": 3.1471476554870605, | |
| "learning_rate": 1.7154587647268834e-05, | |
| "logits/chosen": 18.366016387939453, | |
| "logits/rejected": 18.327919006347656, | |
| "logps/chosen": -55.4625358581543, | |
| "logps/rejected": -69.14282989501953, | |
| "loss": 0.5265, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.9972825050354004, | |
| "rewards/margins": 0.521903395652771, | |
| "rewards/rejected": -1.5191859006881714, | |
| "step": 1840 | |
| }, | |
| { | |
| "epoch": 0.6604784005712245, | |
| "grad_norm": 12.677329063415527, | |
| "learning_rate": 1.697607997143877e-05, | |
| "logits/chosen": 22.81624984741211, | |
| "logits/rejected": 22.75391960144043, | |
| "logps/chosen": -57.1756706237793, | |
| "logps/rejected": -61.690345764160156, | |
| "loss": 0.4095, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.12716366350650787, | |
| "rewards/margins": 1.8074487447738647, | |
| "rewards/rejected": -1.934612512588501, | |
| "step": 1850 | |
| }, | |
| { | |
| "epoch": 0.6640485540878258, | |
| "grad_norm": 12.960306167602539, | |
| "learning_rate": 1.679757229560871e-05, | |
| "logits/chosen": 23.8206844329834, | |
| "logits/rejected": 23.760665893554688, | |
| "logps/chosen": -41.61978530883789, | |
| "logps/rejected": -53.76505661010742, | |
| "loss": 0.4355, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -0.2521887421607971, | |
| "rewards/margins": 1.3217939138412476, | |
| "rewards/rejected": -1.5739825963974, | |
| "step": 1860 | |
| }, | |
| { | |
| "epoch": 0.667618707604427, | |
| "grad_norm": 16.337543487548828, | |
| "learning_rate": 1.6619064619778652e-05, | |
| "logits/chosen": 16.793262481689453, | |
| "logits/rejected": 16.742605209350586, | |
| "logps/chosen": -48.788875579833984, | |
| "logps/rejected": -43.506622314453125, | |
| "loss": 0.4937, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.416334331035614, | |
| "rewards/margins": 1.9437252283096313, | |
| "rewards/rejected": -2.3600597381591797, | |
| "step": 1870 | |
| }, | |
| { | |
| "epoch": 0.6711888611210282, | |
| "grad_norm": 10.352446556091309, | |
| "learning_rate": 1.644055694394859e-05, | |
| "logits/chosen": 20.40374183654785, | |
| "logits/rejected": 20.37020492553711, | |
| "logps/chosen": -58.631561279296875, | |
| "logps/rejected": -57.344520568847656, | |
| "loss": 0.5564, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.8105586767196655, | |
| "rewards/margins": 0.7123308777809143, | |
| "rewards/rejected": -1.5228896141052246, | |
| "step": 1880 | |
| }, | |
| { | |
| "epoch": 0.6747590146376294, | |
| "grad_norm": 5.631677627563477, | |
| "learning_rate": 1.626204926811853e-05, | |
| "logits/chosen": 23.19020652770996, | |
| "logits/rejected": 23.061664581298828, | |
| "logps/chosen": -61.78633499145508, | |
| "logps/rejected": -35.33949279785156, | |
| "loss": 0.4221, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.6975164413452148, | |
| "rewards/margins": 1.0272176265716553, | |
| "rewards/rejected": -1.7247339487075806, | |
| "step": 1890 | |
| }, | |
| { | |
| "epoch": 0.6783291681542306, | |
| "grad_norm": 4.499826431274414, | |
| "learning_rate": 1.608354159228847e-05, | |
| "logits/chosen": 24.53340721130371, | |
| "logits/rejected": 24.41737937927246, | |
| "logps/chosen": -38.38304901123047, | |
| "logps/rejected": -41.079261779785156, | |
| "loss": 0.5325, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.7768689393997192, | |
| "rewards/margins": 1.1220049858093262, | |
| "rewards/rejected": -1.898874044418335, | |
| "step": 1900 | |
| }, | |
| { | |
| "epoch": 0.6783291681542306, | |
| "eval_logits/chosen": 21.38434600830078, | |
| "eval_logits/rejected": 21.400619506835938, | |
| "eval_logps/chosen": -46.102115631103516, | |
| "eval_logps/rejected": -50.66127395629883, | |
| "eval_loss": 0.4424135386943817, | |
| "eval_rewards/accuracies": 0.8166666626930237, | |
| "eval_rewards/chosen": -0.4462220072746277, | |
| "eval_rewards/margins": 1.41618013381958, | |
| "eval_rewards/rejected": -1.862402319908142, | |
| "eval_runtime": 97.7047, | |
| "eval_samples_per_second": 14.656, | |
| "eval_steps_per_second": 0.921, | |
| "step": 1900 | |
| }, | |
| { | |
| "epoch": 0.6818993216708319, | |
| "grad_norm": 7.356407165527344, | |
| "learning_rate": 1.5905033916458408e-05, | |
| "logits/chosen": 22.510066986083984, | |
| "logits/rejected": 22.577346801757812, | |
| "logps/chosen": -35.227081298828125, | |
| "logps/rejected": -55.93198776245117, | |
| "loss": 0.5062, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.9009400606155396, | |
| "rewards/margins": 1.257596731185913, | |
| "rewards/rejected": -2.158536911010742, | |
| "step": 1910 | |
| }, | |
| { | |
| "epoch": 0.6854694751874331, | |
| "grad_norm": 3.671786069869995, | |
| "learning_rate": 1.572652624062835e-05, | |
| "logits/chosen": 20.359420776367188, | |
| "logits/rejected": 20.034189224243164, | |
| "logps/chosen": -63.456451416015625, | |
| "logps/rejected": -51.79132080078125, | |
| "loss": 0.4089, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.6570903658866882, | |
| "rewards/margins": 1.6083368062973022, | |
| "rewards/rejected": -2.265427350997925, | |
| "step": 1920 | |
| }, | |
| { | |
| "epoch": 0.6890396287040342, | |
| "grad_norm": 10.456278800964355, | |
| "learning_rate": 1.5548018564798286e-05, | |
| "logits/chosen": 23.675350189208984, | |
| "logits/rejected": 23.863216400146484, | |
| "logps/chosen": -26.890039443969727, | |
| "logps/rejected": -62.48893356323242, | |
| "loss": 0.5818, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -1.0192630290985107, | |
| "rewards/margins": 1.5647847652435303, | |
| "rewards/rejected": -2.584047794342041, | |
| "step": 1930 | |
| }, | |
| { | |
| "epoch": 0.6926097822206355, | |
| "grad_norm": 9.162726402282715, | |
| "learning_rate": 1.5369510888968227e-05, | |
| "logits/chosen": 23.965158462524414, | |
| "logits/rejected": 23.98688316345215, | |
| "logps/chosen": -21.68771743774414, | |
| "logps/rejected": -23.776859283447266, | |
| "loss": 0.3677, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.539600133895874, | |
| "rewards/margins": 0.6697312593460083, | |
| "rewards/rejected": -1.2093312740325928, | |
| "step": 1940 | |
| }, | |
| { | |
| "epoch": 0.6961799357372367, | |
| "grad_norm": 3.7567951679229736, | |
| "learning_rate": 1.5191003213138166e-05, | |
| "logits/chosen": 24.331890106201172, | |
| "logits/rejected": 24.351388931274414, | |
| "logps/chosen": -18.292221069335938, | |
| "logps/rejected": -28.088825225830078, | |
| "loss": 0.4699, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.5271309614181519, | |
| "rewards/margins": 1.1870901584625244, | |
| "rewards/rejected": -1.7142210006713867, | |
| "step": 1950 | |
| }, | |
| { | |
| "epoch": 0.699750089253838, | |
| "grad_norm": 10.841853141784668, | |
| "learning_rate": 1.5012495537308105e-05, | |
| "logits/chosen": 24.288837432861328, | |
| "logits/rejected": 24.071971893310547, | |
| "logps/chosen": -47.823211669921875, | |
| "logps/rejected": -52.2962532043457, | |
| "loss": 0.4207, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.8029332160949707, | |
| "rewards/margins": 1.4008492231369019, | |
| "rewards/rejected": -2.203782558441162, | |
| "step": 1960 | |
| }, | |
| { | |
| "epoch": 0.7033202427704391, | |
| "grad_norm": 3.0264902114868164, | |
| "learning_rate": 1.4833987861478044e-05, | |
| "logits/chosen": 19.48323631286621, | |
| "logits/rejected": 19.590219497680664, | |
| "logps/chosen": -42.73374557495117, | |
| "logps/rejected": -58.14360427856445, | |
| "loss": 0.4225, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.6881227493286133, | |
| "rewards/margins": 1.56888747215271, | |
| "rewards/rejected": -2.2570104598999023, | |
| "step": 1970 | |
| }, | |
| { | |
| "epoch": 0.7068903962870403, | |
| "grad_norm": 4.581413745880127, | |
| "learning_rate": 1.4655480185647983e-05, | |
| "logits/chosen": 23.510986328125, | |
| "logits/rejected": 23.64889144897461, | |
| "logps/chosen": -50.88288116455078, | |
| "logps/rejected": -30.534610748291016, | |
| "loss": 0.3636, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.262825071811676, | |
| "rewards/margins": 1.9995416402816772, | |
| "rewards/rejected": -2.262366771697998, | |
| "step": 1980 | |
| }, | |
| { | |
| "epoch": 0.7104605498036416, | |
| "grad_norm": 15.385260581970215, | |
| "learning_rate": 1.4476972509817924e-05, | |
| "logits/chosen": 22.773696899414062, | |
| "logits/rejected": 22.666271209716797, | |
| "logps/chosen": -40.43771743774414, | |
| "logps/rejected": -64.87566375732422, | |
| "loss": 0.4299, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -1.787145972251892, | |
| "rewards/margins": 1.5212252140045166, | |
| "rewards/rejected": -3.3083713054656982, | |
| "step": 1990 | |
| }, | |
| { | |
| "epoch": 0.7140307033202428, | |
| "grad_norm": 20.59077262878418, | |
| "learning_rate": 1.4298464833987863e-05, | |
| "logits/chosen": 14.725168228149414, | |
| "logits/rejected": 15.075048446655273, | |
| "logps/chosen": -58.4405517578125, | |
| "logps/rejected": -52.311119079589844, | |
| "loss": 0.5773, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -1.5943628549575806, | |
| "rewards/margins": -0.12944069504737854, | |
| "rewards/rejected": -1.4649221897125244, | |
| "step": 2000 | |
| }, | |
| { | |
| "epoch": 0.7140307033202428, | |
| "eval_logits/chosen": 20.44503402709961, | |
| "eval_logits/rejected": 20.467418670654297, | |
| "eval_logps/chosen": -47.48448944091797, | |
| "eval_logps/rejected": -52.786041259765625, | |
| "eval_loss": 0.440356969833374, | |
| "eval_rewards/accuracies": 0.8277778029441833, | |
| "eval_rewards/chosen": -0.5844593644142151, | |
| "eval_rewards/margins": 1.4904190301895142, | |
| "eval_rewards/rejected": -2.074878454208374, | |
| "eval_runtime": 97.8926, | |
| "eval_samples_per_second": 14.628, | |
| "eval_steps_per_second": 0.919, | |
| "step": 2000 | |
| }, | |
| { | |
| "epoch": 0.7176008568368439, | |
| "grad_norm": 10.820709228515625, | |
| "learning_rate": 1.4119957158157802e-05, | |
| "logits/chosen": 23.97198486328125, | |
| "logits/rejected": 23.974132537841797, | |
| "logps/chosen": -42.35189437866211, | |
| "logps/rejected": -57.66106414794922, | |
| "loss": 0.5486, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.9043490290641785, | |
| "rewards/margins": 1.2323650121688843, | |
| "rewards/rejected": -2.136713981628418, | |
| "step": 2010 | |
| }, | |
| { | |
| "epoch": 0.7211710103534452, | |
| "grad_norm": 4.93384313583374, | |
| "learning_rate": 1.394144948232774e-05, | |
| "logits/chosen": 21.256620407104492, | |
| "logits/rejected": 20.852317810058594, | |
| "logps/chosen": -27.885059356689453, | |
| "logps/rejected": -38.469505310058594, | |
| "loss": 0.4361, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.8113853335380554, | |
| "rewards/margins": 1.1693836450576782, | |
| "rewards/rejected": -1.9807687997817993, | |
| "step": 2020 | |
| }, | |
| { | |
| "epoch": 0.7247411638700464, | |
| "grad_norm": 8.402702331542969, | |
| "learning_rate": 1.3762941806497681e-05, | |
| "logits/chosen": 22.48406982421875, | |
| "logits/rejected": 22.659347534179688, | |
| "logps/chosen": -30.04986572265625, | |
| "logps/rejected": -39.668617248535156, | |
| "loss": 0.3847, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.43164730072021484, | |
| "rewards/margins": 1.5857521295547485, | |
| "rewards/rejected": -2.017399549484253, | |
| "step": 2030 | |
| }, | |
| { | |
| "epoch": 0.7283113173866477, | |
| "grad_norm": 5.305787563323975, | |
| "learning_rate": 1.358443413066762e-05, | |
| "logits/chosen": 24.485246658325195, | |
| "logits/rejected": 24.59568214416504, | |
| "logps/chosen": -61.41462326049805, | |
| "logps/rejected": -70.96296691894531, | |
| "loss": 0.5182, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.9171468019485474, | |
| "rewards/margins": 0.8545171618461609, | |
| "rewards/rejected": -1.7716642618179321, | |
| "step": 2040 | |
| }, | |
| { | |
| "epoch": 0.7318814709032488, | |
| "grad_norm": 7.894079685211182, | |
| "learning_rate": 1.340592645483756e-05, | |
| "logits/chosen": 18.658212661743164, | |
| "logits/rejected": 18.898082733154297, | |
| "logps/chosen": -29.949609756469727, | |
| "logps/rejected": -52.39385986328125, | |
| "loss": 0.349, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.49497169256210327, | |
| "rewards/margins": 1.6922725439071655, | |
| "rewards/rejected": -2.187243938446045, | |
| "step": 2050 | |
| }, | |
| { | |
| "epoch": 0.73545162441985, | |
| "grad_norm": 7.683061599731445, | |
| "learning_rate": 1.3227418779007496e-05, | |
| "logits/chosen": 21.266719818115234, | |
| "logits/rejected": 21.34499740600586, | |
| "logps/chosen": -37.20063018798828, | |
| "logps/rejected": -41.06432342529297, | |
| "loss": 0.4469, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.48325157165527344, | |
| "rewards/margins": 0.4681263566017151, | |
| "rewards/rejected": -0.9513779878616333, | |
| "step": 2060 | |
| }, | |
| { | |
| "epoch": 0.7390217779364513, | |
| "grad_norm": 9.4763765335083, | |
| "learning_rate": 1.3048911103177439e-05, | |
| "logits/chosen": 21.261632919311523, | |
| "logits/rejected": 21.200904846191406, | |
| "logps/chosen": -37.98931121826172, | |
| "logps/rejected": -50.971412658691406, | |
| "loss": 0.4712, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.8451720476150513, | |
| "rewards/margins": 1.3964030742645264, | |
| "rewards/rejected": -2.241574764251709, | |
| "step": 2070 | |
| }, | |
| { | |
| "epoch": 0.7425919314530525, | |
| "grad_norm": 12.83144474029541, | |
| "learning_rate": 1.2870403427347378e-05, | |
| "logits/chosen": 26.042606353759766, | |
| "logits/rejected": 25.88700294494629, | |
| "logps/chosen": -62.49781036376953, | |
| "logps/rejected": -77.66305541992188, | |
| "loss": 0.4901, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -1.1033135652542114, | |
| "rewards/margins": 1.2711939811706543, | |
| "rewards/rejected": -2.374507427215576, | |
| "step": 2080 | |
| }, | |
| { | |
| "epoch": 0.7461620849696537, | |
| "grad_norm": 4.098615646362305, | |
| "learning_rate": 1.2691895751517315e-05, | |
| "logits/chosen": 22.199886322021484, | |
| "logits/rejected": 22.246328353881836, | |
| "logps/chosen": -53.25043869018555, | |
| "logps/rejected": -53.634849548339844, | |
| "loss": 0.447, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.8297556042671204, | |
| "rewards/margins": 1.6126811504364014, | |
| "rewards/rejected": -2.442436695098877, | |
| "step": 2090 | |
| }, | |
| { | |
| "epoch": 0.7497322384862549, | |
| "grad_norm": 7.720545291900635, | |
| "learning_rate": 1.2513388075687254e-05, | |
| "logits/chosen": 23.562522888183594, | |
| "logits/rejected": 23.22288703918457, | |
| "logps/chosen": -79.41565704345703, | |
| "logps/rejected": -45.413997650146484, | |
| "loss": 0.4843, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.48383426666259766, | |
| "rewards/margins": 1.4371916055679321, | |
| "rewards/rejected": -1.9210258722305298, | |
| "step": 2100 | |
| }, | |
| { | |
| "epoch": 0.7497322384862549, | |
| "eval_logits/chosen": 20.6312198638916, | |
| "eval_logits/rejected": 20.657325744628906, | |
| "eval_logps/chosen": -47.1960334777832, | |
| "eval_logps/rejected": -52.259559631347656, | |
| "eval_loss": 0.4409657418727875, | |
| "eval_rewards/accuracies": 0.8333333134651184, | |
| "eval_rewards/chosen": -0.5556142330169678, | |
| "eval_rewards/margins": 1.4666165113449097, | |
| "eval_rewards/rejected": -2.022230625152588, | |
| "eval_runtime": 97.8612, | |
| "eval_samples_per_second": 14.633, | |
| "eval_steps_per_second": 0.92, | |
| "step": 2100 | |
| }, | |
| { | |
| "epoch": 0.7533023920028561, | |
| "grad_norm": 11.717628479003906, | |
| "learning_rate": 1.2334880399857195e-05, | |
| "logits/chosen": 19.04401969909668, | |
| "logits/rejected": 19.13692855834961, | |
| "logps/chosen": -56.0210075378418, | |
| "logps/rejected": -46.79364013671875, | |
| "loss": 0.3983, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.6606385707855225, | |
| "rewards/margins": 1.5798553228378296, | |
| "rewards/rejected": -2.2404940128326416, | |
| "step": 2110 | |
| }, | |
| { | |
| "epoch": 0.7568725455194574, | |
| "grad_norm": 6.157966136932373, | |
| "learning_rate": 1.2156372724027134e-05, | |
| "logits/chosen": 17.96826171875, | |
| "logits/rejected": 18.081592559814453, | |
| "logps/chosen": -42.12479782104492, | |
| "logps/rejected": -44.53673553466797, | |
| "loss": 0.3718, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.36954426765441895, | |
| "rewards/margins": 1.775122880935669, | |
| "rewards/rejected": -2.144667148590088, | |
| "step": 2120 | |
| }, | |
| { | |
| "epoch": 0.7604426990360585, | |
| "grad_norm": 3.8417954444885254, | |
| "learning_rate": 1.1977865048197073e-05, | |
| "logits/chosen": 20.064462661743164, | |
| "logits/rejected": 19.88588523864746, | |
| "logps/chosen": -44.317867279052734, | |
| "logps/rejected": -63.211326599121094, | |
| "loss": 0.5168, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -1.0525072813034058, | |
| "rewards/margins": 1.6052157878875732, | |
| "rewards/rejected": -2.6577229499816895, | |
| "step": 2130 | |
| }, | |
| { | |
| "epoch": 0.7640128525526597, | |
| "grad_norm": 3.4509365558624268, | |
| "learning_rate": 1.1799357372367012e-05, | |
| "logits/chosen": 22.211759567260742, | |
| "logits/rejected": 22.149486541748047, | |
| "logps/chosen": -37.72747039794922, | |
| "logps/rejected": -46.89754867553711, | |
| "loss": 0.3956, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -1.0622680187225342, | |
| "rewards/margins": 1.1859983205795288, | |
| "rewards/rejected": -2.2482664585113525, | |
| "step": 2140 | |
| }, | |
| { | |
| "epoch": 0.767583006069261, | |
| "grad_norm": 13.607682228088379, | |
| "learning_rate": 1.1620849696536952e-05, | |
| "logits/chosen": 23.63653564453125, | |
| "logits/rejected": 23.572277069091797, | |
| "logps/chosen": -41.84113311767578, | |
| "logps/rejected": -45.45246887207031, | |
| "loss": 0.5191, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.7334656715393066, | |
| "rewards/margins": 1.2255051136016846, | |
| "rewards/rejected": -1.9589707851409912, | |
| "step": 2150 | |
| }, | |
| { | |
| "epoch": 0.7711531595858622, | |
| "grad_norm": 12.166585922241211, | |
| "learning_rate": 1.1442342020706891e-05, | |
| "logits/chosen": 23.18703842163086, | |
| "logits/rejected": 23.008901596069336, | |
| "logps/chosen": -48.22418975830078, | |
| "logps/rejected": -39.83467102050781, | |
| "loss": 0.4619, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -1.0254920721054077, | |
| "rewards/margins": 1.014207124710083, | |
| "rewards/rejected": -2.039699077606201, | |
| "step": 2160 | |
| }, | |
| { | |
| "epoch": 0.7747233131024635, | |
| "grad_norm": 4.827226161956787, | |
| "learning_rate": 1.126383434487683e-05, | |
| "logits/chosen": 19.71803092956543, | |
| "logits/rejected": 19.300630569458008, | |
| "logps/chosen": -55.87500762939453, | |
| "logps/rejected": -42.52462387084961, | |
| "loss": 0.4894, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.7345470190048218, | |
| "rewards/margins": 0.6044321656227112, | |
| "rewards/rejected": -1.3389792442321777, | |
| "step": 2170 | |
| }, | |
| { | |
| "epoch": 0.7782934666190646, | |
| "grad_norm": 9.447345733642578, | |
| "learning_rate": 1.108532666904677e-05, | |
| "logits/chosen": 19.782794952392578, | |
| "logits/rejected": 19.624738693237305, | |
| "logps/chosen": -45.2284049987793, | |
| "logps/rejected": -68.25267791748047, | |
| "loss": 0.443, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.21583375334739685, | |
| "rewards/margins": 0.9129629135131836, | |
| "rewards/rejected": -1.1287966966629028, | |
| "step": 2180 | |
| }, | |
| { | |
| "epoch": 0.7818636201356658, | |
| "grad_norm": 6.821338176727295, | |
| "learning_rate": 1.090681899321671e-05, | |
| "logits/chosen": 22.291385650634766, | |
| "logits/rejected": 22.491024017333984, | |
| "logps/chosen": -35.10066223144531, | |
| "logps/rejected": -53.37052536010742, | |
| "loss": 0.4743, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.867012619972229, | |
| "rewards/margins": 0.7674096822738647, | |
| "rewards/rejected": -1.6344223022460938, | |
| "step": 2190 | |
| }, | |
| { | |
| "epoch": 0.7854337736522671, | |
| "grad_norm": 13.862120628356934, | |
| "learning_rate": 1.0728311317386647e-05, | |
| "logits/chosen": 24.0218448638916, | |
| "logits/rejected": 23.70220947265625, | |
| "logps/chosen": -70.60053253173828, | |
| "logps/rejected": -49.052589416503906, | |
| "loss": 0.5865, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.9927393794059753, | |
| "rewards/margins": 0.9688960313796997, | |
| "rewards/rejected": -1.9616353511810303, | |
| "step": 2200 | |
| }, | |
| { | |
| "epoch": 0.7854337736522671, | |
| "eval_logits/chosen": 20.780187606811523, | |
| "eval_logits/rejected": 20.791622161865234, | |
| "eval_logps/chosen": -47.48575210571289, | |
| "eval_logps/rejected": -52.499168395996094, | |
| "eval_loss": 0.43738028407096863, | |
| "eval_rewards/accuracies": 0.8166666626930237, | |
| "eval_rewards/chosen": -0.5845859050750732, | |
| "eval_rewards/margins": 1.4616053104400635, | |
| "eval_rewards/rejected": -2.0461909770965576, | |
| "eval_runtime": 97.9018, | |
| "eval_samples_per_second": 14.627, | |
| "eval_steps_per_second": 0.919, | |
| "step": 2200 | |
| }, | |
| { | |
| "epoch": 0.7890039271688682, | |
| "grad_norm": 9.391676902770996, | |
| "learning_rate": 1.0549803641556586e-05, | |
| "logits/chosen": 19.061254501342773, | |
| "logits/rejected": 18.95170021057129, | |
| "logps/chosen": -24.127471923828125, | |
| "logps/rejected": -29.48319435119629, | |
| "loss": 0.3811, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -1.1544402837753296, | |
| "rewards/margins": 0.755560040473938, | |
| "rewards/rejected": -1.9100004434585571, | |
| "step": 2210 | |
| }, | |
| { | |
| "epoch": 0.7925740806854695, | |
| "grad_norm": 15.025814056396484, | |
| "learning_rate": 1.0371295965726527e-05, | |
| "logits/chosen": 22.466449737548828, | |
| "logits/rejected": 22.482467651367188, | |
| "logps/chosen": -66.04009246826172, | |
| "logps/rejected": -77.5560073852539, | |
| "loss": 0.5311, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.26618120074272156, | |
| "rewards/margins": 1.5414941310882568, | |
| "rewards/rejected": -1.8076753616333008, | |
| "step": 2220 | |
| }, | |
| { | |
| "epoch": 0.7961442342020707, | |
| "grad_norm": 4.339944839477539, | |
| "learning_rate": 1.0192788289896466e-05, | |
| "logits/chosen": 22.064773559570312, | |
| "logits/rejected": 22.116168975830078, | |
| "logps/chosen": -46.800048828125, | |
| "logps/rejected": -47.54998016357422, | |
| "loss": 0.4871, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -1.3751857280731201, | |
| "rewards/margins": 0.8837114572525024, | |
| "rewards/rejected": -2.258897304534912, | |
| "step": 2230 | |
| }, | |
| { | |
| "epoch": 0.7997143877186719, | |
| "grad_norm": 6.989345550537109, | |
| "learning_rate": 1.0014280614066405e-05, | |
| "logits/chosen": 21.530237197875977, | |
| "logits/rejected": 21.780250549316406, | |
| "logps/chosen": -20.928579330444336, | |
| "logps/rejected": -34.38627243041992, | |
| "loss": 0.3897, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.6259101629257202, | |
| "rewards/margins": 1.1122276782989502, | |
| "rewards/rejected": -1.7381378412246704, | |
| "step": 2240 | |
| }, | |
| { | |
| "epoch": 0.8032845412352732, | |
| "grad_norm": 10.125934600830078, | |
| "learning_rate": 9.835772938236344e-06, | |
| "logits/chosen": 19.070201873779297, | |
| "logits/rejected": 19.056577682495117, | |
| "logps/chosen": -57.353233337402344, | |
| "logps/rejected": -45.93163299560547, | |
| "loss": 0.6086, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.8172680735588074, | |
| "rewards/margins": 1.0970861911773682, | |
| "rewards/rejected": -1.9143543243408203, | |
| "step": 2250 | |
| }, | |
| { | |
| "epoch": 0.8068546947518743, | |
| "grad_norm": 3.600658655166626, | |
| "learning_rate": 9.657265262406285e-06, | |
| "logits/chosen": 25.529754638671875, | |
| "logits/rejected": 25.521421432495117, | |
| "logps/chosen": -34.79078674316406, | |
| "logps/rejected": -43.617095947265625, | |
| "loss": 0.513, | |
| "rewards/accuracies": 0.4000000059604645, | |
| "rewards/chosen": -0.6766828894615173, | |
| "rewards/margins": -0.25649863481521606, | |
| "rewards/rejected": -0.42018431425094604, | |
| "step": 2260 | |
| }, | |
| { | |
| "epoch": 0.8104248482684755, | |
| "grad_norm": 9.139204025268555, | |
| "learning_rate": 9.478757586576224e-06, | |
| "logits/chosen": 19.52242660522461, | |
| "logits/rejected": 19.63443946838379, | |
| "logps/chosen": -33.019832611083984, | |
| "logps/rejected": -49.0518684387207, | |
| "loss": 0.4795, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.5696147084236145, | |
| "rewards/margins": 1.7224088907241821, | |
| "rewards/rejected": -2.2920238971710205, | |
| "step": 2270 | |
| }, | |
| { | |
| "epoch": 0.8139950017850768, | |
| "grad_norm": 8.38289737701416, | |
| "learning_rate": 9.300249910746163e-06, | |
| "logits/chosen": 21.76547622680664, | |
| "logits/rejected": 22.202754974365234, | |
| "logps/chosen": -57.740699768066406, | |
| "logps/rejected": -56.75148391723633, | |
| "loss": 0.3864, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.994134247303009, | |
| "rewards/margins": 1.7650930881500244, | |
| "rewards/rejected": -2.7592272758483887, | |
| "step": 2280 | |
| }, | |
| { | |
| "epoch": 0.817565155301678, | |
| "grad_norm": 7.5334272384643555, | |
| "learning_rate": 9.121742234916102e-06, | |
| "logits/chosen": 24.34808349609375, | |
| "logits/rejected": 24.858325958251953, | |
| "logps/chosen": -25.712127685546875, | |
| "logps/rejected": -42.50498962402344, | |
| "loss": 0.6181, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.5895062685012817, | |
| "rewards/margins": 1.4605395793914795, | |
| "rewards/rejected": -2.0500457286834717, | |
| "step": 2290 | |
| }, | |
| { | |
| "epoch": 0.8211353088182792, | |
| "grad_norm": 6.730588912963867, | |
| "learning_rate": 8.943234559086042e-06, | |
| "logits/chosen": 23.88949203491211, | |
| "logits/rejected": 23.867450714111328, | |
| "logps/chosen": -29.585073471069336, | |
| "logps/rejected": -44.84501647949219, | |
| "loss": 0.4544, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": -1.01124107837677, | |
| "rewards/margins": 1.032428503036499, | |
| "rewards/rejected": -2.0436697006225586, | |
| "step": 2300 | |
| }, | |
| { | |
| "epoch": 0.8211353088182792, | |
| "eval_logits/chosen": 20.524213790893555, | |
| "eval_logits/rejected": 20.531003952026367, | |
| "eval_logps/chosen": -47.56471252441406, | |
| "eval_logps/rejected": -52.853515625, | |
| "eval_loss": 0.4356454610824585, | |
| "eval_rewards/accuracies": 0.8333333134651184, | |
| "eval_rewards/chosen": -0.5924822688102722, | |
| "eval_rewards/margins": 1.4891438484191895, | |
| "eval_rewards/rejected": -2.0816261768341064, | |
| "eval_runtime": 97.6895, | |
| "eval_samples_per_second": 14.659, | |
| "eval_steps_per_second": 0.921, | |
| "step": 2300 | |
| }, | |
| { | |
| "epoch": 0.8247054623348804, | |
| "grad_norm": 7.662333011627197, | |
| "learning_rate": 8.76472688325598e-06, | |
| "logits/chosen": 19.167724609375, | |
| "logits/rejected": 19.311237335205078, | |
| "logps/chosen": -60.642913818359375, | |
| "logps/rejected": -61.336265563964844, | |
| "loss": 0.4462, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.7695547342300415, | |
| "rewards/margins": 1.0697910785675049, | |
| "rewards/rejected": -1.839345932006836, | |
| "step": 2310 | |
| }, | |
| { | |
| "epoch": 0.8282756158514816, | |
| "grad_norm": 13.9297513961792, | |
| "learning_rate": 8.58621920742592e-06, | |
| "logits/chosen": 24.825382232666016, | |
| "logits/rejected": 24.4403076171875, | |
| "logps/chosen": -50.69221115112305, | |
| "logps/rejected": -43.589820861816406, | |
| "loss": 0.472, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.14803439378738403, | |
| "rewards/margins": 2.013376474380493, | |
| "rewards/rejected": -2.1614108085632324, | |
| "step": 2320 | |
| }, | |
| { | |
| "epoch": 0.8318457693680829, | |
| "grad_norm": 7.068613529205322, | |
| "learning_rate": 8.40771153159586e-06, | |
| "logits/chosen": 20.52377700805664, | |
| "logits/rejected": 20.742507934570312, | |
| "logps/chosen": -48.990299224853516, | |
| "logps/rejected": -56.431175231933594, | |
| "loss": 0.5638, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.6809730529785156, | |
| "rewards/margins": 1.5615565776824951, | |
| "rewards/rejected": -2.2425293922424316, | |
| "step": 2330 | |
| }, | |
| { | |
| "epoch": 0.835415922884684, | |
| "grad_norm": 6.40026330947876, | |
| "learning_rate": 8.229203855765798e-06, | |
| "logits/chosen": 20.41210174560547, | |
| "logits/rejected": 20.379867553710938, | |
| "logps/chosen": -59.4276123046875, | |
| "logps/rejected": -71.07743072509766, | |
| "loss": 0.5384, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -1.5872411727905273, | |
| "rewards/margins": 0.9024609327316284, | |
| "rewards/rejected": -2.4897022247314453, | |
| "step": 2340 | |
| }, | |
| { | |
| "epoch": 0.8389860764012853, | |
| "grad_norm": 9.892749786376953, | |
| "learning_rate": 8.050696179935737e-06, | |
| "logits/chosen": 22.273807525634766, | |
| "logits/rejected": 22.05521011352539, | |
| "logps/chosen": -37.72688293457031, | |
| "logps/rejected": -37.593387603759766, | |
| "loss": 0.4539, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -1.0601468086242676, | |
| "rewards/margins": 0.7513321042060852, | |
| "rewards/rejected": -1.811478853225708, | |
| "step": 2350 | |
| }, | |
| { | |
| "epoch": 0.8425562299178865, | |
| "grad_norm": 19.193857192993164, | |
| "learning_rate": 7.872188504105676e-06, | |
| "logits/chosen": 21.529739379882812, | |
| "logits/rejected": 21.197284698486328, | |
| "logps/chosen": -25.73797607421875, | |
| "logps/rejected": -45.44214630126953, | |
| "loss": 0.5542, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.01898249424993992, | |
| "rewards/margins": 1.9577863216400146, | |
| "rewards/rejected": -1.9388039112091064, | |
| "step": 2360 | |
| }, | |
| { | |
| "epoch": 0.8461263834344876, | |
| "grad_norm": 9.629522323608398, | |
| "learning_rate": 7.693680828275617e-06, | |
| "logits/chosen": 13.054910659790039, | |
| "logits/rejected": 12.92231559753418, | |
| "logps/chosen": -28.6776180267334, | |
| "logps/rejected": -39.813411712646484, | |
| "loss": 0.5216, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -1.1042935848236084, | |
| "rewards/margins": 1.2019240856170654, | |
| "rewards/rejected": -2.306217670440674, | |
| "step": 2370 | |
| }, | |
| { | |
| "epoch": 0.8496965369510889, | |
| "grad_norm": 8.702773094177246, | |
| "learning_rate": 7.515173152445555e-06, | |
| "logits/chosen": 24.35626792907715, | |
| "logits/rejected": 24.593463897705078, | |
| "logps/chosen": -44.167537689208984, | |
| "logps/rejected": -70.86328125, | |
| "loss": 0.4955, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -1.1845743656158447, | |
| "rewards/margins": 1.8204050064086914, | |
| "rewards/rejected": -3.004979372024536, | |
| "step": 2380 | |
| }, | |
| { | |
| "epoch": 0.8532666904676901, | |
| "grad_norm": 11.280304908752441, | |
| "learning_rate": 7.336665476615495e-06, | |
| "logits/chosen": 21.997047424316406, | |
| "logits/rejected": 22.113534927368164, | |
| "logps/chosen": -57.37868118286133, | |
| "logps/rejected": -64.85008239746094, | |
| "loss": 0.5263, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -1.2687997817993164, | |
| "rewards/margins": 0.5181077718734741, | |
| "rewards/rejected": -1.7869075536727905, | |
| "step": 2390 | |
| }, | |
| { | |
| "epoch": 0.8568368439842913, | |
| "grad_norm": 7.5682854652404785, | |
| "learning_rate": 7.158157800785434e-06, | |
| "logits/chosen": 23.916316986083984, | |
| "logits/rejected": 23.814477920532227, | |
| "logps/chosen": -60.103416442871094, | |
| "logps/rejected": -71.51342010498047, | |
| "loss": 0.4501, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.49635568261146545, | |
| "rewards/margins": 1.9991073608398438, | |
| "rewards/rejected": -2.4954631328582764, | |
| "step": 2400 | |
| }, | |
| { | |
| "epoch": 0.8568368439842913, | |
| "eval_logits/chosen": 20.50776481628418, | |
| "eval_logits/rejected": 20.51170539855957, | |
| "eval_logps/chosen": -47.33561706542969, | |
| "eval_logps/rejected": -52.59519958496094, | |
| "eval_loss": 0.43392252922058105, | |
| "eval_rewards/accuracies": 0.8444444537162781, | |
| "eval_rewards/chosen": -0.5695720911026001, | |
| "eval_rewards/margins": 1.4862221479415894, | |
| "eval_rewards/rejected": -2.0557942390441895, | |
| "eval_runtime": 97.6403, | |
| "eval_samples_per_second": 14.666, | |
| "eval_steps_per_second": 0.922, | |
| "step": 2400 | |
| }, | |
| { | |
| "epoch": 0.8604069975008926, | |
| "grad_norm": 5.012470722198486, | |
| "learning_rate": 6.979650124955374e-06, | |
| "logits/chosen": 21.85761260986328, | |
| "logits/rejected": 21.804880142211914, | |
| "logps/chosen": -31.270885467529297, | |
| "logps/rejected": -52.126930236816406, | |
| "loss": 0.478, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.35933440923690796, | |
| "rewards/margins": 1.06755530834198, | |
| "rewards/rejected": -1.4268897771835327, | |
| "step": 2410 | |
| }, | |
| { | |
| "epoch": 0.8639771510174937, | |
| "grad_norm": 5.9783854484558105, | |
| "learning_rate": 6.801142449125313e-06, | |
| "logits/chosen": 19.979433059692383, | |
| "logits/rejected": 20.035179138183594, | |
| "logps/chosen": -51.7042121887207, | |
| "logps/rejected": -57.767913818359375, | |
| "loss": 0.4853, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.6708565354347229, | |
| "rewards/margins": 1.6906753778457642, | |
| "rewards/rejected": -2.3615317344665527, | |
| "step": 2420 | |
| }, | |
| { | |
| "epoch": 0.867547304534095, | |
| "grad_norm": 7.05979061126709, | |
| "learning_rate": 6.6226347732952526e-06, | |
| "logits/chosen": 15.46339225769043, | |
| "logits/rejected": 15.658937454223633, | |
| "logps/chosen": -40.01195526123047, | |
| "logps/rejected": -46.074501037597656, | |
| "loss": 0.5628, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.18836592137813568, | |
| "rewards/margins": 1.5064311027526855, | |
| "rewards/rejected": -1.6947968006134033, | |
| "step": 2430 | |
| }, | |
| { | |
| "epoch": 0.8711174580506962, | |
| "grad_norm": 12.882414817810059, | |
| "learning_rate": 6.4441270974651915e-06, | |
| "logits/chosen": 21.048954010009766, | |
| "logits/rejected": 21.059772491455078, | |
| "logps/chosen": -53.65748977661133, | |
| "logps/rejected": -48.547874450683594, | |
| "loss": 0.4885, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -1.3018702268600464, | |
| "rewards/margins": 1.0138559341430664, | |
| "rewards/rejected": -2.3157262802124023, | |
| "step": 2440 | |
| }, | |
| { | |
| "epoch": 0.8746876115672974, | |
| "grad_norm": 11.930545806884766, | |
| "learning_rate": 6.26561942163513e-06, | |
| "logits/chosen": 20.654542922973633, | |
| "logits/rejected": 20.686420440673828, | |
| "logps/chosen": -39.971920013427734, | |
| "logps/rejected": -35.28145217895508, | |
| "loss": 0.5033, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.6068005561828613, | |
| "rewards/margins": 1.3182332515716553, | |
| "rewards/rejected": -1.9250338077545166, | |
| "step": 2450 | |
| }, | |
| { | |
| "epoch": 0.8782577650838986, | |
| "grad_norm": 9.403008460998535, | |
| "learning_rate": 6.08711174580507e-06, | |
| "logits/chosen": 22.882566452026367, | |
| "logits/rejected": 22.231182098388672, | |
| "logps/chosen": -52.44697952270508, | |
| "logps/rejected": -47.005287170410156, | |
| "loss": 0.5709, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.8271516561508179, | |
| "rewards/margins": 0.7867218852043152, | |
| "rewards/rejected": -1.6138734817504883, | |
| "step": 2460 | |
| }, | |
| { | |
| "epoch": 0.8818279186004998, | |
| "grad_norm": 23.268875122070312, | |
| "learning_rate": 5.908604069975009e-06, | |
| "logits/chosen": 21.904462814331055, | |
| "logits/rejected": 22.555965423583984, | |
| "logps/chosen": -63.27705001831055, | |
| "logps/rejected": -87.00935363769531, | |
| "loss": 0.383, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -1.2159323692321777, | |
| "rewards/margins": 1.8115613460540771, | |
| "rewards/rejected": -3.027493715286255, | |
| "step": 2470 | |
| }, | |
| { | |
| "epoch": 0.8853980721171011, | |
| "grad_norm": 16.253660202026367, | |
| "learning_rate": 5.730096394144948e-06, | |
| "logits/chosen": 22.083019256591797, | |
| "logits/rejected": 22.175024032592773, | |
| "logps/chosen": -43.36615753173828, | |
| "logps/rejected": -42.4653205871582, | |
| "loss": 0.531, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -1.2614014148712158, | |
| "rewards/margins": 0.07179747521877289, | |
| "rewards/rejected": -1.33319890499115, | |
| "step": 2480 | |
| }, | |
| { | |
| "epoch": 0.8889682256337023, | |
| "grad_norm": 8.766143798828125, | |
| "learning_rate": 5.551588718314888e-06, | |
| "logits/chosen": 22.502155303955078, | |
| "logits/rejected": 22.834453582763672, | |
| "logps/chosen": -57.746620178222656, | |
| "logps/rejected": -73.5748519897461, | |
| "loss": 0.386, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.6636447906494141, | |
| "rewards/margins": 1.0091074705123901, | |
| "rewards/rejected": -1.6727523803710938, | |
| "step": 2490 | |
| }, | |
| { | |
| "epoch": 0.8925383791503034, | |
| "grad_norm": 9.421289443969727, | |
| "learning_rate": 5.373081042484827e-06, | |
| "logits/chosen": 22.923538208007812, | |
| "logits/rejected": 23.32149887084961, | |
| "logps/chosen": -63.822181701660156, | |
| "logps/rejected": -57.5202522277832, | |
| "loss": 0.5171, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -1.152776837348938, | |
| "rewards/margins": 0.4619804322719574, | |
| "rewards/rejected": -1.6147572994232178, | |
| "step": 2500 | |
| }, | |
| { | |
| "epoch": 0.8925383791503034, | |
| "eval_logits/chosen": 20.416898727416992, | |
| "eval_logits/rejected": 20.427398681640625, | |
| "eval_logps/chosen": -47.32997131347656, | |
| "eval_logps/rejected": -52.617210388183594, | |
| "eval_loss": 0.4359460473060608, | |
| "eval_rewards/accuracies": 0.8333333134651184, | |
| "eval_rewards/chosen": -0.5690072774887085, | |
| "eval_rewards/margins": 1.4889880418777466, | |
| "eval_rewards/rejected": -2.057995319366455, | |
| "eval_runtime": 98.594, | |
| "eval_samples_per_second": 14.524, | |
| "eval_steps_per_second": 0.913, | |
| "step": 2500 | |
| }, | |
| { | |
| "epoch": 0.8961085326669047, | |
| "grad_norm": 10.914727210998535, | |
| "learning_rate": 5.194573366654766e-06, | |
| "logits/chosen": 27.528060913085938, | |
| "logits/rejected": 27.470294952392578, | |
| "logps/chosen": -26.07723045349121, | |
| "logps/rejected": -25.864681243896484, | |
| "loss": 0.4914, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.3140444755554199, | |
| "rewards/margins": 0.8342065811157227, | |
| "rewards/rejected": -1.1482508182525635, | |
| "step": 2510 | |
| }, | |
| { | |
| "epoch": 0.8996786861835059, | |
| "grad_norm": 4.41356897354126, | |
| "learning_rate": 5.016065690824705e-06, | |
| "logits/chosen": 21.382648468017578, | |
| "logits/rejected": 21.26772689819336, | |
| "logps/chosen": -65.88087463378906, | |
| "logps/rejected": -38.547142028808594, | |
| "loss": 0.5372, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.7923516631126404, | |
| "rewards/margins": 0.6893059015274048, | |
| "rewards/rejected": -1.4816573858261108, | |
| "step": 2520 | |
| }, | |
| { | |
| "epoch": 0.903248839700107, | |
| "grad_norm": 14.50999641418457, | |
| "learning_rate": 4.837558014994645e-06, | |
| "logits/chosen": 24.574230194091797, | |
| "logits/rejected": 24.432174682617188, | |
| "logps/chosen": -40.37361145019531, | |
| "logps/rejected": -61.320648193359375, | |
| "loss": 0.59, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.6994588971138, | |
| "rewards/margins": 1.3289554119110107, | |
| "rewards/rejected": -2.028414487838745, | |
| "step": 2530 | |
| }, | |
| { | |
| "epoch": 0.9068189932167083, | |
| "grad_norm": 3.6535849571228027, | |
| "learning_rate": 4.659050339164584e-06, | |
| "logits/chosen": 22.09380340576172, | |
| "logits/rejected": 22.342832565307617, | |
| "logps/chosen": -43.704811096191406, | |
| "logps/rejected": -57.10416793823242, | |
| "loss": 0.3651, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": -0.5590577721595764, | |
| "rewards/margins": 2.217897891998291, | |
| "rewards/rejected": -2.7769553661346436, | |
| "step": 2540 | |
| }, | |
| { | |
| "epoch": 0.9103891467333095, | |
| "grad_norm": 12.662737846374512, | |
| "learning_rate": 4.480542663334524e-06, | |
| "logits/chosen": 26.030466079711914, | |
| "logits/rejected": 26.27060317993164, | |
| "logps/chosen": -45.546714782714844, | |
| "logps/rejected": -56.18561935424805, | |
| "loss": 0.631, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.6547432541847229, | |
| "rewards/margins": 1.1012141704559326, | |
| "rewards/rejected": -1.7559573650360107, | |
| "step": 2550 | |
| }, | |
| { | |
| "epoch": 0.9139593002499108, | |
| "grad_norm": 12.380247116088867, | |
| "learning_rate": 4.302034987504463e-06, | |
| "logits/chosen": 26.792774200439453, | |
| "logits/rejected": 26.5482120513916, | |
| "logps/chosen": -56.17375564575195, | |
| "logps/rejected": -41.74138641357422, | |
| "loss": 0.5118, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -1.4354368448257446, | |
| "rewards/margins": 0.7226849794387817, | |
| "rewards/rejected": -2.1581218242645264, | |
| "step": 2560 | |
| }, | |
| { | |
| "epoch": 0.917529453766512, | |
| "grad_norm": 14.337486267089844, | |
| "learning_rate": 4.123527311674403e-06, | |
| "logits/chosen": 20.274442672729492, | |
| "logits/rejected": 20.080303192138672, | |
| "logps/chosen": -64.95756530761719, | |
| "logps/rejected": -54.136146545410156, | |
| "loss": 0.4504, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.9958245158195496, | |
| "rewards/margins": 1.3853988647460938, | |
| "rewards/rejected": -2.381223440170288, | |
| "step": 2570 | |
| }, | |
| { | |
| "epoch": 0.9210996072831131, | |
| "grad_norm": 5.5064616203308105, | |
| "learning_rate": 3.945019635844342e-06, | |
| "logits/chosen": 23.955215454101562, | |
| "logits/rejected": 23.814537048339844, | |
| "logps/chosen": -47.43402862548828, | |
| "logps/rejected": -75.19926452636719, | |
| "loss": 0.3622, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -1.0230860710144043, | |
| "rewards/margins": 1.3491160869598389, | |
| "rewards/rejected": -2.372201919555664, | |
| "step": 2580 | |
| }, | |
| { | |
| "epoch": 0.9246697607997144, | |
| "grad_norm": 8.065810203552246, | |
| "learning_rate": 3.766511960014281e-06, | |
| "logits/chosen": 24.445186614990234, | |
| "logits/rejected": 24.539066314697266, | |
| "logps/chosen": -54.058349609375, | |
| "logps/rejected": -57.293846130371094, | |
| "loss": 0.5582, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.8545524477958679, | |
| "rewards/margins": 0.8429762721061707, | |
| "rewards/rejected": -1.697528600692749, | |
| "step": 2590 | |
| }, | |
| { | |
| "epoch": 0.9282399143163156, | |
| "grad_norm": 8.037162780761719, | |
| "learning_rate": 3.5880042841842204e-06, | |
| "logits/chosen": 22.97157859802246, | |
| "logits/rejected": 22.82067108154297, | |
| "logps/chosen": -36.165348052978516, | |
| "logps/rejected": -39.38242721557617, | |
| "loss": 0.4775, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -1.074142336845398, | |
| "rewards/margins": 0.5587809681892395, | |
| "rewards/rejected": -1.6329231262207031, | |
| "step": 2600 | |
| }, | |
| { | |
| "epoch": 0.9282399143163156, | |
| "eval_logits/chosen": 20.513593673706055, | |
| "eval_logits/rejected": 20.519195556640625, | |
| "eval_logps/chosen": -47.434913635253906, | |
| "eval_logps/rejected": -52.493404388427734, | |
| "eval_loss": 0.43444961309432983, | |
| "eval_rewards/accuracies": 0.8333333134651184, | |
| "eval_rewards/chosen": -0.5795022249221802, | |
| "eval_rewards/margins": 1.4661126136779785, | |
| "eval_rewards/rejected": -2.045614719390869, | |
| "eval_runtime": 97.6912, | |
| "eval_samples_per_second": 14.658, | |
| "eval_steps_per_second": 0.921, | |
| "step": 2600 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 2801, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 100, | |
| "total_flos": 0.0, | |
| "train_batch_size": 1, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |