diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,3106 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.05592059275828324, + "eval_steps": 50, + "global_step": 200, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0002796029637914162, + "grad_norm": 25.76739404251824, + "learning_rate": 5e-08, + "logits/chosen": -0.9193593263626099, + "logits/rejected": -1.1508712768554688, + "logps/chosen": -0.9903213977813721, + "logps/rejected": -0.9371722936630249, + "loss": 1.683, + "rewards/accuracies": 0.5, + "rewards/chosen": -2.4758033752441406, + "rewards/margins": -0.13287273049354553, + "rewards/rejected": -2.342930793762207, + "step": 1 + }, + { + "epoch": 0.0005592059275828324, + "grad_norm": 45.297284297927725, + "learning_rate": 1e-07, + "logits/chosen": -0.8994183540344238, + "logits/rejected": -1.2252119779586792, + "logps/chosen": -1.4980652332305908, + "logps/rejected": -1.216281533241272, + "loss": 1.7319, + "rewards/accuracies": 0.25, + "rewards/chosen": -3.7451629638671875, + "rewards/margins": -0.7044591903686523, + "rewards/rejected": -3.040703773498535, + "step": 2 + }, + { + "epoch": 0.0008388088913742485, + "grad_norm": 31.873718310326044, + "learning_rate": 1.5e-07, + "logits/chosen": -0.5185738205909729, + "logits/rejected": -1.3382601737976074, + "logps/chosen": -1.3677482604980469, + "logps/rejected": -1.4751423597335815, + "loss": 1.4124, + "rewards/accuracies": 0.5, + "rewards/chosen": -3.419370412826538, + "rewards/margins": 0.26848527789115906, + "rewards/rejected": -3.6878557205200195, + "step": 3 + }, + { + "epoch": 0.0011184118551656648, + "grad_norm": 48.966957322673935, + "learning_rate": 2e-07, + "logits/chosen": -0.6579636931419373, + "logits/rejected": -1.3838244676589966, + "logps/chosen": -0.9612038731575012, + "logps/rejected": -1.2441552877426147, + "loss": 1.4082, + "rewards/accuracies": 0.5, + "rewards/chosen": -2.4030096530914307, + "rewards/margins": 0.7073784470558167, + "rewards/rejected": -3.1103880405426025, + "step": 4 + }, + { + "epoch": 0.001398014818957081, + "grad_norm": 46.91034256260659, + "learning_rate": 2.5e-07, + "logits/chosen": -1.2698190212249756, + "logits/rejected": -1.5386033058166504, + "logps/chosen": -1.6302634477615356, + "logps/rejected": -1.7429945468902588, + "loss": 1.6189, + "rewards/accuracies": 0.5, + "rewards/chosen": -4.075658798217773, + "rewards/margins": 0.28182780742645264, + "rewards/rejected": -4.357486248016357, + "step": 5 + }, + { + "epoch": 0.001677617782748497, + "grad_norm": 73.65526070267136, + "learning_rate": 3e-07, + "logits/chosen": -0.8474517464637756, + "logits/rejected": -0.7566354274749756, + "logps/chosen": -1.1040312051773071, + "logps/rejected": -1.1213470697402954, + "loss": 1.3294, + "rewards/accuracies": 0.5, + "rewards/chosen": -2.760077953338623, + "rewards/margins": 0.04328957200050354, + "rewards/rejected": -2.8033676147460938, + "step": 6 + }, + { + "epoch": 0.001957220746539913, + "grad_norm": 24.84640983932336, + "learning_rate": 3.5e-07, + "logits/chosen": -0.8542510271072388, + "logits/rejected": -0.34997448325157166, + "logps/chosen": -0.7964720726013184, + "logps/rejected": -0.8464012145996094, + "loss": 1.583, + "rewards/accuracies": 0.5, + "rewards/chosen": -1.991180181503296, + "rewards/margins": 0.12482279539108276, + "rewards/rejected": -2.1160030364990234, + "step": 7 + }, + { + "epoch": 0.0022368237103313295, + "grad_norm": 22.841316781980318, + "learning_rate": 4e-07, + "logits/chosen": -0.7528856992721558, + "logits/rejected": -1.2337926626205444, + "logps/chosen": -1.1317272186279297, + "logps/rejected": -1.4922257661819458, + "loss": 1.3675, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.829317808151245, + "rewards/margins": 0.9012465476989746, + "rewards/rejected": -3.7305643558502197, + "step": 8 + }, + { + "epoch": 0.002516426674122746, + "grad_norm": 87.07632980072657, + "learning_rate": 4.5e-07, + "logits/chosen": -0.9475423693656921, + "logits/rejected": -1.4221036434173584, + "logps/chosen": -1.2074404954910278, + "logps/rejected": -0.9064420461654663, + "loss": 1.7839, + "rewards/accuracies": 0.25, + "rewards/chosen": -3.018601417541504, + "rewards/margins": -0.7524962425231934, + "rewards/rejected": -2.2661051750183105, + "step": 9 + }, + { + "epoch": 0.002796029637914162, + "grad_norm": 27.009928075261808, + "learning_rate": 5e-07, + "logits/chosen": -0.6325011253356934, + "logits/rejected": -1.1582834720611572, + "logps/chosen": -0.7608655691146851, + "logps/rejected": -0.8033685088157654, + "loss": 1.3733, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.9021639823913574, + "rewards/margins": 0.10625733435153961, + "rewards/rejected": -2.0084214210510254, + "step": 10 + }, + { + "epoch": 0.003075632601705578, + "grad_norm": 30.088075877499065, + "learning_rate": 5.5e-07, + "logits/chosen": -1.018723964691162, + "logits/rejected": -1.0942765474319458, + "logps/chosen": -0.9146077632904053, + "logps/rejected": -0.8910419344902039, + "loss": 1.2264, + "rewards/accuracies": 0.5, + "rewards/chosen": -2.2865195274353027, + "rewards/margins": -0.058914512395858765, + "rewards/rejected": -2.227604866027832, + "step": 11 + }, + { + "epoch": 0.003355235565496994, + "grad_norm": 69.74430608906756, + "learning_rate": 6e-07, + "logits/chosen": -1.0309629440307617, + "logits/rejected": -1.2155206203460693, + "logps/chosen": -1.868607521057129, + "logps/rejected": -2.074054002761841, + "loss": 1.3981, + "rewards/accuracies": 0.5, + "rewards/chosen": -4.6715192794799805, + "rewards/margins": 0.5136159062385559, + "rewards/rejected": -5.1851348876953125, + "step": 12 + }, + { + "epoch": 0.0036348385292884105, + "grad_norm": 33.254043165620104, + "learning_rate": 6.5e-07, + "logits/chosen": -1.0877211093902588, + "logits/rejected": -1.1691397428512573, + "logps/chosen": -1.4225409030914307, + "logps/rejected": -1.5891215801239014, + "loss": 1.9712, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.556352138519287, + "rewards/margins": 0.4164515733718872, + "rewards/rejected": -3.9728035926818848, + "step": 13 + }, + { + "epoch": 0.003914441493079826, + "grad_norm": 21.755018885778018, + "learning_rate": 7e-07, + "logits/chosen": -0.5653956532478333, + "logits/rejected": -1.170688271522522, + "logps/chosen": -1.041928768157959, + "logps/rejected": -1.546263337135315, + "loss": 1.4989, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.6048219203948975, + "rewards/margins": 1.2608366012573242, + "rewards/rejected": -3.8656585216522217, + "step": 14 + }, + { + "epoch": 0.004194044456871243, + "grad_norm": 69.81299826798852, + "learning_rate": 7.5e-07, + "logits/chosen": -0.4804845452308655, + "logits/rejected": -0.8382622003555298, + "logps/chosen": -1.0265494585037231, + "logps/rejected": -1.0567755699157715, + "loss": 1.4168, + "rewards/accuracies": 0.5, + "rewards/chosen": -2.566373586654663, + "rewards/margins": 0.07556554675102234, + "rewards/rejected": -2.641939163208008, + "step": 15 + }, + { + "epoch": 0.004473647420662659, + "grad_norm": 24.193090245288765, + "learning_rate": 8e-07, + "logits/chosen": -0.6370764970779419, + "logits/rejected": -1.0934374332427979, + "logps/chosen": -0.71354079246521, + "logps/rejected": -1.11177396774292, + "loss": 0.9962, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.783851981163025, + "rewards/margins": 0.9955829977989197, + "rewards/rejected": -2.7794349193573, + "step": 16 + }, + { + "epoch": 0.004753250384454075, + "grad_norm": 25.26345718560276, + "learning_rate": 8.499999999999999e-07, + "logits/chosen": -0.9751855134963989, + "logits/rejected": -1.1418442726135254, + "logps/chosen": -1.2313238382339478, + "logps/rejected": -1.6264705657958984, + "loss": 1.4442, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.0783097743988037, + "rewards/margins": 0.9878668189048767, + "rewards/rejected": -4.066176414489746, + "step": 17 + }, + { + "epoch": 0.005032853348245492, + "grad_norm": 15.702824197515223, + "learning_rate": 9e-07, + "logits/chosen": -0.9074665904045105, + "logits/rejected": -1.053283929824829, + "logps/chosen": -0.8471686244010925, + "logps/rejected": -1.086425542831421, + "loss": 1.4019, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.1179215908050537, + "rewards/margins": 0.5981425046920776, + "rewards/rejected": -2.716063976287842, + "step": 18 + }, + { + "epoch": 0.005312456312036908, + "grad_norm": 26.78497329037122, + "learning_rate": 9.499999999999999e-07, + "logits/chosen": -0.8664759993553162, + "logits/rejected": -0.9702961444854736, + "logps/chosen": -0.9274067878723145, + "logps/rejected": -1.054181694984436, + "loss": 1.1298, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.318516969680786, + "rewards/margins": 0.31693723797798157, + "rewards/rejected": -2.6354541778564453, + "step": 19 + }, + { + "epoch": 0.005592059275828324, + "grad_norm": 43.484383450898136, + "learning_rate": 1e-06, + "logits/chosen": -0.9217432141304016, + "logits/rejected": -1.0792617797851562, + "logps/chosen": -1.1316174268722534, + "logps/rejected": -1.3112761974334717, + "loss": 1.6817, + "rewards/accuracies": 0.5, + "rewards/chosen": -2.8290436267852783, + "rewards/margins": 0.44914710521698, + "rewards/rejected": -3.2781906127929688, + "step": 20 + }, + { + "epoch": 0.0058716622396197396, + "grad_norm": 23.72254452641318, + "learning_rate": 9.999238475781957e-07, + "logits/chosen": -0.892957866191864, + "logits/rejected": -0.896960973739624, + "logps/chosen": -1.0195448398590088, + "logps/rejected": -1.172441005706787, + "loss": 1.4088, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.5488619804382324, + "rewards/margins": 0.3822404146194458, + "rewards/rejected": -2.9311022758483887, + "step": 21 + }, + { + "epoch": 0.006151265203411156, + "grad_norm": 19.983043621451948, + "learning_rate": 9.996954135095478e-07, + "logits/chosen": -0.815680742263794, + "logits/rejected": -1.0532113313674927, + "logps/chosen": -1.1461201906204224, + "logps/rejected": -1.190143346786499, + "loss": 1.4757, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.865300416946411, + "rewards/margins": 0.11005795001983643, + "rewards/rejected": -2.975358486175537, + "step": 22 + }, + { + "epoch": 0.006430868167202572, + "grad_norm": 130.05670927021893, + "learning_rate": 9.99314767377287e-07, + "logits/chosen": -1.0124224424362183, + "logits/rejected": -1.3622022867202759, + "logps/chosen": -1.240678310394287, + "logps/rejected": -1.411242961883545, + "loss": 1.4839, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.1016955375671387, + "rewards/margins": 0.42641204595565796, + "rewards/rejected": -3.5281076431274414, + "step": 23 + }, + { + "epoch": 0.006710471130993988, + "grad_norm": 22.122176134369795, + "learning_rate": 9.98782025129912e-07, + "logits/chosen": -1.276932716369629, + "logits/rejected": -1.717280387878418, + "logps/chosen": -1.536954402923584, + "logps/rejected": -2.9943461418151855, + "loss": 0.6631, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.84238600730896, + "rewards/margins": 3.643479108810425, + "rewards/rejected": -7.485865592956543, + "step": 24 + }, + { + "epoch": 0.006990074094785405, + "grad_norm": 25.203974129539585, + "learning_rate": 9.980973490458728e-07, + "logits/chosen": -1.0255053043365479, + "logits/rejected": -1.122296690940857, + "logps/chosen": -1.0807580947875977, + "logps/rejected": -1.0660605430603027, + "loss": 1.6138, + "rewards/accuracies": 0.25, + "rewards/chosen": -2.701895236968994, + "rewards/margins": -0.036744117736816406, + "rewards/rejected": -2.6651511192321777, + "step": 25 + }, + { + "epoch": 0.007269677058576821, + "grad_norm": 27.512055310670853, + "learning_rate": 9.972609476841365e-07, + "logits/chosen": -1.2338676452636719, + "logits/rejected": -1.3265719413757324, + "logps/chosen": -1.2775177955627441, + "logps/rejected": -1.486391544342041, + "loss": 1.4048, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.1937947273254395, + "rewards/margins": 0.5221841335296631, + "rewards/rejected": -3.7159788608551025, + "step": 26 + }, + { + "epoch": 0.007549280022368237, + "grad_norm": 137.0687602353488, + "learning_rate": 9.96273075820661e-07, + "logits/chosen": -0.8894615173339844, + "logits/rejected": -1.346874713897705, + "logps/chosen": -0.8310048580169678, + "logps/rejected": -1.700638771057129, + "loss": 1.2364, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.077512264251709, + "rewards/margins": 2.1740849018096924, + "rewards/rejected": -4.2515974044799805, + "step": 27 + }, + { + "epoch": 0.007828882986159653, + "grad_norm": 17.619065074739723, + "learning_rate": 9.95134034370785e-07, + "logits/chosen": -1.0894310474395752, + "logits/rejected": -1.202174425125122, + "logps/chosen": -1.1609501838684082, + "logps/rejected": -1.407175064086914, + "loss": 1.4807, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.9023752212524414, + "rewards/margins": 0.6155624389648438, + "rewards/rejected": -3.517937660217285, + "step": 28 + }, + { + "epoch": 0.00810848594995107, + "grad_norm": 110.88293466564413, + "learning_rate": 9.938441702975689e-07, + "logits/chosen": -0.601264476776123, + "logits/rejected": -0.8674606084823608, + "logps/chosen": -1.1357152462005615, + "logps/rejected": -1.3777132034301758, + "loss": 1.6135, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.839287757873535, + "rewards/margins": 0.6049953699111938, + "rewards/rejected": -3.4442832469940186, + "step": 29 + }, + { + "epoch": 0.008388088913742486, + "grad_norm": 28.486621774152123, + "learning_rate": 9.92403876506104e-07, + "logits/chosen": -0.8283142447471619, + "logits/rejected": -1.3725578784942627, + "logps/chosen": -0.8853375911712646, + "logps/rejected": -1.7784080505371094, + "loss": 1.4251, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.213344097137451, + "rewards/margins": 2.2326760292053223, + "rewards/rejected": -4.446020126342773, + "step": 30 + }, + { + "epoch": 0.008667691877533901, + "grad_norm": 40.53817922951689, + "learning_rate": 9.90813591723832e-07, + "logits/chosen": -1.3251934051513672, + "logits/rejected": -1.2978826761245728, + "logps/chosen": -1.1201034784317017, + "logps/rejected": -1.6758404970169067, + "loss": 1.1433, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.8002586364746094, + "rewards/margins": 1.3893425464630127, + "rewards/rejected": -4.189601421356201, + "step": 31 + }, + { + "epoch": 0.008947294841325318, + "grad_norm": 30.2037356357338, + "learning_rate": 9.890738003669027e-07, + "logits/chosen": -0.8872109055519104, + "logits/rejected": -1.0567409992218018, + "logps/chosen": -1.3809734582901, + "logps/rejected": -1.5892744064331055, + "loss": 2.2798, + "rewards/accuracies": 0.5, + "rewards/chosen": -3.4524335861206055, + "rewards/margins": 0.5207523703575134, + "rewards/rejected": -3.9731857776641846, + "step": 32 + }, + { + "epoch": 0.009226897805116735, + "grad_norm": 22.08377585136328, + "learning_rate": 9.871850323926177e-07, + "logits/chosen": -0.6034784913063049, + "logits/rejected": -1.3737246990203857, + "logps/chosen": -0.7753689885139465, + "logps/rejected": -1.4922962188720703, + "loss": 1.186, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.938422441482544, + "rewards/margins": 1.7923178672790527, + "rewards/rejected": -3.7307403087615967, + "step": 33 + }, + { + "epoch": 0.00950650076890815, + "grad_norm": 44.83487292747772, + "learning_rate": 9.851478631379982e-07, + "logits/chosen": -1.5287456512451172, + "logits/rejected": -1.604817509651184, + "logps/chosen": -1.2651934623718262, + "logps/rejected": -1.606818675994873, + "loss": 1.2006, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.1629838943481445, + "rewards/margins": 0.8540627360343933, + "rewards/rejected": -4.0170464515686035, + "step": 34 + }, + { + "epoch": 0.009786103732699567, + "grad_norm": 33.18197115037115, + "learning_rate": 9.82962913144534e-07, + "logits/chosen": -0.9402756094932556, + "logits/rejected": -1.3055014610290527, + "logps/chosen": -1.1110124588012695, + "logps/rejected": -1.350355625152588, + "loss": 1.2388, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.777531147003174, + "rewards/margins": 0.5983579158782959, + "rewards/rejected": -3.3758890628814697, + "step": 35 + }, + { + "epoch": 0.010065706696490984, + "grad_norm": 28.70546678811581, + "learning_rate": 9.806308479691594e-07, + "logits/chosen": -1.2600600719451904, + "logits/rejected": -1.17176353931427, + "logps/chosen": -0.9965181350708008, + "logps/rejected": -1.3181788921356201, + "loss": 1.4099, + "rewards/accuracies": 0.5, + "rewards/chosen": -2.491295337677002, + "rewards/margins": 0.8041520118713379, + "rewards/rejected": -3.29544734954834, + "step": 36 + }, + { + "epoch": 0.010345309660282399, + "grad_norm": 21.048051889450377, + "learning_rate": 9.781523779815178e-07, + "logits/chosen": -0.8446118831634521, + "logits/rejected": -0.9734996557235718, + "logps/chosen": -1.013080358505249, + "logps/rejected": -1.1583538055419922, + "loss": 1.5673, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.532700777053833, + "rewards/margins": 0.3631836771965027, + "rewards/rejected": -2.8958845138549805, + "step": 37 + }, + { + "epoch": 0.010624912624073815, + "grad_norm": 23.549736337048298, + "learning_rate": 9.755282581475767e-07, + "logits/chosen": -0.772028386592865, + "logits/rejected": -0.821832001209259, + "logps/chosen": -1.1050798892974854, + "logps/rejected": -1.126805305480957, + "loss": 1.6001, + "rewards/accuracies": 0.5, + "rewards/chosen": -2.762699842453003, + "rewards/margins": 0.0543133020401001, + "rewards/rejected": -2.8170132637023926, + "step": 38 + }, + { + "epoch": 0.010904515587865232, + "grad_norm": 31.678751895679902, + "learning_rate": 9.727592877996584e-07, + "logits/chosen": -1.2923513650894165, + "logits/rejected": -0.9500499963760376, + "logps/chosen": -1.3736703395843506, + "logps/rejected": -1.8005601167678833, + "loss": 1.3401, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.434175968170166, + "rewards/margins": 1.0672242641448975, + "rewards/rejected": -4.501400470733643, + "step": 39 + }, + { + "epoch": 0.011184118551656647, + "grad_norm": 42.14350999647913, + "learning_rate": 9.698463103929541e-07, + "logits/chosen": -0.9315403699874878, + "logits/rejected": -0.9749379754066467, + "logps/chosen": -1.1820948123931885, + "logps/rejected": -1.1733828783035278, + "loss": 1.1077, + "rewards/accuracies": 0.5, + "rewards/chosen": -2.9552369117736816, + "rewards/margins": -0.021779626607894897, + "rewards/rejected": -2.933457374572754, + "step": 40 + }, + { + "epoch": 0.011463721515448064, + "grad_norm": 28.8262776267842, + "learning_rate": 9.667902132486008e-07, + "logits/chosen": -1.2946910858154297, + "logits/rejected": -1.3090943098068237, + "logps/chosen": -1.1675629615783691, + "logps/rejected": -1.324582576751709, + "loss": 1.4377, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.918907403945923, + "rewards/margins": 0.39254891872406006, + "rewards/rejected": -3.3114564418792725, + "step": 41 + }, + { + "epoch": 0.011743324479239479, + "grad_norm": 106.3933529697075, + "learning_rate": 9.635919272833937e-07, + "logits/chosen": -0.8468477725982666, + "logits/rejected": -0.87651127576828, + "logps/chosen": -1.5985316038131714, + "logps/rejected": -1.475963830947876, + "loss": 1.5327, + "rewards/accuracies": 0.25, + "rewards/chosen": -3.9963290691375732, + "rewards/margins": -0.3064194321632385, + "rewards/rejected": -3.6899096965789795, + "step": 42 + }, + { + "epoch": 0.012022927443030896, + "grad_norm": 16.873173603232406, + "learning_rate": 9.602524267262202e-07, + "logits/chosen": -0.6168065667152405, + "logits/rejected": -0.7960647940635681, + "logps/chosen": -1.0298948287963867, + "logps/rejected": -1.0422132015228271, + "loss": 1.0135, + "rewards/accuracies": 0.25, + "rewards/chosen": -2.574737310409546, + "rewards/margins": 0.030795753002166748, + "rewards/rejected": -2.6055331230163574, + "step": 43 + }, + { + "epoch": 0.012302530406822313, + "grad_norm": 22.346008825908783, + "learning_rate": 9.567727288213004e-07, + "logits/chosen": -0.8090029954910278, + "logits/rejected": -0.7225205898284912, + "logps/chosen": -1.0612010955810547, + "logps/rejected": -1.0335237979888916, + "loss": 1.4244, + "rewards/accuracies": 0.25, + "rewards/chosen": -2.6530027389526367, + "rewards/margins": -0.06919330358505249, + "rewards/rejected": -2.5838096141815186, + "step": 44 + }, + { + "epoch": 0.012582133370613728, + "grad_norm": 23.528648829320613, + "learning_rate": 9.531538935183249e-07, + "logits/chosen": -1.1082680225372314, + "logits/rejected": -1.311267614364624, + "logps/chosen": -1.0698087215423584, + "logps/rejected": -1.3603800535202026, + "loss": 1.1636, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.6745219230651855, + "rewards/margins": 0.7264281511306763, + "rewards/rejected": -3.4009499549865723, + "step": 45 + }, + { + "epoch": 0.012861736334405145, + "grad_norm": 36.88944148482263, + "learning_rate": 9.493970231495834e-07, + "logits/chosen": -0.9574987292289734, + "logits/rejected": -1.4005671739578247, + "logps/chosen": -1.1250247955322266, + "logps/rejected": -1.6039255857467651, + "loss": 1.3524, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.8125619888305664, + "rewards/margins": 1.1972520351409912, + "rewards/rejected": -4.009814262390137, + "step": 46 + }, + { + "epoch": 0.013141339298196561, + "grad_norm": 34.07967217969598, + "learning_rate": 9.455032620941839e-07, + "logits/chosen": -0.575081467628479, + "logits/rejected": -1.0301098823547363, + "logps/chosen": -0.6966181993484497, + "logps/rejected": -0.7595060467720032, + "loss": 1.1816, + "rewards/accuracies": 0.75, + "rewards/chosen": -1.7415456771850586, + "rewards/margins": 0.15721949934959412, + "rewards/rejected": -1.8987650871276855, + "step": 47 + }, + { + "epoch": 0.013420942261987976, + "grad_norm": 37.33338028904019, + "learning_rate": 9.414737964294634e-07, + "logits/chosen": -0.48685717582702637, + "logits/rejected": -0.9498811960220337, + "logps/chosen": -0.8811389207839966, + "logps/rejected": -1.4033122062683105, + "loss": 0.9144, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.202847480773926, + "rewards/margins": 1.3054332733154297, + "rewards/rejected": -3.5082805156707764, + "step": 48 + }, + { + "epoch": 0.013700545225779393, + "grad_norm": 20.112352494523353, + "learning_rate": 9.373098535696978e-07, + "logits/chosen": -0.9682779908180237, + "logits/rejected": -1.1488547325134277, + "logps/chosen": -1.1651062965393066, + "logps/rejected": -1.3421710729599, + "loss": 1.5188, + "rewards/accuracies": 0.5, + "rewards/chosen": -2.9127655029296875, + "rewards/margins": 0.4426620602607727, + "rewards/rejected": -3.3554277420043945, + "step": 49 + }, + { + "epoch": 0.01398014818957081, + "grad_norm": 33.55406806691841, + "learning_rate": 9.330127018922193e-07, + "logits/chosen": -1.2889659404754639, + "logits/rejected": -0.9354056119918823, + "logps/chosen": -1.3529675006866455, + "logps/rejected": -1.1981086730957031, + "loss": 1.645, + "rewards/accuracies": 0.5, + "rewards/chosen": -3.3824188709259033, + "rewards/margins": -0.38714712858200073, + "rewards/rejected": -2.995271682739258, + "step": 50 + }, + { + "epoch": 0.01398014818957081, + "eval_logits/chosen": -1.184069275856018, + "eval_logits/rejected": -1.346756100654602, + "eval_logps/chosen": -1.1177828311920166, + "eval_logps/rejected": -1.4929856061935425, + "eval_loss": 1.2563124895095825, + "eval_rewards/accuracies": 0.7027027010917664, + "eval_rewards/chosen": -2.794457197189331, + "eval_rewards/margins": 0.9380068182945251, + "eval_rewards/rejected": -3.73246431350708, + "eval_runtime": 345.1606, + "eval_samples_per_second": 3.847, + "eval_steps_per_second": 0.322, + "step": 50 + }, + { + "epoch": 0.014259751153362225, + "grad_norm": 22.5880454834462, + "learning_rate": 9.285836503510562e-07, + "logits/chosen": -0.8200809955596924, + "logits/rejected": -0.8582978248596191, + "logps/chosen": -1.1703073978424072, + "logps/rejected": -1.458601951599121, + "loss": 0.9907, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.9257683753967285, + "rewards/margins": 0.7207364439964294, + "rewards/rejected": -3.6465048789978027, + "step": 51 + }, + { + "epoch": 0.014539354117153642, + "grad_norm": 45.098078544435936, + "learning_rate": 9.240240480782129e-07, + "logits/chosen": -0.7370530962944031, + "logits/rejected": -1.368110179901123, + "logps/chosen": -0.9416132569313049, + "logps/rejected": -1.0832104682922363, + "loss": 1.697, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.3540329933166504, + "rewards/margins": 0.35399293899536133, + "rewards/rejected": -2.708026170730591, + "step": 52 + }, + { + "epoch": 0.014818957080945059, + "grad_norm": 101.10764848570363, + "learning_rate": 9.19335283972712e-07, + "logits/chosen": -0.5560005903244019, + "logits/rejected": -0.7855780124664307, + "logps/chosen": -1.0373799800872803, + "logps/rejected": -1.0468213558197021, + "loss": 1.3264, + "rewards/accuracies": 0.5, + "rewards/chosen": -2.5934500694274902, + "rewards/margins": 0.023603498935699463, + "rewards/rejected": -2.617053508758545, + "step": 53 + }, + { + "epoch": 0.015098560044736474, + "grad_norm": 64.45755933217201, + "learning_rate": 9.145187862775208e-07, + "logits/chosen": -1.421772837638855, + "logits/rejected": -1.4179284572601318, + "logps/chosen": -1.214949607849121, + "logps/rejected": -2.0096287727355957, + "loss": 1.2303, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.0373740196228027, + "rewards/margins": 1.9866979122161865, + "rewards/rejected": -5.02407169342041, + "step": 54 + }, + { + "epoch": 0.01537816300852789, + "grad_norm": 24.722247357252545, + "learning_rate": 9.095760221444959e-07, + "logits/chosen": -1.1340514421463013, + "logits/rejected": -1.4643547534942627, + "logps/chosen": -1.0777888298034668, + "logps/rejected": -1.2248834371566772, + "loss": 1.163, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.694471836090088, + "rewards/margins": 0.3677366375923157, + "rewards/rejected": -3.062208652496338, + "step": 55 + }, + { + "epoch": 0.015657765972319306, + "grad_norm": 88.82834717216237, + "learning_rate": 9.045084971874737e-07, + "logits/chosen": -1.0740933418273926, + "logits/rejected": -0.9167161583900452, + "logps/chosen": -1.2093803882598877, + "logps/rejected": -1.6125764846801758, + "loss": 1.2562, + "rewards/accuracies": 0.5, + "rewards/chosen": -3.023451089859009, + "rewards/margins": 1.0079903602600098, + "rewards/rejected": -4.031441688537598, + "step": 56 + }, + { + "epoch": 0.015937368936110724, + "grad_norm": 18.31655486222315, + "learning_rate": 8.993177550236463e-07, + "logits/chosen": -0.6805390119552612, + "logits/rejected": -0.7955325841903687, + "logps/chosen": -1.2940752506256104, + "logps/rejected": -2.059095859527588, + "loss": 1.2213, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.2351880073547363, + "rewards/margins": 1.912551760673523, + "rewards/rejected": -5.147739410400391, + "step": 57 + }, + { + "epoch": 0.01621697189990214, + "grad_norm": 22.784684254467784, + "learning_rate": 8.940053768033608e-07, + "logits/chosen": -0.5054349303245544, + "logits/rejected": -1.086111068725586, + "logps/chosen": -0.8294824957847595, + "logps/rejected": -1.2600980997085571, + "loss": 1.1366, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.0737061500549316, + "rewards/margins": 1.0765390396118164, + "rewards/rejected": -3.150245189666748, + "step": 58 + }, + { + "epoch": 0.016496574863693554, + "grad_norm": 24.23188699617836, + "learning_rate": 8.885729807284854e-07, + "logits/chosen": -0.9133630990982056, + "logits/rejected": -1.0163180828094482, + "logps/chosen": -1.049289345741272, + "logps/rejected": -1.134240984916687, + "loss": 1.0682, + "rewards/accuracies": 0.5, + "rewards/chosen": -2.623223304748535, + "rewards/margins": 0.21237897872924805, + "rewards/rejected": -2.8356025218963623, + "step": 59 + }, + { + "epoch": 0.016776177827484973, + "grad_norm": 28.93680920583823, + "learning_rate": 8.83022221559489e-07, + "logits/chosen": -1.0600225925445557, + "logits/rejected": -1.2378225326538086, + "logps/chosen": -1.2599135637283325, + "logps/rejected": -1.3674981594085693, + "loss": 0.9747, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.1497838497161865, + "rewards/margins": 0.2689616084098816, + "rewards/rejected": -3.418745517730713, + "step": 60 + }, + { + "epoch": 0.017055780791276388, + "grad_norm": 29.885312194374713, + "learning_rate": 8.773547901113861e-07, + "logits/chosen": -0.7806551456451416, + "logits/rejected": -1.628957748413086, + "logps/chosen": -0.7070074677467346, + "logps/rejected": -1.1015799045562744, + "loss": 0.7853, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.7675187587738037, + "rewards/margins": 0.9864310026168823, + "rewards/rejected": -2.7539496421813965, + "step": 61 + }, + { + "epoch": 0.017335383755067803, + "grad_norm": 24.972205075082606, + "learning_rate": 8.71572412738697e-07, + "logits/chosen": -1.0657659769058228, + "logits/rejected": -1.4624931812286377, + "logps/chosen": -1.1949583292007446, + "logps/rejected": -1.8615357875823975, + "loss": 0.631, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.987395763397217, + "rewards/margins": 1.6664433479309082, + "rewards/rejected": -4.653839111328125, + "step": 62 + }, + { + "epoch": 0.01761498671885922, + "grad_norm": 21.933003035104587, + "learning_rate": 8.656768508095852e-07, + "logits/chosen": -1.1191598176956177, + "logits/rejected": -1.2448678016662598, + "logps/chosen": -1.0887205600738525, + "logps/rejected": -1.6507292985916138, + "loss": 0.887, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.7218017578125, + "rewards/margins": 1.4050216674804688, + "rewards/rejected": -4.126823425292969, + "step": 63 + }, + { + "epoch": 0.017894589682650636, + "grad_norm": 24.060712897137755, + "learning_rate": 8.596699001693255e-07, + "logits/chosen": -0.798321008682251, + "logits/rejected": -1.0583261251449585, + "logps/chosen": -0.7978274822235107, + "logps/rejected": -1.1801316738128662, + "loss": 1.3542, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.9945688247680664, + "rewards/margins": 0.9557601809501648, + "rewards/rejected": -2.950329065322876, + "step": 64 + }, + { + "epoch": 0.01817419264644205, + "grad_norm": 35.64070939669877, + "learning_rate": 8.535533905932737e-07, + "logits/chosen": -0.9407416582107544, + "logits/rejected": -1.2474677562713623, + "logps/chosen": -1.1173073053359985, + "logps/rejected": -1.4939210414886475, + "loss": 1.1944, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.7932682037353516, + "rewards/margins": 0.9415344595909119, + "rewards/rejected": -3.734802722930908, + "step": 65 + }, + { + "epoch": 0.01845379561023347, + "grad_norm": 23.204374003534983, + "learning_rate": 8.473291852294986e-07, + "logits/chosen": -0.7950379252433777, + "logits/rejected": -1.1784076690673828, + "logps/chosen": -0.8873813152313232, + "logps/rejected": -1.0771222114562988, + "loss": 1.291, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.2184531688690186, + "rewards/margins": 0.4743524193763733, + "rewards/rejected": -2.692805767059326, + "step": 66 + }, + { + "epoch": 0.018733398574024885, + "grad_norm": 22.342402065645, + "learning_rate": 8.409991800312492e-07, + "logits/chosen": -0.6992224454879761, + "logits/rejected": -1.0948431491851807, + "logps/chosen": -1.1709400415420532, + "logps/rejected": -1.746230959892273, + "loss": 1.0885, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.9273500442504883, + "rewards/margins": 1.4382274150848389, + "rewards/rejected": -4.365577220916748, + "step": 67 + }, + { + "epoch": 0.0190130015378163, + "grad_norm": 26.635960985863385, + "learning_rate": 8.34565303179429e-07, + "logits/chosen": -0.9120014905929565, + "logits/rejected": -1.2207849025726318, + "logps/chosen": -1.022168755531311, + "logps/rejected": -1.3500474691390991, + "loss": 1.0317, + "rewards/accuracies": 0.5, + "rewards/chosen": -2.555421829223633, + "rewards/margins": 0.819696843624115, + "rewards/rejected": -3.3751187324523926, + "step": 68 + }, + { + "epoch": 0.01929260450160772, + "grad_norm": 216.32442664714605, + "learning_rate": 8.280295144952536e-07, + "logits/chosen": -0.9009850025177002, + "logits/rejected": -0.9338705539703369, + "logps/chosen": -1.7400050163269043, + "logps/rejected": -3.0889453887939453, + "loss": 1.6279, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.350012302398682, + "rewards/margins": 3.3723511695861816, + "rewards/rejected": -7.722363471984863, + "step": 69 + }, + { + "epoch": 0.019572207465399134, + "grad_norm": 33.24100380368566, + "learning_rate": 8.213938048432696e-07, + "logits/chosen": -1.1465697288513184, + "logits/rejected": -1.5220608711242676, + "logps/chosen": -1.5942764282226562, + "logps/rejected": -2.3567376136779785, + "loss": 0.9686, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.9856910705566406, + "rewards/margins": 1.9061533212661743, + "rewards/rejected": -5.891844272613525, + "step": 70 + }, + { + "epoch": 0.01985181042919055, + "grad_norm": 17.831061474851452, + "learning_rate": 8.146601955249187e-07, + "logits/chosen": -0.7144116163253784, + "logits/rejected": -1.260697603225708, + "logps/chosen": -0.7787595987319946, + "logps/rejected": -1.2630029916763306, + "loss": 1.2457, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.9468989372253418, + "rewards/margins": 1.2106084823608398, + "rewards/rejected": -3.1575074195861816, + "step": 71 + }, + { + "epoch": 0.020131413392981967, + "grad_norm": 32.219733178240375, + "learning_rate": 8.07830737662829e-07, + "logits/chosen": -1.14467191696167, + "logits/rejected": -0.7406169176101685, + "logps/chosen": -1.5345901250839233, + "logps/rejected": -1.2437353134155273, + "loss": 1.2542, + "rewards/accuracies": 0.25, + "rewards/chosen": -3.836475372314453, + "rewards/margins": -0.7271367907524109, + "rewards/rejected": -3.1093382835388184, + "step": 72 + }, + { + "epoch": 0.020411016356773382, + "grad_norm": 28.263777851446193, + "learning_rate": 8.009075115760242e-07, + "logits/chosen": -0.8964699506759644, + "logits/rejected": -1.3684449195861816, + "logps/chosen": -0.913366436958313, + "logps/rejected": -1.5431159734725952, + "loss": 1.0506, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.2834160327911377, + "rewards/margins": 1.5743739604949951, + "rewards/rejected": -3.857790231704712, + "step": 73 + }, + { + "epoch": 0.020690619320564797, + "grad_norm": 33.62623441801084, + "learning_rate": 7.938926261462365e-07, + "logits/chosen": -1.5208332538604736, + "logits/rejected": -1.4947750568389893, + "logps/chosen": -1.6027369499206543, + "logps/rejected": -2.2491843700408936, + "loss": 1.1593, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.006842136383057, + "rewards/margins": 1.6161189079284668, + "rewards/rejected": -5.622961044311523, + "step": 74 + }, + { + "epoch": 0.020970222284356216, + "grad_norm": 19.73073540290438, + "learning_rate": 7.86788218175523e-07, + "logits/chosen": -0.8135213851928711, + "logits/rejected": -1.1303890943527222, + "logps/chosen": -1.1947009563446045, + "logps/rejected": -1.178377628326416, + "loss": 1.0676, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.986752510070801, + "rewards/margins": -0.04080846905708313, + "rewards/rejected": -2.945943832397461, + "step": 75 + }, + { + "epoch": 0.02124982524814763, + "grad_norm": 30.725692444381682, + "learning_rate": 7.795964517353733e-07, + "logits/chosen": -0.5928746461868286, + "logits/rejected": -1.3386216163635254, + "logps/chosen": -1.1336582899093628, + "logps/rejected": -2.486417770385742, + "loss": 1.2958, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.8341457843780518, + "rewards/margins": 3.3818984031677246, + "rewards/rejected": -6.2160444259643555, + "step": 76 + }, + { + "epoch": 0.021529428211939046, + "grad_norm": 19.993312804225585, + "learning_rate": 7.723195175075135e-07, + "logits/chosen": -1.0590918064117432, + "logits/rejected": -1.4984095096588135, + "logps/chosen": -1.341080904006958, + "logps/rejected": -2.226268768310547, + "loss": 0.7399, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.3527023792266846, + "rewards/margins": 2.212970018386841, + "rewards/rejected": -5.565672397613525, + "step": 77 + }, + { + "epoch": 0.021809031175730464, + "grad_norm": 27.58218498128165, + "learning_rate": 7.649596321166024e-07, + "logits/chosen": -0.903189480304718, + "logits/rejected": -1.4045133590698242, + "logps/chosen": -1.2869590520858765, + "logps/rejected": -2.1190972328186035, + "loss": 1.1142, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.217397689819336, + "rewards/margins": 2.080345392227173, + "rewards/rejected": -5.29774284362793, + "step": 78 + }, + { + "epoch": 0.02208863413952188, + "grad_norm": 33.76140161445492, + "learning_rate": 7.575190374550271e-07, + "logits/chosen": -1.1929422616958618, + "logits/rejected": -1.402430534362793, + "logps/chosen": -1.4159026145935059, + "logps/rejected": -1.8726705312728882, + "loss": 1.0192, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.5397567749023438, + "rewards/margins": 1.1419193744659424, + "rewards/rejected": -4.681675910949707, + "step": 79 + }, + { + "epoch": 0.022368237103313295, + "grad_norm": 23.027800645867977, + "learning_rate": 7.5e-07, + "logits/chosen": -0.6945698857307434, + "logits/rejected": -1.40185546875, + "logps/chosen": -1.4425044059753418, + "logps/rejected": -2.2865071296691895, + "loss": 0.9812, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.6062612533569336, + "rewards/margins": 2.110006809234619, + "rewards/rejected": -5.7162675857543945, + "step": 80 + }, + { + "epoch": 0.02264784006710471, + "grad_norm": 38.27741144565889, + "learning_rate": 7.424048101231686e-07, + "logits/chosen": -1.2587817907333374, + "logits/rejected": -1.0885329246520996, + "logps/chosen": -1.2346124649047852, + "logps/rejected": -1.6860368251800537, + "loss": 0.7012, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.086531162261963, + "rewards/margins": 1.1285609006881714, + "rewards/rejected": -4.215092182159424, + "step": 81 + }, + { + "epoch": 0.022927443030896128, + "grad_norm": 90.96248649883246, + "learning_rate": 7.347357813929454e-07, + "logits/chosen": -1.2520544528961182, + "logits/rejected": -1.0464751720428467, + "logps/chosen": -2.1459150314331055, + "logps/rejected": -2.046426296234131, + "loss": 1.5015, + "rewards/accuracies": 0.5, + "rewards/chosen": -5.364788055419922, + "rewards/margins": -0.24872243404388428, + "rewards/rejected": -5.116065502166748, + "step": 82 + }, + { + "epoch": 0.023207045994687543, + "grad_norm": 100.9433648678187, + "learning_rate": 7.269952498697734e-07, + "logits/chosen": -1.059312105178833, + "logits/rejected": -0.9108483791351318, + "logps/chosen": -1.4121086597442627, + "logps/rejected": -1.6452100276947021, + "loss": 1.3536, + "rewards/accuracies": 0.5, + "rewards/chosen": -3.530271530151367, + "rewards/margins": 0.5827533602714539, + "rewards/rejected": -4.113024711608887, + "step": 83 + }, + { + "epoch": 0.023486648958478958, + "grad_norm": 173.9823816167412, + "learning_rate": 7.191855733945386e-07, + "logits/chosen": -0.9303680658340454, + "logits/rejected": -1.1252155303955078, + "logps/chosen": -1.284766674041748, + "logps/rejected": -1.3653523921966553, + "loss": 2.4014, + "rewards/accuracies": 0.5, + "rewards/chosen": -3.211916923522949, + "rewards/margins": 0.2014639973640442, + "rewards/rejected": -3.4133810997009277, + "step": 84 + }, + { + "epoch": 0.023766251922270377, + "grad_norm": 28.477804453675674, + "learning_rate": 7.113091308703497e-07, + "logits/chosen": -0.7400285601615906, + "logits/rejected": -0.847143292427063, + "logps/chosen": -1.8564958572387695, + "logps/rejected": -2.197305917739868, + "loss": 1.3282, + "rewards/accuracies": 0.5, + "rewards/chosen": -4.641240119934082, + "rewards/margins": 0.8520248532295227, + "rewards/rejected": -5.493264675140381, + "step": 85 + }, + { + "epoch": 0.024045854886061792, + "grad_norm": 158.57710825565218, + "learning_rate": 7.033683215379002e-07, + "logits/chosen": -1.1289122104644775, + "logits/rejected": -1.7158691883087158, + "logps/chosen": -1.2116670608520508, + "logps/rejected": -2.2744312286376953, + "loss": 1.1088, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.029167652130127, + "rewards/margins": 2.6569106578826904, + "rewards/rejected": -5.686078071594238, + "step": 86 + }, + { + "epoch": 0.024325457849853207, + "grad_norm": 14.96513597899858, + "learning_rate": 6.953655642446367e-07, + "logits/chosen": -0.6412773728370667, + "logits/rejected": -1.4269566535949707, + "logps/chosen": -0.985595703125, + "logps/rejected": -2.8099608421325684, + "loss": 0.6137, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.4639892578125, + "rewards/margins": 4.5609130859375, + "rewards/rejected": -7.02490234375, + "step": 87 + }, + { + "epoch": 0.024605060813644625, + "grad_norm": 69.31853470031655, + "learning_rate": 6.87303296707956e-07, + "logits/chosen": -1.252925157546997, + "logits/rejected": -1.3233586549758911, + "logps/chosen": -1.3767462968826294, + "logps/rejected": -1.7665659189224243, + "loss": 1.2815, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.441865921020508, + "rewards/margins": 0.9745489358901978, + "rewards/rejected": -4.416414737701416, + "step": 88 + }, + { + "epoch": 0.02488466377743604, + "grad_norm": 27.097255688460947, + "learning_rate": 6.7918397477265e-07, + "logits/chosen": -0.8277978301048279, + "logits/rejected": -1.0574264526367188, + "logps/chosen": -1.1671627759933472, + "logps/rejected": -2.2953076362609863, + "loss": 1.0056, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.9179069995880127, + "rewards/margins": 2.820362091064453, + "rewards/rejected": -5.738268852233887, + "step": 89 + }, + { + "epoch": 0.025164266741227456, + "grad_norm": 52.839387510999444, + "learning_rate": 6.710100716628344e-07, + "logits/chosen": -1.40108323097229, + "logits/rejected": -1.4667222499847412, + "logps/chosen": -1.4003353118896484, + "logps/rejected": -1.6744577884674072, + "loss": 1.3688, + "rewards/accuracies": 0.5, + "rewards/chosen": -3.500838279724121, + "rewards/margins": 0.6853060126304626, + "rewards/rejected": -4.1861443519592285, + "step": 90 + }, + { + "epoch": 0.025443869705018874, + "grad_norm": 21.925148847234777, + "learning_rate": 6.627840772285783e-07, + "logits/chosen": -1.0636601448059082, + "logits/rejected": -1.0789152383804321, + "logps/chosen": -1.1292977333068848, + "logps/rejected": -1.3410239219665527, + "loss": 1.4889, + "rewards/accuracies": 0.5, + "rewards/chosen": -2.823244094848633, + "rewards/margins": 0.5293155908584595, + "rewards/rejected": -3.3525595664978027, + "step": 91 + }, + { + "epoch": 0.02572347266881029, + "grad_norm": 26.167972640340828, + "learning_rate": 6.545084971874736e-07, + "logits/chosen": -0.6311432719230652, + "logits/rejected": -1.1260812282562256, + "logps/chosen": -1.1044063568115234, + "logps/rejected": -1.6531506776809692, + "loss": 1.2341, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.7610158920288086, + "rewards/margins": 1.3718609809875488, + "rewards/rejected": -4.132876873016357, + "step": 92 + }, + { + "epoch": 0.026003075632601704, + "grad_norm": 19.955245570389728, + "learning_rate": 6.461858523613684e-07, + "logits/chosen": -0.9683464765548706, + "logits/rejected": -0.9185085296630859, + "logps/chosen": -0.9664384722709656, + "logps/rejected": -1.6319332122802734, + "loss": 0.8303, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.4160962104797363, + "rewards/margins": 1.6637368202209473, + "rewards/rejected": -4.079833030700684, + "step": 93 + }, + { + "epoch": 0.026282678596393123, + "grad_norm": 38.679628570882045, + "learning_rate": 6.378186779084995e-07, + "logits/chosen": -0.8759285807609558, + "logits/rejected": -0.6558728218078613, + "logps/chosen": -1.1541013717651367, + "logps/rejected": -1.992755651473999, + "loss": 0.598, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.885253429412842, + "rewards/margins": 2.0966360569000244, + "rewards/rejected": -4.981889247894287, + "step": 94 + }, + { + "epoch": 0.026562281560184538, + "grad_norm": 43.75173307796015, + "learning_rate": 6.294095225512604e-07, + "logits/chosen": -0.9964724779129028, + "logits/rejected": -0.7991459369659424, + "logps/chosen": -1.0949715375900269, + "logps/rejected": -1.5087236166000366, + "loss": 1.0437, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.737428665161133, + "rewards/margins": 1.034380316734314, + "rewards/rejected": -3.7718091011047363, + "step": 95 + }, + { + "epoch": 0.026841884523975953, + "grad_norm": 22.420799786964153, + "learning_rate": 6.209609477998338e-07, + "logits/chosen": -0.4085179269313812, + "logits/rejected": -0.7962716221809387, + "logps/chosen": -1.0953500270843506, + "logps/rejected": -1.046759843826294, + "loss": 1.5788, + "rewards/accuracies": 0.5, + "rewards/chosen": -2.738374710083008, + "rewards/margins": -0.12147501111030579, + "rewards/rejected": -2.6168999671936035, + "step": 96 + }, + { + "epoch": 0.02712148748776737, + "grad_norm": 28.1032386388583, + "learning_rate": 6.124755271719326e-07, + "logits/chosen": -0.96661376953125, + "logits/rejected": -1.131087064743042, + "logps/chosen": -1.2935943603515625, + "logps/rejected": -1.4725990295410156, + "loss": 1.1007, + "rewards/accuracies": 0.5, + "rewards/chosen": -3.233985662460327, + "rewards/margins": 0.4475119113922119, + "rewards/rejected": -3.681497573852539, + "step": 97 + }, + { + "epoch": 0.027401090451558786, + "grad_norm": 51.25959864452938, + "learning_rate": 6.039558454088795e-07, + "logits/chosen": -1.0819038152694702, + "logits/rejected": -1.3961520195007324, + "logps/chosen": -1.1702494621276855, + "logps/rejected": -2.0412793159484863, + "loss": 0.8748, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.925623655319214, + "rewards/margins": 2.1775741577148438, + "rewards/rejected": -5.103198051452637, + "step": 98 + }, + { + "epoch": 0.0276806934153502, + "grad_norm": 44.86957712355451, + "learning_rate": 5.954044976882723e-07, + "logits/chosen": -0.4054487347602844, + "logits/rejected": -0.9896742105484009, + "logps/chosen": -0.8635058999061584, + "logps/rejected": -1.2418280839920044, + "loss": 1.0457, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.1587648391723633, + "rewards/margins": 0.9458054900169373, + "rewards/rejected": -3.1045703887939453, + "step": 99 + }, + { + "epoch": 0.02796029637914162, + "grad_norm": 28.90572894211588, + "learning_rate": 5.868240888334652e-07, + "logits/chosen": -0.7194627523422241, + "logits/rejected": -1.274187445640564, + "logps/chosen": -0.822409987449646, + "logps/rejected": -1.5885546207427979, + "loss": 0.8722, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.0560250282287598, + "rewards/margins": 1.9153616428375244, + "rewards/rejected": -3.971386432647705, + "step": 100 + }, + { + "epoch": 0.02796029637914162, + "eval_logits/chosen": -1.2195580005645752, + "eval_logits/rejected": -1.3816862106323242, + "eval_logps/chosen": -1.2307511568069458, + "eval_logps/rejected": -1.8937082290649414, + "eval_loss": 1.0619382858276367, + "eval_rewards/accuracies": 0.7319819927215576, + "eval_rewards/chosen": -3.0768775939941406, + "eval_rewards/margins": 1.6573928594589233, + "eval_rewards/rejected": -4.7342705726623535, + "eval_runtime": 343.8898, + "eval_samples_per_second": 3.862, + "eval_steps_per_second": 0.323, + "step": 100 + }, + { + "epoch": 0.028239899342933035, + "grad_norm": 23.014771898473285, + "learning_rate": 5.782172325201155e-07, + "logits/chosen": -1.019005298614502, + "logits/rejected": -1.587357759475708, + "logps/chosen": -0.8991877436637878, + "logps/rejected": -2.31307053565979, + "loss": 1.1455, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.247969388961792, + "rewards/margins": 3.5347068309783936, + "rewards/rejected": -5.7826762199401855, + "step": 101 + }, + { + "epoch": 0.02851950230672445, + "grad_norm": 129.77518240929777, + "learning_rate": 5.695865504800327e-07, + "logits/chosen": -0.8380779027938843, + "logits/rejected": -1.340008020401001, + "logps/chosen": -1.1220433712005615, + "logps/rejected": -1.7773716449737549, + "loss": 0.8694, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.8051083087921143, + "rewards/margins": 1.6383205652236938, + "rewards/rejected": -4.443428993225098, + "step": 102 + }, + { + "epoch": 0.02879910527051587, + "grad_norm": 172.3569574432423, + "learning_rate": 5.609346717025737e-07, + "logits/chosen": -1.052437424659729, + "logits/rejected": -1.4004924297332764, + "logps/chosen": -0.9889116287231445, + "logps/rejected": -1.5602869987487793, + "loss": 1.7629, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.4722790718078613, + "rewards/margins": 1.428438425064087, + "rewards/rejected": -3.9007177352905273, + "step": 103 + }, + { + "epoch": 0.029078708234307284, + "grad_norm": 36.623523960138435, + "learning_rate": 5.522642316338268e-07, + "logits/chosen": -1.1740097999572754, + "logits/rejected": -0.980044424533844, + "logps/chosen": -1.2009364366531372, + "logps/rejected": -1.3514597415924072, + "loss": 1.2361, + "rewards/accuracies": 0.5, + "rewards/chosen": -3.0023412704467773, + "rewards/margins": 0.3763081431388855, + "rewards/rejected": -3.3786492347717285, + "step": 104 + }, + { + "epoch": 0.0293583111980987, + "grad_norm": 77.37683529524608, + "learning_rate": 5.435778713738292e-07, + "logits/chosen": -1.2499001026153564, + "logits/rejected": -1.2681267261505127, + "logps/chosen": -1.7606796026229858, + "logps/rejected": -2.930025577545166, + "loss": 1.1121, + "rewards/accuracies": 0.5, + "rewards/chosen": -4.401699066162109, + "rewards/margins": 2.9233648777008057, + "rewards/rejected": -7.325064182281494, + "step": 105 + }, + { + "epoch": 0.029637914161890117, + "grad_norm": 147.76082371794118, + "learning_rate": 5.348782368720625e-07, + "logits/chosen": -0.7781787514686584, + "logits/rejected": -1.1500039100646973, + "logps/chosen": -1.7407994270324707, + "logps/rejected": -5.4726667404174805, + "loss": 1.2898, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.351998805999756, + "rewards/margins": 9.329668045043945, + "rewards/rejected": -13.68166732788086, + "step": 106 + }, + { + "epoch": 0.029917517125681532, + "grad_norm": 23.97972069856743, + "learning_rate": 5.26167978121472e-07, + "logits/chosen": -0.7871793508529663, + "logits/rejected": -1.1882908344268799, + "logps/chosen": -0.5037890672683716, + "logps/rejected": -1.5715763568878174, + "loss": 0.6191, + "rewards/accuracies": 1.0, + "rewards/chosen": -1.2594726085662842, + "rewards/margins": 2.669468402862549, + "rewards/rejected": -3.928941249847412, + "step": 107 + }, + { + "epoch": 0.030197120089472947, + "grad_norm": 26.15247415705612, + "learning_rate": 5.174497483512505e-07, + "logits/chosen": -1.1062333583831787, + "logits/rejected": -1.6174620389938354, + "logps/chosen": -1.3177549839019775, + "logps/rejected": -1.8563097715377808, + "loss": 0.941, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.2943873405456543, + "rewards/margins": 1.3463870286941528, + "rewards/rejected": -4.640774726867676, + "step": 108 + }, + { + "epoch": 0.030476723053264366, + "grad_norm": 36.12906136492366, + "learning_rate": 5.087262032186418e-07, + "logits/chosen": -1.320471167564392, + "logits/rejected": -1.2250688076019287, + "logps/chosen": -1.1929512023925781, + "logps/rejected": -1.7405405044555664, + "loss": 0.6462, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.9823780059814453, + "rewards/margins": 1.3689732551574707, + "rewards/rejected": -4.351351261138916, + "step": 109 + }, + { + "epoch": 0.03075632601705578, + "grad_norm": 21.26364702784651, + "learning_rate": 5e-07, + "logits/chosen": -0.6797312498092651, + "logits/rejected": -1.517352819442749, + "logps/chosen": -1.253048062324524, + "logps/rejected": -2.614882707595825, + "loss": 0.7045, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.132620334625244, + "rewards/margins": 3.4045865535736084, + "rewards/rejected": -6.537206649780273, + "step": 110 + }, + { + "epoch": 0.031035928980847196, + "grad_norm": 44.22722460589391, + "learning_rate": 4.912737967813582e-07, + "logits/chosen": -0.8700337409973145, + "logits/rejected": -1.606900930404663, + "logps/chosen": -1.6089668273925781, + "logps/rejected": -2.9295287132263184, + "loss": 0.8187, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.022417068481445, + "rewards/margins": 3.3014047145843506, + "rewards/rejected": -7.323822021484375, + "step": 111 + }, + { + "epoch": 0.03131553194463861, + "grad_norm": 76.80253216440869, + "learning_rate": 4.825502516487496e-07, + "logits/chosen": -0.8108619451522827, + "logits/rejected": -1.4683247804641724, + "logps/chosen": -1.2750149965286255, + "logps/rejected": -1.875692367553711, + "loss": 1.1174, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.187537670135498, + "rewards/margins": 1.501693606376648, + "rewards/rejected": -4.689230918884277, + "step": 112 + }, + { + "epoch": 0.03159513490843003, + "grad_norm": 39.5980607890587, + "learning_rate": 4.7383202187852804e-07, + "logits/chosen": -1.134104609489441, + "logits/rejected": -1.3199536800384521, + "logps/chosen": -1.1517322063446045, + "logps/rejected": -2.180983543395996, + "loss": 1.4293, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.879330635070801, + "rewards/margins": 2.5731282234191895, + "rewards/rejected": -5.45245885848999, + "step": 113 + }, + { + "epoch": 0.03187473787222145, + "grad_norm": 159.282123008416, + "learning_rate": 4.6512176312793735e-07, + "logits/chosen": -1.2752604484558105, + "logits/rejected": -1.6413378715515137, + "logps/chosen": -0.9552395343780518, + "logps/rejected": -2.1065025329589844, + "loss": 1.8414, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.38809871673584, + "rewards/margins": 2.878157377243042, + "rewards/rejected": -5.266256332397461, + "step": 114 + }, + { + "epoch": 0.03215434083601286, + "grad_norm": 23.36900105938972, + "learning_rate": 4.5642212862617085e-07, + "logits/chosen": -1.099194049835205, + "logits/rejected": -1.1428616046905518, + "logps/chosen": -1.1194630861282349, + "logps/rejected": -1.9647703170776367, + "loss": 1.211, + "rewards/accuracies": 0.5, + "rewards/chosen": -2.7986578941345215, + "rewards/margins": 2.1132683753967285, + "rewards/rejected": -4.911925792694092, + "step": 115 + }, + { + "epoch": 0.03243394379980428, + "grad_norm": 22.810174582850898, + "learning_rate": 4.477357683661733e-07, + "logits/chosen": -1.2737977504730225, + "logits/rejected": -1.3265190124511719, + "logps/chosen": -1.2955751419067383, + "logps/rejected": -2.3141541481018066, + "loss": 1.1194, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.2389378547668457, + "rewards/margins": 2.546447515487671, + "rewards/rejected": -5.785386085510254, + "step": 116 + }, + { + "epoch": 0.0327135467635957, + "grad_norm": 155.39469755085068, + "learning_rate": 4.390653282974263e-07, + "logits/chosen": -1.007920503616333, + "logits/rejected": -1.2567858695983887, + "logps/chosen": -1.3624416589736938, + "logps/rejected": -1.7886241674423218, + "loss": 1.4514, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.406104326248169, + "rewards/margins": 1.0654561519622803, + "rewards/rejected": -4.471560478210449, + "step": 117 + }, + { + "epoch": 0.03299314972738711, + "grad_norm": 28.639395965705603, + "learning_rate": 4.304134495199674e-07, + "logits/chosen": -0.8960734605789185, + "logits/rejected": -0.5021066069602966, + "logps/chosen": -0.928629994392395, + "logps/rejected": -1.1891580820083618, + "loss": 0.8406, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.3215749263763428, + "rewards/margins": 0.651320219039917, + "rewards/rejected": -2.9728951454162598, + "step": 118 + }, + { + "epoch": 0.03327275269117853, + "grad_norm": 43.81099025759765, + "learning_rate": 4.2178276747988444e-07, + "logits/chosen": -1.010365605354309, + "logits/rejected": -1.6597810983657837, + "logps/chosen": -1.1012680530548096, + "logps/rejected": -3.7689075469970703, + "loss": 1.0053, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.7531704902648926, + "rewards/margins": 6.669097900390625, + "rewards/rejected": -9.422268867492676, + "step": 119 + }, + { + "epoch": 0.033552355654969945, + "grad_norm": 50.65816817248901, + "learning_rate": 4.131759111665348e-07, + "logits/chosen": -0.8199977874755859, + "logits/rejected": -1.105729341506958, + "logps/chosen": -1.1784420013427734, + "logps/rejected": -1.7748521566390991, + "loss": 1.0382, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.9461052417755127, + "rewards/margins": 1.4910252094268799, + "rewards/rejected": -4.437130928039551, + "step": 120 + }, + { + "epoch": 0.03383195861876136, + "grad_norm": 135.269469018394, + "learning_rate": 4.0459550231172755e-07, + "logits/chosen": -1.0090067386627197, + "logits/rejected": -1.2103219032287598, + "logps/chosen": -1.3218321800231934, + "logps/rejected": -1.8139877319335938, + "loss": 1.3587, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.3045804500579834, + "rewards/margins": 1.23038911819458, + "rewards/rejected": -4.534969806671143, + "step": 121 + }, + { + "epoch": 0.034111561582552775, + "grad_norm": 39.54642498391402, + "learning_rate": 3.960441545911204e-07, + "logits/chosen": -0.7739511132240295, + "logits/rejected": -1.1406965255737305, + "logps/chosen": -1.6368045806884766, + "logps/rejected": -1.7438288927078247, + "loss": 1.1795, + "rewards/accuracies": 0.5, + "rewards/chosen": -4.092011451721191, + "rewards/margins": 0.2675609588623047, + "rewards/rejected": -4.359572410583496, + "step": 122 + }, + { + "epoch": 0.034391164546344194, + "grad_norm": 100.27265763472896, + "learning_rate": 3.8752447282806755e-07, + "logits/chosen": -1.0782852172851562, + "logits/rejected": -1.3873653411865234, + "logps/chosen": -1.1215283870697021, + "logps/rejected": -2.216892719268799, + "loss": 1.3417, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.803821086883545, + "rewards/margins": 2.738410472869873, + "rewards/rejected": -5.542231559753418, + "step": 123 + }, + { + "epoch": 0.034670767510135606, + "grad_norm": 58.20614675830375, + "learning_rate": 3.790390522001662e-07, + "logits/chosen": -0.778974711894989, + "logits/rejected": -1.2259283065795898, + "logps/chosen": -1.4575412273406982, + "logps/rejected": -2.0206165313720703, + "loss": 0.7091, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.643852949142456, + "rewards/margins": 1.4076881408691406, + "rewards/rejected": -5.051541328430176, + "step": 124 + }, + { + "epoch": 0.034950370473927024, + "grad_norm": 14.601199122985598, + "learning_rate": 3.7059047744873955e-07, + "logits/chosen": -0.8316655158996582, + "logits/rejected": -1.5047085285186768, + "logps/chosen": -1.2063120603561401, + "logps/rejected": -2.6588189601898193, + "loss": 0.4785, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.015780448913574, + "rewards/margins": 3.6312668323516846, + "rewards/rejected": -6.64704704284668, + "step": 125 + }, + { + "epoch": 0.03522997343771844, + "grad_norm": 27.93055728093027, + "learning_rate": 3.621813220915004e-07, + "logits/chosen": -1.099701166152954, + "logits/rejected": -1.5104920864105225, + "logps/chosen": -1.2890055179595947, + "logps/rejected": -2.336859703063965, + "loss": 0.6663, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.2225139141082764, + "rewards/margins": 2.6196351051330566, + "rewards/rejected": -5.842149257659912, + "step": 126 + }, + { + "epoch": 0.035509576401509854, + "grad_norm": 38.5610788021146, + "learning_rate": 3.5381414763863163e-07, + "logits/chosen": -1.0062170028686523, + "logits/rejected": -1.459620714187622, + "logps/chosen": -1.9963696002960205, + "logps/rejected": -2.577173948287964, + "loss": 1.0607, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.990923881530762, + "rewards/margins": 1.4520111083984375, + "rewards/rejected": -6.442934989929199, + "step": 127 + }, + { + "epoch": 0.03578917936530127, + "grad_norm": 64.49312626407205, + "learning_rate": 3.454915028125263e-07, + "logits/chosen": -0.9179176092147827, + "logits/rejected": -1.1796026229858398, + "logps/chosen": -1.013648271560669, + "logps/rejected": -1.823269248008728, + "loss": 1.0723, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.534120559692383, + "rewards/margins": 2.024052143096924, + "rewards/rejected": -4.558172702789307, + "step": 128 + }, + { + "epoch": 0.03606878232909269, + "grad_norm": 36.00805300468051, + "learning_rate": 3.3721592277142173e-07, + "logits/chosen": -1.0871798992156982, + "logits/rejected": -1.1635856628417969, + "logps/chosen": -1.6440746784210205, + "logps/rejected": -2.153071641921997, + "loss": 1.0645, + "rewards/accuracies": 0.5, + "rewards/chosen": -4.110186576843262, + "rewards/margins": 1.2724924087524414, + "rewards/rejected": -5.382678985595703, + "step": 129 + }, + { + "epoch": 0.0363483852928841, + "grad_norm": 18.299398093476555, + "learning_rate": 3.2898992833716563e-07, + "logits/chosen": -0.852850079536438, + "logits/rejected": -1.2836261987686157, + "logps/chosen": -1.0674067735671997, + "logps/rejected": -2.2412667274475098, + "loss": 0.8613, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.6685171127319336, + "rewards/margins": 2.934649705886841, + "rewards/rejected": -5.603166580200195, + "step": 130 + }, + { + "epoch": 0.03662798825667552, + "grad_norm": 52.07726812332011, + "learning_rate": 3.2081602522734985e-07, + "logits/chosen": -1.0853538513183594, + "logits/rejected": -1.274430274963379, + "logps/chosen": -1.4096601009368896, + "logps/rejected": -1.5677156448364258, + "loss": 1.65, + "rewards/accuracies": 0.5, + "rewards/chosen": -3.5241503715515137, + "rewards/margins": 0.3951385021209717, + "rewards/rejected": -3.9192888736724854, + "step": 131 + }, + { + "epoch": 0.03690759122046694, + "grad_norm": 17.512442562016194, + "learning_rate": 3.1269670329204393e-07, + "logits/chosen": -1.1719293594360352, + "logits/rejected": -0.9959172606468201, + "logps/chosen": -1.1080865859985352, + "logps/rejected": -2.0133416652679443, + "loss": 0.7374, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.770216464996338, + "rewards/margins": 2.2631378173828125, + "rewards/rejected": -5.033354759216309, + "step": 132 + }, + { + "epoch": 0.03718719418425835, + "grad_norm": 684.8675181225219, + "learning_rate": 3.0463443575536317e-07, + "logits/chosen": -1.3861017227172852, + "logits/rejected": -1.1517585515975952, + "logps/chosen": -2.434314250946045, + "logps/rejected": -1.6802737712860107, + "loss": 2.4108, + "rewards/accuracies": 0.25, + "rewards/chosen": -6.085785388946533, + "rewards/margins": -1.8851011991500854, + "rewards/rejected": -4.200684070587158, + "step": 133 + }, + { + "epoch": 0.03746679714804977, + "grad_norm": 203.35746911786714, + "learning_rate": 2.9663167846209996e-07, + "logits/chosen": -0.6332231163978577, + "logits/rejected": -1.2013721466064453, + "logps/chosen": -0.8640561699867249, + "logps/rejected": -1.9115948677062988, + "loss": 1.0114, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.1601402759552, + "rewards/margins": 2.618846893310547, + "rewards/rejected": -4.778987407684326, + "step": 134 + }, + { + "epoch": 0.03774640011184119, + "grad_norm": 48.99403271441584, + "learning_rate": 2.8869086912965036e-07, + "logits/chosen": -1.1275267601013184, + "logits/rejected": -1.376509189605713, + "logps/chosen": -1.562412977218628, + "logps/rejected": -3.6972084045410156, + "loss": 0.8296, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.9060323238372803, + "rewards/margins": 5.3369879722595215, + "rewards/rejected": -9.243021011352539, + "step": 135 + }, + { + "epoch": 0.0380260030756326, + "grad_norm": 27.827124034061338, + "learning_rate": 2.808144266054612e-07, + "logits/chosen": -0.7335802912712097, + "logits/rejected": -1.3153456449508667, + "logps/chosen": -1.1802624464035034, + "logps/rejected": -2.093515157699585, + "loss": 1.0364, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.9506561756134033, + "rewards/margins": 2.2831313610076904, + "rewards/rejected": -5.233787536621094, + "step": 136 + }, + { + "epoch": 0.03830560603942402, + "grad_norm": 41.51238681345992, + "learning_rate": 2.730047501302266e-07, + "logits/chosen": -0.9400544166564941, + "logits/rejected": -1.42076575756073, + "logps/chosen": -1.4838162660598755, + "logps/rejected": -2.123250722885132, + "loss": 1.4503, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.709540843963623, + "rewards/margins": 1.598586082458496, + "rewards/rejected": -5.308126926422119, + "step": 137 + }, + { + "epoch": 0.03858520900321544, + "grad_norm": 22.114098288460713, + "learning_rate": 2.6526421860705473e-07, + "logits/chosen": -0.8485064506530762, + "logits/rejected": -1.127300500869751, + "logps/chosen": -1.4366434812545776, + "logps/rejected": -1.9873528480529785, + "loss": 0.6841, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.591608762741089, + "rewards/margins": 1.3767732381820679, + "rewards/rejected": -4.968381881713867, + "step": 138 + }, + { + "epoch": 0.03886481196700685, + "grad_norm": 20.628330217139027, + "learning_rate": 2.575951898768315e-07, + "logits/chosen": -1.1259043216705322, + "logits/rejected": -1.3020617961883545, + "logps/chosen": -1.4316058158874512, + "logps/rejected": -2.6916065216064453, + "loss": 0.845, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.579014301300049, + "rewards/margins": 3.1500015258789062, + "rewards/rejected": -6.729016304016113, + "step": 139 + }, + { + "epoch": 0.03914441493079827, + "grad_norm": 21.796146073482507, + "learning_rate": 2.500000000000001e-07, + "logits/chosen": -1.3260260820388794, + "logits/rejected": -1.5544166564941406, + "logps/chosen": -2.0802931785583496, + "logps/rejected": -2.217221260070801, + "loss": 1.0337, + "rewards/accuracies": 0.5, + "rewards/chosen": -5.200733661651611, + "rewards/margins": 0.3423195481300354, + "rewards/rejected": -5.543053150177002, + "step": 140 + }, + { + "epoch": 0.039424017894589686, + "grad_norm": 18.878252064796683, + "learning_rate": 2.424809625449729e-07, + "logits/chosen": -0.8346128463745117, + "logits/rejected": -1.0135633945465088, + "logps/chosen": -1.6469535827636719, + "logps/rejected": -2.0207669734954834, + "loss": 0.379, + "rewards/accuracies": 1.0, + "rewards/chosen": -4.11738395690918, + "rewards/margins": 0.9345333576202393, + "rewards/rejected": -5.05191707611084, + "step": 141 + }, + { + "epoch": 0.0397036208583811, + "grad_norm": 15.618943765371364, + "learning_rate": 2.350403678833976e-07, + "logits/chosen": -1.1391526460647583, + "logits/rejected": -1.3611162900924683, + "logps/chosen": -1.1667499542236328, + "logps/rejected": -2.9371497631073, + "loss": 0.363, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.916875123977661, + "rewards/margins": 4.425999164581299, + "rewards/rejected": -7.342874526977539, + "step": 142 + }, + { + "epoch": 0.039983223822172516, + "grad_norm": 67.95024500967284, + "learning_rate": 2.2768048249248644e-07, + "logits/chosen": -0.9159025549888611, + "logits/rejected": -1.5903668403625488, + "logps/chosen": -1.0491520166397095, + "logps/rejected": -3.3000054359436035, + "loss": 0.7062, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.622879981994629, + "rewards/margins": 5.627133846282959, + "rewards/rejected": -8.250014305114746, + "step": 143 + }, + { + "epoch": 0.040262826785963934, + "grad_norm": 24.48650844009346, + "learning_rate": 2.2040354826462664e-07, + "logits/chosen": -0.948284387588501, + "logits/rejected": -1.2763409614562988, + "logps/chosen": -1.060044527053833, + "logps/rejected": -2.7557332515716553, + "loss": 0.4762, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.650111436843872, + "rewards/margins": 4.239221572875977, + "rewards/rejected": -6.8893327713012695, + "step": 144 + }, + { + "epoch": 0.040542429749755346, + "grad_norm": 22.244412776379363, + "learning_rate": 2.1321178182447709e-07, + "logits/chosen": -0.8840318322181702, + "logits/rejected": -0.11764158308506012, + "logps/chosen": -1.242844581604004, + "logps/rejected": -1.281248927116394, + "loss": 0.9514, + "rewards/accuracies": 0.5, + "rewards/chosen": -3.1071114540100098, + "rewards/margins": 0.09601074457168579, + "rewards/rejected": -3.20312237739563, + "step": 145 + }, + { + "epoch": 0.040822032713546764, + "grad_norm": 33.28893137790503, + "learning_rate": 2.0610737385376348e-07, + "logits/chosen": -1.2970675230026245, + "logits/rejected": -1.1824913024902344, + "logps/chosen": -1.404200553894043, + "logps/rejected": -2.006523609161377, + "loss": 0.9091, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.5105013847351074, + "rewards/margins": 1.505807638168335, + "rewards/rejected": -5.016308784484863, + "step": 146 + }, + { + "epoch": 0.04110163567733818, + "grad_norm": 62.45940480786672, + "learning_rate": 1.990924884239758e-07, + "logits/chosen": -0.7262787222862244, + "logits/rejected": -0.9536744952201843, + "logps/chosen": -1.3462529182434082, + "logps/rejected": -2.189612865447998, + "loss": 1.1719, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.3656325340270996, + "rewards/margins": 2.1083993911743164, + "rewards/rejected": -5.474031448364258, + "step": 147 + }, + { + "epoch": 0.041381238641129595, + "grad_norm": 14.195827731761723, + "learning_rate": 1.9216926233717084e-07, + "logits/chosen": -0.7084841132164001, + "logits/rejected": -1.271304965019226, + "logps/chosen": -1.093338131904602, + "logps/rejected": -2.5351109504699707, + "loss": 0.4317, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.7333450317382812, + "rewards/margins": 3.6044318675994873, + "rewards/rejected": -6.337777137756348, + "step": 148 + }, + { + "epoch": 0.04166084160492101, + "grad_norm": 30.52003756938978, + "learning_rate": 1.8533980447508135e-07, + "logits/chosen": -0.8182432055473328, + "logits/rejected": -1.2552411556243896, + "logps/chosen": -1.5775116682052612, + "logps/rejected": -1.9700381755828857, + "loss": 0.9821, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.9437789916992188, + "rewards/margins": 0.9813161492347717, + "rewards/rejected": -4.925095558166504, + "step": 149 + }, + { + "epoch": 0.04194044456871243, + "grad_norm": 53.98320994568875, + "learning_rate": 1.7860619515673032e-07, + "logits/chosen": -0.9220970869064331, + "logits/rejected": -0.8754558563232422, + "logps/chosen": -1.3505525588989258, + "logps/rejected": -1.8634326457977295, + "loss": 1.0404, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.3763811588287354, + "rewards/margins": 1.282200574874878, + "rewards/rejected": -4.658581733703613, + "step": 150 + }, + { + "epoch": 0.04194044456871243, + "eval_logits/chosen": -1.208183765411377, + "eval_logits/rejected": -1.363861083984375, + "eval_logps/chosen": -1.3818128108978271, + "eval_logps/rejected": -2.246399164199829, + "eval_loss": 0.9883071780204773, + "eval_rewards/accuracies": 0.7545045018196106, + "eval_rewards/chosen": -3.4545319080352783, + "eval_rewards/margins": 2.161465644836426, + "eval_rewards/rejected": -5.615997791290283, + "eval_runtime": 343.7097, + "eval_samples_per_second": 3.864, + "eval_steps_per_second": 0.323, + "step": 150 + }, + { + "epoch": 0.04222004753250384, + "grad_norm": 220.36984630306912, + "learning_rate": 1.7197048550474641e-07, + "logits/chosen": -0.6710684299468994, + "logits/rejected": -0.9249241948127747, + "logps/chosen": -1.366889476776123, + "logps/rejected": -3.0947508811950684, + "loss": 1.8539, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.4172236919403076, + "rewards/margins": 4.3196539878845215, + "rewards/rejected": -7.73687744140625, + "step": 151 + }, + { + "epoch": 0.04249965049629526, + "grad_norm": 70.6167555600515, + "learning_rate": 1.6543469682057104e-07, + "logits/chosen": -1.052169919013977, + "logits/rejected": -1.4687204360961914, + "logps/chosen": -1.1982924938201904, + "logps/rejected": -1.9964299201965332, + "loss": 0.7174, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.9957308769226074, + "rewards/margins": 1.9953436851501465, + "rewards/rejected": -4.991075038909912, + "step": 152 + }, + { + "epoch": 0.04277925346008668, + "grad_norm": 14.934556421376993, + "learning_rate": 1.590008199687508e-07, + "logits/chosen": -1.2788257598876953, + "logits/rejected": -1.3499538898468018, + "logps/chosen": -1.4407329559326172, + "logps/rejected": -2.7239699363708496, + "loss": 0.964, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.601832389831543, + "rewards/margins": 3.208092212677002, + "rewards/rejected": -6.809924602508545, + "step": 153 + }, + { + "epoch": 0.04305885642387809, + "grad_norm": 34.17784688286548, + "learning_rate": 1.5267081477050131e-07, + "logits/chosen": -0.9582281112670898, + "logits/rejected": -1.459122896194458, + "logps/chosen": -1.7836196422576904, + "logps/rejected": -1.8749334812164307, + "loss": 0.774, + "rewards/accuracies": 0.5, + "rewards/chosen": -4.459049224853516, + "rewards/margins": 0.22828447818756104, + "rewards/rejected": -4.687334060668945, + "step": 154 + }, + { + "epoch": 0.04333845938766951, + "grad_norm": 16.121695263258527, + "learning_rate": 1.4644660940672627e-07, + "logits/chosen": -0.6466392874717712, + "logits/rejected": -1.4516868591308594, + "logps/chosen": -1.236275315284729, + "logps/rejected": -3.074626922607422, + "loss": 0.8325, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.0906882286071777, + "rewards/margins": 4.595878601074219, + "rewards/rejected": -7.686567306518555, + "step": 155 + }, + { + "epoch": 0.04361806235146093, + "grad_norm": 59.092861255129954, + "learning_rate": 1.4033009983067452e-07, + "logits/chosen": -1.0997824668884277, + "logits/rejected": -0.8644530773162842, + "logps/chosen": -1.5466465950012207, + "logps/rejected": -1.9648511409759521, + "loss": 1.2282, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.8666162490844727, + "rewards/margins": 1.0455113649368286, + "rewards/rejected": -4.91212797164917, + "step": 156 + }, + { + "epoch": 0.04389766531525234, + "grad_norm": 124.7156154616911, + "learning_rate": 1.3432314919041477e-07, + "logits/chosen": -1.2434520721435547, + "logits/rejected": -0.9779855608940125, + "logps/chosen": -2.0951521396636963, + "logps/rejected": -1.2733700275421143, + "loss": 2.0045, + "rewards/accuracies": 0.25, + "rewards/chosen": -5.237880706787109, + "rewards/margins": -2.054455280303955, + "rewards/rejected": -3.183424949645996, + "step": 157 + }, + { + "epoch": 0.04417726827904376, + "grad_norm": 17.359377980307887, + "learning_rate": 1.284275872613028e-07, + "logits/chosen": -0.8045494556427002, + "logits/rejected": -1.271446943283081, + "logps/chosen": -1.310563087463379, + "logps/rejected": -2.273303508758545, + "loss": 0.6038, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.2764077186584473, + "rewards/margins": 2.406850814819336, + "rewards/rejected": -5.683258533477783, + "step": 158 + }, + { + "epoch": 0.04445687124283517, + "grad_norm": 41.376354047075345, + "learning_rate": 1.22645209888614e-07, + "logits/chosen": -0.7052321434020996, + "logits/rejected": -1.4795576333999634, + "logps/chosen": -1.3022897243499756, + "logps/rejected": -2.3771719932556152, + "loss": 0.9528, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.2557244300842285, + "rewards/margins": 2.6872057914733887, + "rewards/rejected": -5.942930221557617, + "step": 159 + }, + { + "epoch": 0.04473647420662659, + "grad_norm": 45.99787159929172, + "learning_rate": 1.1697777844051104e-07, + "logits/chosen": -0.9138689041137695, + "logits/rejected": -1.5214874744415283, + "logps/chosen": -1.44037663936615, + "logps/rejected": -2.119154214859009, + "loss": 1.4147, + "rewards/accuracies": 0.5, + "rewards/chosen": -3.6009416580200195, + "rewards/margins": 1.696943759918213, + "rewards/rejected": -5.297885417938232, + "step": 160 + }, + { + "epoch": 0.04501607717041801, + "grad_norm": 39.8925791182464, + "learning_rate": 1.1142701927151454e-07, + "logits/chosen": -1.4353740215301514, + "logits/rejected": -1.4286375045776367, + "logps/chosen": -1.4594804048538208, + "logps/rejected": -2.6095705032348633, + "loss": 0.7317, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.6487011909484863, + "rewards/margins": 2.87522554397583, + "rewards/rejected": -6.523926734924316, + "step": 161 + }, + { + "epoch": 0.04529568013420942, + "grad_norm": 31.35281628244166, + "learning_rate": 1.0599462319663904e-07, + "logits/chosen": -1.1067039966583252, + "logits/rejected": -1.1960399150848389, + "logps/chosen": -1.6614606380462646, + "logps/rejected": -2.211236000061035, + "loss": 0.9855, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.153651714324951, + "rewards/margins": 1.374437928199768, + "rewards/rejected": -5.52808952331543, + "step": 162 + }, + { + "epoch": 0.04557528309800084, + "grad_norm": 75.21576214554203, + "learning_rate": 1.0068224497635369e-07, + "logits/chosen": -1.2265667915344238, + "logits/rejected": -1.0434250831604004, + "logps/chosen": -1.1752564907073975, + "logps/rejected": -2.1756644248962402, + "loss": 0.7941, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.9381415843963623, + "rewards/margins": 2.501019239425659, + "rewards/rejected": -5.4391608238220215, + "step": 163 + }, + { + "epoch": 0.045854886061792256, + "grad_norm": 93.40285204475232, + "learning_rate": 9.549150281252632e-08, + "logits/chosen": -0.831551194190979, + "logits/rejected": -0.7609004974365234, + "logps/chosen": -1.2593779563903809, + "logps/rejected": -1.790838599205017, + "loss": 1.5802, + "rewards/accuracies": 0.25, + "rewards/chosen": -3.1484451293945312, + "rewards/margins": 1.3286514282226562, + "rewards/rejected": -4.4770965576171875, + "step": 164 + }, + { + "epoch": 0.04613448902558367, + "grad_norm": 41.48157239220995, + "learning_rate": 9.042397785550404e-08, + "logits/chosen": -0.6658945679664612, + "logits/rejected": -0.9514182806015015, + "logps/chosen": -1.3134242296218872, + "logps/rejected": -1.9456892013549805, + "loss": 0.5563, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.2835607528686523, + "rewards/margins": 1.580662488937378, + "rewards/rejected": -4.864223003387451, + "step": 165 + }, + { + "epoch": 0.046414091989375086, + "grad_norm": 49.840157280127464, + "learning_rate": 8.548121372247919e-08, + "logits/chosen": -0.9251920580863953, + "logits/rejected": -1.4082067012786865, + "logps/chosen": -1.3225380182266235, + "logps/rejected": -3.200735092163086, + "loss": 0.7022, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.306344985961914, + "rewards/margins": 4.695493221282959, + "rewards/rejected": -8.001838684082031, + "step": 166 + }, + { + "epoch": 0.046693694953166505, + "grad_norm": 118.05732935497699, + "learning_rate": 8.066471602728803e-08, + "logits/chosen": -1.2742832899093628, + "logits/rejected": -1.172432541847229, + "logps/chosen": -1.6827492713928223, + "logps/rejected": -1.412048578262329, + "loss": 1.6709, + "rewards/accuracies": 0.5, + "rewards/chosen": -4.206873416900635, + "rewards/margins": -0.6767516732215881, + "rewards/rejected": -3.5301215648651123, + "step": 167 + }, + { + "epoch": 0.046973297916957917, + "grad_norm": 16.707525761245098, + "learning_rate": 7.597595192178702e-08, + "logits/chosen": -1.2708326578140259, + "logits/rejected": -1.250439167022705, + "logps/chosen": -0.9043286442756653, + "logps/rejected": -2.5250792503356934, + "loss": 0.4602, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.2608213424682617, + "rewards/margins": 4.051877021789551, + "rewards/rejected": -6.3126983642578125, + "step": 168 + }, + { + "epoch": 0.047252900880749335, + "grad_norm": 43.907524013668834, + "learning_rate": 7.141634964894388e-08, + "logits/chosen": -0.9978333711624146, + "logits/rejected": -1.234339714050293, + "logps/chosen": -1.7665379047393799, + "logps/rejected": -2.1084702014923096, + "loss": 0.9423, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.41634464263916, + "rewards/margins": 0.8548306226730347, + "rewards/rejected": -5.271175384521484, + "step": 169 + }, + { + "epoch": 0.047532503844540754, + "grad_norm": 88.65537871518796, + "learning_rate": 6.698729810778064e-08, + "logits/chosen": -0.8296370506286621, + "logits/rejected": -0.7557154297828674, + "logps/chosen": -1.1506996154785156, + "logps/rejected": -1.4387410879135132, + "loss": 1.2953, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.876749038696289, + "rewards/margins": 0.7201035022735596, + "rewards/rejected": -3.5968525409698486, + "step": 170 + }, + { + "epoch": 0.047812106808332165, + "grad_norm": 35.86474595345899, + "learning_rate": 6.269014643030212e-08, + "logits/chosen": -1.3039546012878418, + "logits/rejected": -1.6143985986709595, + "logps/chosen": -2.294726610183716, + "logps/rejected": -2.714628219604492, + "loss": 0.951, + "rewards/accuracies": 0.5, + "rewards/chosen": -5.73681640625, + "rewards/margins": 1.0497541427612305, + "rewards/rejected": -6.7865705490112305, + "step": 171 + }, + { + "epoch": 0.048091709772123584, + "grad_norm": 47.72518452969935, + "learning_rate": 5.8526203570536504e-08, + "logits/chosen": -1.2507563829421997, + "logits/rejected": -1.449178695678711, + "logps/chosen": -1.8437649011611938, + "logps/rejected": -3.4956912994384766, + "loss": 1.271, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.60941219329834, + "rewards/margins": 4.129816055297852, + "rewards/rejected": -8.739228248596191, + "step": 172 + }, + { + "epoch": 0.048371312735915, + "grad_norm": 45.10015403750292, + "learning_rate": 5.44967379058161e-08, + "logits/chosen": -1.5573780536651611, + "logits/rejected": -1.247377634048462, + "logps/chosen": -0.8558065891265869, + "logps/rejected": -2.1532318592071533, + "loss": 0.7389, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.1395163536071777, + "rewards/margins": 3.243563175201416, + "rewards/rejected": -5.383079528808594, + "step": 173 + }, + { + "epoch": 0.048650915699706414, + "grad_norm": 126.90491606998833, + "learning_rate": 5.060297685041659e-08, + "logits/chosen": -1.0825918912887573, + "logits/rejected": -1.1961251497268677, + "logps/chosen": -1.5061373710632324, + "logps/rejected": -2.8128504753112793, + "loss": 1.415, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.76534366607666, + "rewards/margins": 3.266782283782959, + "rewards/rejected": -7.032125949859619, + "step": 174 + }, + { + "epoch": 0.04893051866349783, + "grad_norm": 104.83325539903379, + "learning_rate": 4.684610648167503e-08, + "logits/chosen": -0.2594190239906311, + "logits/rejected": -1.2538113594055176, + "logps/chosen": -0.8970815539360046, + "logps/rejected": -3.357142925262451, + "loss": 0.7837, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.242703914642334, + "rewards/margins": 6.150154113769531, + "rewards/rejected": -8.392858505249023, + "step": 175 + }, + { + "epoch": 0.04921012162728925, + "grad_norm": 25.391300351126056, + "learning_rate": 4.322727117869951e-08, + "logits/chosen": -1.0320615768432617, + "logits/rejected": -1.3764903545379639, + "logps/chosen": -1.2408421039581299, + "logps/rejected": -3.5107479095458984, + "loss": 0.701, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.102104902267456, + "rewards/margins": 5.674765110015869, + "rewards/rejected": -8.776870727539062, + "step": 176 + }, + { + "epoch": 0.04948972459108066, + "grad_norm": 100.44588632718167, + "learning_rate": 3.974757327377981e-08, + "logits/chosen": -1.1055123805999756, + "logits/rejected": -0.956331729888916, + "logps/chosen": -1.6190693378448486, + "logps/rejected": -2.0674889087677, + "loss": 0.8604, + "rewards/accuracies": 0.75, + "rewards/chosen": -4.047673225402832, + "rewards/margins": 1.1210490465164185, + "rewards/rejected": -5.168722629547119, + "step": 177 + }, + { + "epoch": 0.04976932755487208, + "grad_norm": 23.973360234647668, + "learning_rate": 3.6408072716606345e-08, + "logits/chosen": -0.6234188079833984, + "logits/rejected": -0.7233335375785828, + "logps/chosen": -1.2935428619384766, + "logps/rejected": -1.6678102016448975, + "loss": 1.0504, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.2338571548461914, + "rewards/margins": 0.9356681704521179, + "rewards/rejected": -4.169525146484375, + "step": 178 + }, + { + "epoch": 0.0500489305186635, + "grad_norm": 128.60647901400966, + "learning_rate": 3.3209786751399184e-08, + "logits/chosen": -0.6642372608184814, + "logits/rejected": -1.3921313285827637, + "logps/chosen": -1.1783480644226074, + "logps/rejected": -4.460213661193848, + "loss": 1.2305, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.9458699226379395, + "rewards/margins": 8.20466423034668, + "rewards/rejected": -11.150533676147461, + "step": 179 + }, + { + "epoch": 0.05032853348245491, + "grad_norm": 45.48438050770713, + "learning_rate": 3.015368960704584e-08, + "logits/chosen": -0.753233790397644, + "logits/rejected": -0.880287766456604, + "logps/chosen": -1.1847925186157227, + "logps/rejected": -2.643932342529297, + "loss": 1.6424, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.9619812965393066, + "rewards/margins": 3.6478493213653564, + "rewards/rejected": -6.609830856323242, + "step": 180 + }, + { + "epoch": 0.05060813644624633, + "grad_norm": 43.3062398465464, + "learning_rate": 2.7240712200341576e-08, + "logits/chosen": -1.2592175006866455, + "logits/rejected": -1.2679963111877441, + "logps/chosen": -1.8530656099319458, + "logps/rejected": -1.859635353088379, + "loss": 1.8695, + "rewards/accuracies": 0.5, + "rewards/chosen": -4.632663726806641, + "rewards/margins": 0.01642441749572754, + "rewards/rejected": -4.649088382720947, + "step": 181 + }, + { + "epoch": 0.05088773941003775, + "grad_norm": 34.019147990522235, + "learning_rate": 2.4471741852423233e-08, + "logits/chosen": -1.0714620351791382, + "logits/rejected": -1.2085020542144775, + "logps/chosen": -1.3924332857131958, + "logps/rejected": -1.9933589696884155, + "loss": 1.1236, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.481083393096924, + "rewards/margins": 1.5023138523101807, + "rewards/rejected": -4.983397483825684, + "step": 182 + }, + { + "epoch": 0.05116734237382916, + "grad_norm": 38.25765530969567, + "learning_rate": 2.1847622018482282e-08, + "logits/chosen": -1.323840856552124, + "logits/rejected": -1.2305158376693726, + "logps/chosen": -1.737386703491211, + "logps/rejected": -2.449878692626953, + "loss": 1.3231, + "rewards/accuracies": 0.5, + "rewards/chosen": -4.343466758728027, + "rewards/margins": 1.7812304496765137, + "rewards/rejected": -6.124697208404541, + "step": 183 + }, + { + "epoch": 0.05144694533762058, + "grad_norm": 21.832900395742424, + "learning_rate": 1.936915203084055e-08, + "logits/chosen": -1.102403163909912, + "logits/rejected": -0.8794164061546326, + "logps/chosen": -1.1961630582809448, + "logps/rejected": -2.0587728023529053, + "loss": 0.5583, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.990407705307007, + "rewards/margins": 2.156524181365967, + "rewards/rejected": -5.146932125091553, + "step": 184 + }, + { + "epoch": 0.051726548301412, + "grad_norm": 97.17645669564288, + "learning_rate": 1.7037086855465898e-08, + "logits/chosen": -0.8995164036750793, + "logits/rejected": -1.224024772644043, + "logps/chosen": -1.5391126871109009, + "logps/rejected": -3.2247936725616455, + "loss": 1.0714, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.8477818965911865, + "rewards/margins": 4.214201927185059, + "rewards/rejected": -8.061984062194824, + "step": 185 + }, + { + "epoch": 0.05200615126520341, + "grad_norm": 46.501514059890596, + "learning_rate": 1.4852136862001763e-08, + "logits/chosen": -1.3521168231964111, + "logits/rejected": -1.7020211219787598, + "logps/chosen": -1.7328448295593262, + "logps/rejected": -2.3388752937316895, + "loss": 1.4423, + "rewards/accuracies": 0.5, + "rewards/chosen": -4.332111835479736, + "rewards/margins": 1.5150765180587769, + "rewards/rejected": -5.847188472747803, + "step": 186 + }, + { + "epoch": 0.05228575422899483, + "grad_norm": 43.90284836540054, + "learning_rate": 1.2814967607382432e-08, + "logits/chosen": -0.7001566886901855, + "logits/rejected": -0.8944656848907471, + "logps/chosen": -1.2563323974609375, + "logps/rejected": -1.8060386180877686, + "loss": 0.4237, + "rewards/accuracies": 1.0, + "rewards/chosen": -3.1408309936523438, + "rewards/margins": 1.374265432357788, + "rewards/rejected": -4.515096664428711, + "step": 187 + }, + { + "epoch": 0.052565357192786245, + "grad_norm": 66.58807618416985, + "learning_rate": 1.0926199633097154e-08, + "logits/chosen": -1.131943941116333, + "logits/rejected": -1.2059730291366577, + "logps/chosen": -1.1455706357955933, + "logps/rejected": -1.936645746231079, + "loss": 0.7375, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.863926649093628, + "rewards/margins": 1.9776880741119385, + "rewards/rejected": -4.841614723205566, + "step": 188 + }, + { + "epoch": 0.05284496015657766, + "grad_norm": 36.45672997200926, + "learning_rate": 9.186408276168011e-09, + "logits/chosen": -0.9764798879623413, + "logits/rejected": -1.0748597383499146, + "logps/chosen": -1.0145312547683716, + "logps/rejected": -2.6933655738830566, + "loss": 1.0416, + "rewards/accuracies": 0.5, + "rewards/chosen": -2.5363283157348633, + "rewards/margins": 4.197086334228516, + "rewards/rejected": -6.733414649963379, + "step": 189 + }, + { + "epoch": 0.053124563120369075, + "grad_norm": 23.88892255070114, + "learning_rate": 7.59612349389599e-09, + "logits/chosen": -0.809168815612793, + "logits/rejected": -1.1629016399383545, + "logps/chosen": -0.9837249517440796, + "logps/rejected": -1.9994207620620728, + "loss": 0.603, + "rewards/accuracies": 1.0, + "rewards/chosen": -2.4593124389648438, + "rewards/margins": 2.5392391681671143, + "rewards/rejected": -4.998551845550537, + "step": 190 + }, + { + "epoch": 0.053404166084160494, + "grad_norm": 38.039713749484235, + "learning_rate": 6.15582970243117e-09, + "logits/chosen": -1.061342716217041, + "logits/rejected": -1.2929720878601074, + "logps/chosen": -1.4455804824829102, + "logps/rejected": -1.7912752628326416, + "loss": 1.2204, + "rewards/accuracies": 0.5, + "rewards/chosen": -3.6139512062072754, + "rewards/margins": 0.8642373085021973, + "rewards/rejected": -4.478188514709473, + "step": 191 + }, + { + "epoch": 0.053683769047951906, + "grad_norm": 33.26330215420443, + "learning_rate": 4.865965629214819e-09, + "logits/chosen": -0.7917881608009338, + "logits/rejected": -1.4771734476089478, + "logps/chosen": -1.3420734405517578, + "logps/rejected": -2.956531047821045, + "loss": 1.2029, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.3551833629608154, + "rewards/margins": 4.036144256591797, + "rewards/rejected": -7.391327381134033, + "step": 192 + }, + { + "epoch": 0.053963372011743324, + "grad_norm": 50.418507707166555, + "learning_rate": 3.7269241793390084e-09, + "logits/chosen": -0.7339926362037659, + "logits/rejected": -0.8001390695571899, + "logps/chosen": -1.375474214553833, + "logps/rejected": -1.5513949394226074, + "loss": 1.4448, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.438685655593872, + "rewards/margins": 0.43980175256729126, + "rewards/rejected": -3.8784873485565186, + "step": 193 + }, + { + "epoch": 0.05424297497553474, + "grad_norm": 46.99311043278277, + "learning_rate": 2.739052315863355e-09, + "logits/chosen": -1.1927969455718994, + "logits/rejected": -1.1135954856872559, + "logps/chosen": -1.4356844425201416, + "logps/rejected": -1.4172120094299316, + "loss": 1.2511, + "rewards/accuracies": 0.25, + "rewards/chosen": -3.5892109870910645, + "rewards/margins": -0.046180903911590576, + "rewards/rejected": -3.54302978515625, + "step": 194 + }, + { + "epoch": 0.054522577939326154, + "grad_norm": 50.89269657493953, + "learning_rate": 1.9026509541272273e-09, + "logits/chosen": -1.0880765914916992, + "logits/rejected": -1.1170682907104492, + "logps/chosen": -1.426069974899292, + "logps/rejected": -2.2877893447875977, + "loss": 1.1481, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.5651750564575195, + "rewards/margins": 2.1542983055114746, + "rewards/rejected": -5.719472885131836, + "step": 195 + }, + { + "epoch": 0.05480218090311757, + "grad_norm": 119.28075830322265, + "learning_rate": 1.217974870087901e-09, + "logits/chosen": -1.1547623872756958, + "logits/rejected": -1.3626446723937988, + "logps/chosen": -1.1919413805007935, + "logps/rejected": -2.2724056243896484, + "loss": 1.446, + "rewards/accuracies": 0.75, + "rewards/chosen": -2.979853391647339, + "rewards/margins": 2.7011606693267822, + "rewards/rejected": -5.681014060974121, + "step": 196 + }, + { + "epoch": 0.05508178386690899, + "grad_norm": 48.79918657535743, + "learning_rate": 6.852326227130833e-10, + "logits/chosen": -1.2602670192718506, + "logits/rejected": -1.142798662185669, + "logps/chosen": -1.5842217206954956, + "logps/rejected": -1.682457685470581, + "loss": 0.7233, + "rewards/accuracies": 0.5, + "rewards/chosen": -3.9605541229248047, + "rewards/margins": 0.2455899715423584, + "rewards/rejected": -4.206143856048584, + "step": 197 + }, + { + "epoch": 0.0553613868307004, + "grad_norm": 20.938919943310346, + "learning_rate": 3.0458649045211894e-10, + "logits/chosen": -0.6655712127685547, + "logits/rejected": -0.7740037441253662, + "logps/chosen": -1.6575719118118286, + "logps/rejected": -1.8176357746124268, + "loss": 1.2588, + "rewards/accuracies": 0.5, + "rewards/chosen": -4.143929958343506, + "rewards/margins": 0.4001593291759491, + "rewards/rejected": -4.544089317321777, + "step": 198 + }, + { + "epoch": 0.05564098979449182, + "grad_norm": 25.414963350937853, + "learning_rate": 7.615242180436521e-11, + "logits/chosen": -1.2654355764389038, + "logits/rejected": -1.2140486240386963, + "logps/chosen": -1.4302688837051392, + "logps/rejected": -2.340785026550293, + "loss": 0.8395, + "rewards/accuracies": 0.75, + "rewards/chosen": -3.575672149658203, + "rewards/margins": 2.276289939880371, + "rewards/rejected": -5.851962089538574, + "step": 199 + }, + { + "epoch": 0.05592059275828324, + "grad_norm": 82.36002044117453, + "learning_rate": 0.0, + "logits/chosen": -1.2721612453460693, + "logits/rejected": -0.8177624940872192, + "logps/chosen": -1.314448595046997, + "logps/rejected": -1.1094343662261963, + "loss": 1.4672, + "rewards/accuracies": 0.25, + "rewards/chosen": -3.286121368408203, + "rewards/margins": -0.5125355124473572, + "rewards/rejected": -2.773585796356201, + "step": 200 + }, + { + "epoch": 0.05592059275828324, + "eval_logits/chosen": -1.2117185592651367, + "eval_logits/rejected": -1.3683098554611206, + "eval_logps/chosen": -1.3815547227859497, + "eval_logps/rejected": -2.264155149459839, + "eval_loss": 0.9751168489456177, + "eval_rewards/accuracies": 0.7612612843513489, + "eval_rewards/chosen": -3.4538867473602295, + "eval_rewards/margins": 2.206501007080078, + "eval_rewards/rejected": -5.66038703918457, + "eval_runtime": 344.9671, + "eval_samples_per_second": 3.85, + "eval_steps_per_second": 0.322, + "step": 200 + }, + { + "epoch": 0.05592059275828324, + "step": 200, + "total_flos": 0.0, + "train_loss": 1.163502004146576, + "train_runtime": 4144.9448, + "train_samples_per_second": 0.579, + "train_steps_per_second": 0.048 + } + ], + "logging_steps": 1, + "max_steps": 200, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 10000, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": false, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +}