| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.2616431187859759, | |
| "eval_steps": 500, | |
| "global_step": 500, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.0, | |
| "grad_norm": 25.513092629771464, | |
| "kl": 0.0, | |
| "learning_rate": 4.99738356881214e-07, | |
| "logps/chosen": -255.54945373535156, | |
| "logps/rejected": -223.1166229248047, | |
| "loss": 0.2812, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "grad_norm": 19.05975346317889, | |
| "kl": 0.0, | |
| "learning_rate": 4.99476713762428e-07, | |
| "logps/chosen": -284.558837890625, | |
| "logps/rejected": -214.7106475830078, | |
| "loss": 0.2015, | |
| "rewards/chosen": 0.34634318947792053, | |
| "rewards/margins": 0.3400447368621826, | |
| "rewards/rejected": 0.006298447493463755, | |
| "step": 2 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "grad_norm": 16.41510277772972, | |
| "kl": 2.259248733520508, | |
| "learning_rate": 4.992150706436421e-07, | |
| "logps/chosen": -191.8242645263672, | |
| "logps/rejected": -164.48486328125, | |
| "loss": 0.2416, | |
| "rewards/chosen": 0.3580303192138672, | |
| "rewards/margins": 0.17033512890338898, | |
| "rewards/rejected": 0.1876951903104782, | |
| "step": 3 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "grad_norm": 18.59921957448892, | |
| "kl": 1.6011241674423218, | |
| "learning_rate": 4.989534275248561e-07, | |
| "logps/chosen": -191.90174865722656, | |
| "logps/rejected": -228.51425170898438, | |
| "loss": 0.2344, | |
| "rewards/chosen": 0.14850404858589172, | |
| "rewards/margins": 0.18605005741119385, | |
| "rewards/rejected": -0.03754601255059242, | |
| "step": 4 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "grad_norm": 19.28790409947415, | |
| "kl": 0.6965503692626953, | |
| "learning_rate": 4.986917844060701e-07, | |
| "logps/chosen": -213.99278259277344, | |
| "logps/rejected": -244.74432373046875, | |
| "loss": 0.2692, | |
| "rewards/chosen": 0.08441495150327682, | |
| "rewards/margins": -0.010940708220005035, | |
| "rewards/rejected": 0.09535565972328186, | |
| "step": 5 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "grad_norm": 16.851732715209323, | |
| "kl": 0.0, | |
| "learning_rate": 4.984301412872841e-07, | |
| "logps/chosen": -235.20309448242188, | |
| "logps/rejected": -261.23590087890625, | |
| "loss": 0.1815, | |
| "rewards/chosen": 0.035614676773548126, | |
| "rewards/margins": 0.42004701495170593, | |
| "rewards/rejected": -0.3844323456287384, | |
| "step": 6 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "grad_norm": 16.58819557091471, | |
| "kl": 0.0, | |
| "learning_rate": 4.981684981684982e-07, | |
| "logps/chosen": -193.42202758789062, | |
| "logps/rejected": -225.8984832763672, | |
| "loss": 0.1914, | |
| "rewards/chosen": -0.15666261315345764, | |
| "rewards/margins": 0.47593703866004944, | |
| "rewards/rejected": -0.6325996518135071, | |
| "step": 7 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "grad_norm": 15.607759793866348, | |
| "kl": 0.0, | |
| "learning_rate": 4.979068550497122e-07, | |
| "logps/chosen": -246.3267059326172, | |
| "logps/rejected": -197.67164611816406, | |
| "loss": 0.1615, | |
| "rewards/chosen": 0.23949605226516724, | |
| "rewards/margins": 0.5585669279098511, | |
| "rewards/rejected": -0.3190709054470062, | |
| "step": 8 | |
| }, | |
| { | |
| "epoch": 0.0, | |
| "grad_norm": 17.644608030177945, | |
| "kl": 0.0, | |
| "learning_rate": 4.976452119309262e-07, | |
| "logps/chosen": -270.65924072265625, | |
| "logps/rejected": -205.1195068359375, | |
| "loss": 0.2314, | |
| "rewards/chosen": -0.07605857402086258, | |
| "rewards/margins": 0.15047526359558105, | |
| "rewards/rejected": -0.22653384506702423, | |
| "step": 9 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 16.618428473287977, | |
| "kl": 0.0, | |
| "learning_rate": 4.973835688121403e-07, | |
| "logps/chosen": -174.2410888671875, | |
| "logps/rejected": -181.774169921875, | |
| "loss": 0.2923, | |
| "rewards/chosen": -0.08562064915895462, | |
| "rewards/margins": 0.02823999524116516, | |
| "rewards/rejected": -0.11386064440011978, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 17.243866067758734, | |
| "kl": 0.0, | |
| "learning_rate": 4.971219256933543e-07, | |
| "logps/chosen": -198.24940490722656, | |
| "logps/rejected": -215.78672790527344, | |
| "loss": 0.2918, | |
| "rewards/chosen": 0.13005053997039795, | |
| "rewards/margins": 0.43379873037338257, | |
| "rewards/rejected": -0.3037481904029846, | |
| "step": 11 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 15.745968199362501, | |
| "kl": 0.0, | |
| "learning_rate": 4.968602825745683e-07, | |
| "logps/chosen": -205.6284637451172, | |
| "logps/rejected": -214.56349182128906, | |
| "loss": 0.2833, | |
| "rewards/chosen": -0.3640637993812561, | |
| "rewards/margins": 0.07524752616882324, | |
| "rewards/rejected": -0.43931132555007935, | |
| "step": 12 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 15.030324414091437, | |
| "kl": 0.2821946144104004, | |
| "learning_rate": 4.965986394557823e-07, | |
| "logps/chosen": -286.0364990234375, | |
| "logps/rejected": -206.61985778808594, | |
| "loss": 0.1812, | |
| "rewards/chosen": 0.5389153361320496, | |
| "rewards/margins": 0.35903000831604004, | |
| "rewards/rejected": 0.17988532781600952, | |
| "step": 13 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 16.769570066708727, | |
| "kl": 0.0, | |
| "learning_rate": 4.963369963369964e-07, | |
| "logps/chosen": -198.4276580810547, | |
| "logps/rejected": -259.9801940917969, | |
| "loss": 0.2128, | |
| "rewards/chosen": 0.21227878332138062, | |
| "rewards/margins": 0.3703112006187439, | |
| "rewards/rejected": -0.1580324023962021, | |
| "step": 14 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 17.041145148374916, | |
| "kl": 1.2743568420410156, | |
| "learning_rate": 4.960753532182104e-07, | |
| "logps/chosen": -229.1648712158203, | |
| "logps/rejected": -253.59913635253906, | |
| "loss": 0.2295, | |
| "rewards/chosen": 0.6795628070831299, | |
| "rewards/margins": 0.9129478335380554, | |
| "rewards/rejected": -0.23338502645492554, | |
| "step": 15 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 15.28869972325163, | |
| "kl": 0.0, | |
| "learning_rate": 4.958137100994244e-07, | |
| "logps/chosen": -187.63201904296875, | |
| "logps/rejected": -241.87457275390625, | |
| "loss": 0.1656, | |
| "rewards/chosen": 0.1542114019393921, | |
| "rewards/margins": 0.32588493824005127, | |
| "rewards/rejected": -0.17167352139949799, | |
| "step": 16 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 16.352437946436485, | |
| "kl": 0.0, | |
| "learning_rate": 4.955520669806384e-07, | |
| "logps/chosen": -285.8620910644531, | |
| "logps/rejected": -275.36328125, | |
| "loss": 0.229, | |
| "rewards/chosen": 0.02602279745042324, | |
| "rewards/margins": 0.5908135175704956, | |
| "rewards/rejected": -0.5647907257080078, | |
| "step": 17 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 14.496172300986403, | |
| "kl": 0.0, | |
| "learning_rate": 4.952904238618525e-07, | |
| "logps/chosen": -236.38160705566406, | |
| "logps/rejected": -206.48521423339844, | |
| "loss": 0.127, | |
| "rewards/chosen": 0.3640650808811188, | |
| "rewards/margins": 1.3152188062667847, | |
| "rewards/rejected": -0.9511536955833435, | |
| "step": 18 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 15.755801807248107, | |
| "kl": 0.0, | |
| "learning_rate": 4.950287807430665e-07, | |
| "logps/chosen": -261.5793151855469, | |
| "logps/rejected": -213.00888061523438, | |
| "loss": 0.2456, | |
| "rewards/chosen": 0.2305096834897995, | |
| "rewards/margins": 1.4204968214035034, | |
| "rewards/rejected": -1.1899871826171875, | |
| "step": 19 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 14.150683957656698, | |
| "kl": 0.0, | |
| "learning_rate": 4.947671376242805e-07, | |
| "logps/chosen": -250.30348205566406, | |
| "logps/rejected": -237.50193786621094, | |
| "loss": 0.184, | |
| "rewards/chosen": -0.09291169792413712, | |
| "rewards/margins": 0.669346809387207, | |
| "rewards/rejected": -0.7622585296630859, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 16.71784084154209, | |
| "kl": 0.0, | |
| "learning_rate": 4.945054945054945e-07, | |
| "logps/chosen": -250.10418701171875, | |
| "logps/rejected": -258.0350341796875, | |
| "loss": 0.2064, | |
| "rewards/chosen": -0.44092392921447754, | |
| "rewards/margins": 0.4753909707069397, | |
| "rewards/rejected": -0.9163148999214172, | |
| "step": 21 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 14.027074569038245, | |
| "kl": 0.0, | |
| "learning_rate": 4.942438513867084e-07, | |
| "logps/chosen": -238.38027954101562, | |
| "logps/rejected": -237.553955078125, | |
| "loss": 0.2034, | |
| "rewards/chosen": -0.11189079284667969, | |
| "rewards/margins": 1.4180889129638672, | |
| "rewards/rejected": -1.5299797058105469, | |
| "step": 22 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 13.242349649129327, | |
| "kl": 0.0, | |
| "learning_rate": 4.939822082679226e-07, | |
| "logps/chosen": -211.67172241210938, | |
| "logps/rejected": -220.88316345214844, | |
| "loss": 0.1695, | |
| "rewards/chosen": -0.0019831450190395117, | |
| "rewards/margins": 1.3524924516677856, | |
| "rewards/rejected": -1.3544756174087524, | |
| "step": 23 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 14.558526366105085, | |
| "kl": 0.0, | |
| "learning_rate": 4.937205651491365e-07, | |
| "logps/chosen": -183.22792053222656, | |
| "logps/rejected": -288.7313537597656, | |
| "loss": 0.3057, | |
| "rewards/chosen": -0.08994026482105255, | |
| "rewards/margins": 1.0271720886230469, | |
| "rewards/rejected": -1.117112398147583, | |
| "step": 24 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 14.612720259085432, | |
| "kl": 0.0, | |
| "learning_rate": 4.934589220303505e-07, | |
| "logps/chosen": -183.35086059570312, | |
| "logps/rejected": -287.0506896972656, | |
| "loss": 0.1631, | |
| "rewards/chosen": -0.2845403850078583, | |
| "rewards/margins": 0.9921867847442627, | |
| "rewards/rejected": -1.2767271995544434, | |
| "step": 25 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 14.1871906272162, | |
| "kl": 0.0, | |
| "learning_rate": 4.931972789115645e-07, | |
| "logps/chosen": -260.46820068359375, | |
| "logps/rejected": -231.28530883789062, | |
| "loss": 0.1819, | |
| "rewards/chosen": -0.6221957802772522, | |
| "rewards/margins": 0.9863255620002747, | |
| "rewards/rejected": -1.6085213422775269, | |
| "step": 26 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 12.28801127212002, | |
| "kl": 0.0, | |
| "learning_rate": 4.929356357927786e-07, | |
| "logps/chosen": -218.0127410888672, | |
| "logps/rejected": -236.330078125, | |
| "loss": 0.2623, | |
| "rewards/chosen": -1.2400844097137451, | |
| "rewards/margins": 1.2188379764556885, | |
| "rewards/rejected": -2.4589223861694336, | |
| "step": 27 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 13.969382177172214, | |
| "kl": 0.0, | |
| "learning_rate": 4.926739926739926e-07, | |
| "logps/chosen": -262.0902099609375, | |
| "logps/rejected": -233.80523681640625, | |
| "loss": 0.2037, | |
| "rewards/chosen": -1.2043147087097168, | |
| "rewards/margins": 1.4312412738800049, | |
| "rewards/rejected": -2.6355559825897217, | |
| "step": 28 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 14.346116577373035, | |
| "kl": 0.0, | |
| "learning_rate": 4.924123495552066e-07, | |
| "logps/chosen": -192.3516387939453, | |
| "logps/rejected": -301.5596923828125, | |
| "loss": 0.2649, | |
| "rewards/chosen": -0.6799588799476624, | |
| "rewards/margins": 0.6100900769233704, | |
| "rewards/rejected": -1.2900489568710327, | |
| "step": 29 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 14.447170251019777, | |
| "kl": 0.0, | |
| "learning_rate": 4.921507064364207e-07, | |
| "logps/chosen": -246.61720275878906, | |
| "logps/rejected": -279.7056884765625, | |
| "loss": 0.1786, | |
| "rewards/chosen": -0.16088435053825378, | |
| "rewards/margins": 2.5958094596862793, | |
| "rewards/rejected": -2.7566938400268555, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 11.638201730563553, | |
| "kl": 0.0, | |
| "learning_rate": 4.918890633176347e-07, | |
| "logps/chosen": -269.7441101074219, | |
| "logps/rejected": -227.7078094482422, | |
| "loss": 0.1939, | |
| "rewards/chosen": 0.17156387865543365, | |
| "rewards/margins": 2.2280192375183105, | |
| "rewards/rejected": -2.056455373764038, | |
| "step": 31 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 12.742789811988482, | |
| "kl": 0.0, | |
| "learning_rate": 4.916274201988487e-07, | |
| "logps/chosen": -276.08001708984375, | |
| "logps/rejected": -285.455078125, | |
| "loss": 0.1504, | |
| "rewards/chosen": -0.9163249135017395, | |
| "rewards/margins": 1.5270984172821045, | |
| "rewards/rejected": -2.443423271179199, | |
| "step": 32 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 11.788909838526948, | |
| "kl": 0.0, | |
| "learning_rate": 4.913657770800627e-07, | |
| "logps/chosen": -205.1074981689453, | |
| "logps/rejected": -295.74462890625, | |
| "loss": 0.225, | |
| "rewards/chosen": -0.3685793876647949, | |
| "rewards/margins": 1.5645514726638794, | |
| "rewards/rejected": -1.9331308603286743, | |
| "step": 33 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 12.82701185590145, | |
| "kl": 0.0, | |
| "learning_rate": 4.911041339612768e-07, | |
| "logps/chosen": -215.7355499267578, | |
| "logps/rejected": -270.3883056640625, | |
| "loss": 0.1812, | |
| "rewards/chosen": -0.46088603138923645, | |
| "rewards/margins": 1.8914861679077148, | |
| "rewards/rejected": -2.352372169494629, | |
| "step": 34 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 12.279622843857423, | |
| "kl": 0.0, | |
| "learning_rate": 4.908424908424908e-07, | |
| "logps/chosen": -229.2891387939453, | |
| "logps/rejected": -190.41712951660156, | |
| "loss": 0.2008, | |
| "rewards/chosen": 0.11880671977996826, | |
| "rewards/margins": 1.337594985961914, | |
| "rewards/rejected": -1.2187882661819458, | |
| "step": 35 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 12.833830578324568, | |
| "kl": 0.0, | |
| "learning_rate": 4.905808477237048e-07, | |
| "logps/chosen": -265.9535827636719, | |
| "logps/rejected": -229.44387817382812, | |
| "loss": 0.2017, | |
| "rewards/chosen": 0.7549086213111877, | |
| "rewards/margins": 2.0256097316741943, | |
| "rewards/rejected": -1.2707011699676514, | |
| "step": 36 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 12.878415132245943, | |
| "kl": 0.0, | |
| "learning_rate": 4.903192046049188e-07, | |
| "logps/chosen": -223.6148681640625, | |
| "logps/rejected": -232.38751220703125, | |
| "loss": 0.1578, | |
| "rewards/chosen": 0.42322105169296265, | |
| "rewards/margins": 1.5377566814422607, | |
| "rewards/rejected": -1.1145355701446533, | |
| "step": 37 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 12.016348202428658, | |
| "kl": 0.0, | |
| "learning_rate": 4.900575614861329e-07, | |
| "logps/chosen": -221.3059844970703, | |
| "logps/rejected": -233.40020751953125, | |
| "loss": 0.153, | |
| "rewards/chosen": 0.6136862635612488, | |
| "rewards/margins": 2.5119822025299072, | |
| "rewards/rejected": -1.8982958793640137, | |
| "step": 38 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 11.696306326828102, | |
| "kl": 0.0, | |
| "learning_rate": 4.897959183673469e-07, | |
| "logps/chosen": -263.5332336425781, | |
| "logps/rejected": -223.24720764160156, | |
| "loss": 0.2326, | |
| "rewards/chosen": -0.20513269305229187, | |
| "rewards/margins": 2.1170685291290283, | |
| "rewards/rejected": -2.3222012519836426, | |
| "step": 39 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 12.797741214492445, | |
| "kl": 0.0, | |
| "learning_rate": 4.895342752485609e-07, | |
| "logps/chosen": -241.70838928222656, | |
| "logps/rejected": -242.095703125, | |
| "loss": 0.2954, | |
| "rewards/chosen": -0.5702841281890869, | |
| "rewards/margins": 0.34226876497268677, | |
| "rewards/rejected": -0.9125528931617737, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 12.245463459701265, | |
| "kl": 0.0, | |
| "learning_rate": 4.89272632129775e-07, | |
| "logps/chosen": -200.02525329589844, | |
| "logps/rejected": -240.9892578125, | |
| "loss": 0.2966, | |
| "rewards/chosen": -1.1106632947921753, | |
| "rewards/margins": -0.09493815898895264, | |
| "rewards/rejected": -1.0157251358032227, | |
| "step": 41 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 12.419341321902724, | |
| "kl": 0.0, | |
| "learning_rate": 4.890109890109889e-07, | |
| "logps/chosen": -224.20755004882812, | |
| "logps/rejected": -281.16717529296875, | |
| "loss": 0.1364, | |
| "rewards/chosen": -0.4932700991630554, | |
| "rewards/margins": 1.310615062713623, | |
| "rewards/rejected": -1.8038851022720337, | |
| "step": 42 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 11.043427588426145, | |
| "kl": 0.0, | |
| "learning_rate": 4.88749345892203e-07, | |
| "logps/chosen": -233.59933471679688, | |
| "logps/rejected": -260.2566833496094, | |
| "loss": 0.2067, | |
| "rewards/chosen": -0.8917900323867798, | |
| "rewards/margins": 1.0747038125991821, | |
| "rewards/rejected": -1.966493844985962, | |
| "step": 43 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 12.44162754838529, | |
| "kl": 0.0, | |
| "learning_rate": 4.88487702773417e-07, | |
| "logps/chosen": -216.97845458984375, | |
| "logps/rejected": -269.1575012207031, | |
| "loss": 0.2059, | |
| "rewards/chosen": 0.21655617654323578, | |
| "rewards/margins": 2.175200939178467, | |
| "rewards/rejected": -1.9586448669433594, | |
| "step": 44 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 13.803513332755253, | |
| "kl": 0.0, | |
| "learning_rate": 4.88226059654631e-07, | |
| "logps/chosen": -265.6169128417969, | |
| "logps/rejected": -308.48760986328125, | |
| "loss": 0.1974, | |
| "rewards/chosen": -0.7583664059638977, | |
| "rewards/margins": 1.8766412734985352, | |
| "rewards/rejected": -2.635007619857788, | |
| "step": 45 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 13.259849756023506, | |
| "kl": 0.0, | |
| "learning_rate": 4.879644165358451e-07, | |
| "logps/chosen": -216.37884521484375, | |
| "logps/rejected": -263.8242492675781, | |
| "loss": 0.1924, | |
| "rewards/chosen": -0.5627312660217285, | |
| "rewards/margins": 1.7108609676361084, | |
| "rewards/rejected": -2.273592233657837, | |
| "step": 46 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 11.401346938362929, | |
| "kl": 0.0, | |
| "learning_rate": 4.877027734170591e-07, | |
| "logps/chosen": -196.87896728515625, | |
| "logps/rejected": -168.52236938476562, | |
| "loss": 0.3164, | |
| "rewards/chosen": -0.26614972949028015, | |
| "rewards/margins": 1.1080857515335083, | |
| "rewards/rejected": -1.3742355108261108, | |
| "step": 47 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 10.108076253718023, | |
| "kl": 0.0, | |
| "learning_rate": 4.874411302982731e-07, | |
| "logps/chosen": -207.7807159423828, | |
| "logps/rejected": -219.1448974609375, | |
| "loss": 0.1305, | |
| "rewards/chosen": -1.0533264875411987, | |
| "rewards/margins": 1.3320115804672241, | |
| "rewards/rejected": -2.385338068008423, | |
| "step": 48 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 10.762036932864307, | |
| "kl": 0.0, | |
| "learning_rate": 4.871794871794871e-07, | |
| "logps/chosen": -236.81471252441406, | |
| "logps/rejected": -228.0085906982422, | |
| "loss": 0.1264, | |
| "rewards/chosen": 0.9120302796363831, | |
| "rewards/margins": 2.634833335876465, | |
| "rewards/rejected": -1.722802996635437, | |
| "step": 49 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 11.066959845009587, | |
| "kl": 0.0, | |
| "learning_rate": 4.869178440607012e-07, | |
| "logps/chosen": -252.11825561523438, | |
| "logps/rejected": -209.72384643554688, | |
| "loss": 0.2249, | |
| "rewards/chosen": -0.5552583336830139, | |
| "rewards/margins": 1.0669043064117432, | |
| "rewards/rejected": -1.6221626996994019, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 11.898163000668239, | |
| "kl": 0.0, | |
| "learning_rate": 4.866562009419152e-07, | |
| "logps/chosen": -229.1280975341797, | |
| "logps/rejected": -239.55064392089844, | |
| "loss": 0.2366, | |
| "rewards/chosen": -0.3932199478149414, | |
| "rewards/margins": 0.8869756460189819, | |
| "rewards/rejected": -1.2801955938339233, | |
| "step": 51 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 11.53373054095599, | |
| "kl": 0.0, | |
| "learning_rate": 4.863945578231292e-07, | |
| "logps/chosen": -282.8030090332031, | |
| "logps/rejected": -215.99546813964844, | |
| "loss": 0.176, | |
| "rewards/chosen": -0.04715963453054428, | |
| "rewards/margins": 1.483907699584961, | |
| "rewards/rejected": -1.5310673713684082, | |
| "step": 52 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 10.818173532721506, | |
| "kl": 0.0, | |
| "learning_rate": 4.861329147043432e-07, | |
| "logps/chosen": -170.08155822753906, | |
| "logps/rejected": -266.9096984863281, | |
| "loss": 0.196, | |
| "rewards/chosen": -0.43234577775001526, | |
| "rewards/margins": 2.175133466720581, | |
| "rewards/rejected": -2.6074793338775635, | |
| "step": 53 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 12.389896501721763, | |
| "kl": 0.0, | |
| "learning_rate": 4.858712715855573e-07, | |
| "logps/chosen": -188.79185485839844, | |
| "logps/rejected": -217.9726104736328, | |
| "loss": 0.1855, | |
| "rewards/chosen": -0.7043643593788147, | |
| "rewards/margins": 1.4354064464569092, | |
| "rewards/rejected": -2.139770746231079, | |
| "step": 54 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 10.719323563675456, | |
| "kl": 0.0, | |
| "learning_rate": 4.856096284667713e-07, | |
| "logps/chosen": -235.02947998046875, | |
| "logps/rejected": -264.5641174316406, | |
| "loss": 0.217, | |
| "rewards/chosen": -1.3881831169128418, | |
| "rewards/margins": 2.237257480621338, | |
| "rewards/rejected": -3.6254405975341797, | |
| "step": 55 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 11.632198403788388, | |
| "kl": 0.0, | |
| "learning_rate": 4.853479853479853e-07, | |
| "logps/chosen": -281.6618347167969, | |
| "logps/rejected": -228.85995483398438, | |
| "loss": 0.0917, | |
| "rewards/chosen": 0.013584318570792675, | |
| "rewards/margins": 3.580209732055664, | |
| "rewards/rejected": -3.5666253566741943, | |
| "step": 56 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 12.518341857853112, | |
| "kl": 0.0, | |
| "learning_rate": 4.850863422291994e-07, | |
| "logps/chosen": -159.27957153320312, | |
| "logps/rejected": -251.3924560546875, | |
| "loss": 0.21, | |
| "rewards/chosen": -0.8034593462944031, | |
| "rewards/margins": 1.3652153015136719, | |
| "rewards/rejected": -2.1686747074127197, | |
| "step": 57 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 11.773486848179154, | |
| "kl": 0.0, | |
| "learning_rate": 4.848246991104133e-07, | |
| "logps/chosen": -163.08563232421875, | |
| "logps/rejected": -225.0583038330078, | |
| "loss": 0.2323, | |
| "rewards/chosen": -0.784336268901825, | |
| "rewards/margins": 0.35770100355148315, | |
| "rewards/rejected": -1.142037272453308, | |
| "step": 58 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 9.98418104323235, | |
| "kl": 0.0, | |
| "learning_rate": 4.845630559916274e-07, | |
| "logps/chosen": -250.50967407226562, | |
| "logps/rejected": -260.6304016113281, | |
| "loss": 0.1682, | |
| "rewards/chosen": -0.046240393072366714, | |
| "rewards/margins": 2.6554722785949707, | |
| "rewards/rejected": -2.7017126083374023, | |
| "step": 59 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 13.45864474945625, | |
| "kl": 0.0, | |
| "learning_rate": 4.843014128728414e-07, | |
| "logps/chosen": -241.01133728027344, | |
| "logps/rejected": -208.72291564941406, | |
| "loss": 0.2261, | |
| "rewards/chosen": -1.1776716709136963, | |
| "rewards/margins": 0.5967569351196289, | |
| "rewards/rejected": -1.7744286060333252, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 10.967311785129077, | |
| "kl": 0.0, | |
| "learning_rate": 4.840397697540555e-07, | |
| "logps/chosen": -240.189453125, | |
| "logps/rejected": -213.64976501464844, | |
| "loss": 0.2169, | |
| "rewards/chosen": -0.17561621963977814, | |
| "rewards/margins": 2.7882721424102783, | |
| "rewards/rejected": -2.96388840675354, | |
| "step": 61 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 10.93191003753912, | |
| "kl": 0.0, | |
| "learning_rate": 4.837781266352695e-07, | |
| "logps/chosen": -227.45742797851562, | |
| "logps/rejected": -187.86070251464844, | |
| "loss": 0.1054, | |
| "rewards/chosen": -0.3846696615219116, | |
| "rewards/margins": 1.533595085144043, | |
| "rewards/rejected": -1.9182647466659546, | |
| "step": 62 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 9.831720988058269, | |
| "kl": 0.0, | |
| "learning_rate": 4.835164835164835e-07, | |
| "logps/chosen": -273.84375, | |
| "logps/rejected": -213.56747436523438, | |
| "loss": 0.1685, | |
| "rewards/chosen": -0.3652818500995636, | |
| "rewards/margins": 2.15742564201355, | |
| "rewards/rejected": -2.522707462310791, | |
| "step": 63 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 12.175263730184788, | |
| "kl": 0.0, | |
| "learning_rate": 4.832548403976975e-07, | |
| "logps/chosen": -244.45236206054688, | |
| "logps/rejected": -183.27474975585938, | |
| "loss": 0.1439, | |
| "rewards/chosen": 0.15830610692501068, | |
| "rewards/margins": 2.1498560905456543, | |
| "rewards/rejected": -1.9915499687194824, | |
| "step": 64 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 12.593612516595744, | |
| "kl": 0.0, | |
| "learning_rate": 4.829931972789115e-07, | |
| "logps/chosen": -265.2001953125, | |
| "logps/rejected": -295.2158508300781, | |
| "loss": 0.1956, | |
| "rewards/chosen": -0.9956868886947632, | |
| "rewards/margins": 0.9066202640533447, | |
| "rewards/rejected": -1.902307152748108, | |
| "step": 65 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 12.056826564998115, | |
| "kl": 0.0, | |
| "learning_rate": 4.827315541601256e-07, | |
| "logps/chosen": -192.40469360351562, | |
| "logps/rejected": -247.81382751464844, | |
| "loss": 0.178, | |
| "rewards/chosen": -0.7804039120674133, | |
| "rewards/margins": 1.2094306945800781, | |
| "rewards/rejected": -1.9898346662521362, | |
| "step": 66 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 13.384738493954586, | |
| "kl": 0.0, | |
| "learning_rate": 4.824699110413396e-07, | |
| "logps/chosen": -159.72280883789062, | |
| "logps/rejected": -328.75616455078125, | |
| "loss": 0.1926, | |
| "rewards/chosen": -0.9741663932800293, | |
| "rewards/margins": 1.3278543949127197, | |
| "rewards/rejected": -2.302020788192749, | |
| "step": 67 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 12.538951694906014, | |
| "kl": 0.0, | |
| "learning_rate": 4.822082679225536e-07, | |
| "logps/chosen": -254.5558319091797, | |
| "logps/rejected": -190.9063262939453, | |
| "loss": 0.2038, | |
| "rewards/chosen": -0.5719714164733887, | |
| "rewards/margins": 1.5436241626739502, | |
| "rewards/rejected": -2.115595579147339, | |
| "step": 68 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 11.328347418708772, | |
| "kl": 0.0, | |
| "learning_rate": 4.819466248037676e-07, | |
| "logps/chosen": -237.08462524414062, | |
| "logps/rejected": -318.36474609375, | |
| "loss": 0.327, | |
| "rewards/chosen": -1.5885767936706543, | |
| "rewards/margins": 1.1602427959442139, | |
| "rewards/rejected": -2.748819589614868, | |
| "step": 69 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 12.400662861104186, | |
| "kl": 0.0, | |
| "learning_rate": 4.816849816849817e-07, | |
| "logps/chosen": -238.31106567382812, | |
| "logps/rejected": -195.43804931640625, | |
| "loss": 0.1737, | |
| "rewards/chosen": -0.6987349987030029, | |
| "rewards/margins": 2.256924629211426, | |
| "rewards/rejected": -2.9556596279144287, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 12.841610853219839, | |
| "kl": 0.0, | |
| "learning_rate": 4.814233385661957e-07, | |
| "logps/chosen": -204.5384521484375, | |
| "logps/rejected": -208.86331176757812, | |
| "loss": 0.2565, | |
| "rewards/chosen": -1.2488123178482056, | |
| "rewards/margins": 1.1284335851669312, | |
| "rewards/rejected": -2.3772459030151367, | |
| "step": 71 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 13.054533599473265, | |
| "kl": 0.0, | |
| "learning_rate": 4.811616954474097e-07, | |
| "logps/chosen": -250.12615966796875, | |
| "logps/rejected": -243.621337890625, | |
| "loss": 0.1785, | |
| "rewards/chosen": -0.7167716026306152, | |
| "rewards/margins": 1.7554688453674316, | |
| "rewards/rejected": -2.472240447998047, | |
| "step": 72 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 11.218204777591135, | |
| "kl": 0.0, | |
| "learning_rate": 4.809000523286237e-07, | |
| "logps/chosen": -202.5470733642578, | |
| "logps/rejected": -251.5446014404297, | |
| "loss": 0.2578, | |
| "rewards/chosen": -0.35193824768066406, | |
| "rewards/margins": 1.2476764917373657, | |
| "rewards/rejected": -1.5996147394180298, | |
| "step": 73 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 10.005076969894088, | |
| "kl": 0.0, | |
| "learning_rate": 4.806384092098378e-07, | |
| "logps/chosen": -240.54025268554688, | |
| "logps/rejected": -238.83575439453125, | |
| "loss": 0.2025, | |
| "rewards/chosen": -1.1610413789749146, | |
| "rewards/margins": 1.6202930212020874, | |
| "rewards/rejected": -2.781334400177002, | |
| "step": 74 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 13.421231936678494, | |
| "kl": 0.0, | |
| "learning_rate": 4.803767660910518e-07, | |
| "logps/chosen": -287.82110595703125, | |
| "logps/rejected": -193.2252960205078, | |
| "loss": 0.2056, | |
| "rewards/chosen": -0.41320356726646423, | |
| "rewards/margins": 2.6422228813171387, | |
| "rewards/rejected": -3.0554263591766357, | |
| "step": 75 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 11.604997350152324, | |
| "kl": 0.0, | |
| "learning_rate": 4.801151229722658e-07, | |
| "logps/chosen": -266.6557922363281, | |
| "logps/rejected": -204.9314727783203, | |
| "loss": 0.1635, | |
| "rewards/chosen": -0.5626986622810364, | |
| "rewards/margins": 1.7505519390106201, | |
| "rewards/rejected": -2.3132505416870117, | |
| "step": 76 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 13.167949690406468, | |
| "kl": 0.0, | |
| "learning_rate": 4.798534798534799e-07, | |
| "logps/chosen": -212.5312957763672, | |
| "logps/rejected": -245.55245971679688, | |
| "loss": 0.2015, | |
| "rewards/chosen": -0.6334313750267029, | |
| "rewards/margins": 0.9760074019432068, | |
| "rewards/rejected": -1.6094387769699097, | |
| "step": 77 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 13.71229302802406, | |
| "kl": 0.0, | |
| "learning_rate": 4.795918367346938e-07, | |
| "logps/chosen": -248.7874755859375, | |
| "logps/rejected": -254.36085510253906, | |
| "loss": 0.2529, | |
| "rewards/chosen": 0.024569960311055183, | |
| "rewards/margins": 1.5464191436767578, | |
| "rewards/rejected": -1.5218491554260254, | |
| "step": 78 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 12.206901910051194, | |
| "kl": 0.0, | |
| "learning_rate": 4.793301936159079e-07, | |
| "logps/chosen": -214.5361328125, | |
| "logps/rejected": -224.99876403808594, | |
| "loss": 0.1976, | |
| "rewards/chosen": -0.3784463107585907, | |
| "rewards/margins": 2.108283281326294, | |
| "rewards/rejected": -2.486729621887207, | |
| "step": 79 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 12.358475151298732, | |
| "kl": 0.0, | |
| "learning_rate": 4.790685504971219e-07, | |
| "logps/chosen": -227.8341064453125, | |
| "logps/rejected": -244.53817749023438, | |
| "loss": 0.1508, | |
| "rewards/chosen": 0.0641309916973114, | |
| "rewards/margins": 2.4745709896087646, | |
| "rewards/rejected": -2.410439968109131, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 10.7530961108445, | |
| "kl": 0.0, | |
| "learning_rate": 4.78806907378336e-07, | |
| "logps/chosen": -197.07553100585938, | |
| "logps/rejected": -282.6841735839844, | |
| "loss": 0.2055, | |
| "rewards/chosen": -1.8003284931182861, | |
| "rewards/margins": 0.7180378437042236, | |
| "rewards/rejected": -2.5183663368225098, | |
| "step": 81 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 15.543960456447536, | |
| "kl": 0.0, | |
| "learning_rate": 4.7854526425955e-07, | |
| "logps/chosen": -286.0069580078125, | |
| "logps/rejected": -333.1952819824219, | |
| "loss": 0.2215, | |
| "rewards/chosen": -0.9929041862487793, | |
| "rewards/margins": 1.838501214981079, | |
| "rewards/rejected": -2.8314054012298584, | |
| "step": 82 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 12.540856467590372, | |
| "kl": 0.0, | |
| "learning_rate": 4.78283621140764e-07, | |
| "logps/chosen": -250.1410369873047, | |
| "logps/rejected": -234.04039001464844, | |
| "loss": 0.2142, | |
| "rewards/chosen": -1.1462880373001099, | |
| "rewards/margins": 1.3604058027267456, | |
| "rewards/rejected": -2.5066938400268555, | |
| "step": 83 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 12.685162706946018, | |
| "kl": 0.0, | |
| "learning_rate": 4.78021978021978e-07, | |
| "logps/chosen": -244.63720703125, | |
| "logps/rejected": -206.22079467773438, | |
| "loss": 0.2197, | |
| "rewards/chosen": -0.5177630186080933, | |
| "rewards/margins": 2.0680246353149414, | |
| "rewards/rejected": -2.585787534713745, | |
| "step": 84 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 10.152423089357503, | |
| "kl": 0.0, | |
| "learning_rate": 4.77760334903192e-07, | |
| "logps/chosen": -208.19064331054688, | |
| "logps/rejected": -250.68597412109375, | |
| "loss": 0.0981, | |
| "rewards/chosen": 0.4294729232788086, | |
| "rewards/margins": 2.881974697113037, | |
| "rewards/rejected": -2.4525017738342285, | |
| "step": 85 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 10.497367332273754, | |
| "kl": 0.0, | |
| "learning_rate": 4.774986917844061e-07, | |
| "logps/chosen": -200.9931640625, | |
| "logps/rejected": -224.72264099121094, | |
| "loss": 0.2253, | |
| "rewards/chosen": -0.9502390623092651, | |
| "rewards/margins": 0.7048424482345581, | |
| "rewards/rejected": -1.6550815105438232, | |
| "step": 86 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 10.454551429296167, | |
| "kl": 0.0, | |
| "learning_rate": 4.772370486656201e-07, | |
| "logps/chosen": -204.69024658203125, | |
| "logps/rejected": -238.31492614746094, | |
| "loss": 0.1596, | |
| "rewards/chosen": 0.6356328725814819, | |
| "rewards/margins": 2.17765474319458, | |
| "rewards/rejected": -1.5420219898223877, | |
| "step": 87 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 11.947921924106597, | |
| "kl": 0.0, | |
| "learning_rate": 4.769754055468341e-07, | |
| "logps/chosen": -233.5601806640625, | |
| "logps/rejected": -232.72801208496094, | |
| "loss": 0.2242, | |
| "rewards/chosen": 0.4331187307834625, | |
| "rewards/margins": 1.7527743577957153, | |
| "rewards/rejected": -1.3196556568145752, | |
| "step": 88 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 12.796884542926785, | |
| "kl": 0.0, | |
| "learning_rate": 4.767137624280481e-07, | |
| "logps/chosen": -229.7658233642578, | |
| "logps/rejected": -266.162841796875, | |
| "loss": 0.2534, | |
| "rewards/chosen": -0.2590081989765167, | |
| "rewards/margins": 1.418634295463562, | |
| "rewards/rejected": -1.6776424646377563, | |
| "step": 89 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 10.31450194916881, | |
| "kl": 0.0, | |
| "learning_rate": 4.764521193092622e-07, | |
| "logps/chosen": -273.9610595703125, | |
| "logps/rejected": -277.6679382324219, | |
| "loss": 0.1663, | |
| "rewards/chosen": -0.03371845558285713, | |
| "rewards/margins": 2.451915740966797, | |
| "rewards/rejected": -2.4856340885162354, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 9.278455988308327, | |
| "kl": 0.0, | |
| "learning_rate": 4.761904761904761e-07, | |
| "logps/chosen": -232.08615112304688, | |
| "logps/rejected": -272.5909118652344, | |
| "loss": 0.184, | |
| "rewards/chosen": -0.7493877410888672, | |
| "rewards/margins": 2.300028085708618, | |
| "rewards/rejected": -3.0494158267974854, | |
| "step": 91 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 8.817410071740532, | |
| "kl": 0.0, | |
| "learning_rate": 4.7592883307169017e-07, | |
| "logps/chosen": -222.3486785888672, | |
| "logps/rejected": -248.85317993164062, | |
| "loss": 0.1731, | |
| "rewards/chosen": -0.2696648836135864, | |
| "rewards/margins": 2.0320448875427246, | |
| "rewards/rejected": -2.3017096519470215, | |
| "step": 92 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 10.59864769694075, | |
| "kl": 0.0, | |
| "learning_rate": 4.756671899529042e-07, | |
| "logps/chosen": -189.18434143066406, | |
| "logps/rejected": -270.7514343261719, | |
| "loss": 0.1508, | |
| "rewards/chosen": 0.2290552258491516, | |
| "rewards/margins": 3.3581156730651855, | |
| "rewards/rejected": -3.1290605068206787, | |
| "step": 93 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 11.988809402174233, | |
| "kl": 0.0, | |
| "learning_rate": 4.7540554683411827e-07, | |
| "logps/chosen": -276.038330078125, | |
| "logps/rejected": -303.55596923828125, | |
| "loss": 0.191, | |
| "rewards/chosen": -0.4145098030567169, | |
| "rewards/margins": 2.4649319648742676, | |
| "rewards/rejected": -2.879441738128662, | |
| "step": 94 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 14.079727761013277, | |
| "kl": 0.0, | |
| "learning_rate": 4.7514390371533227e-07, | |
| "logps/chosen": -257.2515869140625, | |
| "logps/rejected": -244.81591796875, | |
| "loss": 0.2074, | |
| "rewards/chosen": -1.449769139289856, | |
| "rewards/margins": 2.1893882751464844, | |
| "rewards/rejected": -3.63915753364563, | |
| "step": 95 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 12.455104492784045, | |
| "kl": 0.0, | |
| "learning_rate": 4.7488226059654627e-07, | |
| "logps/chosen": -258.3236389160156, | |
| "logps/rejected": -223.65472412109375, | |
| "loss": 0.2373, | |
| "rewards/chosen": -0.474732369184494, | |
| "rewards/margins": 1.6088600158691406, | |
| "rewards/rejected": -2.083592414855957, | |
| "step": 96 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 12.813735596225209, | |
| "kl": 0.0, | |
| "learning_rate": 4.746206174777603e-07, | |
| "logps/chosen": -250.6949005126953, | |
| "logps/rejected": -282.6470031738281, | |
| "loss": 0.2388, | |
| "rewards/chosen": -0.8905995488166809, | |
| "rewards/margins": 1.769092321395874, | |
| "rewards/rejected": -2.65969181060791, | |
| "step": 97 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 11.204915138695135, | |
| "kl": 0.0, | |
| "learning_rate": 4.743589743589743e-07, | |
| "logps/chosen": -176.22486877441406, | |
| "logps/rejected": -309.79229736328125, | |
| "loss": 0.2097, | |
| "rewards/chosen": -1.4110625982284546, | |
| "rewards/margins": 0.993507981300354, | |
| "rewards/rejected": -2.4045705795288086, | |
| "step": 98 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 11.735425350671436, | |
| "kl": 0.0, | |
| "learning_rate": 4.7409733124018836e-07, | |
| "logps/chosen": -224.20626831054688, | |
| "logps/rejected": -246.65956115722656, | |
| "loss": 0.1503, | |
| "rewards/chosen": -0.4009709060192108, | |
| "rewards/margins": 2.9179859161376953, | |
| "rewards/rejected": -3.3189568519592285, | |
| "step": 99 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 9.89695125192596, | |
| "kl": 0.0, | |
| "learning_rate": 4.738356881214024e-07, | |
| "logps/chosen": -217.57763671875, | |
| "logps/rejected": -224.3143768310547, | |
| "loss": 0.1092, | |
| "rewards/chosen": 0.3369201123714447, | |
| "rewards/margins": 2.3455708026885986, | |
| "rewards/rejected": -2.008650779724121, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 10.212358128672067, | |
| "kl": 0.0, | |
| "learning_rate": 4.7357404500261636e-07, | |
| "logps/chosen": -197.8075408935547, | |
| "logps/rejected": -229.45677185058594, | |
| "loss": 0.2126, | |
| "rewards/chosen": -0.7042273879051208, | |
| "rewards/margins": 1.3926599025726318, | |
| "rewards/rejected": -2.0968873500823975, | |
| "step": 101 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 12.604368131047796, | |
| "kl": 0.0, | |
| "learning_rate": 4.733124018838304e-07, | |
| "logps/chosen": -211.51576232910156, | |
| "logps/rejected": -228.77308654785156, | |
| "loss": 0.2363, | |
| "rewards/chosen": -1.1265863180160522, | |
| "rewards/margins": 0.9696308374404907, | |
| "rewards/rejected": -2.096217155456543, | |
| "step": 102 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 11.118276337061534, | |
| "kl": 0.0, | |
| "learning_rate": 4.7305075876504446e-07, | |
| "logps/chosen": -199.48486328125, | |
| "logps/rejected": -370.7545166015625, | |
| "loss": 0.1847, | |
| "rewards/chosen": -0.5413498282432556, | |
| "rewards/margins": 2.3086533546447754, | |
| "rewards/rejected": -2.850003242492676, | |
| "step": 103 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 13.028026388315732, | |
| "kl": 0.0, | |
| "learning_rate": 4.727891156462585e-07, | |
| "logps/chosen": -262.6328430175781, | |
| "logps/rejected": -287.1793518066406, | |
| "loss": 0.2251, | |
| "rewards/chosen": -0.2909429669380188, | |
| "rewards/margins": 2.515258550643921, | |
| "rewards/rejected": -2.806201457977295, | |
| "step": 104 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 11.421812108784723, | |
| "kl": 0.0, | |
| "learning_rate": 4.725274725274725e-07, | |
| "logps/chosen": -234.4532470703125, | |
| "logps/rejected": -272.8806457519531, | |
| "loss": 0.1591, | |
| "rewards/chosen": 0.45129498839378357, | |
| "rewards/margins": 2.3014488220214844, | |
| "rewards/rejected": -1.8501538038253784, | |
| "step": 105 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 10.451980811425736, | |
| "kl": 0.0, | |
| "learning_rate": 4.7226582940868656e-07, | |
| "logps/chosen": -266.4130859375, | |
| "logps/rejected": -256.59039306640625, | |
| "loss": 0.1315, | |
| "rewards/chosen": -0.39178353548049927, | |
| "rewards/margins": 1.9923765659332275, | |
| "rewards/rejected": -2.384160041809082, | |
| "step": 106 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 10.541336180624638, | |
| "kl": 0.0, | |
| "learning_rate": 4.7200418628990056e-07, | |
| "logps/chosen": -253.11045837402344, | |
| "logps/rejected": -257.3990478515625, | |
| "loss": 0.1589, | |
| "rewards/chosen": 0.02405649609863758, | |
| "rewards/margins": 3.012369394302368, | |
| "rewards/rejected": -2.9883129596710205, | |
| "step": 107 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 11.191607092594259, | |
| "kl": 0.0, | |
| "learning_rate": 4.7174254317111455e-07, | |
| "logps/chosen": -219.4788818359375, | |
| "logps/rejected": -251.76841735839844, | |
| "loss": 0.1878, | |
| "rewards/chosen": -0.653347909450531, | |
| "rewards/margins": 1.0217313766479492, | |
| "rewards/rejected": -1.675079345703125, | |
| "step": 108 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 10.465911824092663, | |
| "kl": 0.0, | |
| "learning_rate": 4.714809000523286e-07, | |
| "logps/chosen": -253.79640197753906, | |
| "logps/rejected": -208.65074157714844, | |
| "loss": 0.2581, | |
| "rewards/chosen": -1.6879644393920898, | |
| "rewards/margins": -0.33700597286224365, | |
| "rewards/rejected": -1.3509584665298462, | |
| "step": 109 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 11.000969752280952, | |
| "kl": 0.0, | |
| "learning_rate": 4.7121925693354265e-07, | |
| "logps/chosen": -251.3641815185547, | |
| "logps/rejected": -240.36642456054688, | |
| "loss": 0.1976, | |
| "rewards/chosen": -0.2726837396621704, | |
| "rewards/margins": 2.477322578430176, | |
| "rewards/rejected": -2.7500064373016357, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 14.908402118189745, | |
| "kl": 0.0, | |
| "learning_rate": 4.7095761381475665e-07, | |
| "logps/chosen": -244.45858764648438, | |
| "logps/rejected": -266.1649169921875, | |
| "loss": 0.1959, | |
| "rewards/chosen": -0.5489299893379211, | |
| "rewards/margins": 0.9248166680335999, | |
| "rewards/rejected": -1.473746657371521, | |
| "step": 111 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 10.994589492437557, | |
| "kl": 0.0, | |
| "learning_rate": 4.7069597069597065e-07, | |
| "logps/chosen": -250.3166961669922, | |
| "logps/rejected": -275.6036071777344, | |
| "loss": 0.2578, | |
| "rewards/chosen": -1.3922420740127563, | |
| "rewards/margins": 0.7932580709457397, | |
| "rewards/rejected": -2.185500144958496, | |
| "step": 112 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 12.230772963233404, | |
| "kl": 0.0, | |
| "learning_rate": 4.704343275771847e-07, | |
| "logps/chosen": -251.1761474609375, | |
| "logps/rejected": -354.3590393066406, | |
| "loss": 0.1702, | |
| "rewards/chosen": 0.4129093289375305, | |
| "rewards/margins": 1.9088201522827148, | |
| "rewards/rejected": -1.4959107637405396, | |
| "step": 113 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 11.73992280402579, | |
| "kl": 0.0, | |
| "learning_rate": 4.7017268445839875e-07, | |
| "logps/chosen": -185.67906188964844, | |
| "logps/rejected": -188.59017944335938, | |
| "loss": 0.1331, | |
| "rewards/chosen": -0.09853968024253845, | |
| "rewards/margins": 1.8528465032577515, | |
| "rewards/rejected": -1.9513862133026123, | |
| "step": 114 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 10.142692814638496, | |
| "kl": 0.0, | |
| "learning_rate": 4.6991104133961275e-07, | |
| "logps/chosen": -266.6419982910156, | |
| "logps/rejected": -270.0727233886719, | |
| "loss": 0.1801, | |
| "rewards/chosen": -0.9462317824363708, | |
| "rewards/margins": 1.412889003753662, | |
| "rewards/rejected": -2.3591208457946777, | |
| "step": 115 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 10.967127165803278, | |
| "kl": 0.0, | |
| "learning_rate": 4.696493982208268e-07, | |
| "logps/chosen": -247.25096130371094, | |
| "logps/rejected": -212.33914184570312, | |
| "loss": 0.1917, | |
| "rewards/chosen": -1.022446632385254, | |
| "rewards/margins": -0.13641774654388428, | |
| "rewards/rejected": -0.8860288858413696, | |
| "step": 116 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 12.672315042849133, | |
| "kl": 0.0, | |
| "learning_rate": 4.693877551020408e-07, | |
| "logps/chosen": -251.9243621826172, | |
| "logps/rejected": -278.6723327636719, | |
| "loss": 0.2143, | |
| "rewards/chosen": -0.1397348791360855, | |
| "rewards/margins": 2.0988736152648926, | |
| "rewards/rejected": -2.2386085987091064, | |
| "step": 117 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 9.90707282468987, | |
| "kl": 0.0, | |
| "learning_rate": 4.691261119832548e-07, | |
| "logps/chosen": -198.83712768554688, | |
| "logps/rejected": -277.9150085449219, | |
| "loss": 0.241, | |
| "rewards/chosen": -0.4362916052341461, | |
| "rewards/margins": 2.6319870948791504, | |
| "rewards/rejected": -3.0682787895202637, | |
| "step": 118 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 9.980080260830226, | |
| "kl": 0.0, | |
| "learning_rate": 4.6886446886446884e-07, | |
| "logps/chosen": -222.55442810058594, | |
| "logps/rejected": -239.49298095703125, | |
| "loss": 0.1667, | |
| "rewards/chosen": -0.26491254568099976, | |
| "rewards/margins": 2.8245887756347656, | |
| "rewards/rejected": -3.08950138092041, | |
| "step": 119 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 14.223816837655765, | |
| "kl": 0.0, | |
| "learning_rate": 4.686028257456829e-07, | |
| "logps/chosen": -229.81675720214844, | |
| "logps/rejected": -248.0295867919922, | |
| "loss": 0.2241, | |
| "rewards/chosen": 0.6647676229476929, | |
| "rewards/margins": 2.933067798614502, | |
| "rewards/rejected": -2.2683000564575195, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 11.062546156999915, | |
| "kl": 0.0, | |
| "learning_rate": 4.683411826268969e-07, | |
| "logps/chosen": -215.87423706054688, | |
| "logps/rejected": -205.9690399169922, | |
| "loss": 0.221, | |
| "rewards/chosen": -0.38256943225860596, | |
| "rewards/margins": 1.25730562210083, | |
| "rewards/rejected": -1.639875054359436, | |
| "step": 121 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 11.71515711397141, | |
| "kl": 0.0, | |
| "learning_rate": 4.680795395081109e-07, | |
| "logps/chosen": -195.1471710205078, | |
| "logps/rejected": -204.15228271484375, | |
| "loss": 0.205, | |
| "rewards/chosen": 0.031107014045119286, | |
| "rewards/margins": 1.4260950088500977, | |
| "rewards/rejected": -1.394987940788269, | |
| "step": 122 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 10.668033463026218, | |
| "kl": 0.0, | |
| "learning_rate": 4.6781789638932494e-07, | |
| "logps/chosen": -238.59262084960938, | |
| "logps/rejected": -244.41387939453125, | |
| "loss": 0.2863, | |
| "rewards/chosen": -0.6594109535217285, | |
| "rewards/margins": 0.7834258079528809, | |
| "rewards/rejected": -1.4428367614746094, | |
| "step": 123 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 13.050812790862148, | |
| "kl": 0.0, | |
| "learning_rate": 4.6755625327053893e-07, | |
| "logps/chosen": -208.3390655517578, | |
| "logps/rejected": -246.9788360595703, | |
| "loss": 0.1784, | |
| "rewards/chosen": -0.5635675191879272, | |
| "rewards/margins": 0.9708378314971924, | |
| "rewards/rejected": -1.5344053506851196, | |
| "step": 124 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 11.38382856846076, | |
| "kl": 0.0, | |
| "learning_rate": 4.67294610151753e-07, | |
| "logps/chosen": -211.11561584472656, | |
| "logps/rejected": -253.8684539794922, | |
| "loss": 0.1853, | |
| "rewards/chosen": -0.8592585325241089, | |
| "rewards/margins": 1.5102192163467407, | |
| "rewards/rejected": -2.3694777488708496, | |
| "step": 125 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 11.616081872243173, | |
| "kl": 0.0, | |
| "learning_rate": 4.6703296703296704e-07, | |
| "logps/chosen": -198.94691467285156, | |
| "logps/rejected": -280.16485595703125, | |
| "loss": 0.1689, | |
| "rewards/chosen": -0.6960778832435608, | |
| "rewards/margins": 2.7315070629119873, | |
| "rewards/rejected": -3.4275848865509033, | |
| "step": 126 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 9.393491633545926, | |
| "kl": 0.0, | |
| "learning_rate": 4.6677132391418103e-07, | |
| "logps/chosen": -237.6060333251953, | |
| "logps/rejected": -240.0240020751953, | |
| "loss": 0.2003, | |
| "rewards/chosen": -0.7763846516609192, | |
| "rewards/margins": 1.804600715637207, | |
| "rewards/rejected": -2.5809853076934814, | |
| "step": 127 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 10.566838825942087, | |
| "kl": 0.0, | |
| "learning_rate": 4.6650968079539503e-07, | |
| "logps/chosen": -266.26409912109375, | |
| "logps/rejected": -245.97015380859375, | |
| "loss": 0.179, | |
| "rewards/chosen": -0.013315518386662006, | |
| "rewards/margins": 2.7922699451446533, | |
| "rewards/rejected": -2.8055853843688965, | |
| "step": 128 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 14.386798112694665, | |
| "kl": 0.0, | |
| "learning_rate": 4.662480376766091e-07, | |
| "logps/chosen": -259.3221740722656, | |
| "logps/rejected": -280.12982177734375, | |
| "loss": 0.2262, | |
| "rewards/chosen": 0.06482791900634766, | |
| "rewards/margins": 1.3292893171310425, | |
| "rewards/rejected": -1.2644613981246948, | |
| "step": 129 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 10.761953162868792, | |
| "kl": 0.0, | |
| "learning_rate": 4.6598639455782313e-07, | |
| "logps/chosen": -193.20936584472656, | |
| "logps/rejected": -270.4673767089844, | |
| "loss": 0.1212, | |
| "rewards/chosen": -0.4515216648578644, | |
| "rewards/margins": 2.8607840538024902, | |
| "rewards/rejected": -3.3123056888580322, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 12.751720769362379, | |
| "kl": 0.0, | |
| "learning_rate": 4.6572475143903713e-07, | |
| "logps/chosen": -235.50526428222656, | |
| "logps/rejected": -217.54153442382812, | |
| "loss": 0.2364, | |
| "rewards/chosen": 0.6571396589279175, | |
| "rewards/margins": 2.786506175994873, | |
| "rewards/rejected": -2.129366636276245, | |
| "step": 131 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 10.524940036364148, | |
| "kl": 0.0, | |
| "learning_rate": 4.654631083202512e-07, | |
| "logps/chosen": -200.957275390625, | |
| "logps/rejected": -260.49444580078125, | |
| "loss": 0.1259, | |
| "rewards/chosen": -0.29692986607551575, | |
| "rewards/margins": 2.9654293060302734, | |
| "rewards/rejected": -3.262359142303467, | |
| "step": 132 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 10.735468201198927, | |
| "kl": 0.0, | |
| "learning_rate": 4.652014652014652e-07, | |
| "logps/chosen": -243.689208984375, | |
| "logps/rejected": -259.3601379394531, | |
| "loss": 0.1994, | |
| "rewards/chosen": -0.13563716411590576, | |
| "rewards/margins": 2.890209674835205, | |
| "rewards/rejected": -3.0258467197418213, | |
| "step": 133 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 13.921041963760125, | |
| "kl": 0.0, | |
| "learning_rate": 4.6493982208267917e-07, | |
| "logps/chosen": -260.98651123046875, | |
| "logps/rejected": -302.0372314453125, | |
| "loss": 0.2204, | |
| "rewards/chosen": -0.5615333318710327, | |
| "rewards/margins": 1.4565614461898804, | |
| "rewards/rejected": -2.018094778060913, | |
| "step": 134 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 8.886703505939792, | |
| "kl": 0.0, | |
| "learning_rate": 4.646781789638932e-07, | |
| "logps/chosen": -219.5489501953125, | |
| "logps/rejected": -222.55165100097656, | |
| "loss": 0.2585, | |
| "rewards/chosen": -2.1579434871673584, | |
| "rewards/margins": -0.29236698150634766, | |
| "rewards/rejected": -1.8655765056610107, | |
| "step": 135 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 12.360378101489804, | |
| "kl": 0.0, | |
| "learning_rate": 4.644165358451073e-07, | |
| "logps/chosen": -200.14247131347656, | |
| "logps/rejected": -243.30284118652344, | |
| "loss": 0.2047, | |
| "rewards/chosen": -0.22793890535831451, | |
| "rewards/margins": 2.7516727447509766, | |
| "rewards/rejected": -2.97961163520813, | |
| "step": 136 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 11.520891149644566, | |
| "kl": 0.0, | |
| "learning_rate": 4.641548927263213e-07, | |
| "logps/chosen": -212.42518615722656, | |
| "logps/rejected": -271.6721496582031, | |
| "loss": 0.1469, | |
| "rewards/chosen": -0.4246146082878113, | |
| "rewards/margins": 3.187201499938965, | |
| "rewards/rejected": -3.611816167831421, | |
| "step": 137 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 8.17959898500212, | |
| "kl": 0.0, | |
| "learning_rate": 4.6389324960753527e-07, | |
| "logps/chosen": -195.15431213378906, | |
| "logps/rejected": -255.0457000732422, | |
| "loss": 0.2136, | |
| "rewards/chosen": -0.6788855791091919, | |
| "rewards/margins": 2.8008108139038086, | |
| "rewards/rejected": -3.47969651222229, | |
| "step": 138 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 10.843997894954787, | |
| "kl": 0.0, | |
| "learning_rate": 4.636316064887493e-07, | |
| "logps/chosen": -201.34103393554688, | |
| "logps/rejected": -312.7411804199219, | |
| "loss": 0.186, | |
| "rewards/chosen": -1.7989752292633057, | |
| "rewards/margins": 2.105161428451538, | |
| "rewards/rejected": -3.9041366577148438, | |
| "step": 139 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 12.401485298711776, | |
| "kl": 0.0, | |
| "learning_rate": 4.6336996336996337e-07, | |
| "logps/chosen": -174.31419372558594, | |
| "logps/rejected": -279.531494140625, | |
| "loss": 0.1899, | |
| "rewards/chosen": -0.8827177286148071, | |
| "rewards/margins": 1.6950773000717163, | |
| "rewards/rejected": -2.5777950286865234, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 12.876518397686489, | |
| "kl": 0.0, | |
| "learning_rate": 4.6310832025117737e-07, | |
| "logps/chosen": -219.05679321289062, | |
| "logps/rejected": -270.2620544433594, | |
| "loss": 0.2003, | |
| "rewards/chosen": -0.21344774961471558, | |
| "rewards/margins": 2.0357961654663086, | |
| "rewards/rejected": -2.249243974685669, | |
| "step": 141 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 10.774258331144129, | |
| "kl": 0.0, | |
| "learning_rate": 4.628466771323914e-07, | |
| "logps/chosen": -212.2626495361328, | |
| "logps/rejected": -283.8631286621094, | |
| "loss": 0.1711, | |
| "rewards/chosen": -0.1372440755367279, | |
| "rewards/margins": 2.4757416248321533, | |
| "rewards/rejected": -2.612985610961914, | |
| "step": 142 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 8.790512175321405, | |
| "kl": 0.0, | |
| "learning_rate": 4.625850340136054e-07, | |
| "logps/chosen": -232.13316345214844, | |
| "logps/rejected": -223.1201934814453, | |
| "loss": 0.1773, | |
| "rewards/chosen": -0.856933057308197, | |
| "rewards/margins": 2.2891058921813965, | |
| "rewards/rejected": -3.1460390090942383, | |
| "step": 143 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 10.352466882768905, | |
| "kl": 0.0, | |
| "learning_rate": 4.623233908948194e-07, | |
| "logps/chosen": -184.80145263671875, | |
| "logps/rejected": -193.4806671142578, | |
| "loss": 0.215, | |
| "rewards/chosen": -1.3604167699813843, | |
| "rewards/margins": 0.628121018409729, | |
| "rewards/rejected": -1.9885377883911133, | |
| "step": 144 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 7.163092288844522, | |
| "kl": 0.0, | |
| "learning_rate": 4.6206174777603346e-07, | |
| "logps/chosen": -202.53546142578125, | |
| "logps/rejected": -254.51487731933594, | |
| "loss": 0.0969, | |
| "rewards/chosen": -0.07566285133361816, | |
| "rewards/margins": 4.109183311462402, | |
| "rewards/rejected": -4.1848464012146, | |
| "step": 145 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 12.265660907944843, | |
| "kl": 0.0, | |
| "learning_rate": 4.618001046572475e-07, | |
| "logps/chosen": -265.9979553222656, | |
| "logps/rejected": -339.3370666503906, | |
| "loss": 0.1413, | |
| "rewards/chosen": 0.027865849435329437, | |
| "rewards/margins": 4.2701802253723145, | |
| "rewards/rejected": -4.242314338684082, | |
| "step": 146 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 13.038939425208627, | |
| "kl": 0.0, | |
| "learning_rate": 4.6153846153846156e-07, | |
| "logps/chosen": -262.6026306152344, | |
| "logps/rejected": -261.2329406738281, | |
| "loss": 0.1645, | |
| "rewards/chosen": -0.25406259298324585, | |
| "rewards/margins": 3.3194682598114014, | |
| "rewards/rejected": -3.573530912399292, | |
| "step": 147 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 13.392666406051779, | |
| "kl": 0.0, | |
| "learning_rate": 4.612768184196755e-07, | |
| "logps/chosen": -265.6265563964844, | |
| "logps/rejected": -281.2951354980469, | |
| "loss": 0.2543, | |
| "rewards/chosen": 0.6286525130271912, | |
| "rewards/margins": 3.1757638454437256, | |
| "rewards/rejected": -2.5471112728118896, | |
| "step": 148 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 14.164191990445923, | |
| "kl": 0.0, | |
| "learning_rate": 4.6101517530088956e-07, | |
| "logps/chosen": -280.87567138671875, | |
| "logps/rejected": -298.3075866699219, | |
| "loss": 0.2937, | |
| "rewards/chosen": -1.144802212715149, | |
| "rewards/margins": 0.6895921230316162, | |
| "rewards/rejected": -1.8343943357467651, | |
| "step": 149 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 10.347666390603884, | |
| "kl": 0.0, | |
| "learning_rate": 4.607535321821036e-07, | |
| "logps/chosen": -227.58335876464844, | |
| "logps/rejected": -239.180908203125, | |
| "loss": 0.1543, | |
| "rewards/chosen": -1.3316487073898315, | |
| "rewards/margins": 0.7044416666030884, | |
| "rewards/rejected": -2.03609037399292, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 11.52825621590645, | |
| "kl": 0.0, | |
| "learning_rate": 4.604918890633176e-07, | |
| "logps/chosen": -278.8197326660156, | |
| "logps/rejected": -309.47271728515625, | |
| "loss": 0.2073, | |
| "rewards/chosen": -0.36525189876556396, | |
| "rewards/margins": 3.581759452819824, | |
| "rewards/rejected": -3.9470112323760986, | |
| "step": 151 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 10.00307147974759, | |
| "kl": 0.0, | |
| "learning_rate": 4.6023024594453166e-07, | |
| "logps/chosen": -232.21923828125, | |
| "logps/rejected": -288.6890563964844, | |
| "loss": 0.1359, | |
| "rewards/chosen": -1.7685518264770508, | |
| "rewards/margins": 1.4930996894836426, | |
| "rewards/rejected": -3.2616515159606934, | |
| "step": 152 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 9.048295062554471, | |
| "kl": 0.0, | |
| "learning_rate": 4.5996860282574565e-07, | |
| "logps/chosen": -263.74456787109375, | |
| "logps/rejected": -182.83853149414062, | |
| "loss": 0.1225, | |
| "rewards/chosen": 0.26467177271842957, | |
| "rewards/margins": 3.1499152183532715, | |
| "rewards/rejected": -2.8852434158325195, | |
| "step": 153 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 9.505705109305136, | |
| "kl": 0.0, | |
| "learning_rate": 4.5970695970695965e-07, | |
| "logps/chosen": -239.68685913085938, | |
| "logps/rejected": -284.68927001953125, | |
| "loss": 0.228, | |
| "rewards/chosen": -1.0681113004684448, | |
| "rewards/margins": 0.38854992389678955, | |
| "rewards/rejected": -1.4566612243652344, | |
| "step": 154 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 12.907956927820472, | |
| "kl": 0.0, | |
| "learning_rate": 4.594453165881737e-07, | |
| "logps/chosen": -171.40005493164062, | |
| "logps/rejected": -231.8187255859375, | |
| "loss": 0.2271, | |
| "rewards/chosen": -0.5379889607429504, | |
| "rewards/margins": 2.1988117694854736, | |
| "rewards/rejected": -2.7368006706237793, | |
| "step": 155 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 10.134785294701812, | |
| "kl": 0.0, | |
| "learning_rate": 4.5918367346938775e-07, | |
| "logps/chosen": -206.3992462158203, | |
| "logps/rejected": -202.78799438476562, | |
| "loss": 0.1833, | |
| "rewards/chosen": -0.1859370917081833, | |
| "rewards/margins": 2.8349618911743164, | |
| "rewards/rejected": -3.0208990573883057, | |
| "step": 156 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 10.654899640372967, | |
| "kl": 0.0, | |
| "learning_rate": 4.589220303506018e-07, | |
| "logps/chosen": -226.39356994628906, | |
| "logps/rejected": -253.27488708496094, | |
| "loss": 0.2065, | |
| "rewards/chosen": -0.936338484287262, | |
| "rewards/margins": 1.9939072132110596, | |
| "rewards/rejected": -2.9302456378936768, | |
| "step": 157 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 10.940456012048248, | |
| "kl": 0.0, | |
| "learning_rate": 4.5866038723181575e-07, | |
| "logps/chosen": -224.9565887451172, | |
| "logps/rejected": -265.723388671875, | |
| "loss": 0.2093, | |
| "rewards/chosen": -1.0069631338119507, | |
| "rewards/margins": 1.279760718345642, | |
| "rewards/rejected": -2.2867238521575928, | |
| "step": 158 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 13.331073300797962, | |
| "kl": 0.0, | |
| "learning_rate": 4.583987441130298e-07, | |
| "logps/chosen": -199.77915954589844, | |
| "logps/rejected": -303.58538818359375, | |
| "loss": 0.1757, | |
| "rewards/chosen": -0.519672155380249, | |
| "rewards/margins": 1.5667927265167236, | |
| "rewards/rejected": -2.0864648818969727, | |
| "step": 159 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 12.759132153782163, | |
| "kl": 0.0, | |
| "learning_rate": 4.5813710099424385e-07, | |
| "logps/chosen": -263.2257385253906, | |
| "logps/rejected": -216.82395935058594, | |
| "loss": 0.2497, | |
| "rewards/chosen": -0.19666321575641632, | |
| "rewards/margins": 1.9927500486373901, | |
| "rewards/rejected": -2.18941330909729, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 13.295115337094142, | |
| "kl": 0.0, | |
| "learning_rate": 4.5787545787545784e-07, | |
| "logps/chosen": -228.7009735107422, | |
| "logps/rejected": -295.6015319824219, | |
| "loss": 0.2364, | |
| "rewards/chosen": -0.8198361992835999, | |
| "rewards/margins": 0.8233547806739807, | |
| "rewards/rejected": -1.6431909799575806, | |
| "step": 161 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 14.82680643745037, | |
| "kl": 0.0, | |
| "learning_rate": 4.576138147566719e-07, | |
| "logps/chosen": -260.577880859375, | |
| "logps/rejected": -253.9260711669922, | |
| "loss": 0.2625, | |
| "rewards/chosen": -0.1303100883960724, | |
| "rewards/margins": 2.1792666912078857, | |
| "rewards/rejected": -2.3095767498016357, | |
| "step": 162 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 13.208958389079216, | |
| "kl": 0.0, | |
| "learning_rate": 4.5735217163788594e-07, | |
| "logps/chosen": -190.9998016357422, | |
| "logps/rejected": -276.3247985839844, | |
| "loss": 0.1975, | |
| "rewards/chosen": 0.03442925959825516, | |
| "rewards/margins": 2.56953763961792, | |
| "rewards/rejected": -2.5351083278656006, | |
| "step": 163 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 11.80477720551951, | |
| "kl": 0.0, | |
| "learning_rate": 4.570905285190999e-07, | |
| "logps/chosen": -246.15234375, | |
| "logps/rejected": -248.68072509765625, | |
| "loss": 0.2212, | |
| "rewards/chosen": -0.5984467267990112, | |
| "rewards/margins": 1.7412387132644653, | |
| "rewards/rejected": -2.3396854400634766, | |
| "step": 164 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 9.00263121063917, | |
| "kl": 0.0, | |
| "learning_rate": 4.5682888540031394e-07, | |
| "logps/chosen": -252.92127990722656, | |
| "logps/rejected": -215.0977325439453, | |
| "loss": 0.1732, | |
| "rewards/chosen": -0.49442756175994873, | |
| "rewards/margins": 2.391329288482666, | |
| "rewards/rejected": -2.885756731033325, | |
| "step": 165 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 9.7104117892985, | |
| "kl": 0.0, | |
| "learning_rate": 4.56567242281528e-07, | |
| "logps/chosen": -286.4942626953125, | |
| "logps/rejected": -277.3382873535156, | |
| "loss": 0.1979, | |
| "rewards/chosen": -0.5238850712776184, | |
| "rewards/margins": 2.418689489364624, | |
| "rewards/rejected": -2.9425745010375977, | |
| "step": 166 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 9.591639733233848, | |
| "kl": 0.0, | |
| "learning_rate": 4.56305599162742e-07, | |
| "logps/chosen": -237.4594268798828, | |
| "logps/rejected": -217.5932159423828, | |
| "loss": 0.1465, | |
| "rewards/chosen": -0.8079670071601868, | |
| "rewards/margins": 1.809727668762207, | |
| "rewards/rejected": -2.617694616317749, | |
| "step": 167 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 11.459074832211314, | |
| "kl": 0.0, | |
| "learning_rate": 4.5604395604395604e-07, | |
| "logps/chosen": -220.34298706054688, | |
| "logps/rejected": -245.24441528320312, | |
| "loss": 0.17, | |
| "rewards/chosen": 0.019128350540995598, | |
| "rewards/margins": 2.7078402042388916, | |
| "rewards/rejected": -2.6887118816375732, | |
| "step": 168 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 15.063963939999352, | |
| "kl": 0.0, | |
| "learning_rate": 4.5578231292517003e-07, | |
| "logps/chosen": -297.51300048828125, | |
| "logps/rejected": -251.10533142089844, | |
| "loss": 0.1753, | |
| "rewards/chosen": 0.8247194886207581, | |
| "rewards/margins": 2.683767557144165, | |
| "rewards/rejected": -1.8590481281280518, | |
| "step": 169 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 9.142681234947581, | |
| "kl": 0.0, | |
| "learning_rate": 4.555206698063841e-07, | |
| "logps/chosen": -300.2449951171875, | |
| "logps/rejected": -224.29713439941406, | |
| "loss": 0.2087, | |
| "rewards/chosen": -0.8797904253005981, | |
| "rewards/margins": 1.0547667741775513, | |
| "rewards/rejected": -1.9345571994781494, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 10.227907117502706, | |
| "kl": 0.0, | |
| "learning_rate": 4.552590266875981e-07, | |
| "logps/chosen": -224.4203338623047, | |
| "logps/rejected": -315.0204162597656, | |
| "loss": 0.1784, | |
| "rewards/chosen": -0.9637618064880371, | |
| "rewards/margins": 2.0699081420898438, | |
| "rewards/rejected": -3.033669948577881, | |
| "step": 171 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 11.184644215978503, | |
| "kl": 0.0, | |
| "learning_rate": 4.5499738356881213e-07, | |
| "logps/chosen": -232.22695922851562, | |
| "logps/rejected": -202.02688598632812, | |
| "loss": 0.2471, | |
| "rewards/chosen": -1.0796171426773071, | |
| "rewards/margins": 1.041359305381775, | |
| "rewards/rejected": -2.120976448059082, | |
| "step": 172 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 12.858302396434425, | |
| "kl": 0.0, | |
| "learning_rate": 4.547357404500262e-07, | |
| "logps/chosen": -231.92547607421875, | |
| "logps/rejected": -284.2903137207031, | |
| "loss": 0.1662, | |
| "rewards/chosen": 0.22813068330287933, | |
| "rewards/margins": 2.726292610168457, | |
| "rewards/rejected": -2.498162031173706, | |
| "step": 173 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 10.830471355823923, | |
| "kl": 0.0, | |
| "learning_rate": 4.5447409733124013e-07, | |
| "logps/chosen": -245.82032775878906, | |
| "logps/rejected": -204.56790161132812, | |
| "loss": 0.2141, | |
| "rewards/chosen": 0.45919883251190186, | |
| "rewards/margins": 2.118945598602295, | |
| "rewards/rejected": -1.6597468852996826, | |
| "step": 174 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 11.540820927447694, | |
| "kl": 0.0, | |
| "learning_rate": 4.542124542124542e-07, | |
| "logps/chosen": -221.0320587158203, | |
| "logps/rejected": -244.3851318359375, | |
| "loss": 0.1404, | |
| "rewards/chosen": 0.015136350877583027, | |
| "rewards/margins": 2.4611456394195557, | |
| "rewards/rejected": -2.446009397506714, | |
| "step": 175 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 9.370935336042336, | |
| "kl": 0.0, | |
| "learning_rate": 4.5395081109366823e-07, | |
| "logps/chosen": -148.11834716796875, | |
| "logps/rejected": -254.36962890625, | |
| "loss": 0.1237, | |
| "rewards/chosen": 0.47263994812965393, | |
| "rewards/margins": 3.6257412433624268, | |
| "rewards/rejected": -3.1531012058258057, | |
| "step": 176 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 9.524027879440371, | |
| "kl": 0.0, | |
| "learning_rate": 4.536891679748822e-07, | |
| "logps/chosen": -218.85006713867188, | |
| "logps/rejected": -268.6586608886719, | |
| "loss": 0.1249, | |
| "rewards/chosen": -1.152251958847046, | |
| "rewards/margins": 1.3508758544921875, | |
| "rewards/rejected": -2.5031278133392334, | |
| "step": 177 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 10.097839005642474, | |
| "kl": 0.0, | |
| "learning_rate": 4.534275248560963e-07, | |
| "logps/chosen": -140.55996704101562, | |
| "logps/rejected": -265.65789794921875, | |
| "loss": 0.279, | |
| "rewards/chosen": -0.11726745963096619, | |
| "rewards/margins": 1.562013030052185, | |
| "rewards/rejected": -1.6792805194854736, | |
| "step": 178 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 11.353665853013293, | |
| "kl": 0.0, | |
| "learning_rate": 4.5316588173731027e-07, | |
| "logps/chosen": -168.6680145263672, | |
| "logps/rejected": -309.8199462890625, | |
| "loss": 0.1995, | |
| "rewards/chosen": -0.32298406958580017, | |
| "rewards/margins": 1.8489097356796265, | |
| "rewards/rejected": -2.171893835067749, | |
| "step": 179 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 10.693119887330608, | |
| "kl": 0.0, | |
| "learning_rate": 4.529042386185243e-07, | |
| "logps/chosen": -262.695556640625, | |
| "logps/rejected": -234.45143127441406, | |
| "loss": 0.2569, | |
| "rewards/chosen": -0.22278353571891785, | |
| "rewards/margins": 2.558516502380371, | |
| "rewards/rejected": -2.7813000679016113, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 13.107582728245008, | |
| "kl": 0.0, | |
| "learning_rate": 4.526425954997383e-07, | |
| "logps/chosen": -244.29647827148438, | |
| "logps/rejected": -291.70941162109375, | |
| "loss": 0.2651, | |
| "rewards/chosen": -0.5042144656181335, | |
| "rewards/margins": 2.197082757949829, | |
| "rewards/rejected": -2.7012972831726074, | |
| "step": 181 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 9.773171918884088, | |
| "kl": 0.0, | |
| "learning_rate": 4.5238095238095237e-07, | |
| "logps/chosen": -197.32012939453125, | |
| "logps/rejected": -293.9897766113281, | |
| "loss": 0.1873, | |
| "rewards/chosen": -0.5700012445449829, | |
| "rewards/margins": 4.046950340270996, | |
| "rewards/rejected": -4.6169514656066895, | |
| "step": 182 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 11.20040922510821, | |
| "kl": 0.0, | |
| "learning_rate": 4.521193092621664e-07, | |
| "logps/chosen": -194.4012908935547, | |
| "logps/rejected": -223.0313720703125, | |
| "loss": 0.1483, | |
| "rewards/chosen": 0.47125691175460815, | |
| "rewards/margins": 2.2409911155700684, | |
| "rewards/rejected": -1.7697341442108154, | |
| "step": 183 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 12.989173617269795, | |
| "kl": 0.0, | |
| "learning_rate": 4.5185766614338037e-07, | |
| "logps/chosen": -171.93673706054688, | |
| "logps/rejected": -316.18316650390625, | |
| "loss": 0.216, | |
| "rewards/chosen": -0.1644737273454666, | |
| "rewards/margins": 0.4819384217262268, | |
| "rewards/rejected": -0.6464121341705322, | |
| "step": 184 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 13.329019681893538, | |
| "kl": 0.0, | |
| "learning_rate": 4.515960230245944e-07, | |
| "logps/chosen": -215.2859649658203, | |
| "logps/rejected": -258.4815368652344, | |
| "loss": 0.1873, | |
| "rewards/chosen": 0.19089780747890472, | |
| "rewards/margins": 3.2682807445526123, | |
| "rewards/rejected": -3.077383041381836, | |
| "step": 185 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 9.903441953636646, | |
| "kl": 0.0, | |
| "learning_rate": 4.5133437990580847e-07, | |
| "logps/chosen": -203.1886444091797, | |
| "logps/rejected": -219.4387969970703, | |
| "loss": 0.1725, | |
| "rewards/chosen": 0.7246238589286804, | |
| "rewards/margins": 1.6935207843780518, | |
| "rewards/rejected": -0.9688969254493713, | |
| "step": 186 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 10.026171474980584, | |
| "kl": 0.0, | |
| "learning_rate": 4.5107273678702246e-07, | |
| "logps/chosen": -237.39012145996094, | |
| "logps/rejected": -313.46038818359375, | |
| "loss": 0.1848, | |
| "rewards/chosen": 0.13696634769439697, | |
| "rewards/margins": 2.4037017822265625, | |
| "rewards/rejected": -2.266735315322876, | |
| "step": 187 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 13.743945884823951, | |
| "kl": 0.0, | |
| "learning_rate": 4.508110936682365e-07, | |
| "logps/chosen": -226.96961975097656, | |
| "logps/rejected": -239.95713806152344, | |
| "loss": 0.0994, | |
| "rewards/chosen": 0.7345108389854431, | |
| "rewards/margins": 2.190957546234131, | |
| "rewards/rejected": -1.4564467668533325, | |
| "step": 188 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 11.111688156500954, | |
| "kl": 0.0, | |
| "learning_rate": 4.5054945054945056e-07, | |
| "logps/chosen": -228.42758178710938, | |
| "logps/rejected": -203.33209228515625, | |
| "loss": 0.1395, | |
| "rewards/chosen": 0.4875214099884033, | |
| "rewards/margins": 1.7979084253311157, | |
| "rewards/rejected": -1.3103870153427124, | |
| "step": 189 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 9.470046480370337, | |
| "kl": 0.0, | |
| "learning_rate": 4.5028780743066456e-07, | |
| "logps/chosen": -207.3546905517578, | |
| "logps/rejected": -231.11886596679688, | |
| "loss": 0.1262, | |
| "rewards/chosen": -0.461222380399704, | |
| "rewards/margins": 1.9130089282989502, | |
| "rewards/rejected": -2.3742313385009766, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 12.783425591778233, | |
| "kl": 0.0, | |
| "learning_rate": 4.5002616431187856e-07, | |
| "logps/chosen": -212.4601593017578, | |
| "logps/rejected": -252.49920654296875, | |
| "loss": 0.1552, | |
| "rewards/chosen": 0.14723794162273407, | |
| "rewards/margins": 1.8059498071670532, | |
| "rewards/rejected": -1.6587119102478027, | |
| "step": 191 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 11.876829640439261, | |
| "kl": 0.0, | |
| "learning_rate": 4.497645211930926e-07, | |
| "logps/chosen": -331.9737548828125, | |
| "logps/rejected": -216.70932006835938, | |
| "loss": 0.1592, | |
| "rewards/chosen": 0.7978419065475464, | |
| "rewards/margins": 3.4641337394714355, | |
| "rewards/rejected": -2.6662919521331787, | |
| "step": 192 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 11.18786423314498, | |
| "kl": 0.0, | |
| "learning_rate": 4.4950287807430666e-07, | |
| "logps/chosen": -210.8509521484375, | |
| "logps/rejected": -276.6178283691406, | |
| "loss": 0.1855, | |
| "rewards/chosen": 0.7132515907287598, | |
| "rewards/margins": 3.511439561843872, | |
| "rewards/rejected": -2.7981879711151123, | |
| "step": 193 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 11.51339539474961, | |
| "kl": 0.0, | |
| "learning_rate": 4.4924123495552066e-07, | |
| "logps/chosen": -193.66836547851562, | |
| "logps/rejected": -206.64353942871094, | |
| "loss": 0.2209, | |
| "rewards/chosen": -0.7599809765815735, | |
| "rewards/margins": 0.2910911440849304, | |
| "rewards/rejected": -1.051072120666504, | |
| "step": 194 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 9.775297052636276, | |
| "kl": 0.0, | |
| "learning_rate": 4.4897959183673465e-07, | |
| "logps/chosen": -210.68817138671875, | |
| "logps/rejected": -269.3539123535156, | |
| "loss": 0.1624, | |
| "rewards/chosen": 0.5124328136444092, | |
| "rewards/margins": 3.528744697570801, | |
| "rewards/rejected": -3.0163118839263916, | |
| "step": 195 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 13.388916890187613, | |
| "kl": 0.0, | |
| "learning_rate": 4.487179487179487e-07, | |
| "logps/chosen": -286.09918212890625, | |
| "logps/rejected": -238.72901916503906, | |
| "loss": 0.2343, | |
| "rewards/chosen": -0.26804402470588684, | |
| "rewards/margins": 1.2514140605926514, | |
| "rewards/rejected": -1.5194580554962158, | |
| "step": 196 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 11.098868865803937, | |
| "kl": 0.0, | |
| "learning_rate": 4.484563055991627e-07, | |
| "logps/chosen": -247.14202880859375, | |
| "logps/rejected": -196.82537841796875, | |
| "loss": 0.157, | |
| "rewards/chosen": -0.33019348978996277, | |
| "rewards/margins": 2.0103812217712402, | |
| "rewards/rejected": -2.3405747413635254, | |
| "step": 197 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 14.223147581105783, | |
| "kl": 0.0, | |
| "learning_rate": 4.4819466248037675e-07, | |
| "logps/chosen": -204.22280883789062, | |
| "logps/rejected": -286.9610900878906, | |
| "loss": 0.237, | |
| "rewards/chosen": -0.013581500388681889, | |
| "rewards/margins": 1.4753634929656982, | |
| "rewards/rejected": -1.4889450073242188, | |
| "step": 198 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 14.41947331635678, | |
| "kl": 0.0, | |
| "learning_rate": 4.479330193615908e-07, | |
| "logps/chosen": -213.51507568359375, | |
| "logps/rejected": -270.748046875, | |
| "loss": 0.1949, | |
| "rewards/chosen": -0.7232166528701782, | |
| "rewards/margins": 2.0983409881591797, | |
| "rewards/rejected": -2.8215577602386475, | |
| "step": 199 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 9.05336870531292, | |
| "kl": 0.0, | |
| "learning_rate": 4.476713762428048e-07, | |
| "logps/chosen": -266.99261474609375, | |
| "logps/rejected": -215.208740234375, | |
| "loss": 0.164, | |
| "rewards/chosen": 0.1612011045217514, | |
| "rewards/margins": 1.967577338218689, | |
| "rewards/rejected": -1.8063762187957764, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 12.145917811167815, | |
| "kl": 0.0, | |
| "learning_rate": 4.474097331240188e-07, | |
| "logps/chosen": -311.310546875, | |
| "logps/rejected": -207.57504272460938, | |
| "loss": 0.198, | |
| "rewards/chosen": 0.33403000235557556, | |
| "rewards/margins": 2.6162540912628174, | |
| "rewards/rejected": -2.282224178314209, | |
| "step": 201 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 10.620881317672168, | |
| "kl": 0.0, | |
| "learning_rate": 4.4714809000523285e-07, | |
| "logps/chosen": -250.87953186035156, | |
| "logps/rejected": -275.86480712890625, | |
| "loss": 0.1359, | |
| "rewards/chosen": 0.3162269592285156, | |
| "rewards/margins": 1.990512728691101, | |
| "rewards/rejected": -1.6742857694625854, | |
| "step": 202 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 10.496041019849555, | |
| "kl": 0.0, | |
| "learning_rate": 4.468864468864469e-07, | |
| "logps/chosen": -247.9547119140625, | |
| "logps/rejected": -217.61874389648438, | |
| "loss": 0.203, | |
| "rewards/chosen": -1.653024673461914, | |
| "rewards/margins": 0.7888014316558838, | |
| "rewards/rejected": -2.441826105117798, | |
| "step": 203 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 11.99618040191761, | |
| "kl": 0.0, | |
| "learning_rate": 4.466248037676609e-07, | |
| "logps/chosen": -150.46995544433594, | |
| "logps/rejected": -319.3192138671875, | |
| "loss": 0.2054, | |
| "rewards/chosen": -0.28798094391822815, | |
| "rewards/margins": 1.984279751777649, | |
| "rewards/rejected": -2.2722606658935547, | |
| "step": 204 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 11.06635292149166, | |
| "kl": 0.0, | |
| "learning_rate": 4.463631606488749e-07, | |
| "logps/chosen": -206.5174560546875, | |
| "logps/rejected": -254.26016235351562, | |
| "loss": 0.1693, | |
| "rewards/chosen": -0.9574307799339294, | |
| "rewards/margins": 2.373122453689575, | |
| "rewards/rejected": -3.3305532932281494, | |
| "step": 205 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 9.1313149728796, | |
| "kl": 0.0, | |
| "learning_rate": 4.4610151753008894e-07, | |
| "logps/chosen": -211.5845489501953, | |
| "logps/rejected": -266.97222900390625, | |
| "loss": 0.2363, | |
| "rewards/chosen": -0.3957107961177826, | |
| "rewards/margins": 3.1181278228759766, | |
| "rewards/rejected": -3.513838529586792, | |
| "step": 206 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 13.851054073575309, | |
| "kl": 0.0, | |
| "learning_rate": 4.4583987441130294e-07, | |
| "logps/chosen": -266.7466125488281, | |
| "logps/rejected": -223.49757385253906, | |
| "loss": 0.2842, | |
| "rewards/chosen": -1.3039528131484985, | |
| "rewards/margins": 0.7924333810806274, | |
| "rewards/rejected": -2.096386194229126, | |
| "step": 207 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 11.650743591966863, | |
| "kl": 0.0, | |
| "learning_rate": 4.45578231292517e-07, | |
| "logps/chosen": -183.24017333984375, | |
| "logps/rejected": -276.4227294921875, | |
| "loss": 0.1615, | |
| "rewards/chosen": -0.3646261990070343, | |
| "rewards/margins": 2.279031276702881, | |
| "rewards/rejected": -2.6436574459075928, | |
| "step": 208 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 13.362859226103993, | |
| "kl": 0.0, | |
| "learning_rate": 4.4531658817373104e-07, | |
| "logps/chosen": -205.38035583496094, | |
| "logps/rejected": -269.58526611328125, | |
| "loss": 0.2259, | |
| "rewards/chosen": -0.2013118416070938, | |
| "rewards/margins": 2.4193832874298096, | |
| "rewards/rejected": -2.620695114135742, | |
| "step": 209 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 12.32754631235818, | |
| "kl": 0.0, | |
| "learning_rate": 4.45054945054945e-07, | |
| "logps/chosen": -268.922607421875, | |
| "logps/rejected": -284.7842102050781, | |
| "loss": 0.2121, | |
| "rewards/chosen": -1.0982351303100586, | |
| "rewards/margins": 1.700211524963379, | |
| "rewards/rejected": -2.7984466552734375, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 11.666014991292931, | |
| "kl": 0.0, | |
| "learning_rate": 4.4479330193615904e-07, | |
| "logps/chosen": -273.0299987792969, | |
| "logps/rejected": -326.8552551269531, | |
| "loss": 0.139, | |
| "rewards/chosen": -0.8367617726325989, | |
| "rewards/margins": 2.7738115787506104, | |
| "rewards/rejected": -3.6105732917785645, | |
| "step": 211 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 12.85199720190171, | |
| "kl": 0.0, | |
| "learning_rate": 4.445316588173731e-07, | |
| "logps/chosen": -267.0897216796875, | |
| "logps/rejected": -248.22039794921875, | |
| "loss": 0.2089, | |
| "rewards/chosen": -1.914868950843811, | |
| "rewards/margins": 1.703445553779602, | |
| "rewards/rejected": -3.618314504623413, | |
| "step": 212 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 13.279155862550748, | |
| "kl": 0.0, | |
| "learning_rate": 4.4427001569858714e-07, | |
| "logps/chosen": -270.5106201171875, | |
| "logps/rejected": -323.9206848144531, | |
| "loss": 0.1508, | |
| "rewards/chosen": 0.43147632479667664, | |
| "rewards/margins": 4.1532816886901855, | |
| "rewards/rejected": -3.7218053340911865, | |
| "step": 213 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 10.842517342661287, | |
| "kl": 0.0, | |
| "learning_rate": 4.4400837257980113e-07, | |
| "logps/chosen": -251.76222229003906, | |
| "logps/rejected": -204.5667724609375, | |
| "loss": 0.2109, | |
| "rewards/chosen": -0.23020866513252258, | |
| "rewards/margins": 1.9914796352386475, | |
| "rewards/rejected": -2.2216882705688477, | |
| "step": 214 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 11.17043596208898, | |
| "kl": 0.0, | |
| "learning_rate": 4.4374672946101513e-07, | |
| "logps/chosen": -203.4984588623047, | |
| "logps/rejected": -343.187255859375, | |
| "loss": 0.2139, | |
| "rewards/chosen": -0.3769696354866028, | |
| "rewards/margins": 2.5140483379364014, | |
| "rewards/rejected": -2.8910179138183594, | |
| "step": 215 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 11.293416573180265, | |
| "kl": 0.0, | |
| "learning_rate": 4.434850863422292e-07, | |
| "logps/chosen": -294.1488037109375, | |
| "logps/rejected": -282.24981689453125, | |
| "loss": 0.1393, | |
| "rewards/chosen": 0.7586994767189026, | |
| "rewards/margins": 4.094053268432617, | |
| "rewards/rejected": -3.3353536128997803, | |
| "step": 216 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 13.608688968178855, | |
| "kl": 0.0, | |
| "learning_rate": 4.432234432234432e-07, | |
| "logps/chosen": -274.1988220214844, | |
| "logps/rejected": -293.933837890625, | |
| "loss": 0.1346, | |
| "rewards/chosen": -0.13219358026981354, | |
| "rewards/margins": 2.8667314052581787, | |
| "rewards/rejected": -2.998924970626831, | |
| "step": 217 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 10.83390830521355, | |
| "kl": 0.0, | |
| "learning_rate": 4.4296180010465723e-07, | |
| "logps/chosen": -222.037353515625, | |
| "logps/rejected": -319.1642150878906, | |
| "loss": 0.1958, | |
| "rewards/chosen": -0.27647799253463745, | |
| "rewards/margins": 2.2584426403045654, | |
| "rewards/rejected": -2.5349206924438477, | |
| "step": 218 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 11.966693070507677, | |
| "kl": 0.0, | |
| "learning_rate": 4.427001569858713e-07, | |
| "logps/chosen": -191.46580505371094, | |
| "logps/rejected": -366.8327941894531, | |
| "loss": 0.1951, | |
| "rewards/chosen": -0.4248928129673004, | |
| "rewards/margins": 2.9257259368896484, | |
| "rewards/rejected": -3.350618839263916, | |
| "step": 219 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 9.986050551901583, | |
| "kl": 0.0, | |
| "learning_rate": 4.424385138670852e-07, | |
| "logps/chosen": -235.12484741210938, | |
| "logps/rejected": -248.2144012451172, | |
| "loss": 0.2251, | |
| "rewards/chosen": -1.9966895580291748, | |
| "rewards/margins": 1.9721267223358154, | |
| "rewards/rejected": -3.9688162803649902, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 10.554269588738498, | |
| "kl": 0.0, | |
| "learning_rate": 4.421768707482993e-07, | |
| "logps/chosen": -239.76820373535156, | |
| "logps/rejected": -218.79449462890625, | |
| "loss": 0.2125, | |
| "rewards/chosen": -1.0601166486740112, | |
| "rewards/margins": 2.6879591941833496, | |
| "rewards/rejected": -3.7480757236480713, | |
| "step": 221 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 11.807560445400602, | |
| "kl": 0.0, | |
| "learning_rate": 4.419152276295133e-07, | |
| "logps/chosen": -226.56256103515625, | |
| "logps/rejected": -218.861328125, | |
| "loss": 0.1385, | |
| "rewards/chosen": -0.5115461349487305, | |
| "rewards/margins": 2.6464967727661133, | |
| "rewards/rejected": -3.1580429077148438, | |
| "step": 222 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 11.670267600602166, | |
| "kl": 0.0, | |
| "learning_rate": 4.416535845107274e-07, | |
| "logps/chosen": -282.7430725097656, | |
| "logps/rejected": -211.0267791748047, | |
| "loss": 0.2221, | |
| "rewards/chosen": -1.8116750717163086, | |
| "rewards/margins": 0.58609938621521, | |
| "rewards/rejected": -2.3977744579315186, | |
| "step": 223 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 11.238377690831841, | |
| "kl": 0.0, | |
| "learning_rate": 4.4139194139194137e-07, | |
| "logps/chosen": -218.9561004638672, | |
| "logps/rejected": -232.92832946777344, | |
| "loss": 0.1533, | |
| "rewards/chosen": -0.23154856264591217, | |
| "rewards/margins": 1.915379285812378, | |
| "rewards/rejected": -2.146927833557129, | |
| "step": 224 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 13.477552695461938, | |
| "kl": 0.0, | |
| "learning_rate": 4.411302982731554e-07, | |
| "logps/chosen": -241.8193359375, | |
| "logps/rejected": -243.86459350585938, | |
| "loss": 0.2513, | |
| "rewards/chosen": -0.6799749135971069, | |
| "rewards/margins": 0.9313141107559204, | |
| "rewards/rejected": -1.6112890243530273, | |
| "step": 225 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 10.248025744939582, | |
| "kl": 0.0, | |
| "learning_rate": 4.408686551543694e-07, | |
| "logps/chosen": -255.42225646972656, | |
| "logps/rejected": -354.8009948730469, | |
| "loss": 0.1809, | |
| "rewards/chosen": -0.15073581039905548, | |
| "rewards/margins": 3.6917855739593506, | |
| "rewards/rejected": -3.8425214290618896, | |
| "step": 226 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 15.177673567635171, | |
| "kl": 0.0, | |
| "learning_rate": 4.406070120355834e-07, | |
| "logps/chosen": -219.69955444335938, | |
| "logps/rejected": -257.246337890625, | |
| "loss": 0.2311, | |
| "rewards/chosen": -0.5061897039413452, | |
| "rewards/margins": 1.860742449760437, | |
| "rewards/rejected": -2.3669321537017822, | |
| "step": 227 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 10.212253558418395, | |
| "kl": 0.0, | |
| "learning_rate": 4.4034536891679747e-07, | |
| "logps/chosen": -216.00762939453125, | |
| "logps/rejected": -271.8702392578125, | |
| "loss": 0.1099, | |
| "rewards/chosen": -0.29265689849853516, | |
| "rewards/margins": 2.9313840866088867, | |
| "rewards/rejected": -3.224040985107422, | |
| "step": 228 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 11.734827553729616, | |
| "kl": 0.0, | |
| "learning_rate": 4.400837257980115e-07, | |
| "logps/chosen": -217.1133270263672, | |
| "logps/rejected": -310.7548522949219, | |
| "loss": 0.2233, | |
| "rewards/chosen": -0.2567431330680847, | |
| "rewards/margins": 2.2228996753692627, | |
| "rewards/rejected": -2.479642868041992, | |
| "step": 229 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 9.41912765209931, | |
| "kl": 0.0, | |
| "learning_rate": 4.398220826792255e-07, | |
| "logps/chosen": -194.0414581298828, | |
| "logps/rejected": -252.26522827148438, | |
| "loss": 0.1112, | |
| "rewards/chosen": 0.5574043989181519, | |
| "rewards/margins": 4.030994892120361, | |
| "rewards/rejected": -3.47359037399292, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 10.488015000282592, | |
| "kl": 0.0, | |
| "learning_rate": 4.395604395604395e-07, | |
| "logps/chosen": -188.4119873046875, | |
| "logps/rejected": -282.2178039550781, | |
| "loss": 0.1811, | |
| "rewards/chosen": -0.3539419174194336, | |
| "rewards/margins": 1.9429447650909424, | |
| "rewards/rejected": -2.296886682510376, | |
| "step": 231 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 10.39486238771726, | |
| "kl": 0.0, | |
| "learning_rate": 4.3929879644165356e-07, | |
| "logps/chosen": -223.63308715820312, | |
| "logps/rejected": -240.90345764160156, | |
| "loss": 0.1975, | |
| "rewards/chosen": -0.5462809205055237, | |
| "rewards/margins": 1.494751214981079, | |
| "rewards/rejected": -2.041032075881958, | |
| "step": 232 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 11.215577164704541, | |
| "kl": 0.0, | |
| "learning_rate": 4.390371533228676e-07, | |
| "logps/chosen": -208.15684509277344, | |
| "logps/rejected": -315.0474548339844, | |
| "loss": 0.2079, | |
| "rewards/chosen": -1.009997010231018, | |
| "rewards/margins": 1.5491665601730347, | |
| "rewards/rejected": -2.5591635704040527, | |
| "step": 233 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 10.766769653642761, | |
| "kl": 0.0, | |
| "learning_rate": 4.387755102040816e-07, | |
| "logps/chosen": -183.52944946289062, | |
| "logps/rejected": -254.58596801757812, | |
| "loss": 0.1539, | |
| "rewards/chosen": -0.7764175534248352, | |
| "rewards/margins": 2.6363468170166016, | |
| "rewards/rejected": -3.412764310836792, | |
| "step": 234 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 11.608047188492522, | |
| "kl": 0.0, | |
| "learning_rate": 4.3851386708529566e-07, | |
| "logps/chosen": -210.39288330078125, | |
| "logps/rejected": -248.83990478515625, | |
| "loss": 0.1721, | |
| "rewards/chosen": -0.28012871742248535, | |
| "rewards/margins": 2.2773919105529785, | |
| "rewards/rejected": -2.557520627975464, | |
| "step": 235 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 13.662061510468588, | |
| "kl": 0.0, | |
| "learning_rate": 4.3825222396650966e-07, | |
| "logps/chosen": -257.81976318359375, | |
| "logps/rejected": -294.05328369140625, | |
| "loss": 0.1683, | |
| "rewards/chosen": 0.09654220193624496, | |
| "rewards/margins": 2.7226510047912598, | |
| "rewards/rejected": -2.6261088848114014, | |
| "step": 236 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 12.161426046134066, | |
| "kl": 0.0, | |
| "learning_rate": 4.3799058084772366e-07, | |
| "logps/chosen": -248.7794647216797, | |
| "logps/rejected": -260.9866943359375, | |
| "loss": 0.1904, | |
| "rewards/chosen": 0.21598704159259796, | |
| "rewards/margins": 3.1339871883392334, | |
| "rewards/rejected": -2.9180002212524414, | |
| "step": 237 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 10.031617578976944, | |
| "kl": 0.0, | |
| "learning_rate": 4.377289377289377e-07, | |
| "logps/chosen": -239.06961059570312, | |
| "logps/rejected": -262.04071044921875, | |
| "loss": 0.1708, | |
| "rewards/chosen": 0.007866961881518364, | |
| "rewards/margins": 2.7547590732574463, | |
| "rewards/rejected": -2.746892213821411, | |
| "step": 238 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 14.122345962730769, | |
| "kl": 0.0, | |
| "learning_rate": 4.3746729461015176e-07, | |
| "logps/chosen": -192.8285369873047, | |
| "logps/rejected": -284.4400634765625, | |
| "loss": 0.2025, | |
| "rewards/chosen": -0.6923890709877014, | |
| "rewards/margins": 1.7750904560089111, | |
| "rewards/rejected": -2.4674794673919678, | |
| "step": 239 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 10.12440657869901, | |
| "kl": 0.0, | |
| "learning_rate": 4.3720565149136575e-07, | |
| "logps/chosen": -254.7532958984375, | |
| "logps/rejected": -278.2186279296875, | |
| "loss": 0.2537, | |
| "rewards/chosen": -1.5660591125488281, | |
| "rewards/margins": 1.244922161102295, | |
| "rewards/rejected": -2.810981273651123, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 9.097833834404545, | |
| "kl": 0.0, | |
| "learning_rate": 4.3694400837257975e-07, | |
| "logps/chosen": -190.60516357421875, | |
| "logps/rejected": -281.48699951171875, | |
| "loss": 0.1973, | |
| "rewards/chosen": -0.27278468012809753, | |
| "rewards/margins": 2.6037862300872803, | |
| "rewards/rejected": -2.8765709400177, | |
| "step": 241 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 10.170278536494735, | |
| "kl": 0.0, | |
| "learning_rate": 4.366823652537938e-07, | |
| "logps/chosen": -241.40150451660156, | |
| "logps/rejected": -271.16192626953125, | |
| "loss": 0.1001, | |
| "rewards/chosen": 0.05600461736321449, | |
| "rewards/margins": 4.060368537902832, | |
| "rewards/rejected": -4.004364013671875, | |
| "step": 242 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 10.968657094423396, | |
| "kl": 0.0, | |
| "learning_rate": 4.3642072213500785e-07, | |
| "logps/chosen": -218.7855987548828, | |
| "logps/rejected": -231.13519287109375, | |
| "loss": 0.1814, | |
| "rewards/chosen": 0.2664739489555359, | |
| "rewards/margins": 1.7813398838043213, | |
| "rewards/rejected": -1.5148659944534302, | |
| "step": 243 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 8.862306842445157, | |
| "kl": 0.0, | |
| "learning_rate": 4.3615907901622185e-07, | |
| "logps/chosen": -195.7134552001953, | |
| "logps/rejected": -222.33502197265625, | |
| "loss": 0.096, | |
| "rewards/chosen": 0.3085781931877136, | |
| "rewards/margins": 2.4294745922088623, | |
| "rewards/rejected": -2.120896339416504, | |
| "step": 244 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 10.488199983055214, | |
| "kl": 0.0, | |
| "learning_rate": 4.358974358974359e-07, | |
| "logps/chosen": -213.25262451171875, | |
| "logps/rejected": -245.7234344482422, | |
| "loss": 0.1646, | |
| "rewards/chosen": 0.47856229543685913, | |
| "rewards/margins": 4.145866870880127, | |
| "rewards/rejected": -3.667304754257202, | |
| "step": 245 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 11.576912231416909, | |
| "kl": 0.0, | |
| "learning_rate": 4.3563579277864995e-07, | |
| "logps/chosen": -215.87538146972656, | |
| "logps/rejected": -262.62786865234375, | |
| "loss": 0.2232, | |
| "rewards/chosen": 0.43594250082969666, | |
| "rewards/margins": 1.7815001010894775, | |
| "rewards/rejected": -1.3455575704574585, | |
| "step": 246 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 11.472489865502666, | |
| "kl": 0.0, | |
| "learning_rate": 4.353741496598639e-07, | |
| "logps/chosen": -298.4697570800781, | |
| "logps/rejected": -223.03550720214844, | |
| "loss": 0.1638, | |
| "rewards/chosen": 0.32377567887306213, | |
| "rewards/margins": 1.912637710571289, | |
| "rewards/rejected": -1.5888620615005493, | |
| "step": 247 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 9.245318090630139, | |
| "kl": 0.0, | |
| "learning_rate": 4.3511250654107794e-07, | |
| "logps/chosen": -160.44625854492188, | |
| "logps/rejected": -268.469482421875, | |
| "loss": 0.1647, | |
| "rewards/chosen": 0.2935628294944763, | |
| "rewards/margins": 4.304611682891846, | |
| "rewards/rejected": -4.011048793792725, | |
| "step": 248 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 9.203079637794323, | |
| "kl": 0.0, | |
| "learning_rate": 4.34850863422292e-07, | |
| "logps/chosen": -248.113037109375, | |
| "logps/rejected": -241.7339630126953, | |
| "loss": 0.1833, | |
| "rewards/chosen": 1.2330036163330078, | |
| "rewards/margins": 3.8498387336730957, | |
| "rewards/rejected": -2.616835117340088, | |
| "step": 249 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 9.144257332584273, | |
| "kl": 0.0, | |
| "learning_rate": 4.34589220303506e-07, | |
| "logps/chosen": -196.73558044433594, | |
| "logps/rejected": -234.81517028808594, | |
| "loss": 0.19, | |
| "rewards/chosen": -0.7745031714439392, | |
| "rewards/margins": 1.9112095832824707, | |
| "rewards/rejected": -2.6857128143310547, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 10.977345554295628, | |
| "kl": 0.0, | |
| "learning_rate": 4.3432757718472004e-07, | |
| "logps/chosen": -270.6989440917969, | |
| "logps/rejected": -296.6756896972656, | |
| "loss": 0.1977, | |
| "rewards/chosen": 0.5559626221656799, | |
| "rewards/margins": 2.0558934211730957, | |
| "rewards/rejected": -1.499930739402771, | |
| "step": 251 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 11.195892622420404, | |
| "kl": 0.0, | |
| "learning_rate": 4.3406593406593404e-07, | |
| "logps/chosen": -324.94915771484375, | |
| "logps/rejected": -269.9296569824219, | |
| "loss": 0.1905, | |
| "rewards/chosen": -0.5299755334854126, | |
| "rewards/margins": 2.1768460273742676, | |
| "rewards/rejected": -2.7068214416503906, | |
| "step": 252 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 13.09663182243734, | |
| "kl": 0.0, | |
| "learning_rate": 4.3380429094714804e-07, | |
| "logps/chosen": -209.63999938964844, | |
| "logps/rejected": -290.00927734375, | |
| "loss": 0.1509, | |
| "rewards/chosen": 0.9980623722076416, | |
| "rewards/margins": 4.123095512390137, | |
| "rewards/rejected": -3.125032901763916, | |
| "step": 253 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 16.101792755673987, | |
| "kl": 0.0, | |
| "learning_rate": 4.335426478283621e-07, | |
| "logps/chosen": -178.9565887451172, | |
| "logps/rejected": -214.99896240234375, | |
| "loss": 0.1981, | |
| "rewards/chosen": -0.7419480085372925, | |
| "rewards/margins": 1.410202145576477, | |
| "rewards/rejected": -2.1521501541137695, | |
| "step": 254 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 9.260673089248163, | |
| "kl": 0.0, | |
| "learning_rate": 4.3328100470957614e-07, | |
| "logps/chosen": -269.7500915527344, | |
| "logps/rejected": -240.69114685058594, | |
| "loss": 0.1547, | |
| "rewards/chosen": -1.514892578125, | |
| "rewards/margins": 1.8574752807617188, | |
| "rewards/rejected": -3.3723678588867188, | |
| "step": 255 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 8.74207098415269, | |
| "kl": 0.0, | |
| "learning_rate": 4.330193615907902e-07, | |
| "logps/chosen": -238.84718322753906, | |
| "logps/rejected": -185.5977325439453, | |
| "loss": 0.2231, | |
| "rewards/chosen": 0.872572660446167, | |
| "rewards/margins": 2.413464069366455, | |
| "rewards/rejected": -1.5408915281295776, | |
| "step": 256 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 13.06989888703404, | |
| "kl": 0.0, | |
| "learning_rate": 4.3275771847200413e-07, | |
| "logps/chosen": -198.09527587890625, | |
| "logps/rejected": -241.32310485839844, | |
| "loss": 0.293, | |
| "rewards/chosen": -0.5270448327064514, | |
| "rewards/margins": 1.5255239009857178, | |
| "rewards/rejected": -2.0525686740875244, | |
| "step": 257 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 9.146455056162033, | |
| "kl": 0.0, | |
| "learning_rate": 4.324960753532182e-07, | |
| "logps/chosen": -158.8040771484375, | |
| "logps/rejected": -317.1865539550781, | |
| "loss": 0.1985, | |
| "rewards/chosen": -0.44321534037590027, | |
| "rewards/margins": 2.745103359222412, | |
| "rewards/rejected": -3.1883187294006348, | |
| "step": 258 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 11.496122217797181, | |
| "kl": 0.0, | |
| "learning_rate": 4.3223443223443223e-07, | |
| "logps/chosen": -195.80255126953125, | |
| "logps/rejected": -251.7202911376953, | |
| "loss": 0.2603, | |
| "rewards/chosen": 0.14385934174060822, | |
| "rewards/margins": 3.09423565864563, | |
| "rewards/rejected": -2.950376272201538, | |
| "step": 259 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 10.14234334654825, | |
| "kl": 0.0, | |
| "learning_rate": 4.3197278911564623e-07, | |
| "logps/chosen": -194.34381103515625, | |
| "logps/rejected": -210.70053100585938, | |
| "loss": 0.2247, | |
| "rewards/chosen": -0.5755199193954468, | |
| "rewards/margins": 1.121391773223877, | |
| "rewards/rejected": -1.6969116926193237, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 10.972480345195148, | |
| "kl": 0.0, | |
| "learning_rate": 4.317111459968603e-07, | |
| "logps/chosen": -190.80508422851562, | |
| "logps/rejected": -250.46139526367188, | |
| "loss": 0.2264, | |
| "rewards/chosen": 0.17243629693984985, | |
| "rewards/margins": 3.771756887435913, | |
| "rewards/rejected": -3.599320650100708, | |
| "step": 261 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 12.562599017075712, | |
| "kl": 0.0, | |
| "learning_rate": 4.314495028780743e-07, | |
| "logps/chosen": -251.84373474121094, | |
| "logps/rejected": -245.03099060058594, | |
| "loss": 0.1727, | |
| "rewards/chosen": 0.41944611072540283, | |
| "rewards/margins": 2.5281949043273926, | |
| "rewards/rejected": -2.1087489128112793, | |
| "step": 262 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 11.382077716702321, | |
| "kl": 0.0, | |
| "learning_rate": 4.311878597592883e-07, | |
| "logps/chosen": -213.92518615722656, | |
| "logps/rejected": -231.60177612304688, | |
| "loss": 0.2323, | |
| "rewards/chosen": -0.6165014505386353, | |
| "rewards/margins": 1.244750738143921, | |
| "rewards/rejected": -1.8612521886825562, | |
| "step": 263 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 9.731491692873679, | |
| "kl": 0.0, | |
| "learning_rate": 4.309262166405023e-07, | |
| "logps/chosen": -212.86976623535156, | |
| "logps/rejected": -261.46868896484375, | |
| "loss": 0.1888, | |
| "rewards/chosen": -0.4610190689563751, | |
| "rewards/margins": 2.1575005054473877, | |
| "rewards/rejected": -2.6185195446014404, | |
| "step": 264 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 11.409608574249868, | |
| "kl": 0.0, | |
| "learning_rate": 4.306645735217164e-07, | |
| "logps/chosen": -224.6321258544922, | |
| "logps/rejected": -252.78160095214844, | |
| "loss": 0.2112, | |
| "rewards/chosen": 0.9657794833183289, | |
| "rewards/margins": 2.610295534133911, | |
| "rewards/rejected": -1.6445159912109375, | |
| "step": 265 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 9.856888071447521, | |
| "kl": 0.0, | |
| "learning_rate": 4.3040293040293043e-07, | |
| "logps/chosen": -222.96458435058594, | |
| "logps/rejected": -214.23255920410156, | |
| "loss": 0.2125, | |
| "rewards/chosen": -0.6956588625907898, | |
| "rewards/margins": 1.7711460590362549, | |
| "rewards/rejected": -2.4668049812316895, | |
| "step": 266 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 12.139684536745287, | |
| "kl": 0.0, | |
| "learning_rate": 4.3014128728414437e-07, | |
| "logps/chosen": -227.04371643066406, | |
| "logps/rejected": -338.61358642578125, | |
| "loss": 0.1782, | |
| "rewards/chosen": -0.1494772583246231, | |
| "rewards/margins": 2.7432305812835693, | |
| "rewards/rejected": -2.8927078247070312, | |
| "step": 267 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 11.372427757605506, | |
| "kl": 0.0, | |
| "learning_rate": 4.298796441653584e-07, | |
| "logps/chosen": -269.498291015625, | |
| "logps/rejected": -285.0457763671875, | |
| "loss": 0.2566, | |
| "rewards/chosen": -0.7780652642250061, | |
| "rewards/margins": 2.5229103565216064, | |
| "rewards/rejected": -3.3009755611419678, | |
| "step": 268 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 10.418607245045079, | |
| "kl": 0.0, | |
| "learning_rate": 4.2961800104657247e-07, | |
| "logps/chosen": -206.6187744140625, | |
| "logps/rejected": -178.76629638671875, | |
| "loss": 0.1973, | |
| "rewards/chosen": -0.2997088134288788, | |
| "rewards/margins": 2.2873473167419434, | |
| "rewards/rejected": -2.5870561599731445, | |
| "step": 269 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 8.700783144862982, | |
| "kl": 0.0, | |
| "learning_rate": 4.2935635792778647e-07, | |
| "logps/chosen": -259.9353942871094, | |
| "logps/rejected": -190.39688110351562, | |
| "loss": 0.1152, | |
| "rewards/chosen": 0.6818735003471375, | |
| "rewards/margins": 2.6719298362731934, | |
| "rewards/rejected": -1.9900562763214111, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 11.113524020838248, | |
| "kl": 0.0, | |
| "learning_rate": 4.290947148090005e-07, | |
| "logps/chosen": -278.58087158203125, | |
| "logps/rejected": -203.67617797851562, | |
| "loss": 0.1375, | |
| "rewards/chosen": 0.04530330374836922, | |
| "rewards/margins": 3.534538507461548, | |
| "rewards/rejected": -3.4892351627349854, | |
| "step": 271 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 8.408405701936438, | |
| "kl": 0.0, | |
| "learning_rate": 4.288330716902145e-07, | |
| "logps/chosen": -167.8460693359375, | |
| "logps/rejected": -220.5739288330078, | |
| "loss": 0.1761, | |
| "rewards/chosen": -0.373342365026474, | |
| "rewards/margins": 2.532893657684326, | |
| "rewards/rejected": -2.906235933303833, | |
| "step": 272 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 13.207340486836898, | |
| "kl": 0.0, | |
| "learning_rate": 4.285714285714285e-07, | |
| "logps/chosen": -271.5782165527344, | |
| "logps/rejected": -229.74559020996094, | |
| "loss": 0.1837, | |
| "rewards/chosen": -0.2260454148054123, | |
| "rewards/margins": 1.8243522644042969, | |
| "rewards/rejected": -2.0503976345062256, | |
| "step": 273 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 9.722222876772891, | |
| "kl": 0.0, | |
| "learning_rate": 4.2830978545264256e-07, | |
| "logps/chosen": -263.2925109863281, | |
| "logps/rejected": -304.8329772949219, | |
| "loss": 0.1408, | |
| "rewards/chosen": -0.7855675220489502, | |
| "rewards/margins": 1.542686939239502, | |
| "rewards/rejected": -2.328254461288452, | |
| "step": 274 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 11.917040600069175, | |
| "kl": 0.0, | |
| "learning_rate": 4.280481423338566e-07, | |
| "logps/chosen": -200.11573791503906, | |
| "logps/rejected": -282.0390930175781, | |
| "loss": 0.1633, | |
| "rewards/chosen": 0.374490350484848, | |
| "rewards/margins": 3.0798637866973877, | |
| "rewards/rejected": -2.705373525619507, | |
| "step": 275 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 14.079400827374393, | |
| "kl": 0.0, | |
| "learning_rate": 4.2778649921507067e-07, | |
| "logps/chosen": -226.02078247070312, | |
| "logps/rejected": -285.05780029296875, | |
| "loss": 0.239, | |
| "rewards/chosen": 0.22204646468162537, | |
| "rewards/margins": 2.745478630065918, | |
| "rewards/rejected": -2.5234322547912598, | |
| "step": 276 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 12.40231967594057, | |
| "kl": 0.0, | |
| "learning_rate": 4.2752485609628466e-07, | |
| "logps/chosen": -233.96995544433594, | |
| "logps/rejected": -245.3420867919922, | |
| "loss": 0.2526, | |
| "rewards/chosen": -0.20722751319408417, | |
| "rewards/margins": 1.8177690505981445, | |
| "rewards/rejected": -2.024996519088745, | |
| "step": 277 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 11.912714615917263, | |
| "kl": 0.0, | |
| "learning_rate": 4.2726321297749866e-07, | |
| "logps/chosen": -242.40673828125, | |
| "logps/rejected": -333.7016906738281, | |
| "loss": 0.2233, | |
| "rewards/chosen": -0.6541883945465088, | |
| "rewards/margins": 1.2289773225784302, | |
| "rewards/rejected": -1.883165717124939, | |
| "step": 278 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 12.620911514266433, | |
| "kl": 0.0, | |
| "learning_rate": 4.270015698587127e-07, | |
| "logps/chosen": -158.87258911132812, | |
| "logps/rejected": -180.81192016601562, | |
| "loss": 0.1446, | |
| "rewards/chosen": -0.11502157896757126, | |
| "rewards/margins": 1.9224528074264526, | |
| "rewards/rejected": -2.0374743938446045, | |
| "step": 279 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 11.601791307438955, | |
| "kl": 0.0, | |
| "learning_rate": 4.267399267399267e-07, | |
| "logps/chosen": -245.42486572265625, | |
| "logps/rejected": -260.4696960449219, | |
| "loss": 0.1418, | |
| "rewards/chosen": 0.19518660008907318, | |
| "rewards/margins": 2.677379846572876, | |
| "rewards/rejected": -2.4821932315826416, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 12.839705365539672, | |
| "kl": 0.0, | |
| "learning_rate": 4.2647828362114076e-07, | |
| "logps/chosen": -256.3037109375, | |
| "logps/rejected": -272.7105712890625, | |
| "loss": 0.2253, | |
| "rewards/chosen": 0.4001716375350952, | |
| "rewards/margins": 2.9852728843688965, | |
| "rewards/rejected": -2.585101366043091, | |
| "step": 281 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 11.039693610954819, | |
| "kl": 0.0, | |
| "learning_rate": 4.262166405023548e-07, | |
| "logps/chosen": -256.00457763671875, | |
| "logps/rejected": -206.69845581054688, | |
| "loss": 0.1776, | |
| "rewards/chosen": 0.3543960452079773, | |
| "rewards/margins": 2.914128303527832, | |
| "rewards/rejected": -2.55973219871521, | |
| "step": 282 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 9.759460114772729, | |
| "kl": 0.0, | |
| "learning_rate": 4.2595499738356875e-07, | |
| "logps/chosen": -273.74395751953125, | |
| "logps/rejected": -195.93484497070312, | |
| "loss": 0.2458, | |
| "rewards/chosen": 1.1952847242355347, | |
| "rewards/margins": 2.3585853576660156, | |
| "rewards/rejected": -1.1633005142211914, | |
| "step": 283 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 10.249902760795546, | |
| "kl": 0.0, | |
| "learning_rate": 4.256933542647828e-07, | |
| "logps/chosen": -250.92709350585938, | |
| "logps/rejected": -251.54632568359375, | |
| "loss": 0.2445, | |
| "rewards/chosen": -1.2972968816757202, | |
| "rewards/margins": 1.1200555562973022, | |
| "rewards/rejected": -2.4173524379730225, | |
| "step": 284 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 11.519889613409475, | |
| "kl": 0.0, | |
| "learning_rate": 4.2543171114599685e-07, | |
| "logps/chosen": -241.36473083496094, | |
| "logps/rejected": -217.71559143066406, | |
| "loss": 0.1655, | |
| "rewards/chosen": -0.08067867159843445, | |
| "rewards/margins": 2.2561678886413574, | |
| "rewards/rejected": -2.3368465900421143, | |
| "step": 285 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 12.116227642012538, | |
| "kl": 0.0, | |
| "learning_rate": 4.2517006802721085e-07, | |
| "logps/chosen": -227.16310119628906, | |
| "logps/rejected": -254.51699829101562, | |
| "loss": 0.1795, | |
| "rewards/chosen": -0.5490508675575256, | |
| "rewards/margins": 2.050340175628662, | |
| "rewards/rejected": -2.599390983581543, | |
| "step": 286 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 10.761723965602993, | |
| "kl": 0.0, | |
| "learning_rate": 4.249084249084249e-07, | |
| "logps/chosen": -213.15887451171875, | |
| "logps/rejected": -229.43199157714844, | |
| "loss": 0.2217, | |
| "rewards/chosen": -0.36798590421676636, | |
| "rewards/margins": 0.6998893618583679, | |
| "rewards/rejected": -1.0678752660751343, | |
| "step": 287 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 9.96614506259357, | |
| "kl": 0.0, | |
| "learning_rate": 4.246467817896389e-07, | |
| "logps/chosen": -224.59458923339844, | |
| "logps/rejected": -168.9609375, | |
| "loss": 0.2067, | |
| "rewards/chosen": 0.40960344672203064, | |
| "rewards/margins": 1.400376319885254, | |
| "rewards/rejected": -0.9907728433609009, | |
| "step": 288 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 10.590063389968323, | |
| "kl": 0.0, | |
| "learning_rate": 4.2438513867085295e-07, | |
| "logps/chosen": -230.34326171875, | |
| "logps/rejected": -194.3732452392578, | |
| "loss": 0.1615, | |
| "rewards/chosen": -0.03669742867350578, | |
| "rewards/margins": 1.9274276494979858, | |
| "rewards/rejected": -1.9641250371932983, | |
| "step": 289 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 12.341220101671244, | |
| "kl": 0.0, | |
| "learning_rate": 4.2412349555206695e-07, | |
| "logps/chosen": -261.1521911621094, | |
| "logps/rejected": -184.7381591796875, | |
| "loss": 0.1907, | |
| "rewards/chosen": 0.5180913209915161, | |
| "rewards/margins": 1.7361607551574707, | |
| "rewards/rejected": -1.2180694341659546, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 13.945755502648126, | |
| "kl": 0.0, | |
| "learning_rate": 4.23861852433281e-07, | |
| "logps/chosen": -318.291015625, | |
| "logps/rejected": -216.2500457763672, | |
| "loss": 0.2175, | |
| "rewards/chosen": -0.4215258061885834, | |
| "rewards/margins": 0.7515729665756226, | |
| "rewards/rejected": -1.1730988025665283, | |
| "step": 291 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 11.888573752629188, | |
| "kl": 0.0, | |
| "learning_rate": 4.2360020931449505e-07, | |
| "logps/chosen": -213.46910095214844, | |
| "logps/rejected": -188.46728515625, | |
| "loss": 0.2024, | |
| "rewards/chosen": 0.058938559144735336, | |
| "rewards/margins": 1.171943187713623, | |
| "rewards/rejected": -1.1130046844482422, | |
| "step": 292 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 16.17419414361992, | |
| "kl": 0.0, | |
| "learning_rate": 4.23338566195709e-07, | |
| "logps/chosen": -192.65357971191406, | |
| "logps/rejected": -261.0958251953125, | |
| "loss": 0.2653, | |
| "rewards/chosen": 0.42944175004959106, | |
| "rewards/margins": 1.398432970046997, | |
| "rewards/rejected": -0.9689911603927612, | |
| "step": 293 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 9.577005409188526, | |
| "kl": 0.0, | |
| "learning_rate": 4.2307692307692304e-07, | |
| "logps/chosen": -214.690185546875, | |
| "logps/rejected": -195.5751953125, | |
| "loss": 0.1916, | |
| "rewards/chosen": 0.16525867581367493, | |
| "rewards/margins": 1.8478789329528809, | |
| "rewards/rejected": -1.6826202869415283, | |
| "step": 294 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 13.024241787173485, | |
| "kl": 0.0, | |
| "learning_rate": 4.228152799581371e-07, | |
| "logps/chosen": -269.0843505859375, | |
| "logps/rejected": -258.8379821777344, | |
| "loss": 0.1626, | |
| "rewards/chosen": 0.0963350236415863, | |
| "rewards/margins": 1.255967617034912, | |
| "rewards/rejected": -1.1596325635910034, | |
| "step": 295 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 8.85778497897942, | |
| "kl": 0.0, | |
| "learning_rate": 4.225536368393511e-07, | |
| "logps/chosen": -217.76644897460938, | |
| "logps/rejected": -309.886962890625, | |
| "loss": 0.112, | |
| "rewards/chosen": 0.8563817739486694, | |
| "rewards/margins": 3.093313217163086, | |
| "rewards/rejected": -2.236931562423706, | |
| "step": 296 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 12.645142193880144, | |
| "kl": 0.0, | |
| "learning_rate": 4.2229199372056514e-07, | |
| "logps/chosen": -181.10629272460938, | |
| "logps/rejected": -232.3704071044922, | |
| "loss": 0.2186, | |
| "rewards/chosen": -0.1876239776611328, | |
| "rewards/margins": 1.843381643295288, | |
| "rewards/rejected": -2.031005620956421, | |
| "step": 297 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 11.248387892063585, | |
| "kl": 0.0, | |
| "learning_rate": 4.2203035060177914e-07, | |
| "logps/chosen": -248.957763671875, | |
| "logps/rejected": -215.41151428222656, | |
| "loss": 0.1798, | |
| "rewards/chosen": 0.26162558794021606, | |
| "rewards/margins": 2.0767014026641846, | |
| "rewards/rejected": -1.8150758743286133, | |
| "step": 298 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 13.744936576539429, | |
| "kl": 0.0, | |
| "learning_rate": 4.217687074829932e-07, | |
| "logps/chosen": -221.40310668945312, | |
| "logps/rejected": -279.14703369140625, | |
| "loss": 0.2128, | |
| "rewards/chosen": 0.0988471731543541, | |
| "rewards/margins": 2.282918691635132, | |
| "rewards/rejected": -2.1840715408325195, | |
| "step": 299 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 11.139613060769499, | |
| "kl": 0.0, | |
| "learning_rate": 4.215070643642072e-07, | |
| "logps/chosen": -219.13638305664062, | |
| "logps/rejected": -316.4018249511719, | |
| "loss": 0.1851, | |
| "rewards/chosen": -0.3484467566013336, | |
| "rewards/margins": 2.9719343185424805, | |
| "rewards/rejected": -3.3203811645507812, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 11.23566454390853, | |
| "kl": 0.0, | |
| "learning_rate": 4.2124542124542123e-07, | |
| "logps/chosen": -264.99456787109375, | |
| "logps/rejected": -212.24009704589844, | |
| "loss": 0.2013, | |
| "rewards/chosen": 0.5730603337287903, | |
| "rewards/margins": 2.691094398498535, | |
| "rewards/rejected": -2.1180341243743896, | |
| "step": 301 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 9.57651530765622, | |
| "kl": 0.0, | |
| "learning_rate": 4.209837781266353e-07, | |
| "logps/chosen": -220.54957580566406, | |
| "logps/rejected": -235.98797607421875, | |
| "loss": 0.1582, | |
| "rewards/chosen": 0.9783976674079895, | |
| "rewards/margins": 2.1505749225616455, | |
| "rewards/rejected": -1.1721771955490112, | |
| "step": 302 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 11.869078446374544, | |
| "kl": 0.0, | |
| "learning_rate": 4.2072213500784923e-07, | |
| "logps/chosen": -275.3968505859375, | |
| "logps/rejected": -187.9066925048828, | |
| "loss": 0.1583, | |
| "rewards/chosen": 0.454397588968277, | |
| "rewards/margins": 1.851892352104187, | |
| "rewards/rejected": -1.3974947929382324, | |
| "step": 303 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 11.706252453621756, | |
| "kl": 0.0, | |
| "learning_rate": 4.204604918890633e-07, | |
| "logps/chosen": -310.59661865234375, | |
| "logps/rejected": -275.7345275878906, | |
| "loss": 0.2664, | |
| "rewards/chosen": -0.8155102133750916, | |
| "rewards/margins": 0.8342878222465515, | |
| "rewards/rejected": -1.649798035621643, | |
| "step": 304 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 11.279199741590181, | |
| "kl": 0.0, | |
| "learning_rate": 4.2019884877027733e-07, | |
| "logps/chosen": -238.77163696289062, | |
| "logps/rejected": -267.1834716796875, | |
| "loss": 0.1667, | |
| "rewards/chosen": 0.08637741953134537, | |
| "rewards/margins": 2.7349460124969482, | |
| "rewards/rejected": -2.648568630218506, | |
| "step": 305 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 10.974707777807534, | |
| "kl": 0.0, | |
| "learning_rate": 4.1993720565149133e-07, | |
| "logps/chosen": -249.7024383544922, | |
| "logps/rejected": -290.3708190917969, | |
| "loss": 0.2169, | |
| "rewards/chosen": 0.3819645047187805, | |
| "rewards/margins": 2.038438558578491, | |
| "rewards/rejected": -1.6564741134643555, | |
| "step": 306 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 15.241423459831774, | |
| "kl": 0.0, | |
| "learning_rate": 4.196755625327054e-07, | |
| "logps/chosen": -256.07049560546875, | |
| "logps/rejected": -295.1194152832031, | |
| "loss": 0.1785, | |
| "rewards/chosen": 0.4329548180103302, | |
| "rewards/margins": 2.4334588050842285, | |
| "rewards/rejected": -2.0005040168762207, | |
| "step": 307 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 12.073965597469332, | |
| "kl": 0.0, | |
| "learning_rate": 4.1941391941391943e-07, | |
| "logps/chosen": -233.57545471191406, | |
| "logps/rejected": -308.6473388671875, | |
| "loss": 0.1959, | |
| "rewards/chosen": -0.34189051389694214, | |
| "rewards/margins": 1.3108210563659668, | |
| "rewards/rejected": -1.6527115106582642, | |
| "step": 308 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 10.835558830180704, | |
| "kl": 0.0, | |
| "learning_rate": 4.191522762951334e-07, | |
| "logps/chosen": -221.1939239501953, | |
| "logps/rejected": -216.98193359375, | |
| "loss": 0.1861, | |
| "rewards/chosen": 0.3302954435348511, | |
| "rewards/margins": 1.9516035318374634, | |
| "rewards/rejected": -1.6213080883026123, | |
| "step": 309 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 10.5831850564, | |
| "kl": 0.0, | |
| "learning_rate": 4.188906331763474e-07, | |
| "logps/chosen": -193.68711853027344, | |
| "logps/rejected": -257.5124206542969, | |
| "loss": 0.2299, | |
| "rewards/chosen": -0.3544810116291046, | |
| "rewards/margins": 1.7205111980438232, | |
| "rewards/rejected": -2.0749921798706055, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 12.791438944238683, | |
| "kl": 0.0, | |
| "learning_rate": 4.1862899005756147e-07, | |
| "logps/chosen": -259.3917236328125, | |
| "logps/rejected": -346.5878601074219, | |
| "loss": 0.2063, | |
| "rewards/chosen": -1.0072615146636963, | |
| "rewards/margins": 0.5092824697494507, | |
| "rewards/rejected": -1.516543984413147, | |
| "step": 311 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 8.990082192323051, | |
| "kl": 0.0, | |
| "learning_rate": 4.183673469387755e-07, | |
| "logps/chosen": -185.4383544921875, | |
| "logps/rejected": -225.95294189453125, | |
| "loss": 0.1446, | |
| "rewards/chosen": 0.1784064769744873, | |
| "rewards/margins": 2.15664005279541, | |
| "rewards/rejected": -1.9782336950302124, | |
| "step": 312 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 9.526272579899544, | |
| "kl": 0.0, | |
| "learning_rate": 4.181057038199895e-07, | |
| "logps/chosen": -220.56004333496094, | |
| "logps/rejected": -190.05809020996094, | |
| "loss": 0.1914, | |
| "rewards/chosen": 1.1748487949371338, | |
| "rewards/margins": 3.3340234756469727, | |
| "rewards/rejected": -2.159174680709839, | |
| "step": 313 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 11.139585767174607, | |
| "kl": 0.0, | |
| "learning_rate": 4.178440607012035e-07, | |
| "logps/chosen": -268.95770263671875, | |
| "logps/rejected": -222.8634033203125, | |
| "loss": 0.2051, | |
| "rewards/chosen": -0.40160444378852844, | |
| "rewards/margins": 2.409646987915039, | |
| "rewards/rejected": -2.811251401901245, | |
| "step": 314 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 10.949601505354943, | |
| "kl": 0.0, | |
| "learning_rate": 4.1758241758241757e-07, | |
| "logps/chosen": -186.237060546875, | |
| "logps/rejected": -245.27206420898438, | |
| "loss": 0.249, | |
| "rewards/chosen": -0.8131721019744873, | |
| "rewards/margins": 1.2247133255004883, | |
| "rewards/rejected": -2.0378854274749756, | |
| "step": 315 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "grad_norm": 11.351891085819371, | |
| "kl": 0.0, | |
| "learning_rate": 4.1732077446363157e-07, | |
| "logps/chosen": -201.30223083496094, | |
| "logps/rejected": -243.75341796875, | |
| "loss": 0.1895, | |
| "rewards/chosen": 0.3100842535495758, | |
| "rewards/margins": 2.5161705017089844, | |
| "rewards/rejected": -2.2060861587524414, | |
| "step": 316 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "grad_norm": 12.221976691002848, | |
| "kl": 0.0, | |
| "learning_rate": 4.170591313448456e-07, | |
| "logps/chosen": -232.7662353515625, | |
| "logps/rejected": -362.24041748046875, | |
| "loss": 0.1972, | |
| "rewards/chosen": 0.6142377853393555, | |
| "rewards/margins": 3.4600791931152344, | |
| "rewards/rejected": -2.845841407775879, | |
| "step": 317 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "grad_norm": 10.754382204616256, | |
| "kl": 0.0, | |
| "learning_rate": 4.1679748822605967e-07, | |
| "logps/chosen": -215.42227172851562, | |
| "logps/rejected": -278.8962707519531, | |
| "loss": 0.1188, | |
| "rewards/chosen": 0.1050441712141037, | |
| "rewards/margins": 3.3125483989715576, | |
| "rewards/rejected": -3.2075042724609375, | |
| "step": 318 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "grad_norm": 12.987540188284319, | |
| "kl": 0.0, | |
| "learning_rate": 4.1653584510727366e-07, | |
| "logps/chosen": -244.31643676757812, | |
| "logps/rejected": -323.1125183105469, | |
| "loss": 0.2359, | |
| "rewards/chosen": -0.4736923575401306, | |
| "rewards/margins": 2.1419007778167725, | |
| "rewards/rejected": -2.615593194961548, | |
| "step": 319 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "grad_norm": 8.80349986892105, | |
| "kl": 0.0, | |
| "learning_rate": 4.1627420198848766e-07, | |
| "logps/chosen": -223.25929260253906, | |
| "logps/rejected": -236.41934204101562, | |
| "loss": 0.1387, | |
| "rewards/chosen": -0.46407273411750793, | |
| "rewards/margins": 1.7442665100097656, | |
| "rewards/rejected": -2.208339214324951, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "grad_norm": 13.1846568437176, | |
| "kl": 0.0, | |
| "learning_rate": 4.160125588697017e-07, | |
| "logps/chosen": -213.67198181152344, | |
| "logps/rejected": -295.5606384277344, | |
| "loss": 0.2656, | |
| "rewards/chosen": -0.33450353145599365, | |
| "rewards/margins": 1.9383219480514526, | |
| "rewards/rejected": -2.2728254795074463, | |
| "step": 321 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "grad_norm": 13.253044198831535, | |
| "kl": 0.0, | |
| "learning_rate": 4.1575091575091576e-07, | |
| "logps/chosen": -231.430908203125, | |
| "logps/rejected": -249.1555938720703, | |
| "loss": 0.1841, | |
| "rewards/chosen": 0.48544788360595703, | |
| "rewards/margins": 1.7013275623321533, | |
| "rewards/rejected": -1.2158796787261963, | |
| "step": 322 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "grad_norm": 11.874190668515645, | |
| "kl": 0.0, | |
| "learning_rate": 4.1548927263212976e-07, | |
| "logps/chosen": -208.916015625, | |
| "logps/rejected": -248.04067993164062, | |
| "loss": 0.1922, | |
| "rewards/chosen": -0.29332485795021057, | |
| "rewards/margins": 2.6810503005981445, | |
| "rewards/rejected": -2.9743752479553223, | |
| "step": 323 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "grad_norm": 9.788377245433486, | |
| "kl": 0.0, | |
| "learning_rate": 4.1522762951334376e-07, | |
| "logps/chosen": -237.29441833496094, | |
| "logps/rejected": -246.01272583007812, | |
| "loss": 0.2483, | |
| "rewards/chosen": -1.0158889293670654, | |
| "rewards/margins": 2.2013275623321533, | |
| "rewards/rejected": -3.2172164916992188, | |
| "step": 324 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "grad_norm": 10.674006139111874, | |
| "kl": 0.0, | |
| "learning_rate": 4.149659863945578e-07, | |
| "logps/chosen": -240.6201171875, | |
| "logps/rejected": -255.26345825195312, | |
| "loss": 0.2101, | |
| "rewards/chosen": -0.13020078837871552, | |
| "rewards/margins": 3.523922920227051, | |
| "rewards/rejected": -3.6541237831115723, | |
| "step": 325 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "grad_norm": 10.0670473259245, | |
| "kl": 0.0, | |
| "learning_rate": 4.147043432757718e-07, | |
| "logps/chosen": -229.05381774902344, | |
| "logps/rejected": -261.8768005371094, | |
| "loss": 0.1158, | |
| "rewards/chosen": -0.555793821811676, | |
| "rewards/margins": 3.118558645248413, | |
| "rewards/rejected": -3.6743524074554443, | |
| "step": 326 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "grad_norm": 8.521982319858306, | |
| "kl": 0.0, | |
| "learning_rate": 4.1444270015698585e-07, | |
| "logps/chosen": -220.83934020996094, | |
| "logps/rejected": -249.3174285888672, | |
| "loss": 0.1808, | |
| "rewards/chosen": 0.1714603751897812, | |
| "rewards/margins": 2.402540922164917, | |
| "rewards/rejected": -2.2310805320739746, | |
| "step": 327 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "grad_norm": 12.228304372001576, | |
| "kl": 0.0, | |
| "learning_rate": 4.141810570381999e-07, | |
| "logps/chosen": -286.3045654296875, | |
| "logps/rejected": -269.97601318359375, | |
| "loss": 0.1993, | |
| "rewards/chosen": -0.551383912563324, | |
| "rewards/margins": 1.8383245468139648, | |
| "rewards/rejected": -2.3897085189819336, | |
| "step": 328 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "grad_norm": 11.897560403128791, | |
| "kl": 0.0, | |
| "learning_rate": 4.1391941391941385e-07, | |
| "logps/chosen": -154.19729614257812, | |
| "logps/rejected": -238.20199584960938, | |
| "loss": 0.2342, | |
| "rewards/chosen": 0.1875210553407669, | |
| "rewards/margins": 1.9252369403839111, | |
| "rewards/rejected": -1.7377158403396606, | |
| "step": 329 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "grad_norm": 10.436673403488303, | |
| "kl": 0.0, | |
| "learning_rate": 4.136577708006279e-07, | |
| "logps/chosen": -209.2281036376953, | |
| "logps/rejected": -269.2527160644531, | |
| "loss": 0.1088, | |
| "rewards/chosen": 0.6344658732414246, | |
| "rewards/margins": 3.8878352642059326, | |
| "rewards/rejected": -3.2533693313598633, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "grad_norm": 8.255234754549457, | |
| "kl": 0.0, | |
| "learning_rate": 4.1339612768184195e-07, | |
| "logps/chosen": -212.30836486816406, | |
| "logps/rejected": -201.14207458496094, | |
| "loss": 0.174, | |
| "rewards/chosen": 0.6615203619003296, | |
| "rewards/margins": 2.292316436767578, | |
| "rewards/rejected": -1.630796194076538, | |
| "step": 331 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "grad_norm": 11.884943779747632, | |
| "kl": 0.0, | |
| "learning_rate": 4.13134484563056e-07, | |
| "logps/chosen": -211.07350158691406, | |
| "logps/rejected": -211.02023315429688, | |
| "loss": 0.2495, | |
| "rewards/chosen": 0.40837398171424866, | |
| "rewards/margins": 2.4228506088256836, | |
| "rewards/rejected": -2.0144765377044678, | |
| "step": 332 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "grad_norm": 11.994293665472632, | |
| "kl": 0.0, | |
| "learning_rate": 4.1287284144427e-07, | |
| "logps/chosen": -289.47589111328125, | |
| "logps/rejected": -228.80950927734375, | |
| "loss": 0.1505, | |
| "rewards/chosen": -0.011884563602507114, | |
| "rewards/margins": 1.357936143875122, | |
| "rewards/rejected": -1.3698207139968872, | |
| "step": 333 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "grad_norm": 12.497465243458791, | |
| "kl": 0.0, | |
| "learning_rate": 4.1261119832548405e-07, | |
| "logps/chosen": -214.80462646484375, | |
| "logps/rejected": -285.0194396972656, | |
| "loss": 0.2098, | |
| "rewards/chosen": -0.4318108856678009, | |
| "rewards/margins": 1.8095088005065918, | |
| "rewards/rejected": -2.2413196563720703, | |
| "step": 334 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "grad_norm": 12.597201422275237, | |
| "kl": 0.0, | |
| "learning_rate": 4.1234955520669805e-07, | |
| "logps/chosen": -173.9023895263672, | |
| "logps/rejected": -242.03558349609375, | |
| "loss": 0.1763, | |
| "rewards/chosen": -0.03515807166695595, | |
| "rewards/margins": 1.9462581872940063, | |
| "rewards/rejected": -1.9814162254333496, | |
| "step": 335 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "grad_norm": 11.29563689087554, | |
| "kl": 0.0, | |
| "learning_rate": 4.1208791208791204e-07, | |
| "logps/chosen": -224.08148193359375, | |
| "logps/rejected": -253.4733123779297, | |
| "loss": 0.1798, | |
| "rewards/chosen": 0.09991315752267838, | |
| "rewards/margins": 2.7761764526367188, | |
| "rewards/rejected": -2.6762633323669434, | |
| "step": 336 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "grad_norm": 9.798183455958176, | |
| "kl": 0.0, | |
| "learning_rate": 4.118262689691261e-07, | |
| "logps/chosen": -226.20651245117188, | |
| "logps/rejected": -261.2646789550781, | |
| "loss": 0.2247, | |
| "rewards/chosen": -0.36128562688827515, | |
| "rewards/margins": 2.0138986110687256, | |
| "rewards/rejected": -2.3751842975616455, | |
| "step": 337 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "grad_norm": 10.59217786675228, | |
| "kl": 0.0, | |
| "learning_rate": 4.1156462585034014e-07, | |
| "logps/chosen": -191.44393920898438, | |
| "logps/rejected": -222.71336364746094, | |
| "loss": 0.2218, | |
| "rewards/chosen": -0.10784044116735458, | |
| "rewards/margins": 1.7394548654556274, | |
| "rewards/rejected": -1.8472952842712402, | |
| "step": 338 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "grad_norm": 9.929264582178304, | |
| "kl": 0.0, | |
| "learning_rate": 4.1130298273155414e-07, | |
| "logps/chosen": -211.10133361816406, | |
| "logps/rejected": -235.98374938964844, | |
| "loss": 0.1557, | |
| "rewards/chosen": -0.17055515944957733, | |
| "rewards/margins": 3.654494285583496, | |
| "rewards/rejected": -3.82504940032959, | |
| "step": 339 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "grad_norm": 12.576617441384204, | |
| "kl": 0.0, | |
| "learning_rate": 4.1104133961276814e-07, | |
| "logps/chosen": -245.1005096435547, | |
| "logps/rejected": -265.1275939941406, | |
| "loss": 0.1974, | |
| "rewards/chosen": 0.29450762271881104, | |
| "rewards/margins": 1.8650548458099365, | |
| "rewards/rejected": -1.5705472230911255, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "grad_norm": 12.893258406638443, | |
| "kl": 0.0, | |
| "learning_rate": 4.107796964939822e-07, | |
| "logps/chosen": -283.796630859375, | |
| "logps/rejected": -255.04974365234375, | |
| "loss": 0.1138, | |
| "rewards/chosen": 1.2097806930541992, | |
| "rewards/margins": 3.861506223678589, | |
| "rewards/rejected": -2.6517255306243896, | |
| "step": 341 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "grad_norm": 9.442037096083547, | |
| "kl": 0.0, | |
| "learning_rate": 4.1051805337519624e-07, | |
| "logps/chosen": -193.82659912109375, | |
| "logps/rejected": -236.00262451171875, | |
| "loss": 0.169, | |
| "rewards/chosen": 0.25398653745651245, | |
| "rewards/margins": 3.0244903564453125, | |
| "rewards/rejected": -2.7705037593841553, | |
| "step": 342 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "grad_norm": 10.498722183093273, | |
| "kl": 0.0, | |
| "learning_rate": 4.1025641025641024e-07, | |
| "logps/chosen": -337.9830017089844, | |
| "logps/rejected": -247.3750457763672, | |
| "loss": 0.1899, | |
| "rewards/chosen": -1.3225940465927124, | |
| "rewards/margins": 1.0586594343185425, | |
| "rewards/rejected": -2.381253480911255, | |
| "step": 343 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "grad_norm": 10.172536259980642, | |
| "kl": 0.0, | |
| "learning_rate": 4.099947671376243e-07, | |
| "logps/chosen": -250.67410278320312, | |
| "logps/rejected": -271.3428039550781, | |
| "loss": 0.2109, | |
| "rewards/chosen": -0.8246206045150757, | |
| "rewards/margins": 2.084084987640381, | |
| "rewards/rejected": -2.908705472946167, | |
| "step": 344 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "grad_norm": 12.383051195247807, | |
| "kl": 0.0, | |
| "learning_rate": 4.097331240188383e-07, | |
| "logps/chosen": -263.4064025878906, | |
| "logps/rejected": -282.1274108886719, | |
| "loss": 0.2069, | |
| "rewards/chosen": -0.49988675117492676, | |
| "rewards/margins": 1.9791779518127441, | |
| "rewards/rejected": -2.479064702987671, | |
| "step": 345 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "grad_norm": 11.923267130594034, | |
| "kl": 0.0, | |
| "learning_rate": 4.094714809000523e-07, | |
| "logps/chosen": -215.8848419189453, | |
| "logps/rejected": -253.63076782226562, | |
| "loss": 0.1745, | |
| "rewards/chosen": -0.4700072109699249, | |
| "rewards/margins": 1.989243507385254, | |
| "rewards/rejected": -2.4592506885528564, | |
| "step": 346 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "grad_norm": 8.982041772561747, | |
| "kl": 0.0, | |
| "learning_rate": 4.0920983778126633e-07, | |
| "logps/chosen": -225.7939910888672, | |
| "logps/rejected": -244.41358947753906, | |
| "loss": 0.1869, | |
| "rewards/chosen": -1.364567756652832, | |
| "rewards/margins": 0.951836109161377, | |
| "rewards/rejected": -2.316403865814209, | |
| "step": 347 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "grad_norm": 10.837403601758242, | |
| "kl": 0.0, | |
| "learning_rate": 4.089481946624804e-07, | |
| "logps/chosen": -249.8412628173828, | |
| "logps/rejected": -241.59420776367188, | |
| "loss": 0.1471, | |
| "rewards/chosen": -0.19725684821605682, | |
| "rewards/margins": 2.119018077850342, | |
| "rewards/rejected": -2.316274881362915, | |
| "step": 348 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "grad_norm": 11.217732507310354, | |
| "kl": 0.0, | |
| "learning_rate": 4.086865515436944e-07, | |
| "logps/chosen": -257.2194519042969, | |
| "logps/rejected": -180.22837829589844, | |
| "loss": 0.1527, | |
| "rewards/chosen": 0.24126391112804413, | |
| "rewards/margins": 2.3532466888427734, | |
| "rewards/rejected": -2.111982822418213, | |
| "step": 349 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "grad_norm": 10.502546710908188, | |
| "kl": 0.0, | |
| "learning_rate": 4.084249084249084e-07, | |
| "logps/chosen": -227.2938232421875, | |
| "logps/rejected": -300.4139709472656, | |
| "loss": 0.1961, | |
| "rewards/chosen": -0.11248449236154556, | |
| "rewards/margins": 2.296675205230713, | |
| "rewards/rejected": -2.4091596603393555, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "grad_norm": 11.149043285356687, | |
| "kl": 0.0, | |
| "learning_rate": 4.0816326530612243e-07, | |
| "logps/chosen": -235.18333435058594, | |
| "logps/rejected": -206.1359100341797, | |
| "loss": 0.1829, | |
| "rewards/chosen": 0.5669969320297241, | |
| "rewards/margins": 3.09303617477417, | |
| "rewards/rejected": -2.5260391235351562, | |
| "step": 351 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "grad_norm": 11.899081712479965, | |
| "kl": 0.0, | |
| "learning_rate": 4.079016221873365e-07, | |
| "logps/chosen": -247.81387329101562, | |
| "logps/rejected": -268.5088195800781, | |
| "loss": 0.2918, | |
| "rewards/chosen": -0.36709415912628174, | |
| "rewards/margins": 0.9215909242630005, | |
| "rewards/rejected": -1.2886850833892822, | |
| "step": 352 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "grad_norm": 11.107384586901443, | |
| "kl": 0.0, | |
| "learning_rate": 4.076399790685505e-07, | |
| "logps/chosen": -206.99432373046875, | |
| "logps/rejected": -212.78196716308594, | |
| "loss": 0.2387, | |
| "rewards/chosen": -0.985337495803833, | |
| "rewards/margins": 1.2888743877410889, | |
| "rewards/rejected": -2.274211883544922, | |
| "step": 353 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "grad_norm": 10.235471285138322, | |
| "kl": 0.0, | |
| "learning_rate": 4.073783359497645e-07, | |
| "logps/chosen": -236.5115203857422, | |
| "logps/rejected": -272.5962219238281, | |
| "loss": 0.1737, | |
| "rewards/chosen": -0.7294275164604187, | |
| "rewards/margins": 2.0560388565063477, | |
| "rewards/rejected": -2.785466432571411, | |
| "step": 354 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "grad_norm": 11.272141410111164, | |
| "kl": 0.0, | |
| "learning_rate": 4.071166928309785e-07, | |
| "logps/chosen": -222.7964324951172, | |
| "logps/rejected": -287.06475830078125, | |
| "loss": 0.1308, | |
| "rewards/chosen": 0.3911401629447937, | |
| "rewards/margins": 2.989321231842041, | |
| "rewards/rejected": -2.5981810092926025, | |
| "step": 355 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "grad_norm": 7.964274506087736, | |
| "kl": 0.0, | |
| "learning_rate": 4.068550497121925e-07, | |
| "logps/chosen": -170.38925170898438, | |
| "logps/rejected": -205.2819366455078, | |
| "loss": 0.1071, | |
| "rewards/chosen": -0.18005786836147308, | |
| "rewards/margins": 2.6861469745635986, | |
| "rewards/rejected": -2.8662047386169434, | |
| "step": 356 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "grad_norm": 11.501254991289231, | |
| "kl": 0.0, | |
| "learning_rate": 4.0659340659340657e-07, | |
| "logps/chosen": -246.70559692382812, | |
| "logps/rejected": -238.14466857910156, | |
| "loss": 0.1549, | |
| "rewards/chosen": 0.33830830454826355, | |
| "rewards/margins": 2.2392632961273193, | |
| "rewards/rejected": -1.9009549617767334, | |
| "step": 357 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "grad_norm": 10.23573665557948, | |
| "kl": 0.0, | |
| "learning_rate": 4.063317634746206e-07, | |
| "logps/chosen": -200.4901123046875, | |
| "logps/rejected": -207.58871459960938, | |
| "loss": 0.1599, | |
| "rewards/chosen": 1.0193722248077393, | |
| "rewards/margins": 2.878417491912842, | |
| "rewards/rejected": -1.859045147895813, | |
| "step": 358 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "grad_norm": 11.84162361137601, | |
| "kl": 0.0, | |
| "learning_rate": 4.060701203558346e-07, | |
| "logps/chosen": -253.09637451171875, | |
| "logps/rejected": -247.7617645263672, | |
| "loss": 0.2949, | |
| "rewards/chosen": -1.3407307863235474, | |
| "rewards/margins": 0.6386681795120239, | |
| "rewards/rejected": -1.9793989658355713, | |
| "step": 359 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "grad_norm": 12.38802746375023, | |
| "kl": 0.0, | |
| "learning_rate": 4.058084772370486e-07, | |
| "logps/chosen": -241.19613647460938, | |
| "logps/rejected": -237.13682556152344, | |
| "loss": 0.1108, | |
| "rewards/chosen": -0.31301483511924744, | |
| "rewards/margins": 2.1045053005218506, | |
| "rewards/rejected": -2.417520046234131, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "grad_norm": 12.348300253030986, | |
| "kl": 0.0, | |
| "learning_rate": 4.0554683411826267e-07, | |
| "logps/chosen": -253.8113555908203, | |
| "logps/rejected": -250.69007873535156, | |
| "loss": 0.2231, | |
| "rewards/chosen": -0.6661314964294434, | |
| "rewards/margins": 1.135571002960205, | |
| "rewards/rejected": -1.8017024993896484, | |
| "step": 361 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "grad_norm": 9.813621316611632, | |
| "kl": 0.0, | |
| "learning_rate": 4.052851909994767e-07, | |
| "logps/chosen": -183.4186248779297, | |
| "logps/rejected": -226.8902130126953, | |
| "loss": 0.2452, | |
| "rewards/chosen": 0.09793578833341599, | |
| "rewards/margins": 3.1266567707061768, | |
| "rewards/rejected": -3.0287210941314697, | |
| "step": 362 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "grad_norm": 10.410184074704667, | |
| "kl": 0.0, | |
| "learning_rate": 4.050235478806907e-07, | |
| "logps/chosen": -218.2381134033203, | |
| "logps/rejected": -223.76805114746094, | |
| "loss": 0.2223, | |
| "rewards/chosen": -0.6022669672966003, | |
| "rewards/margins": 2.0103299617767334, | |
| "rewards/rejected": -2.6125969886779785, | |
| "step": 363 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "grad_norm": 12.01700363931878, | |
| "kl": 0.0, | |
| "learning_rate": 4.0476190476190476e-07, | |
| "logps/chosen": -232.75802612304688, | |
| "logps/rejected": -237.35382080078125, | |
| "loss": 0.2363, | |
| "rewards/chosen": 0.516566812992096, | |
| "rewards/margins": 2.019991874694824, | |
| "rewards/rejected": -1.5034250020980835, | |
| "step": 364 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "grad_norm": 10.262930905107156, | |
| "kl": 0.0, | |
| "learning_rate": 4.045002616431188e-07, | |
| "logps/chosen": -204.19009399414062, | |
| "logps/rejected": -235.92129516601562, | |
| "loss": 0.1872, | |
| "rewards/chosen": 0.05859723314642906, | |
| "rewards/margins": 2.741896629333496, | |
| "rewards/rejected": -2.6832993030548096, | |
| "step": 365 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "grad_norm": 11.30426818476053, | |
| "kl": 0.0, | |
| "learning_rate": 4.0423861852433276e-07, | |
| "logps/chosen": -166.32923889160156, | |
| "logps/rejected": -281.9306945800781, | |
| "loss": 0.2366, | |
| "rewards/chosen": -0.21419693529605865, | |
| "rewards/margins": 2.5291309356689453, | |
| "rewards/rejected": -2.7433278560638428, | |
| "step": 366 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "grad_norm": 10.791375928218388, | |
| "kl": 0.0, | |
| "learning_rate": 4.039769754055468e-07, | |
| "logps/chosen": -174.7226104736328, | |
| "logps/rejected": -268.91180419921875, | |
| "loss": 0.1773, | |
| "rewards/chosen": 0.03391220420598984, | |
| "rewards/margins": 3.911714792251587, | |
| "rewards/rejected": -3.877802610397339, | |
| "step": 367 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "grad_norm": 12.229778476365636, | |
| "kl": 0.0, | |
| "learning_rate": 4.0371533228676086e-07, | |
| "logps/chosen": -227.41676330566406, | |
| "logps/rejected": -253.11061096191406, | |
| "loss": 0.2316, | |
| "rewards/chosen": -1.3165072202682495, | |
| "rewards/margins": 2.440537452697754, | |
| "rewards/rejected": -3.757044553756714, | |
| "step": 368 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "grad_norm": 9.70449239918135, | |
| "kl": 0.0, | |
| "learning_rate": 4.0345368916797486e-07, | |
| "logps/chosen": -206.78543090820312, | |
| "logps/rejected": -193.22666931152344, | |
| "loss": 0.152, | |
| "rewards/chosen": -0.005291328299790621, | |
| "rewards/margins": 2.320930004119873, | |
| "rewards/rejected": -2.326221227645874, | |
| "step": 369 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "grad_norm": 12.529777714925808, | |
| "kl": 0.0, | |
| "learning_rate": 4.031920460491889e-07, | |
| "logps/chosen": -260.2597961425781, | |
| "logps/rejected": -245.1582794189453, | |
| "loss": 0.2229, | |
| "rewards/chosen": -0.34088611602783203, | |
| "rewards/margins": 2.3002560138702393, | |
| "rewards/rejected": -2.6411421298980713, | |
| "step": 370 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "grad_norm": 12.126319059662272, | |
| "kl": 0.0, | |
| "learning_rate": 4.029304029304029e-07, | |
| "logps/chosen": -159.91371154785156, | |
| "logps/rejected": -215.52037048339844, | |
| "loss": 0.1604, | |
| "rewards/chosen": -0.13704350590705872, | |
| "rewards/margins": 2.6966662406921387, | |
| "rewards/rejected": -2.833709716796875, | |
| "step": 371 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "grad_norm": 12.2279596400823, | |
| "kl": 0.0, | |
| "learning_rate": 4.026687598116169e-07, | |
| "logps/chosen": -203.8690643310547, | |
| "logps/rejected": -248.30355834960938, | |
| "loss": 0.2764, | |
| "rewards/chosen": -1.4684327840805054, | |
| "rewards/margins": 0.25741446018218994, | |
| "rewards/rejected": -1.7258472442626953, | |
| "step": 372 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "grad_norm": 11.93017592019336, | |
| "kl": 0.0, | |
| "learning_rate": 4.0240711669283095e-07, | |
| "logps/chosen": -312.0763244628906, | |
| "logps/rejected": -262.0274963378906, | |
| "loss": 0.1746, | |
| "rewards/chosen": -1.0104618072509766, | |
| "rewards/margins": 2.119619369506836, | |
| "rewards/rejected": -3.1300811767578125, | |
| "step": 373 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "grad_norm": 10.026240675416432, | |
| "kl": 0.0, | |
| "learning_rate": 4.02145473574045e-07, | |
| "logps/chosen": -220.4754638671875, | |
| "logps/rejected": -265.4399719238281, | |
| "loss": 0.1718, | |
| "rewards/chosen": 0.43962326645851135, | |
| "rewards/margins": 3.062368392944336, | |
| "rewards/rejected": -2.6227450370788574, | |
| "step": 374 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "grad_norm": 7.862119363862361, | |
| "kl": 0.0, | |
| "learning_rate": 4.0188383045525905e-07, | |
| "logps/chosen": -225.05450439453125, | |
| "logps/rejected": -323.68798828125, | |
| "loss": 0.2182, | |
| "rewards/chosen": -0.6643018126487732, | |
| "rewards/margins": 1.284275770187378, | |
| "rewards/rejected": -1.9485775232315063, | |
| "step": 375 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "grad_norm": 10.113068546407243, | |
| "kl": 0.0, | |
| "learning_rate": 4.01622187336473e-07, | |
| "logps/chosen": -238.6414031982422, | |
| "logps/rejected": -226.67205810546875, | |
| "loss": 0.1479, | |
| "rewards/chosen": 0.5626780986785889, | |
| "rewards/margins": 3.329695224761963, | |
| "rewards/rejected": -2.767017126083374, | |
| "step": 376 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "grad_norm": 8.742780985362685, | |
| "kl": 0.0, | |
| "learning_rate": 4.0136054421768705e-07, | |
| "logps/chosen": -205.3953857421875, | |
| "logps/rejected": -243.7183074951172, | |
| "loss": 0.1611, | |
| "rewards/chosen": 0.14700005948543549, | |
| "rewards/margins": 3.2417192459106445, | |
| "rewards/rejected": -3.094719171524048, | |
| "step": 377 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "grad_norm": 11.28132011147294, | |
| "kl": 0.0, | |
| "learning_rate": 4.010989010989011e-07, | |
| "logps/chosen": -214.97244262695312, | |
| "logps/rejected": -216.71173095703125, | |
| "loss": 0.1858, | |
| "rewards/chosen": -0.015776043757796288, | |
| "rewards/margins": 2.4526312351226807, | |
| "rewards/rejected": -2.468407392501831, | |
| "step": 378 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "grad_norm": 12.778638599311456, | |
| "kl": 0.0, | |
| "learning_rate": 4.008372579801151e-07, | |
| "logps/chosen": -281.7500915527344, | |
| "logps/rejected": -205.56297302246094, | |
| "loss": 0.1866, | |
| "rewards/chosen": -0.7977088093757629, | |
| "rewards/margins": 2.43152117729187, | |
| "rewards/rejected": -3.2292299270629883, | |
| "step": 379 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "grad_norm": 13.038247255191017, | |
| "kl": 0.0, | |
| "learning_rate": 4.0057561486132915e-07, | |
| "logps/chosen": -278.4736328125, | |
| "logps/rejected": -237.22613525390625, | |
| "loss": 0.172, | |
| "rewards/chosen": 0.31080275774002075, | |
| "rewards/margins": 2.6367199420928955, | |
| "rewards/rejected": -2.3259172439575195, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "grad_norm": 12.071589262477481, | |
| "kl": 0.0, | |
| "learning_rate": 4.0031397174254314e-07, | |
| "logps/chosen": -236.46629333496094, | |
| "logps/rejected": -230.3934326171875, | |
| "loss": 0.2205, | |
| "rewards/chosen": 0.46186843514442444, | |
| "rewards/margins": 2.98122239112854, | |
| "rewards/rejected": -2.5193538665771484, | |
| "step": 381 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "grad_norm": 11.849099983442954, | |
| "kl": 0.0, | |
| "learning_rate": 4.0005232862375714e-07, | |
| "logps/chosen": -225.62376403808594, | |
| "logps/rejected": -240.21817016601562, | |
| "loss": 0.2767, | |
| "rewards/chosen": -0.6141564846038818, | |
| "rewards/margins": 0.870281457901001, | |
| "rewards/rejected": -1.4844379425048828, | |
| "step": 382 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "grad_norm": 10.00253243924632, | |
| "kl": 0.0, | |
| "learning_rate": 3.997906855049712e-07, | |
| "logps/chosen": -253.96109008789062, | |
| "logps/rejected": -244.46368408203125, | |
| "loss": 0.1898, | |
| "rewards/chosen": -0.6670475006103516, | |
| "rewards/margins": 2.5981571674346924, | |
| "rewards/rejected": -3.265204668045044, | |
| "step": 383 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "grad_norm": 9.423555936016577, | |
| "kl": 0.0, | |
| "learning_rate": 3.9952904238618524e-07, | |
| "logps/chosen": -235.05836486816406, | |
| "logps/rejected": -276.4933166503906, | |
| "loss": 0.1754, | |
| "rewards/chosen": -1.0856462717056274, | |
| "rewards/margins": 2.616638660430908, | |
| "rewards/rejected": -3.702285051345825, | |
| "step": 384 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "grad_norm": 9.603542307090086, | |
| "kl": 0.0, | |
| "learning_rate": 3.992673992673993e-07, | |
| "logps/chosen": -218.27806091308594, | |
| "logps/rejected": -227.58969116210938, | |
| "loss": 0.1948, | |
| "rewards/chosen": 0.6791697144508362, | |
| "rewards/margins": 3.0643129348754883, | |
| "rewards/rejected": -2.385143280029297, | |
| "step": 385 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "grad_norm": 11.997346527322566, | |
| "kl": 0.0, | |
| "learning_rate": 3.9900575614861324e-07, | |
| "logps/chosen": -205.48524475097656, | |
| "logps/rejected": -237.6011505126953, | |
| "loss": 0.2379, | |
| "rewards/chosen": 0.24487002193927765, | |
| "rewards/margins": 2.6219630241394043, | |
| "rewards/rejected": -2.3770930767059326, | |
| "step": 386 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "grad_norm": 9.0666967002916, | |
| "kl": 0.0, | |
| "learning_rate": 3.987441130298273e-07, | |
| "logps/chosen": -196.8197479248047, | |
| "logps/rejected": -200.19735717773438, | |
| "loss": 0.1701, | |
| "rewards/chosen": 0.5998373627662659, | |
| "rewards/margins": 2.8779029846191406, | |
| "rewards/rejected": -2.2780656814575195, | |
| "step": 387 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "grad_norm": 11.538513762529194, | |
| "kl": 0.0, | |
| "learning_rate": 3.9848246991104134e-07, | |
| "logps/chosen": -237.28199768066406, | |
| "logps/rejected": -266.018798828125, | |
| "loss": 0.167, | |
| "rewards/chosen": -0.623340904712677, | |
| "rewards/margins": 2.147143602371216, | |
| "rewards/rejected": -2.770484447479248, | |
| "step": 388 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "grad_norm": 11.828887960825947, | |
| "kl": 0.0, | |
| "learning_rate": 3.9822082679225533e-07, | |
| "logps/chosen": -220.56329345703125, | |
| "logps/rejected": -268.723388671875, | |
| "loss": 0.1492, | |
| "rewards/chosen": 0.7558012008666992, | |
| "rewards/margins": 2.6395673751831055, | |
| "rewards/rejected": -1.8837661743164062, | |
| "step": 389 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "grad_norm": 11.47492309378771, | |
| "kl": 0.0, | |
| "learning_rate": 3.979591836734694e-07, | |
| "logps/chosen": -249.0001983642578, | |
| "logps/rejected": -240.189697265625, | |
| "loss": 0.18, | |
| "rewards/chosen": 0.21565745770931244, | |
| "rewards/margins": 2.2755565643310547, | |
| "rewards/rejected": -2.059899091720581, | |
| "step": 390 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "grad_norm": 8.926444305093995, | |
| "kl": 0.0, | |
| "learning_rate": 3.9769754055468343e-07, | |
| "logps/chosen": -200.0030059814453, | |
| "logps/rejected": -206.71624755859375, | |
| "loss": 0.1516, | |
| "rewards/chosen": -0.6393547058105469, | |
| "rewards/margins": 1.0063797235488892, | |
| "rewards/rejected": -1.645734429359436, | |
| "step": 391 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "grad_norm": 11.660568501998092, | |
| "kl": 0.0, | |
| "learning_rate": 3.974358974358974e-07, | |
| "logps/chosen": -223.32652282714844, | |
| "logps/rejected": -243.16845703125, | |
| "loss": 0.2899, | |
| "rewards/chosen": 0.2290610671043396, | |
| "rewards/margins": 1.6543824672698975, | |
| "rewards/rejected": -1.425321340560913, | |
| "step": 392 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "grad_norm": 14.42295762896043, | |
| "kl": 0.0, | |
| "learning_rate": 3.9717425431711143e-07, | |
| "logps/chosen": -289.037353515625, | |
| "logps/rejected": -252.30154418945312, | |
| "loss": 0.2241, | |
| "rewards/chosen": -1.7561075687408447, | |
| "rewards/margins": 0.43781447410583496, | |
| "rewards/rejected": -2.1939220428466797, | |
| "step": 393 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "grad_norm": 11.559459650147774, | |
| "kl": 0.0, | |
| "learning_rate": 3.969126111983255e-07, | |
| "logps/chosen": -245.2276153564453, | |
| "logps/rejected": -241.8197021484375, | |
| "loss": 0.2797, | |
| "rewards/chosen": -1.3280991315841675, | |
| "rewards/margins": 1.9113882780075073, | |
| "rewards/rejected": -3.239487409591675, | |
| "step": 394 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "grad_norm": 14.711575319555203, | |
| "kl": 0.0, | |
| "learning_rate": 3.9665096807953953e-07, | |
| "logps/chosen": -183.3538818359375, | |
| "logps/rejected": -245.3995819091797, | |
| "loss": 0.2133, | |
| "rewards/chosen": -0.0880654826760292, | |
| "rewards/margins": 2.2100796699523926, | |
| "rewards/rejected": -2.298145055770874, | |
| "step": 395 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "grad_norm": 13.58824047165676, | |
| "kl": 0.0, | |
| "learning_rate": 3.9638932496075353e-07, | |
| "logps/chosen": -319.79083251953125, | |
| "logps/rejected": -241.92750549316406, | |
| "loss": 0.2077, | |
| "rewards/chosen": -0.5814463496208191, | |
| "rewards/margins": 1.7764933109283447, | |
| "rewards/rejected": -2.3579397201538086, | |
| "step": 396 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "grad_norm": 10.51670285219746, | |
| "kl": 0.0, | |
| "learning_rate": 3.961276818419675e-07, | |
| "logps/chosen": -187.76580810546875, | |
| "logps/rejected": -264.5431823730469, | |
| "loss": 0.2047, | |
| "rewards/chosen": 0.11567319184541702, | |
| "rewards/margins": 4.132474899291992, | |
| "rewards/rejected": -4.016801834106445, | |
| "step": 397 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "grad_norm": 10.431439920064484, | |
| "kl": 0.0, | |
| "learning_rate": 3.958660387231816e-07, | |
| "logps/chosen": -152.51498413085938, | |
| "logps/rejected": -235.09324645996094, | |
| "loss": 0.2288, | |
| "rewards/chosen": 0.2288106083869934, | |
| "rewards/margins": 2.305171251296997, | |
| "rewards/rejected": -2.0763607025146484, | |
| "step": 398 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "grad_norm": 11.437728669932744, | |
| "kl": 0.0, | |
| "learning_rate": 3.9560439560439557e-07, | |
| "logps/chosen": -198.04653930664062, | |
| "logps/rejected": -216.70425415039062, | |
| "loss": 0.148, | |
| "rewards/chosen": 0.6912394165992737, | |
| "rewards/margins": 2.1111791133880615, | |
| "rewards/rejected": -1.4199397563934326, | |
| "step": 399 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "grad_norm": 10.47479277819163, | |
| "kl": 0.0, | |
| "learning_rate": 3.953427524856096e-07, | |
| "logps/chosen": -218.10205078125, | |
| "logps/rejected": -294.8763427734375, | |
| "loss": 0.1738, | |
| "rewards/chosen": 0.3303108215332031, | |
| "rewards/margins": 2.3888604640960693, | |
| "rewards/rejected": -2.058549642562866, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "grad_norm": 10.343750828726911, | |
| "kl": 0.0, | |
| "learning_rate": 3.9508110936682367e-07, | |
| "logps/chosen": -216.23837280273438, | |
| "logps/rejected": -222.16204833984375, | |
| "loss": 0.1711, | |
| "rewards/chosen": 0.24862945079803467, | |
| "rewards/margins": 2.214453935623169, | |
| "rewards/rejected": -1.9658244848251343, | |
| "step": 401 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "grad_norm": 11.335701218532215, | |
| "kl": 0.0, | |
| "learning_rate": 3.948194662480376e-07, | |
| "logps/chosen": -236.11170959472656, | |
| "logps/rejected": -206.62315368652344, | |
| "loss": 0.2024, | |
| "rewards/chosen": -0.06115901097655296, | |
| "rewards/margins": 1.5972472429275513, | |
| "rewards/rejected": -1.6584062576293945, | |
| "step": 402 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "grad_norm": 12.475003237928307, | |
| "kl": 0.0, | |
| "learning_rate": 3.9455782312925167e-07, | |
| "logps/chosen": -260.5793151855469, | |
| "logps/rejected": -287.6771545410156, | |
| "loss": 0.1706, | |
| "rewards/chosen": -0.39119893312454224, | |
| "rewards/margins": 1.3929164409637451, | |
| "rewards/rejected": -1.7841154336929321, | |
| "step": 403 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "grad_norm": 9.477036698967389, | |
| "kl": 0.0, | |
| "learning_rate": 3.942961800104657e-07, | |
| "logps/chosen": -175.79698181152344, | |
| "logps/rejected": -189.22921752929688, | |
| "loss": 0.1915, | |
| "rewards/chosen": 0.30524417757987976, | |
| "rewards/margins": 2.519529104232788, | |
| "rewards/rejected": -2.214284896850586, | |
| "step": 404 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "grad_norm": 12.239825540829113, | |
| "kl": 0.0, | |
| "learning_rate": 3.9403453689167977e-07, | |
| "logps/chosen": -271.8373718261719, | |
| "logps/rejected": -304.6640319824219, | |
| "loss": 0.1507, | |
| "rewards/chosen": -0.007900595664978027, | |
| "rewards/margins": 3.236818313598633, | |
| "rewards/rejected": -3.2447190284729004, | |
| "step": 405 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "grad_norm": 10.427157164376496, | |
| "kl": 0.0, | |
| "learning_rate": 3.9377289377289377e-07, | |
| "logps/chosen": -215.03746032714844, | |
| "logps/rejected": -255.59893798828125, | |
| "loss": 0.2518, | |
| "rewards/chosen": -0.6349587440490723, | |
| "rewards/margins": 2.439880847930908, | |
| "rewards/rejected": -3.0748395919799805, | |
| "step": 406 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "grad_norm": 11.207119574932996, | |
| "kl": 0.0, | |
| "learning_rate": 3.9351125065410776e-07, | |
| "logps/chosen": -167.9252471923828, | |
| "logps/rejected": -230.44461059570312, | |
| "loss": 0.2153, | |
| "rewards/chosen": -0.5827754139900208, | |
| "rewards/margins": 1.467712640762329, | |
| "rewards/rejected": -2.050487995147705, | |
| "step": 407 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "grad_norm": 12.843002882536329, | |
| "kl": 0.0, | |
| "learning_rate": 3.932496075353218e-07, | |
| "logps/chosen": -274.9442443847656, | |
| "logps/rejected": -291.43450927734375, | |
| "loss": 0.1899, | |
| "rewards/chosen": -0.4634897708892822, | |
| "rewards/margins": 2.9832985401153564, | |
| "rewards/rejected": -3.4467883110046387, | |
| "step": 408 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "grad_norm": 12.301578880284277, | |
| "kl": 0.0, | |
| "learning_rate": 3.929879644165358e-07, | |
| "logps/chosen": -230.11001586914062, | |
| "logps/rejected": -262.1216125488281, | |
| "loss": 0.2163, | |
| "rewards/chosen": -0.47498393058776855, | |
| "rewards/margins": 2.5229318141937256, | |
| "rewards/rejected": -2.997915744781494, | |
| "step": 409 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "grad_norm": 14.17354036141092, | |
| "kl": 0.0, | |
| "learning_rate": 3.9272632129774986e-07, | |
| "logps/chosen": -236.7783203125, | |
| "logps/rejected": -225.37086486816406, | |
| "loss": 0.1465, | |
| "rewards/chosen": 0.534197211265564, | |
| "rewards/margins": 3.3101611137390137, | |
| "rewards/rejected": -2.7759640216827393, | |
| "step": 410 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "grad_norm": 9.994681734611301, | |
| "kl": 0.0, | |
| "learning_rate": 3.924646781789639e-07, | |
| "logps/chosen": -152.39027404785156, | |
| "logps/rejected": -236.91995239257812, | |
| "loss": 0.1985, | |
| "rewards/chosen": -0.45504745841026306, | |
| "rewards/margins": 2.5142714977264404, | |
| "rewards/rejected": -2.9693188667297363, | |
| "step": 411 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "grad_norm": 9.62216210519223, | |
| "kl": 0.0, | |
| "learning_rate": 3.9220303506017786e-07, | |
| "logps/chosen": -222.86874389648438, | |
| "logps/rejected": -318.4061279296875, | |
| "loss": 0.1912, | |
| "rewards/chosen": -0.2937797009944916, | |
| "rewards/margins": 3.895700216293335, | |
| "rewards/rejected": -4.189479827880859, | |
| "step": 412 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "grad_norm": 10.901795525608296, | |
| "kl": 0.0, | |
| "learning_rate": 3.919413919413919e-07, | |
| "logps/chosen": -214.8961639404297, | |
| "logps/rejected": -260.2914733886719, | |
| "loss": 0.2245, | |
| "rewards/chosen": 0.16179826855659485, | |
| "rewards/margins": 2.0142719745635986, | |
| "rewards/rejected": -1.8524737358093262, | |
| "step": 413 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "grad_norm": 13.652487549232319, | |
| "kl": 0.0, | |
| "learning_rate": 3.9167974882260596e-07, | |
| "logps/chosen": -212.1907958984375, | |
| "logps/rejected": -278.97296142578125, | |
| "loss": 0.1698, | |
| "rewards/chosen": -0.08914380520582199, | |
| "rewards/margins": 3.5527141094207764, | |
| "rewards/rejected": -3.641857862472534, | |
| "step": 414 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "grad_norm": 10.750735909217193, | |
| "kl": 0.0, | |
| "learning_rate": 3.9141810570381995e-07, | |
| "logps/chosen": -219.64820861816406, | |
| "logps/rejected": -223.59970092773438, | |
| "loss": 0.1419, | |
| "rewards/chosen": -0.31567585468292236, | |
| "rewards/margins": 1.8360644578933716, | |
| "rewards/rejected": -2.151740312576294, | |
| "step": 415 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "grad_norm": 13.00956558924868, | |
| "kl": 0.0, | |
| "learning_rate": 3.91156462585034e-07, | |
| "logps/chosen": -221.29444885253906, | |
| "logps/rejected": -292.55242919921875, | |
| "loss": 0.2404, | |
| "rewards/chosen": -0.8755928874015808, | |
| "rewards/margins": 2.879188060760498, | |
| "rewards/rejected": -3.7547810077667236, | |
| "step": 416 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "grad_norm": 11.263246462543782, | |
| "kl": 0.0, | |
| "learning_rate": 3.90894819466248e-07, | |
| "logps/chosen": -204.36553955078125, | |
| "logps/rejected": -277.9330749511719, | |
| "loss": 0.2751, | |
| "rewards/chosen": -0.16093337535858154, | |
| "rewards/margins": 1.7333406209945679, | |
| "rewards/rejected": -1.8942739963531494, | |
| "step": 417 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "grad_norm": 10.77749694683554, | |
| "kl": 0.0, | |
| "learning_rate": 3.9063317634746205e-07, | |
| "logps/chosen": -218.55001831054688, | |
| "logps/rejected": -276.6634216308594, | |
| "loss": 0.1298, | |
| "rewards/chosen": 0.9015990495681763, | |
| "rewards/margins": 3.651735305786133, | |
| "rewards/rejected": -2.750136137008667, | |
| "step": 418 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "grad_norm": 10.516288450325918, | |
| "kl": 0.0, | |
| "learning_rate": 3.9037153322867605e-07, | |
| "logps/chosen": -210.36167907714844, | |
| "logps/rejected": -265.4808654785156, | |
| "loss": 0.2017, | |
| "rewards/chosen": -0.17863792181015015, | |
| "rewards/margins": 2.6375937461853027, | |
| "rewards/rejected": -2.8162317276000977, | |
| "step": 419 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "grad_norm": 11.824811555332477, | |
| "kl": 0.0, | |
| "learning_rate": 3.901098901098901e-07, | |
| "logps/chosen": -208.4647674560547, | |
| "logps/rejected": -255.76190185546875, | |
| "loss": 0.199, | |
| "rewards/chosen": 0.15355457365512848, | |
| "rewards/margins": 2.69331955909729, | |
| "rewards/rejected": -2.539764881134033, | |
| "step": 420 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "grad_norm": 10.69862877229561, | |
| "kl": 0.0, | |
| "learning_rate": 3.8984824699110415e-07, | |
| "logps/chosen": -256.15496826171875, | |
| "logps/rejected": -272.2501220703125, | |
| "loss": 0.1392, | |
| "rewards/chosen": 0.36112552881240845, | |
| "rewards/margins": 3.3303704261779785, | |
| "rewards/rejected": -2.969244956970215, | |
| "step": 421 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "grad_norm": 10.317432492795456, | |
| "kl": 0.0, | |
| "learning_rate": 3.895866038723181e-07, | |
| "logps/chosen": -201.2956085205078, | |
| "logps/rejected": -252.26727294921875, | |
| "loss": 0.2015, | |
| "rewards/chosen": -0.19176657497882843, | |
| "rewards/margins": 1.5057237148284912, | |
| "rewards/rejected": -1.6974903345108032, | |
| "step": 422 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "grad_norm": 11.089829796355776, | |
| "kl": 0.0, | |
| "learning_rate": 3.8932496075353214e-07, | |
| "logps/chosen": -251.06739807128906, | |
| "logps/rejected": -207.30560302734375, | |
| "loss": 0.2024, | |
| "rewards/chosen": 0.43589267134666443, | |
| "rewards/margins": 2.9610509872436523, | |
| "rewards/rejected": -2.525158405303955, | |
| "step": 423 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "grad_norm": 13.046985449887648, | |
| "kl": 0.0, | |
| "learning_rate": 3.890633176347462e-07, | |
| "logps/chosen": -191.86936950683594, | |
| "logps/rejected": -282.1868896484375, | |
| "loss": 0.172, | |
| "rewards/chosen": -0.3784329891204834, | |
| "rewards/margins": 2.067564010620117, | |
| "rewards/rejected": -2.4459969997406006, | |
| "step": 424 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "grad_norm": 9.765346020761122, | |
| "kl": 0.0, | |
| "learning_rate": 3.888016745159602e-07, | |
| "logps/chosen": -199.694580078125, | |
| "logps/rejected": -256.0482482910156, | |
| "loss": 0.1586, | |
| "rewards/chosen": -0.6785757541656494, | |
| "rewards/margins": 1.9910736083984375, | |
| "rewards/rejected": -2.669649362564087, | |
| "step": 425 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "grad_norm": 14.196677493751132, | |
| "kl": 0.0, | |
| "learning_rate": 3.8854003139717424e-07, | |
| "logps/chosen": -228.76034545898438, | |
| "logps/rejected": -302.63836669921875, | |
| "loss": 0.1734, | |
| "rewards/chosen": -0.3336869776248932, | |
| "rewards/margins": 2.1534783840179443, | |
| "rewards/rejected": -2.4871654510498047, | |
| "step": 426 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "grad_norm": 11.772671674738493, | |
| "kl": 0.0, | |
| "learning_rate": 3.882783882783883e-07, | |
| "logps/chosen": -234.97866821289062, | |
| "logps/rejected": -277.7060241699219, | |
| "loss": 0.2091, | |
| "rewards/chosen": -0.5110215544700623, | |
| "rewards/margins": 0.7004496455192566, | |
| "rewards/rejected": -1.2114711999893188, | |
| "step": 427 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "grad_norm": 13.524888701607797, | |
| "kl": 0.0, | |
| "learning_rate": 3.880167451596023e-07, | |
| "logps/chosen": -222.4050750732422, | |
| "logps/rejected": -227.77481079101562, | |
| "loss": 0.2568, | |
| "rewards/chosen": 0.0008676449651829898, | |
| "rewards/margins": 1.8511059284210205, | |
| "rewards/rejected": -1.85023832321167, | |
| "step": 428 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "grad_norm": 10.405779261311427, | |
| "kl": 0.0, | |
| "learning_rate": 3.877551020408163e-07, | |
| "logps/chosen": -204.4320068359375, | |
| "logps/rejected": -222.79629516601562, | |
| "loss": 0.0875, | |
| "rewards/chosen": 0.08031473308801651, | |
| "rewards/margins": 2.481466770172119, | |
| "rewards/rejected": -2.4011521339416504, | |
| "step": 429 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "grad_norm": 14.72924638814786, | |
| "kl": 0.0, | |
| "learning_rate": 3.8749345892203034e-07, | |
| "logps/chosen": -289.6814270019531, | |
| "logps/rejected": -331.42816162109375, | |
| "loss": 0.1481, | |
| "rewards/chosen": -0.13640549778938293, | |
| "rewards/margins": 4.516214370727539, | |
| "rewards/rejected": -4.6526198387146, | |
| "step": 430 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "grad_norm": 10.741301498332556, | |
| "kl": 0.0, | |
| "learning_rate": 3.872318158032444e-07, | |
| "logps/chosen": -222.26815795898438, | |
| "logps/rejected": -271.1949462890625, | |
| "loss": 0.197, | |
| "rewards/chosen": 0.39989006519317627, | |
| "rewards/margins": 2.3632915019989014, | |
| "rewards/rejected": -1.963401436805725, | |
| "step": 431 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "grad_norm": 12.112546116412723, | |
| "kl": 0.0, | |
| "learning_rate": 3.869701726844584e-07, | |
| "logps/chosen": -243.7536163330078, | |
| "logps/rejected": -252.95794677734375, | |
| "loss": 0.2012, | |
| "rewards/chosen": -0.38057583570480347, | |
| "rewards/margins": 1.647994041442871, | |
| "rewards/rejected": -2.0285699367523193, | |
| "step": 432 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "grad_norm": 14.742669713065881, | |
| "kl": 0.0, | |
| "learning_rate": 3.867085295656724e-07, | |
| "logps/chosen": -244.42164611816406, | |
| "logps/rejected": -267.96954345703125, | |
| "loss": 0.2001, | |
| "rewards/chosen": 0.29253336787223816, | |
| "rewards/margins": 2.2655460834503174, | |
| "rewards/rejected": -1.9730128049850464, | |
| "step": 433 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "grad_norm": 10.134298526921, | |
| "kl": 0.0, | |
| "learning_rate": 3.8644688644688643e-07, | |
| "logps/chosen": -193.90284729003906, | |
| "logps/rejected": -232.6556396484375, | |
| "loss": 0.2008, | |
| "rewards/chosen": 0.34420499205589294, | |
| "rewards/margins": 3.2133047580718994, | |
| "rewards/rejected": -2.8690998554229736, | |
| "step": 434 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "grad_norm": 11.746178628630169, | |
| "kl": 0.0, | |
| "learning_rate": 3.8618524332810043e-07, | |
| "logps/chosen": -220.666015625, | |
| "logps/rejected": -303.94451904296875, | |
| "loss": 0.1645, | |
| "rewards/chosen": 0.019716663286089897, | |
| "rewards/margins": 2.469646692276001, | |
| "rewards/rejected": -2.44992995262146, | |
| "step": 435 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "grad_norm": 11.618541405718464, | |
| "kl": 0.0, | |
| "learning_rate": 3.859236002093145e-07, | |
| "logps/chosen": -201.02305603027344, | |
| "logps/rejected": -307.0491027832031, | |
| "loss": 0.1626, | |
| "rewards/chosen": -0.7490205764770508, | |
| "rewards/margins": 2.6788406372070312, | |
| "rewards/rejected": -3.427861213684082, | |
| "step": 436 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "grad_norm": 12.168910387907376, | |
| "kl": 0.0, | |
| "learning_rate": 3.8566195709052853e-07, | |
| "logps/chosen": -278.5819396972656, | |
| "logps/rejected": -261.8109130859375, | |
| "loss": 0.0773, | |
| "rewards/chosen": -0.5979146361351013, | |
| "rewards/margins": 2.49701189994812, | |
| "rewards/rejected": -3.094926595687866, | |
| "step": 437 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "grad_norm": 10.604103054919578, | |
| "kl": 0.0, | |
| "learning_rate": 3.8540031397174253e-07, | |
| "logps/chosen": -193.6268768310547, | |
| "logps/rejected": -241.7948455810547, | |
| "loss": 0.1785, | |
| "rewards/chosen": 0.43937230110168457, | |
| "rewards/margins": 1.834383487701416, | |
| "rewards/rejected": -1.3950111865997314, | |
| "step": 438 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "grad_norm": 10.834100142745722, | |
| "kl": 0.0, | |
| "learning_rate": 3.851386708529565e-07, | |
| "logps/chosen": -229.8187713623047, | |
| "logps/rejected": -325.75762939453125, | |
| "loss": 0.1844, | |
| "rewards/chosen": 0.6806701421737671, | |
| "rewards/margins": 4.323229789733887, | |
| "rewards/rejected": -3.64255952835083, | |
| "step": 439 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "grad_norm": 12.704909503144034, | |
| "kl": 0.0, | |
| "learning_rate": 3.848770277341706e-07, | |
| "logps/chosen": -224.91845703125, | |
| "logps/rejected": -340.0356750488281, | |
| "loss": 0.2631, | |
| "rewards/chosen": 0.026007119566202164, | |
| "rewards/margins": 2.283827781677246, | |
| "rewards/rejected": -2.2578206062316895, | |
| "step": 440 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "grad_norm": 10.41021619314634, | |
| "kl": 0.0, | |
| "learning_rate": 3.8461538461538463e-07, | |
| "logps/chosen": -219.94329833984375, | |
| "logps/rejected": -233.54290771484375, | |
| "loss": 0.1716, | |
| "rewards/chosen": -0.22441209852695465, | |
| "rewards/margins": 2.2788405418395996, | |
| "rewards/rejected": -2.5032527446746826, | |
| "step": 441 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "grad_norm": 10.379317080534767, | |
| "kl": 0.0, | |
| "learning_rate": 3.843537414965986e-07, | |
| "logps/chosen": -229.6413116455078, | |
| "logps/rejected": -249.2711639404297, | |
| "loss": 0.1614, | |
| "rewards/chosen": 0.3848222494125366, | |
| "rewards/margins": 1.9725074768066406, | |
| "rewards/rejected": -1.587685227394104, | |
| "step": 442 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "grad_norm": 10.520229093007123, | |
| "kl": 0.0, | |
| "learning_rate": 3.840920983778126e-07, | |
| "logps/chosen": -222.9331817626953, | |
| "logps/rejected": -208.89297485351562, | |
| "loss": 0.1427, | |
| "rewards/chosen": 0.582370400428772, | |
| "rewards/margins": 3.301401138305664, | |
| "rewards/rejected": -2.7190306186676025, | |
| "step": 443 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "grad_norm": 13.275687424817486, | |
| "kl": 0.0, | |
| "learning_rate": 3.8383045525902667e-07, | |
| "logps/chosen": -189.17926025390625, | |
| "logps/rejected": -219.13580322265625, | |
| "loss": 0.2513, | |
| "rewards/chosen": -0.015819329768419266, | |
| "rewards/margins": 1.8377530574798584, | |
| "rewards/rejected": -1.8535723686218262, | |
| "step": 444 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "grad_norm": 10.600324116674757, | |
| "kl": 0.0, | |
| "learning_rate": 3.8356881214024067e-07, | |
| "logps/chosen": -216.97711181640625, | |
| "logps/rejected": -249.7940216064453, | |
| "loss": 0.2049, | |
| "rewards/chosen": 0.5193243622779846, | |
| "rewards/margins": 2.258141040802002, | |
| "rewards/rejected": -1.738816738128662, | |
| "step": 445 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "grad_norm": 10.525139117282963, | |
| "kl": 0.0, | |
| "learning_rate": 3.833071690214547e-07, | |
| "logps/chosen": -222.4760284423828, | |
| "logps/rejected": -239.29403686523438, | |
| "loss": 0.071, | |
| "rewards/chosen": 0.5078960657119751, | |
| "rewards/margins": 2.6218113899230957, | |
| "rewards/rejected": -2.113915205001831, | |
| "step": 446 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "grad_norm": 13.222155048640982, | |
| "kl": 0.0, | |
| "learning_rate": 3.8304552590266877e-07, | |
| "logps/chosen": -200.71603393554688, | |
| "logps/rejected": -267.3768005371094, | |
| "loss": 0.162, | |
| "rewards/chosen": 0.24389874935150146, | |
| "rewards/margins": 3.596365451812744, | |
| "rewards/rejected": -3.3524668216705322, | |
| "step": 447 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "grad_norm": 10.120823838278643, | |
| "kl": 0.0, | |
| "learning_rate": 3.827838827838827e-07, | |
| "logps/chosen": -196.2765350341797, | |
| "logps/rejected": -215.92112731933594, | |
| "loss": 0.1333, | |
| "rewards/chosen": 0.05430547147989273, | |
| "rewards/margins": 1.7621450424194336, | |
| "rewards/rejected": -1.7078396081924438, | |
| "step": 448 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "grad_norm": 11.458337974039024, | |
| "kl": 0.0, | |
| "learning_rate": 3.8252223966509676e-07, | |
| "logps/chosen": -195.0396270751953, | |
| "logps/rejected": -285.74462890625, | |
| "loss": 0.1939, | |
| "rewards/chosen": 0.07916612178087234, | |
| "rewards/margins": 2.918091058731079, | |
| "rewards/rejected": -2.8389248847961426, | |
| "step": 449 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "grad_norm": 12.160182938253959, | |
| "kl": 0.0, | |
| "learning_rate": 3.822605965463108e-07, | |
| "logps/chosen": -193.6781768798828, | |
| "logps/rejected": -243.92469787597656, | |
| "loss": 0.1757, | |
| "rewards/chosen": -0.5648252964019775, | |
| "rewards/margins": 1.6671717166900635, | |
| "rewards/rejected": -2.231997013092041, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "grad_norm": 12.143861035511275, | |
| "kl": 0.0, | |
| "learning_rate": 3.8199895342752486e-07, | |
| "logps/chosen": -208.96681213378906, | |
| "logps/rejected": -243.71438598632812, | |
| "loss": 0.1589, | |
| "rewards/chosen": -0.6121307015419006, | |
| "rewards/margins": 1.7129931449890137, | |
| "rewards/rejected": -2.3251237869262695, | |
| "step": 451 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "grad_norm": 9.91525808504644, | |
| "kl": 0.0, | |
| "learning_rate": 3.8173731030873886e-07, | |
| "logps/chosen": -266.3404541015625, | |
| "logps/rejected": -302.9526672363281, | |
| "loss": 0.1843, | |
| "rewards/chosen": -0.14618553221225739, | |
| "rewards/margins": 2.309356689453125, | |
| "rewards/rejected": -2.4555423259735107, | |
| "step": 452 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "grad_norm": 9.702336587181945, | |
| "kl": 0.0, | |
| "learning_rate": 3.814756671899529e-07, | |
| "logps/chosen": -181.5204620361328, | |
| "logps/rejected": -304.9017639160156, | |
| "loss": 0.1483, | |
| "rewards/chosen": -0.1943751722574234, | |
| "rewards/margins": 3.55947208404541, | |
| "rewards/rejected": -3.753847360610962, | |
| "step": 453 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "grad_norm": 11.965604322215004, | |
| "kl": 0.0, | |
| "learning_rate": 3.812140240711669e-07, | |
| "logps/chosen": -245.6624755859375, | |
| "logps/rejected": -260.020263671875, | |
| "loss": 0.2183, | |
| "rewards/chosen": -1.2427337169647217, | |
| "rewards/margins": 1.6019926071166992, | |
| "rewards/rejected": -2.844726324081421, | |
| "step": 454 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "grad_norm": 10.836340963305517, | |
| "kl": 0.0, | |
| "learning_rate": 3.809523809523809e-07, | |
| "logps/chosen": -237.24041748046875, | |
| "logps/rejected": -235.87216186523438, | |
| "loss": 0.1095, | |
| "rewards/chosen": 0.8246784210205078, | |
| "rewards/margins": 3.6156461238861084, | |
| "rewards/rejected": -2.7909677028656006, | |
| "step": 455 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "grad_norm": 11.897690499315637, | |
| "kl": 0.0, | |
| "learning_rate": 3.8069073783359496e-07, | |
| "logps/chosen": -232.65216064453125, | |
| "logps/rejected": -197.15951538085938, | |
| "loss": 0.0782, | |
| "rewards/chosen": 0.05020507052540779, | |
| "rewards/margins": 3.441476821899414, | |
| "rewards/rejected": -3.3912718296051025, | |
| "step": 456 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "grad_norm": 8.049103139267938, | |
| "kl": 0.0, | |
| "learning_rate": 3.80429094714809e-07, | |
| "logps/chosen": -239.66998291015625, | |
| "logps/rejected": -256.76837158203125, | |
| "loss": 0.1933, | |
| "rewards/chosen": 0.054171182215213776, | |
| "rewards/margins": 3.8353452682495117, | |
| "rewards/rejected": -3.7811741828918457, | |
| "step": 457 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "grad_norm": 13.088097317739122, | |
| "kl": 0.0, | |
| "learning_rate": 3.80167451596023e-07, | |
| "logps/chosen": -205.0490264892578, | |
| "logps/rejected": -241.65966796875, | |
| "loss": 0.1946, | |
| "rewards/chosen": -0.03932800143957138, | |
| "rewards/margins": 2.7577974796295166, | |
| "rewards/rejected": -2.7971255779266357, | |
| "step": 458 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "grad_norm": 13.14569665419969, | |
| "kl": 0.0, | |
| "learning_rate": 3.79905808477237e-07, | |
| "logps/chosen": -298.6637268066406, | |
| "logps/rejected": -285.0900573730469, | |
| "loss": 0.1677, | |
| "rewards/chosen": -0.5165401697158813, | |
| "rewards/margins": 3.1756558418273926, | |
| "rewards/rejected": -3.6921958923339844, | |
| "step": 459 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "grad_norm": 10.434555514022984, | |
| "kl": 0.0, | |
| "learning_rate": 3.7964416535845105e-07, | |
| "logps/chosen": -228.3349609375, | |
| "logps/rejected": -214.828857421875, | |
| "loss": 0.1609, | |
| "rewards/chosen": 0.11236274987459183, | |
| "rewards/margins": 2.25835919380188, | |
| "rewards/rejected": -2.145996332168579, | |
| "step": 460 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "grad_norm": 10.167277710211138, | |
| "kl": 0.0, | |
| "learning_rate": 3.793825222396651e-07, | |
| "logps/chosen": -245.6664581298828, | |
| "logps/rejected": -274.6752624511719, | |
| "loss": 0.1496, | |
| "rewards/chosen": -0.27803435921669006, | |
| "rewards/margins": 1.9288045167922974, | |
| "rewards/rejected": -2.206838846206665, | |
| "step": 461 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "grad_norm": 10.474016211401475, | |
| "kl": 0.0, | |
| "learning_rate": 3.791208791208791e-07, | |
| "logps/chosen": -193.68919372558594, | |
| "logps/rejected": -202.5056610107422, | |
| "loss": 0.1846, | |
| "rewards/chosen": -0.4363885223865509, | |
| "rewards/margins": 2.7338576316833496, | |
| "rewards/rejected": -3.170246124267578, | |
| "step": 462 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "grad_norm": 10.659154195676038, | |
| "kl": 0.0, | |
| "learning_rate": 3.7885923600209315e-07, | |
| "logps/chosen": -242.42816162109375, | |
| "logps/rejected": -282.0824890136719, | |
| "loss": 0.1603, | |
| "rewards/chosen": 0.673112690448761, | |
| "rewards/margins": 3.3613035678863525, | |
| "rewards/rejected": -2.6881909370422363, | |
| "step": 463 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "grad_norm": 12.65492990351007, | |
| "kl": 0.0, | |
| "learning_rate": 3.7859759288330715e-07, | |
| "logps/chosen": -170.49734497070312, | |
| "logps/rejected": -210.32472229003906, | |
| "loss": 0.1996, | |
| "rewards/chosen": 0.2279433161020279, | |
| "rewards/margins": 2.666241407394409, | |
| "rewards/rejected": -2.438297986984253, | |
| "step": 464 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "grad_norm": 10.54524759203527, | |
| "kl": 0.0, | |
| "learning_rate": 3.7833594976452115e-07, | |
| "logps/chosen": -153.5712432861328, | |
| "logps/rejected": -269.2149353027344, | |
| "loss": 0.2098, | |
| "rewards/chosen": -0.019966602325439453, | |
| "rewards/margins": 2.5769615173339844, | |
| "rewards/rejected": -2.596928119659424, | |
| "step": 465 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "grad_norm": 10.51825030268249, | |
| "kl": 0.0, | |
| "learning_rate": 3.780743066457352e-07, | |
| "logps/chosen": -203.99156188964844, | |
| "logps/rejected": -266.8109436035156, | |
| "loss": 0.1801, | |
| "rewards/chosen": 1.9818308353424072, | |
| "rewards/margins": 3.033543109893799, | |
| "rewards/rejected": -1.051712155342102, | |
| "step": 466 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "grad_norm": 9.77436331647093, | |
| "kl": 0.0, | |
| "learning_rate": 3.7781266352694925e-07, | |
| "logps/chosen": -240.96786499023438, | |
| "logps/rejected": -226.21963500976562, | |
| "loss": 0.146, | |
| "rewards/chosen": -0.04570434242486954, | |
| "rewards/margins": 1.6731890439987183, | |
| "rewards/rejected": -1.7188934087753296, | |
| "step": 467 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "grad_norm": 11.33142669534873, | |
| "kl": 0.0, | |
| "learning_rate": 3.7755102040816324e-07, | |
| "logps/chosen": -274.2488708496094, | |
| "logps/rejected": -221.2158203125, | |
| "loss": 0.2061, | |
| "rewards/chosen": -0.03258565813302994, | |
| "rewards/margins": 1.8558272123336792, | |
| "rewards/rejected": -1.8884128332138062, | |
| "step": 468 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "grad_norm": 12.417447396754456, | |
| "kl": 0.0, | |
| "learning_rate": 3.7728937728937724e-07, | |
| "logps/chosen": -241.2676544189453, | |
| "logps/rejected": -312.99432373046875, | |
| "loss": 0.1357, | |
| "rewards/chosen": 1.6663408279418945, | |
| "rewards/margins": 3.927600383758545, | |
| "rewards/rejected": -2.2612595558166504, | |
| "step": 469 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "grad_norm": 11.170512277856618, | |
| "kl": 0.0, | |
| "learning_rate": 3.770277341705913e-07, | |
| "logps/chosen": -233.8650360107422, | |
| "logps/rejected": -197.24072265625, | |
| "loss": 0.1556, | |
| "rewards/chosen": 0.8342301249504089, | |
| "rewards/margins": 2.448633909225464, | |
| "rewards/rejected": -1.6144037246704102, | |
| "step": 470 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "grad_norm": 13.803253821404509, | |
| "kl": 0.0, | |
| "learning_rate": 3.7676609105180534e-07, | |
| "logps/chosen": -194.3175506591797, | |
| "logps/rejected": -353.5451354980469, | |
| "loss": 0.1461, | |
| "rewards/chosen": 0.6489688754081726, | |
| "rewards/margins": 3.0008530616760254, | |
| "rewards/rejected": -2.351884126663208, | |
| "step": 471 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "grad_norm": 11.510104873936742, | |
| "kl": 0.0, | |
| "learning_rate": 3.7650444793301934e-07, | |
| "logps/chosen": -276.4261474609375, | |
| "logps/rejected": -173.75657653808594, | |
| "loss": 0.2384, | |
| "rewards/chosen": -0.2080867886543274, | |
| "rewards/margins": 0.7136066555976868, | |
| "rewards/rejected": -0.9216934442520142, | |
| "step": 472 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "grad_norm": 11.151674395455558, | |
| "kl": 0.0, | |
| "learning_rate": 3.762428048142334e-07, | |
| "logps/chosen": -234.051025390625, | |
| "logps/rejected": -349.57586669921875, | |
| "loss": 0.1555, | |
| "rewards/chosen": 0.1754276305437088, | |
| "rewards/margins": 2.250746965408325, | |
| "rewards/rejected": -2.075319290161133, | |
| "step": 473 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "grad_norm": 10.012473838900807, | |
| "kl": 0.0, | |
| "learning_rate": 3.759811616954474e-07, | |
| "logps/chosen": -206.8048553466797, | |
| "logps/rejected": -245.5289306640625, | |
| "loss": 0.1644, | |
| "rewards/chosen": 0.046800464391708374, | |
| "rewards/margins": 2.0636954307556152, | |
| "rewards/rejected": -2.016895055770874, | |
| "step": 474 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "grad_norm": 12.41419451120897, | |
| "kl": 0.0, | |
| "learning_rate": 3.757195185766614e-07, | |
| "logps/chosen": -197.76080322265625, | |
| "logps/rejected": -243.91220092773438, | |
| "loss": 0.1441, | |
| "rewards/chosen": 0.16484998166561127, | |
| "rewards/margins": 1.870893120765686, | |
| "rewards/rejected": -1.7060431241989136, | |
| "step": 475 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "grad_norm": 12.281595873200606, | |
| "kl": 0.0, | |
| "learning_rate": 3.7545787545787543e-07, | |
| "logps/chosen": -206.76541137695312, | |
| "logps/rejected": -230.58517456054688, | |
| "loss": 0.1835, | |
| "rewards/chosen": 0.500891923904419, | |
| "rewards/margins": 2.8288941383361816, | |
| "rewards/rejected": -2.3280022144317627, | |
| "step": 476 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "grad_norm": 11.477594673948579, | |
| "kl": 0.0, | |
| "learning_rate": 3.751962323390895e-07, | |
| "logps/chosen": -205.23182678222656, | |
| "logps/rejected": -265.99298095703125, | |
| "loss": 0.2508, | |
| "rewards/chosen": -0.7120283842086792, | |
| "rewards/margins": 1.5744119882583618, | |
| "rewards/rejected": -2.286440372467041, | |
| "step": 477 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "grad_norm": 10.389487469578903, | |
| "kl": 0.0, | |
| "learning_rate": 3.749345892203035e-07, | |
| "logps/chosen": -239.08309936523438, | |
| "logps/rejected": -245.90695190429688, | |
| "loss": 0.2403, | |
| "rewards/chosen": 0.03259509429335594, | |
| "rewards/margins": 1.3131746053695679, | |
| "rewards/rejected": -1.2805795669555664, | |
| "step": 478 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "grad_norm": 12.417872067981774, | |
| "kl": 0.0, | |
| "learning_rate": 3.7467294610151753e-07, | |
| "logps/chosen": -256.7392883300781, | |
| "logps/rejected": -276.9643249511719, | |
| "loss": 0.2144, | |
| "rewards/chosen": 0.17145968973636627, | |
| "rewards/margins": 2.2896573543548584, | |
| "rewards/rejected": -2.1181976795196533, | |
| "step": 479 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "grad_norm": 11.327729837593566, | |
| "kl": 0.0, | |
| "learning_rate": 3.7441130298273153e-07, | |
| "logps/chosen": -194.67178344726562, | |
| "logps/rejected": -293.6398620605469, | |
| "loss": 0.2087, | |
| "rewards/chosen": 0.07769111543893814, | |
| "rewards/margins": 2.251216411590576, | |
| "rewards/rejected": -2.173525333404541, | |
| "step": 480 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "grad_norm": 11.641851328355145, | |
| "kl": 0.0, | |
| "learning_rate": 3.741496598639456e-07, | |
| "logps/chosen": -225.63128662109375, | |
| "logps/rejected": -242.99880981445312, | |
| "loss": 0.2983, | |
| "rewards/chosen": -0.29468005895614624, | |
| "rewards/margins": 1.5418860912322998, | |
| "rewards/rejected": -1.8365662097930908, | |
| "step": 481 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "grad_norm": 10.688981054253707, | |
| "kl": 0.0, | |
| "learning_rate": 3.738880167451596e-07, | |
| "logps/chosen": -194.8035125732422, | |
| "logps/rejected": -327.953369140625, | |
| "loss": 0.1177, | |
| "rewards/chosen": 0.040647171437740326, | |
| "rewards/margins": 2.5855929851531982, | |
| "rewards/rejected": -2.54494571685791, | |
| "step": 482 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "grad_norm": 10.545945342907165, | |
| "kl": 0.0, | |
| "learning_rate": 3.7362637362637363e-07, | |
| "logps/chosen": -299.9187927246094, | |
| "logps/rejected": -232.39605712890625, | |
| "loss": 0.142, | |
| "rewards/chosen": 0.8184850811958313, | |
| "rewards/margins": 3.6689701080322266, | |
| "rewards/rejected": -2.85048508644104, | |
| "step": 483 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "grad_norm": 12.552799610744065, | |
| "kl": 0.0, | |
| "learning_rate": 3.733647305075877e-07, | |
| "logps/chosen": -239.55209350585938, | |
| "logps/rejected": -253.3202667236328, | |
| "loss": 0.1485, | |
| "rewards/chosen": -0.060923367738723755, | |
| "rewards/margins": 1.103141188621521, | |
| "rewards/rejected": -1.1640645265579224, | |
| "step": 484 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "grad_norm": 10.114942413331908, | |
| "kl": 0.0, | |
| "learning_rate": 3.731030873888016e-07, | |
| "logps/chosen": -188.8922882080078, | |
| "logps/rejected": -213.3092041015625, | |
| "loss": 0.1095, | |
| "rewards/chosen": 0.31580618023872375, | |
| "rewards/margins": 2.5469398498535156, | |
| "rewards/rejected": -2.2311336994171143, | |
| "step": 485 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "grad_norm": 14.076487746859804, | |
| "kl": 0.0, | |
| "learning_rate": 3.7284144427001567e-07, | |
| "logps/chosen": -252.74615478515625, | |
| "logps/rejected": -206.2298583984375, | |
| "loss": 0.1541, | |
| "rewards/chosen": 0.32222241163253784, | |
| "rewards/margins": 2.381617307662964, | |
| "rewards/rejected": -2.0593948364257812, | |
| "step": 486 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "grad_norm": 13.614522220242236, | |
| "kl": 0.0, | |
| "learning_rate": 3.725798011512297e-07, | |
| "logps/chosen": -233.2279052734375, | |
| "logps/rejected": -309.02667236328125, | |
| "loss": 0.1624, | |
| "rewards/chosen": 0.3280530273914337, | |
| "rewards/margins": 2.868494987487793, | |
| "rewards/rejected": -2.5404419898986816, | |
| "step": 487 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "grad_norm": 13.04079850351043, | |
| "kl": 0.0, | |
| "learning_rate": 3.723181580324437e-07, | |
| "logps/chosen": -236.299560546875, | |
| "logps/rejected": -235.88479614257812, | |
| "loss": 0.2823, | |
| "rewards/chosen": -0.19492840766906738, | |
| "rewards/margins": 2.121401786804199, | |
| "rewards/rejected": -2.3163301944732666, | |
| "step": 488 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "grad_norm": 13.854146801900269, | |
| "kl": 0.0, | |
| "learning_rate": 3.7205651491365777e-07, | |
| "logps/chosen": -220.61044311523438, | |
| "logps/rejected": -256.89471435546875, | |
| "loss": 0.2039, | |
| "rewards/chosen": 0.09335081279277802, | |
| "rewards/margins": 1.3945070505142212, | |
| "rewards/rejected": -1.3011562824249268, | |
| "step": 489 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "grad_norm": 12.146701230600012, | |
| "kl": 0.0, | |
| "learning_rate": 3.7179487179487177e-07, | |
| "logps/chosen": -244.57260131835938, | |
| "logps/rejected": -216.97161865234375, | |
| "loss": 0.1947, | |
| "rewards/chosen": 0.42316800355911255, | |
| "rewards/margins": 1.4870796203613281, | |
| "rewards/rejected": -1.0639115571975708, | |
| "step": 490 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "grad_norm": 10.730648049345493, | |
| "kl": 0.0, | |
| "learning_rate": 3.7153322867608577e-07, | |
| "logps/chosen": -218.7908935546875, | |
| "logps/rejected": -258.998291015625, | |
| "loss": 0.183, | |
| "rewards/chosen": -0.3176134526729584, | |
| "rewards/margins": 1.3556625843048096, | |
| "rewards/rejected": -1.6732760667800903, | |
| "step": 491 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "grad_norm": 10.23814496690706, | |
| "kl": 0.0, | |
| "learning_rate": 3.712715855572998e-07, | |
| "logps/chosen": -223.61936950683594, | |
| "logps/rejected": -253.8956756591797, | |
| "loss": 0.1727, | |
| "rewards/chosen": 0.5035489201545715, | |
| "rewards/margins": 2.772200107574463, | |
| "rewards/rejected": -2.268651247024536, | |
| "step": 492 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "grad_norm": 12.792135627932897, | |
| "kl": 0.0, | |
| "learning_rate": 3.7100994243851387e-07, | |
| "logps/chosen": -254.05380249023438, | |
| "logps/rejected": -232.8262939453125, | |
| "loss": 0.1926, | |
| "rewards/chosen": 1.061255693435669, | |
| "rewards/margins": 2.6245744228363037, | |
| "rewards/rejected": -1.5633187294006348, | |
| "step": 493 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "grad_norm": 11.345332305270757, | |
| "kl": 0.0, | |
| "learning_rate": 3.707482993197279e-07, | |
| "logps/chosen": -232.64930725097656, | |
| "logps/rejected": -215.4774627685547, | |
| "loss": 0.1479, | |
| "rewards/chosen": 0.9204630851745605, | |
| "rewards/margins": 2.5953850746154785, | |
| "rewards/rejected": -1.674921989440918, | |
| "step": 494 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "grad_norm": 12.930924815591139, | |
| "kl": 0.0, | |
| "learning_rate": 3.7048665620094186e-07, | |
| "logps/chosen": -273.54595947265625, | |
| "logps/rejected": -246.6627655029297, | |
| "loss": 0.2992, | |
| "rewards/chosen": -1.0437400341033936, | |
| "rewards/margins": 0.9890058040618896, | |
| "rewards/rejected": -2.032745838165283, | |
| "step": 495 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "grad_norm": 9.386730828304604, | |
| "kl": 0.0, | |
| "learning_rate": 3.702250130821559e-07, | |
| "logps/chosen": -225.7566680908203, | |
| "logps/rejected": -239.1521759033203, | |
| "loss": 0.1486, | |
| "rewards/chosen": 0.671124279499054, | |
| "rewards/margins": 3.3144171237945557, | |
| "rewards/rejected": -2.6432929039001465, | |
| "step": 496 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "grad_norm": 12.454753782186298, | |
| "kl": 0.0, | |
| "learning_rate": 3.6996336996336996e-07, | |
| "logps/chosen": -289.2835693359375, | |
| "logps/rejected": -271.740478515625, | |
| "loss": 0.2592, | |
| "rewards/chosen": -1.0335984230041504, | |
| "rewards/margins": 1.533452033996582, | |
| "rewards/rejected": -2.5670504570007324, | |
| "step": 497 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "grad_norm": 11.975373879855518, | |
| "kl": 0.0, | |
| "learning_rate": 3.6970172684458396e-07, | |
| "logps/chosen": -213.38905334472656, | |
| "logps/rejected": -201.7144012451172, | |
| "loss": 0.1521, | |
| "rewards/chosen": 0.3211534321308136, | |
| "rewards/margins": 1.6761234998703003, | |
| "rewards/rejected": -1.354970097541809, | |
| "step": 498 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "grad_norm": 10.925688675488383, | |
| "kl": 0.0, | |
| "learning_rate": 3.69440083725798e-07, | |
| "logps/chosen": -208.6687774658203, | |
| "logps/rejected": -211.62774658203125, | |
| "loss": 0.1478, | |
| "rewards/chosen": 1.2924268245697021, | |
| "rewards/margins": 3.0114760398864746, | |
| "rewards/rejected": -1.7190492153167725, | |
| "step": 499 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "grad_norm": 12.744456658603918, | |
| "kl": 0.0, | |
| "learning_rate": 3.69178440607012e-07, | |
| "logps/chosen": -195.3672332763672, | |
| "logps/rejected": -232.05824279785156, | |
| "loss": 0.273, | |
| "rewards/chosen": -0.07960432767868042, | |
| "rewards/margins": 2.1937575340270996, | |
| "rewards/rejected": -2.273361921310425, | |
| "step": 500 | |
| } | |
| ], | |
| "logging_steps": 1.0, | |
| "max_steps": 1911, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 500, | |
| "total_flos": 0.0, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |