diff --git "a/checkpoint-500/trainer_state.json" "b/checkpoint-500/trainer_state.json" new file mode 100644--- /dev/null +++ "b/checkpoint-500/trainer_state.json" @@ -0,0 +1,7021 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.6544502617801047, + "eval_steps": 500, + "global_step": 500, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 5.000000000000001e-07, + "logits/chosen": -2.729219675064087, + "logits/rejected": -2.713034152984619, + "logps/chosen": -183.00042724609375, + "logps/rejected": -183.33316040039062, + "loss": 0.6973, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.004850482568144798, + "rewards/margins": -0.007815884426236153, + "rewards/rejected": 0.0029654023237526417, + "step": 1 + }, + { + "epoch": 0.0, + "learning_rate": 1.0000000000000002e-06, + "logits/chosen": -2.480727195739746, + "logits/rejected": -2.563934564590454, + "logps/chosen": -159.55963134765625, + "logps/rejected": -157.36929321289062, + "loss": 0.6902, + "rewards/accuracies": 0.625, + "rewards/chosen": 0.004495501983910799, + "rewards/margins": 0.006143546663224697, + "rewards/rejected": -0.0016480451449751854, + "step": 2 + }, + { + "epoch": 0.0, + "learning_rate": 1.5e-06, + "logits/chosen": -2.856149911880493, + "logits/rejected": -2.8624300956726074, + "logps/chosen": -241.56802368164062, + "logps/rejected": -251.95797729492188, + "loss": 0.6969, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.01090860366821289, + "rewards/margins": -0.007115649990737438, + "rewards/rejected": -0.0037929536774754524, + "step": 3 + }, + { + "epoch": 0.01, + "learning_rate": 2.0000000000000003e-06, + "logits/chosen": -2.473580837249756, + "logits/rejected": -2.6020100116729736, + "logps/chosen": -138.55348205566406, + "logps/rejected": -167.7603759765625, + "loss": 0.6844, + "rewards/accuracies": 0.6875, + "rewards/chosen": 0.01281428337097168, + "rewards/margins": 0.01771531254053116, + "rewards/rejected": -0.0049010273069143295, + "step": 4 + }, + { + "epoch": 0.01, + "learning_rate": 2.5e-06, + "logits/chosen": -2.446133613586426, + "logits/rejected": -2.5022342205047607, + "logps/chosen": -140.56512451171875, + "logps/rejected": -178.04331970214844, + "loss": 0.6995, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.013703584671020508, + "rewards/margins": -0.012241363525390625, + "rewards/rejected": -0.001462221029214561, + "step": 5 + }, + { + "epoch": 0.01, + "learning_rate": 3e-06, + "logits/chosen": -2.5403974056243896, + "logits/rejected": -2.650925874710083, + "logps/chosen": -162.82369995117188, + "logps/rejected": -214.57489013671875, + "loss": 0.6909, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.003328490536659956, + "rewards/margins": 0.004930590279400349, + "rewards/rejected": -0.0016021011397242546, + "step": 6 + }, + { + "epoch": 0.01, + "learning_rate": 3.5000000000000004e-06, + "logits/chosen": -2.6547656059265137, + "logits/rejected": -2.577648162841797, + "logps/chosen": -219.3771514892578, + "logps/rejected": -215.02362060546875, + "loss": 0.6931, + "rewards/accuracies": 0.4375, + "rewards/chosen": 0.0027452707290649414, + "rewards/margins": 0.0002449510502628982, + "rewards/rejected": 0.0025003196205943823, + "step": 7 + }, + { + "epoch": 0.01, + "learning_rate": 4.000000000000001e-06, + "logits/chosen": -2.5475914478302, + "logits/rejected": -2.586148500442505, + "logps/chosen": -214.5223388671875, + "logps/rejected": -236.43626403808594, + "loss": 0.6928, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.007447218522429466, + "rewards/margins": 0.0010163071565330029, + "rewards/rejected": 0.006430912297219038, + "step": 8 + }, + { + "epoch": 0.01, + "learning_rate": 4.5e-06, + "logits/chosen": -2.657726526260376, + "logits/rejected": -2.7398831844329834, + "logps/chosen": -158.13832092285156, + "logps/rejected": -176.91400146484375, + "loss": 0.6877, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.009470987133681774, + "rewards/margins": 0.011165929958224297, + "rewards/rejected": -0.0016949418932199478, + "step": 9 + }, + { + "epoch": 0.01, + "learning_rate": 5e-06, + "logits/chosen": -2.3761770725250244, + "logits/rejected": -2.4064137935638428, + "logps/chosen": -176.59835815429688, + "logps/rejected": -163.67300415039062, + "loss": 0.6909, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.002156305592507124, + "rewards/margins": 0.004867983516305685, + "rewards/rejected": -0.002711677923798561, + "step": 10 + }, + { + "epoch": 0.01, + "learning_rate": 5.500000000000001e-06, + "logits/chosen": -2.6009013652801514, + "logits/rejected": -2.645084857940674, + "logps/chosen": -197.91195678710938, + "logps/rejected": -245.907470703125, + "loss": 0.695, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.001973224338144064, + "rewards/margins": -0.0031423806212842464, + "rewards/rejected": 0.0011691567488014698, + "step": 11 + }, + { + "epoch": 0.02, + "learning_rate": 6e-06, + "logits/chosen": -2.70407772064209, + "logits/rejected": -2.713822364807129, + "logps/chosen": -190.91189575195312, + "logps/rejected": -190.74317932128906, + "loss": 0.6921, + "rewards/accuracies": 0.375, + "rewards/chosen": 0.0004999642260372639, + "rewards/margins": 0.002835321705788374, + "rewards/rejected": -0.00233535747975111, + "step": 12 + }, + { + "epoch": 0.02, + "learning_rate": 6.5000000000000004e-06, + "logits/chosen": -2.585596799850464, + "logits/rejected": -2.6356894969940186, + "logps/chosen": -214.14459228515625, + "logps/rejected": -242.9970245361328, + "loss": 0.687, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.006649780552834272, + "rewards/margins": 0.01303944643586874, + "rewards/rejected": -0.019689226523041725, + "step": 13 + }, + { + "epoch": 0.02, + "learning_rate": 7.000000000000001e-06, + "logits/chosen": -2.7044496536254883, + "logits/rejected": -2.6631391048431396, + "logps/chosen": -183.95582580566406, + "logps/rejected": -169.8933868408203, + "loss": 0.6967, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.01296384446322918, + "rewards/margins": -0.006747078616172075, + "rewards/rejected": -0.0062167649157345295, + "step": 14 + }, + { + "epoch": 0.02, + "learning_rate": 7.5e-06, + "logits/chosen": -2.6691248416900635, + "logits/rejected": -2.6515817642211914, + "logps/chosen": -161.9134979248047, + "logps/rejected": -170.60101318359375, + "loss": 0.6866, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.009864186868071556, + "rewards/margins": 0.01341402530670166, + "rewards/rejected": -0.0035498379729688168, + "step": 15 + }, + { + "epoch": 0.02, + "learning_rate": 8.000000000000001e-06, + "logits/chosen": -2.6293349266052246, + "logits/rejected": -2.652617931365967, + "logps/chosen": -161.3071746826172, + "logps/rejected": -169.07638549804688, + "loss": 0.6808, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.005685711745172739, + "rewards/margins": 0.025187280029058456, + "rewards/rejected": -0.019501566886901855, + "step": 16 + }, + { + "epoch": 0.02, + "learning_rate": 8.500000000000002e-06, + "logits/chosen": -2.6463351249694824, + "logits/rejected": -2.696180582046509, + "logps/chosen": -154.61085510253906, + "logps/rejected": -148.3529510498047, + "loss": 0.6916, + "rewards/accuracies": 0.5625, + "rewards/chosen": 0.0029291389510035515, + "rewards/margins": 0.0034487005323171616, + "rewards/rejected": -0.0005195615813136101, + "step": 17 + }, + { + "epoch": 0.02, + "learning_rate": 9e-06, + "logits/chosen": -2.5302553176879883, + "logits/rejected": -2.4991636276245117, + "logps/chosen": -152.20655822753906, + "logps/rejected": -131.0479278564453, + "loss": 0.6954, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.009920882061123848, + "rewards/margins": -0.004111624322831631, + "rewards/rejected": -0.005809259135276079, + "step": 18 + }, + { + "epoch": 0.02, + "learning_rate": 9.5e-06, + "logits/chosen": -2.5337584018707275, + "logits/rejected": -2.6624388694763184, + "logps/chosen": -160.88140869140625, + "logps/rejected": -195.38058471679688, + "loss": 0.6829, + "rewards/accuracies": 0.75, + "rewards/chosen": 1.540081575512886e-05, + "rewards/margins": 0.021059704944491386, + "rewards/rejected": -0.021044302731752396, + "step": 19 + }, + { + "epoch": 0.03, + "learning_rate": 1e-05, + "logits/chosen": -2.7187581062316895, + "logits/rejected": -2.699402093887329, + "logps/chosen": -184.01405334472656, + "logps/rejected": -200.99124145507812, + "loss": 0.6898, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.007425189018249512, + "rewards/margins": 0.007250881753861904, + "rewards/rejected": -0.014676070772111416, + "step": 20 + }, + { + "epoch": 0.03, + "learning_rate": 1.05e-05, + "logits/chosen": -2.5241212844848633, + "logits/rejected": -2.5988845825195312, + "logps/chosen": -181.3677215576172, + "logps/rejected": -156.8072509765625, + "loss": 0.687, + "rewards/accuracies": 0.6875, + "rewards/chosen": 0.006232499144971371, + "rewards/margins": 0.01287851296365261, + "rewards/rejected": -0.006646013353019953, + "step": 21 + }, + { + "epoch": 0.03, + "learning_rate": 1.1000000000000001e-05, + "logits/chosen": -2.531973361968994, + "logits/rejected": -2.594179153442383, + "logps/chosen": -166.6884307861328, + "logps/rejected": -173.5614013671875, + "loss": 0.6942, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.007440138608217239, + "rewards/margins": -0.0019538167398422956, + "rewards/rejected": -0.0054863216355443, + "step": 22 + }, + { + "epoch": 0.03, + "learning_rate": 1.1500000000000002e-05, + "logits/chosen": -2.4815938472747803, + "logits/rejected": -2.4820916652679443, + "logps/chosen": -131.22227478027344, + "logps/rejected": -131.2086181640625, + "loss": 0.6915, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.0006843092851340771, + "rewards/margins": 0.003669046564027667, + "rewards/rejected": -0.004353356547653675, + "step": 23 + }, + { + "epoch": 0.03, + "learning_rate": 1.2e-05, + "logits/chosen": -2.4899590015411377, + "logits/rejected": -2.5294911861419678, + "logps/chosen": -149.33543395996094, + "logps/rejected": -141.8245391845703, + "loss": 0.6983, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.017502309754490852, + "rewards/margins": -0.010094404220581055, + "rewards/rejected": -0.0074079036712646484, + "step": 24 + }, + { + "epoch": 0.03, + "learning_rate": 1.25e-05, + "logits/chosen": -2.5714809894561768, + "logits/rejected": -2.5547332763671875, + "logps/chosen": -167.8873291015625, + "logps/rejected": -173.3172149658203, + "loss": 0.69, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.00382807245478034, + "rewards/margins": 0.006511807441711426, + "rewards/rejected": -0.010339880362153053, + "step": 25 + }, + { + "epoch": 0.03, + "learning_rate": 1.3000000000000001e-05, + "logits/chosen": -2.470759630203247, + "logits/rejected": -2.4105560779571533, + "logps/chosen": -174.09585571289062, + "logps/rejected": -175.88677978515625, + "loss": 0.6914, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.0010800834279507399, + "rewards/margins": 0.004079150035977364, + "rewards/rejected": -0.0029990668408572674, + "step": 26 + }, + { + "epoch": 0.04, + "learning_rate": 1.3500000000000001e-05, + "logits/chosen": -2.6590945720672607, + "logits/rejected": -2.649517059326172, + "logps/chosen": -174.6903839111328, + "logps/rejected": -165.15533447265625, + "loss": 0.69, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.012396741658449173, + "rewards/margins": 0.006480884738266468, + "rewards/rejected": -0.018877625465393066, + "step": 27 + }, + { + "epoch": 0.04, + "learning_rate": 1.4000000000000001e-05, + "logits/chosen": -2.517886161804199, + "logits/rejected": -2.6399176120758057, + "logps/chosen": -154.28765869140625, + "logps/rejected": -181.42474365234375, + "loss": 0.6867, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.01030280627310276, + "rewards/margins": 0.01349327526986599, + "rewards/rejected": -0.0237960796803236, + "step": 28 + }, + { + "epoch": 0.04, + "learning_rate": 1.45e-05, + "logits/chosen": -2.5826056003570557, + "logits/rejected": -2.5789594650268555, + "logps/chosen": -157.37286376953125, + "logps/rejected": -154.72024536132812, + "loss": 0.6894, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.016228055581450462, + "rewards/margins": 0.007857107557356358, + "rewards/rejected": -0.024085164070129395, + "step": 29 + }, + { + "epoch": 0.04, + "learning_rate": 1.5e-05, + "logits/chosen": -2.6575980186462402, + "logits/rejected": -2.704176664352417, + "logps/chosen": -159.80322265625, + "logps/rejected": -165.82786560058594, + "loss": 0.6829, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.00888748187571764, + "rewards/margins": 0.02125699445605278, + "rewards/rejected": -0.030144479125738144, + "step": 30 + }, + { + "epoch": 0.04, + "learning_rate": 1.55e-05, + "logits/chosen": -2.650428533554077, + "logits/rejected": -2.693262815475464, + "logps/chosen": -164.53692626953125, + "logps/rejected": -174.57235717773438, + "loss": 0.6852, + "rewards/accuracies": 0.5625, + "rewards/chosen": 0.001265859231352806, + "rewards/margins": 0.01664908044040203, + "rewards/rejected": -0.01538322027772665, + "step": 31 + }, + { + "epoch": 0.04, + "learning_rate": 1.6000000000000003e-05, + "logits/chosen": -2.6280901432037354, + "logits/rejected": -2.6152122020721436, + "logps/chosen": -174.9736328125, + "logps/rejected": -209.5363311767578, + "loss": 0.6859, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.013908598572015762, + "rewards/margins": 0.016062045469880104, + "rewards/rejected": -0.029970645904541016, + "step": 32 + }, + { + "epoch": 0.04, + "learning_rate": 1.65e-05, + "logits/chosen": -2.5862505435943604, + "logits/rejected": -2.5280563831329346, + "logps/chosen": -160.01731872558594, + "logps/rejected": -172.03590393066406, + "loss": 0.694, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.03881430625915527, + "rewards/margins": -0.0009270897135138512, + "rewards/rejected": -0.03788721561431885, + "step": 33 + }, + { + "epoch": 0.04, + "learning_rate": 1.7000000000000003e-05, + "logits/chosen": -2.522751808166504, + "logits/rejected": -2.49782657623291, + "logps/chosen": -167.8050537109375, + "logps/rejected": -177.90399169921875, + "loss": 0.7061, + "rewards/accuracies": 0.1875, + "rewards/chosen": -0.031221888959407806, + "rewards/margins": -0.02547764778137207, + "rewards/rejected": -0.005744242575019598, + "step": 34 + }, + { + "epoch": 0.05, + "learning_rate": 1.75e-05, + "logits/chosen": -2.557039260864258, + "logits/rejected": -2.4520561695098877, + "logps/chosen": -143.8395538330078, + "logps/rejected": -140.37631225585938, + "loss": 0.6944, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.039483096450567245, + "rewards/margins": -0.0023098706733435392, + "rewards/rejected": -0.03717322647571564, + "step": 35 + }, + { + "epoch": 0.05, + "learning_rate": 1.8e-05, + "logits/chosen": -2.6500961780548096, + "logits/rejected": -2.5340402126312256, + "logps/chosen": -172.447998046875, + "logps/rejected": -198.5001678466797, + "loss": 0.7036, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.035362813621759415, + "rewards/margins": -0.020008588209748268, + "rewards/rejected": -0.015354226343333721, + "step": 36 + }, + { + "epoch": 0.05, + "learning_rate": 1.85e-05, + "logits/chosen": -2.6206865310668945, + "logits/rejected": -2.6976592540740967, + "logps/chosen": -167.8064422607422, + "logps/rejected": -185.81964111328125, + "loss": 0.6987, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.03894533962011337, + "rewards/margins": -0.009856510907411575, + "rewards/rejected": -0.029088832437992096, + "step": 37 + }, + { + "epoch": 0.05, + "learning_rate": 1.9e-05, + "logits/chosen": -2.434297800064087, + "logits/rejected": -2.651834487915039, + "logps/chosen": -169.50946044921875, + "logps/rejected": -257.7209167480469, + "loss": 0.6941, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.03687644004821777, + "rewards/margins": -0.0012842637952417135, + "rewards/rejected": -0.035592176020145416, + "step": 38 + }, + { + "epoch": 0.05, + "learning_rate": 1.9500000000000003e-05, + "logits/chosen": -2.4155445098876953, + "logits/rejected": -2.487833023071289, + "logps/chosen": -180.0078582763672, + "logps/rejected": -207.1255340576172, + "loss": 0.6978, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.037531279027462006, + "rewards/margins": -0.008796263486146927, + "rewards/rejected": -0.028735019266605377, + "step": 39 + }, + { + "epoch": 0.05, + "learning_rate": 2e-05, + "logits/chosen": -2.6476895809173584, + "logits/rejected": -2.707326650619507, + "logps/chosen": -182.0277862548828, + "logps/rejected": -180.2669677734375, + "loss": 0.6857, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.029536105692386627, + "rewards/margins": 0.015740489587187767, + "rewards/rejected": -0.04527659714221954, + "step": 40 + }, + { + "epoch": 0.05, + "learning_rate": 2.05e-05, + "logits/chosen": -2.691012144088745, + "logits/rejected": -2.688505172729492, + "logps/chosen": -173.92041015625, + "logps/rejected": -208.61599731445312, + "loss": 0.6781, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.01940765418112278, + "rewards/margins": 0.03146040812134743, + "rewards/rejected": -0.05086805671453476, + "step": 41 + }, + { + "epoch": 0.05, + "learning_rate": 2.1e-05, + "logits/chosen": -2.619537353515625, + "logits/rejected": -2.6051697731018066, + "logps/chosen": -193.06666564941406, + "logps/rejected": -178.87713623046875, + "loss": 0.699, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.035286471247673035, + "rewards/margins": -0.010646676644682884, + "rewards/rejected": -0.0246397964656353, + "step": 42 + }, + { + "epoch": 0.06, + "learning_rate": 2.15e-05, + "logits/chosen": -2.611173391342163, + "logits/rejected": -2.6004340648651123, + "logps/chosen": -188.3818817138672, + "logps/rejected": -214.30926513671875, + "loss": 0.6674, + "rewards/accuracies": 0.6875, + "rewards/chosen": 0.0015826929593458772, + "rewards/margins": 0.05373835563659668, + "rewards/rejected": -0.05215566232800484, + "step": 43 + }, + { + "epoch": 0.06, + "learning_rate": 2.2000000000000003e-05, + "logits/chosen": -2.667587995529175, + "logits/rejected": -2.688136339187622, + "logps/chosen": -188.89466857910156, + "logps/rejected": -207.63580322265625, + "loss": 0.6982, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.04630360007286072, + "rewards/margins": -0.008599425666034222, + "rewards/rejected": -0.03770418092608452, + "step": 44 + }, + { + "epoch": 0.06, + "learning_rate": 2.25e-05, + "logits/chosen": -2.5165350437164307, + "logits/rejected": -2.441213607788086, + "logps/chosen": -170.480712890625, + "logps/rejected": -159.08981323242188, + "loss": 0.6945, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.03015182353556156, + "rewards/margins": -0.0018717292696237564, + "rewards/rejected": -0.028280090540647507, + "step": 45 + }, + { + "epoch": 0.06, + "learning_rate": 2.3000000000000003e-05, + "logits/chosen": -2.523725748062134, + "logits/rejected": -2.6105730533599854, + "logps/chosen": -170.0879364013672, + "logps/rejected": -197.1090087890625, + "loss": 0.6884, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.022913338616490364, + "rewards/margins": 0.010608267970383167, + "rewards/rejected": -0.033521607518196106, + "step": 46 + }, + { + "epoch": 0.06, + "learning_rate": 2.35e-05, + "logits/chosen": -2.5311026573181152, + "logits/rejected": -2.54695463180542, + "logps/chosen": -178.0673065185547, + "logps/rejected": -182.32875061035156, + "loss": 0.6816, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.008729481138288975, + "rewards/margins": 0.02449822425842285, + "rewards/rejected": -0.03322770446538925, + "step": 47 + }, + { + "epoch": 0.06, + "learning_rate": 2.4e-05, + "logits/chosen": -2.6589465141296387, + "logits/rejected": -2.6210758686065674, + "logps/chosen": -182.40536499023438, + "logps/rejected": -169.12103271484375, + "loss": 0.6799, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.003579068696126342, + "rewards/margins": 0.02751018851995468, + "rewards/rejected": -0.031089257448911667, + "step": 48 + }, + { + "epoch": 0.06, + "learning_rate": 2.45e-05, + "logits/chosen": -2.513227701187134, + "logits/rejected": -2.543900489807129, + "logps/chosen": -170.51919555664062, + "logps/rejected": -168.29690551757812, + "loss": 0.7047, + "rewards/accuracies": 0.1875, + "rewards/chosen": -0.021081138402223587, + "rewards/margins": -0.022327663376927376, + "rewards/rejected": 0.0012465240433812141, + "step": 49 + }, + { + "epoch": 0.07, + "learning_rate": 2.5e-05, + "logits/chosen": -2.434335470199585, + "logits/rejected": -2.4906935691833496, + "logps/chosen": -162.16989135742188, + "logps/rejected": -212.56082153320312, + "loss": 0.6807, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.027100684121251106, + "rewards/margins": 0.02660501003265381, + "rewards/rejected": -0.053705692291259766, + "step": 50 + }, + { + "epoch": 0.07, + "learning_rate": 2.5500000000000003e-05, + "logits/chosen": -2.536504030227661, + "logits/rejected": -2.6495611667633057, + "logps/chosen": -171.29580688476562, + "logps/rejected": -191.54605102539062, + "loss": 0.6815, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.025197075679898262, + "rewards/margins": 0.024894431233406067, + "rewards/rejected": -0.05009150505065918, + "step": 51 + }, + { + "epoch": 0.07, + "learning_rate": 2.6000000000000002e-05, + "logits/chosen": -2.6077020168304443, + "logits/rejected": -2.564440965652466, + "logps/chosen": -170.64273071289062, + "logps/rejected": -217.3800811767578, + "loss": 0.7054, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.029342200607061386, + "rewards/margins": -0.02205488458275795, + "rewards/rejected": -0.007287311367690563, + "step": 52 + }, + { + "epoch": 0.07, + "learning_rate": 2.6500000000000004e-05, + "logits/chosen": -2.4113476276397705, + "logits/rejected": -2.4167730808258057, + "logps/chosen": -158.03866577148438, + "logps/rejected": -203.66368103027344, + "loss": 0.7, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.03733339160680771, + "rewards/margins": -0.011522197164595127, + "rewards/rejected": -0.025811197236180305, + "step": 53 + }, + { + "epoch": 0.07, + "learning_rate": 2.7000000000000002e-05, + "logits/chosen": -2.5948705673217773, + "logits/rejected": -2.584592342376709, + "logps/chosen": -163.16751098632812, + "logps/rejected": -196.3957061767578, + "loss": 0.687, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.050568290054798126, + "rewards/margins": 0.014712072908878326, + "rewards/rejected": -0.06528037041425705, + "step": 54 + }, + { + "epoch": 0.07, + "learning_rate": 2.7500000000000004e-05, + "logits/chosen": -2.524641513824463, + "logits/rejected": -2.445185661315918, + "logps/chosen": -167.29495239257812, + "logps/rejected": -178.15013122558594, + "loss": 0.694, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.03202950954437256, + "rewards/margins": 0.00026745768263936043, + "rewards/rejected": -0.03229696676135063, + "step": 55 + }, + { + "epoch": 0.07, + "learning_rate": 2.8000000000000003e-05, + "logits/chosen": -2.7647242546081543, + "logits/rejected": -2.721259117126465, + "logps/chosen": -179.40574645996094, + "logps/rejected": -180.4486541748047, + "loss": 0.6771, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.0023113014176487923, + "rewards/margins": 0.03354344516992569, + "rewards/rejected": -0.035854749381542206, + "step": 56 + }, + { + "epoch": 0.07, + "learning_rate": 2.8499999999999998e-05, + "logits/chosen": -2.676018714904785, + "logits/rejected": -2.6648664474487305, + "logps/chosen": -197.95919799804688, + "logps/rejected": -175.45230102539062, + "loss": 0.6764, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.00892419833689928, + "rewards/margins": 0.036644406616687775, + "rewards/rejected": -0.04556860774755478, + "step": 57 + }, + { + "epoch": 0.08, + "learning_rate": 2.9e-05, + "logits/chosen": -2.644045829772949, + "logits/rejected": -2.7341084480285645, + "logps/chosen": -180.7408905029297, + "logps/rejected": -189.23818969726562, + "loss": 0.6963, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.0153807383030653, + "rewards/margins": -0.005131007172167301, + "rewards/rejected": -0.010249733924865723, + "step": 58 + }, + { + "epoch": 0.08, + "learning_rate": 2.95e-05, + "logits/chosen": -2.6532645225524902, + "logits/rejected": -2.552724599838257, + "logps/chosen": -178.77191162109375, + "logps/rejected": -175.42742919921875, + "loss": 0.6841, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.03405280038714409, + "rewards/margins": 0.020096803084015846, + "rewards/rejected": -0.054149605333805084, + "step": 59 + }, + { + "epoch": 0.08, + "learning_rate": 3e-05, + "logits/chosen": -2.5994341373443604, + "logits/rejected": -2.6342806816101074, + "logps/chosen": -176.28402709960938, + "logps/rejected": -210.64498901367188, + "loss": 0.6975, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.025557922199368477, + "rewards/margins": -0.004868890158832073, + "rewards/rejected": -0.02068903297185898, + "step": 60 + }, + { + "epoch": 0.08, + "learning_rate": 3.05e-05, + "logits/chosen": -2.58650279045105, + "logits/rejected": -2.7233827114105225, + "logps/chosen": -184.57443237304688, + "logps/rejected": -175.98129272460938, + "loss": 0.7081, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.06466653198003769, + "rewards/margins": -0.02823822945356369, + "rewards/rejected": -0.0364283062517643, + "step": 61 + }, + { + "epoch": 0.08, + "learning_rate": 3.1e-05, + "logits/chosen": -2.6913092136383057, + "logits/rejected": -2.5653786659240723, + "logps/chosen": -203.69729614257812, + "logps/rejected": -190.5261688232422, + "loss": 0.7161, + "rewards/accuracies": 0.25, + "rewards/chosen": -0.04825315251946449, + "rewards/margins": -0.04361088201403618, + "rewards/rejected": -0.004642271436750889, + "step": 62 + }, + { + "epoch": 0.08, + "learning_rate": 3.15e-05, + "logits/chosen": -2.666816473007202, + "logits/rejected": -2.604631185531616, + "logps/chosen": -225.96981811523438, + "logps/rejected": -193.25982666015625, + "loss": 0.7001, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.040044669061899185, + "rewards/margins": -0.012522673234343529, + "rewards/rejected": -0.027521992102265358, + "step": 63 + }, + { + "epoch": 0.08, + "learning_rate": 3.2000000000000005e-05, + "logits/chosen": -2.6570017337799072, + "logits/rejected": -2.633661985397339, + "logps/chosen": -188.25892639160156, + "logps/rejected": -171.55096435546875, + "loss": 0.7173, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.08987895399332047, + "rewards/margins": -0.04469916597008705, + "rewards/rejected": -0.04517979919910431, + "step": 64 + }, + { + "epoch": 0.09, + "learning_rate": 3.2500000000000004e-05, + "logits/chosen": -2.582909107208252, + "logits/rejected": -2.587308883666992, + "logps/chosen": -168.5953369140625, + "logps/rejected": -180.11024475097656, + "loss": 0.689, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.038271259516477585, + "rewards/margins": 0.01016156654804945, + "rewards/rejected": -0.04843283072113991, + "step": 65 + }, + { + "epoch": 0.09, + "learning_rate": 3.3e-05, + "logits/chosen": -2.7339940071105957, + "logits/rejected": -2.6128218173980713, + "logps/chosen": -187.80320739746094, + "logps/rejected": -176.04495239257812, + "loss": 0.6948, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.08385930210351944, + "rewards/margins": -0.0003412736114114523, + "rewards/rejected": -0.08351802825927734, + "step": 66 + }, + { + "epoch": 0.09, + "learning_rate": 3.35e-05, + "logits/chosen": -2.406219482421875, + "logits/rejected": -2.435150623321533, + "logps/chosen": -150.17405700683594, + "logps/rejected": -174.16119384765625, + "loss": 0.68, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.030688336119055748, + "rewards/margins": 0.027779744938015938, + "rewards/rejected": -0.058468081057071686, + "step": 67 + }, + { + "epoch": 0.09, + "learning_rate": 3.4000000000000007e-05, + "logits/chosen": -2.7922043800354004, + "logits/rejected": -2.8177366256713867, + "logps/chosen": -202.07823181152344, + "logps/rejected": -233.48065185546875, + "loss": 0.688, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.10107097774744034, + "rewards/margins": 0.014442582614719868, + "rewards/rejected": -0.11551356315612793, + "step": 68 + }, + { + "epoch": 0.09, + "learning_rate": 3.45e-05, + "logits/chosen": -2.621925115585327, + "logits/rejected": -2.548135280609131, + "logps/chosen": -163.22723388671875, + "logps/rejected": -183.32742309570312, + "loss": 0.6853, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.009019946679472923, + "rewards/margins": 0.020815372467041016, + "rewards/rejected": -0.029835321009159088, + "step": 69 + }, + { + "epoch": 0.09, + "learning_rate": 3.5e-05, + "logits/chosen": -2.7518372535705566, + "logits/rejected": -2.7260899543762207, + "logps/chosen": -223.34564208984375, + "logps/rejected": -245.5427703857422, + "loss": 0.7156, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.12107516080141068, + "rewards/margins": -0.03887636959552765, + "rewards/rejected": -0.08219879120588303, + "step": 70 + }, + { + "epoch": 0.09, + "learning_rate": 3.55e-05, + "logits/chosen": -2.5108494758605957, + "logits/rejected": -2.4832143783569336, + "logps/chosen": -166.49508666992188, + "logps/rejected": -157.7345428466797, + "loss": 0.6918, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.05453377217054367, + "rewards/margins": 0.0069806561805307865, + "rewards/rejected": -0.06151442974805832, + "step": 71 + }, + { + "epoch": 0.09, + "learning_rate": 3.6e-05, + "logits/chosen": -2.636711359024048, + "logits/rejected": -2.6597518920898438, + "logps/chosen": -174.1949005126953, + "logps/rejected": -189.0026397705078, + "loss": 0.6782, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.005148457363247871, + "rewards/margins": 0.03288703039288521, + "rewards/rejected": -0.03803548216819763, + "step": 72 + }, + { + "epoch": 0.1, + "learning_rate": 3.65e-05, + "logits/chosen": -2.6395423412323, + "logits/rejected": -2.6543948650360107, + "logps/chosen": -146.5878448486328, + "logps/rejected": -180.54176330566406, + "loss": 0.7075, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.0687551274895668, + "rewards/margins": -0.02330932579934597, + "rewards/rejected": -0.045445799827575684, + "step": 73 + }, + { + "epoch": 0.1, + "learning_rate": 3.7e-05, + "logits/chosen": -2.4257636070251465, + "logits/rejected": -2.502183675765991, + "logps/chosen": -179.7578125, + "logps/rejected": -227.62875366210938, + "loss": 0.7125, + "rewards/accuracies": 0.1875, + "rewards/chosen": -0.03154797852039337, + "rewards/margins": -0.03330230712890625, + "rewards/rejected": 0.0017543300054967403, + "step": 74 + }, + { + "epoch": 0.1, + "learning_rate": 3.7500000000000003e-05, + "logits/chosen": -2.3436474800109863, + "logits/rejected": -2.498957633972168, + "logps/chosen": -151.80540466308594, + "logps/rejected": -164.84146118164062, + "loss": 0.6777, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.017311906442046165, + "rewards/margins": 0.03238987550139427, + "rewards/rejected": -0.049701791256666183, + "step": 75 + }, + { + "epoch": 0.1, + "learning_rate": 3.8e-05, + "logits/chosen": -2.558389663696289, + "logits/rejected": -2.6029064655303955, + "logps/chosen": -176.9400177001953, + "logps/rejected": -177.18336486816406, + "loss": 0.7002, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.04031562805175781, + "rewards/margins": -0.009864617139101028, + "rewards/rejected": -0.030451007187366486, + "step": 76 + }, + { + "epoch": 0.1, + "learning_rate": 3.85e-05, + "logits/chosen": -2.651350259780884, + "logits/rejected": -2.7427544593811035, + "logps/chosen": -210.8737335205078, + "logps/rejected": -199.6597137451172, + "loss": 0.6787, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.048238325864076614, + "rewards/margins": 0.031043197959661484, + "rewards/rejected": -0.0792815238237381, + "step": 77 + }, + { + "epoch": 0.1, + "learning_rate": 3.9000000000000006e-05, + "logits/chosen": -2.4323079586029053, + "logits/rejected": -2.576862335205078, + "logps/chosen": -127.96165466308594, + "logps/rejected": -161.3166961669922, + "loss": 0.724, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.1004774421453476, + "rewards/margins": -0.053969353437423706, + "rewards/rejected": -0.046508073806762695, + "step": 78 + }, + { + "epoch": 0.1, + "learning_rate": 3.9500000000000005e-05, + "logits/chosen": -2.526383638381958, + "logits/rejected": -2.552208423614502, + "logps/chosen": -183.66246032714844, + "logps/rejected": -195.2118377685547, + "loss": 0.6934, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.10480672121047974, + "rewards/margins": 0.005925657227635384, + "rewards/rejected": -0.11073236912488937, + "step": 79 + }, + { + "epoch": 0.1, + "learning_rate": 4e-05, + "logits/chosen": -2.635021209716797, + "logits/rejected": -2.7062149047851562, + "logps/chosen": -203.2865753173828, + "logps/rejected": -200.7851104736328, + "loss": 0.6553, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.012298012152314186, + "rewards/margins": 0.08196020871400833, + "rewards/rejected": -0.09425821155309677, + "step": 80 + }, + { + "epoch": 0.11, + "learning_rate": 4.05e-05, + "logits/chosen": -2.48476505279541, + "logits/rejected": -2.613018274307251, + "logps/chosen": -159.0533905029297, + "logps/rejected": -222.8763427734375, + "loss": 0.6488, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.017749834805727005, + "rewards/margins": 0.10052147507667542, + "rewards/rejected": -0.11827130615711212, + "step": 81 + }, + { + "epoch": 0.11, + "learning_rate": 4.1e-05, + "logits/chosen": -2.5390264987945557, + "logits/rejected": -2.5940589904785156, + "logps/chosen": -220.54562377929688, + "logps/rejected": -193.7894744873047, + "loss": 0.7168, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.15124498307704926, + "rewards/margins": -0.04028485342860222, + "rewards/rejected": -0.11096014082431793, + "step": 82 + }, + { + "epoch": 0.11, + "learning_rate": 4.15e-05, + "logits/chosen": -2.6519391536712646, + "logits/rejected": -2.6670496463775635, + "logps/chosen": -174.1206512451172, + "logps/rejected": -182.67996215820312, + "loss": 0.689, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.049321744590997696, + "rewards/margins": 0.012697530910372734, + "rewards/rejected": -0.06201927736401558, + "step": 83 + }, + { + "epoch": 0.11, + "learning_rate": 4.2e-05, + "logits/chosen": -2.717592239379883, + "logits/rejected": -2.7927956581115723, + "logps/chosen": -169.05294799804688, + "logps/rejected": -190.73846435546875, + "loss": 0.664, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.03787894546985626, + "rewards/margins": 0.06431596726179123, + "rewards/rejected": -0.1021949052810669, + "step": 84 + }, + { + "epoch": 0.11, + "learning_rate": 4.25e-05, + "logits/chosen": -2.5930697917938232, + "logits/rejected": -2.509345531463623, + "logps/chosen": -179.2568817138672, + "logps/rejected": -168.377685546875, + "loss": 0.6953, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.05612773820757866, + "rewards/margins": 0.006718709133565426, + "rewards/rejected": -0.06284645199775696, + "step": 85 + }, + { + "epoch": 0.11, + "learning_rate": 4.3e-05, + "logits/chosen": -2.5787692070007324, + "logits/rejected": -2.64978289604187, + "logps/chosen": -172.04168701171875, + "logps/rejected": -160.20840454101562, + "loss": 0.6896, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.11709931492805481, + "rewards/margins": 0.018632344901561737, + "rewards/rejected": -0.13573165237903595, + "step": 86 + }, + { + "epoch": 0.11, + "learning_rate": 4.35e-05, + "logits/chosen": -2.480304002761841, + "logits/rejected": -2.4922451972961426, + "logps/chosen": -200.24014282226562, + "logps/rejected": -218.20352172851562, + "loss": 0.6417, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.08723511546850204, + "rewards/margins": 0.12010292708873749, + "rewards/rejected": -0.20733806490898132, + "step": 87 + }, + { + "epoch": 0.12, + "learning_rate": 4.4000000000000006e-05, + "logits/chosen": -2.7480030059814453, + "logits/rejected": -2.703220844268799, + "logps/chosen": -164.8146514892578, + "logps/rejected": -173.18063354492188, + "loss": 0.7038, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.1111496239900589, + "rewards/margins": -0.00250411219894886, + "rewards/rejected": -0.10864551365375519, + "step": 88 + }, + { + "epoch": 0.12, + "learning_rate": 4.4500000000000004e-05, + "logits/chosen": -2.4365036487579346, + "logits/rejected": -2.5382070541381836, + "logps/chosen": -173.4228515625, + "logps/rejected": -228.0253448486328, + "loss": 0.662, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.022041939198970795, + "rewards/margins": 0.07041654735803604, + "rewards/rejected": -0.09245848655700684, + "step": 89 + }, + { + "epoch": 0.12, + "learning_rate": 4.5e-05, + "logits/chosen": -2.5812180042266846, + "logits/rejected": -2.6432223320007324, + "logps/chosen": -155.36810302734375, + "logps/rejected": -164.1707000732422, + "loss": 0.7, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.2002555876970291, + "rewards/margins": -0.011146757751703262, + "rewards/rejected": -0.18910883367061615, + "step": 90 + }, + { + "epoch": 0.12, + "learning_rate": 4.55e-05, + "logits/chosen": -2.6511974334716797, + "logits/rejected": -2.7204787731170654, + "logps/chosen": -172.86270141601562, + "logps/rejected": -176.37405395507812, + "loss": 0.6919, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.11853408813476562, + "rewards/margins": 0.03132196143269539, + "rewards/rejected": -0.14985604584217072, + "step": 91 + }, + { + "epoch": 0.12, + "learning_rate": 4.600000000000001e-05, + "logits/chosen": -2.753451108932495, + "logits/rejected": -2.6007282733917236, + "logps/chosen": -192.64649963378906, + "logps/rejected": -164.47393798828125, + "loss": 0.7061, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.26501375436782837, + "rewards/margins": -0.007466696202754974, + "rewards/rejected": -0.2575470507144928, + "step": 92 + }, + { + "epoch": 0.12, + "learning_rate": 4.6500000000000005e-05, + "logits/chosen": -2.6831321716308594, + "logits/rejected": -2.657895088195801, + "logps/chosen": -191.9042510986328, + "logps/rejected": -165.17892456054688, + "loss": 0.7455, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.21064424514770508, + "rewards/margins": -0.08327949792146683, + "rewards/rejected": -0.12736473977565765, + "step": 93 + }, + { + "epoch": 0.12, + "learning_rate": 4.7e-05, + "logits/chosen": -2.5191149711608887, + "logits/rejected": -2.6359121799468994, + "logps/chosen": -171.92013549804688, + "logps/rejected": -177.6357421875, + "loss": 0.7085, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.274119108915329, + "rewards/margins": -0.0034166108816862106, + "rewards/rejected": -0.2707024812698364, + "step": 94 + }, + { + "epoch": 0.12, + "learning_rate": 4.75e-05, + "logits/chosen": -2.731271505355835, + "logits/rejected": -2.7019336223602295, + "logps/chosen": -180.59613037109375, + "logps/rejected": -184.8212890625, + "loss": 0.6521, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.14632394909858704, + "rewards/margins": 0.09543509036302567, + "rewards/rejected": -0.2417590469121933, + "step": 95 + }, + { + "epoch": 0.13, + "learning_rate": 4.8e-05, + "logits/chosen": -2.490906000137329, + "logits/rejected": -2.6088736057281494, + "logps/chosen": -162.78256225585938, + "logps/rejected": -176.74588012695312, + "loss": 0.6336, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.14169403910636902, + "rewards/margins": 0.14634834229946136, + "rewards/rejected": -0.2880423963069916, + "step": 96 + }, + { + "epoch": 0.13, + "learning_rate": 4.85e-05, + "logits/chosen": -2.6869685649871826, + "logits/rejected": -2.6618151664733887, + "logps/chosen": -158.98098754882812, + "logps/rejected": -165.8663330078125, + "loss": 0.6656, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.17300596833229065, + "rewards/margins": 0.07225295156240463, + "rewards/rejected": -0.24525892734527588, + "step": 97 + }, + { + "epoch": 0.13, + "learning_rate": 4.9e-05, + "logits/chosen": -2.655244827270508, + "logits/rejected": -2.7159557342529297, + "logps/chosen": -187.57041931152344, + "logps/rejected": -189.180908203125, + "loss": 0.6596, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.16985346376895905, + "rewards/margins": 0.0829225480556488, + "rewards/rejected": -0.25277602672576904, + "step": 98 + }, + { + "epoch": 0.13, + "learning_rate": 4.9500000000000004e-05, + "logits/chosen": -2.5713696479797363, + "logits/rejected": -2.619272470474243, + "logps/chosen": -164.45220947265625, + "logps/rejected": -189.011962890625, + "loss": 0.6332, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.1459609568119049, + "rewards/margins": 0.14748626947402954, + "rewards/rejected": -0.29344722628593445, + "step": 99 + }, + { + "epoch": 0.13, + "learning_rate": 5e-05, + "logits/chosen": -2.7377047538757324, + "logits/rejected": -2.807655096054077, + "logps/chosen": -212.2166748046875, + "logps/rejected": -205.24362182617188, + "loss": 0.7779, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.3374475836753845, + "rewards/margins": -0.14080718159675598, + "rewards/rejected": -0.19664038717746735, + "step": 100 + }, + { + "epoch": 0.13, + "learning_rate": 4.999997432392803e-05, + "logits/chosen": -2.604559898376465, + "logits/rejected": -2.5846409797668457, + "logps/chosen": -182.06011962890625, + "logps/rejected": -185.51695251464844, + "loss": 0.723, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.2137359231710434, + "rewards/margins": -0.0511900931596756, + "rewards/rejected": -0.1625458300113678, + "step": 101 + }, + { + "epoch": 0.13, + "learning_rate": 4.9999897295764844e-05, + "logits/chosen": -2.7066619396209717, + "logits/rejected": -2.6727826595306396, + "logps/chosen": -194.32232666015625, + "logps/rejected": -219.6756134033203, + "loss": 0.6905, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.2863636910915375, + "rewards/margins": 0.04078531265258789, + "rewards/rejected": -0.32714903354644775, + "step": 102 + }, + { + "epoch": 0.13, + "learning_rate": 4.9999768915668665e-05, + "logits/chosen": -2.5064425468444824, + "logits/rejected": -2.49991512298584, + "logps/chosen": -168.347900390625, + "logps/rejected": -165.95089721679688, + "loss": 0.6714, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.28736329078674316, + "rewards/margins": 0.0547042116522789, + "rewards/rejected": -0.3420674800872803, + "step": 103 + }, + { + "epoch": 0.14, + "learning_rate": 4.999958918390321e-05, + "logits/chosen": -2.568999767303467, + "logits/rejected": -2.629004716873169, + "logps/chosen": -186.33514404296875, + "logps/rejected": -215.6304168701172, + "loss": 0.6716, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.23962916433811188, + "rewards/margins": 0.06436805427074432, + "rewards/rejected": -0.3039971888065338, + "step": 104 + }, + { + "epoch": 0.14, + "learning_rate": 4.999935810083766e-05, + "logits/chosen": -2.667313575744629, + "logits/rejected": -2.6559674739837646, + "logps/chosen": -154.65960693359375, + "logps/rejected": -155.13522338867188, + "loss": 0.667, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.2854609489440918, + "rewards/margins": 0.0853797197341919, + "rewards/rejected": -0.3708406686782837, + "step": 105 + }, + { + "epoch": 0.14, + "learning_rate": 4.999907566694667e-05, + "logits/chosen": -2.5488526821136475, + "logits/rejected": -2.485109567642212, + "logps/chosen": -222.78338623046875, + "logps/rejected": -222.6953887939453, + "loss": 0.6656, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.28172624111175537, + "rewards/margins": 0.07234585285186768, + "rewards/rejected": -0.35407203435897827, + "step": 106 + }, + { + "epoch": 0.14, + "learning_rate": 4.9998741882810384e-05, + "logits/chosen": -2.872877597808838, + "logits/rejected": -2.8446714878082275, + "logps/chosen": -182.39260864257812, + "logps/rejected": -213.32289123535156, + "loss": 0.6628, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.2876412272453308, + "rewards/margins": 0.08153553307056427, + "rewards/rejected": -0.3691767454147339, + "step": 107 + }, + { + "epoch": 0.14, + "learning_rate": 4.999835674911443e-05, + "logits/chosen": -2.7004892826080322, + "logits/rejected": -2.6362013816833496, + "logps/chosen": -162.95108032226562, + "logps/rejected": -161.67657470703125, + "loss": 0.6744, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.3595612049102783, + "rewards/margins": 0.053390078246593475, + "rewards/rejected": -0.4129512906074524, + "step": 108 + }, + { + "epoch": 0.14, + "learning_rate": 4.999792026664991e-05, + "logits/chosen": -2.8488755226135254, + "logits/rejected": -2.914703845977783, + "logps/chosen": -173.82363891601562, + "logps/rejected": -182.16908264160156, + "loss": 0.7065, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.48598194122314453, + "rewards/margins": 0.03310241550207138, + "rewards/rejected": -0.5190844535827637, + "step": 109 + }, + { + "epoch": 0.14, + "learning_rate": 4.9997432436313384e-05, + "logits/chosen": -2.5648086071014404, + "logits/rejected": -2.5932183265686035, + "logps/chosen": -185.70846557617188, + "logps/rejected": -189.32725524902344, + "loss": 0.6842, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.4091907739639282, + "rewards/margins": 0.04006391391158104, + "rewards/rejected": -0.4492546617984772, + "step": 110 + }, + { + "epoch": 0.15, + "learning_rate": 4.99968932591069e-05, + "logits/chosen": -2.770993232727051, + "logits/rejected": -2.7647171020507812, + "logps/chosen": -208.0184326171875, + "logps/rejected": -202.5152587890625, + "loss": 0.6867, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.49601614475250244, + "rewards/margins": 0.027304889634251595, + "rewards/rejected": -0.5233210921287537, + "step": 111 + }, + { + "epoch": 0.15, + "learning_rate": 4.999630273613799e-05, + "logits/chosen": -2.4182567596435547, + "logits/rejected": -2.611553430557251, + "logps/chosen": -138.6142578125, + "logps/rejected": -187.48898315429688, + "loss": 0.7333, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.5143216252326965, + "rewards/margins": -0.011710070073604584, + "rewards/rejected": -0.5026116371154785, + "step": 112 + }, + { + "epoch": 0.15, + "learning_rate": 4.999566086861961e-05, + "logits/chosen": -2.7679452896118164, + "logits/rejected": -2.8067147731781006, + "logps/chosen": -186.18055725097656, + "logps/rejected": -198.92605590820312, + "loss": 0.8218, + "rewards/accuracies": 0.1875, + "rewards/chosen": -0.6542762517929077, + "rewards/margins": -0.2115614116191864, + "rewards/rejected": -0.4427148103713989, + "step": 113 + }, + { + "epoch": 0.15, + "learning_rate": 4.999496765787024e-05, + "logits/chosen": -2.842291831970215, + "logits/rejected": -2.8023149967193604, + "logps/chosen": -162.9304962158203, + "logps/rejected": -182.9091796875, + "loss": 0.6623, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5319749116897583, + "rewards/margins": 0.08390979468822479, + "rewards/rejected": -0.6158846616744995, + "step": 114 + }, + { + "epoch": 0.15, + "learning_rate": 4.9994223105313774e-05, + "logits/chosen": -2.980092763900757, + "logits/rejected": -2.9631030559539795, + "logps/chosen": -217.55894470214844, + "logps/rejected": -226.1593780517578, + "loss": 0.6824, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.4032873809337616, + "rewards/margins": 0.0588911734521389, + "rewards/rejected": -0.4621785283088684, + "step": 115 + }, + { + "epoch": 0.15, + "learning_rate": 4.9993427212479606e-05, + "logits/chosen": -2.5391757488250732, + "logits/rejected": -2.725529909133911, + "logps/chosen": -179.54200744628906, + "logps/rejected": -204.42666625976562, + "loss": 0.6975, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.39779067039489746, + "rewards/margins": 0.0027880650013685226, + "rewards/rejected": -0.40057870745658875, + "step": 116 + }, + { + "epoch": 0.15, + "learning_rate": 4.999257998100254e-05, + "logits/chosen": -2.733851671218872, + "logits/rejected": -2.87308931350708, + "logps/chosen": -176.08485412597656, + "logps/rejected": -193.49484252929688, + "loss": 0.6449, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.40423309803009033, + "rewards/margins": 0.11808924376964569, + "rewards/rejected": -0.5223223567008972, + "step": 117 + }, + { + "epoch": 0.15, + "learning_rate": 4.999168141262289e-05, + "logits/chosen": -2.7447009086608887, + "logits/rejected": -2.808476448059082, + "logps/chosen": -224.22067260742188, + "logps/rejected": -265.44964599609375, + "loss": 0.543, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.5002200603485107, + "rewards/margins": 0.40809527039527893, + "rewards/rejected": -0.9083153605461121, + "step": 118 + }, + { + "epoch": 0.16, + "learning_rate": 4.9990731509186376e-05, + "logits/chosen": -2.639643430709839, + "logits/rejected": -2.7004880905151367, + "logps/chosen": -126.65451049804688, + "logps/rejected": -152.20460510253906, + "loss": 0.6935, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.5373567342758179, + "rewards/margins": 0.048874713480472565, + "rewards/rejected": -0.5862314105033875, + "step": 119 + }, + { + "epoch": 0.16, + "learning_rate": 4.998973027264419e-05, + "logits/chosen": -2.6515791416168213, + "logits/rejected": -2.7365236282348633, + "logps/chosen": -174.46641540527344, + "logps/rejected": -225.31158447265625, + "loss": 0.6869, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5558931827545166, + "rewards/margins": 0.04842944070696831, + "rewards/rejected": -0.6043226718902588, + "step": 120 + }, + { + "epoch": 0.16, + "learning_rate": 4.998867770505295e-05, + "logits/chosen": -2.724031686782837, + "logits/rejected": -2.709028482437134, + "logps/chosen": -166.08251953125, + "logps/rejected": -183.7383575439453, + "loss": 0.6879, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.5829455256462097, + "rewards/margins": 0.03844447806477547, + "rewards/rejected": -0.6213899850845337, + "step": 121 + }, + { + "epoch": 0.16, + "learning_rate": 4.9987573808574726e-05, + "logits/chosen": -2.696485996246338, + "logits/rejected": -2.779482364654541, + "logps/chosen": -161.10256958007812, + "logps/rejected": -185.7605743408203, + "loss": 0.5904, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.41624942421913147, + "rewards/margins": 0.23752669990062714, + "rewards/rejected": -0.6537761092185974, + "step": 122 + }, + { + "epoch": 0.16, + "learning_rate": 4.9986418585477016e-05, + "logits/chosen": -2.7433488368988037, + "logits/rejected": -2.8069894313812256, + "logps/chosen": -160.246826171875, + "logps/rejected": -171.8643798828125, + "loss": 0.728, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.4797200560569763, + "rewards/margins": -0.032596245408058167, + "rewards/rejected": -0.44712376594543457, + "step": 123 + }, + { + "epoch": 0.16, + "learning_rate": 4.998521203813274e-05, + "logits/chosen": -2.738048791885376, + "logits/rejected": -2.747162342071533, + "logps/chosen": -190.9744873046875, + "logps/rejected": -181.24745178222656, + "loss": 0.755, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.5226253271102905, + "rewards/margins": -0.04068867489695549, + "rewards/rejected": -0.48193663358688354, + "step": 124 + }, + { + "epoch": 0.16, + "learning_rate": 4.9983954169020256e-05, + "logits/chosen": -2.6100339889526367, + "logits/rejected": -2.634096145629883, + "logps/chosen": -176.01905822753906, + "logps/rejected": -161.22412109375, + "loss": 0.748, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.5368779301643372, + "rewards/margins": -0.09010656177997589, + "rewards/rejected": -0.4467713534832001, + "step": 125 + }, + { + "epoch": 0.16, + "learning_rate": 4.9982644980723334e-05, + "logits/chosen": -2.797285556793213, + "logits/rejected": -2.825375556945801, + "logps/chosen": -141.42039489746094, + "logps/rejected": -148.95436096191406, + "loss": 0.7302, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.5232774615287781, + "rewards/margins": -0.04633237421512604, + "rewards/rejected": -0.47694510221481323, + "step": 126 + }, + { + "epoch": 0.17, + "learning_rate": 4.998128447593117e-05, + "logits/chosen": -2.6646504402160645, + "logits/rejected": -2.7834081649780273, + "logps/chosen": -230.68499755859375, + "logps/rejected": -245.01414489746094, + "loss": 0.661, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.687300980091095, + "rewards/margins": 0.14017513394355774, + "rewards/rejected": -0.8274761438369751, + "step": 127 + }, + { + "epoch": 0.17, + "learning_rate": 4.997987265743834e-05, + "logits/chosen": -2.775637149810791, + "logits/rejected": -2.738879919052124, + "logps/chosen": -166.31509399414062, + "logps/rejected": -169.01321411132812, + "loss": 0.6557, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.4408762454986572, + "rewards/margins": 0.11477227509021759, + "rewards/rejected": -0.5556485652923584, + "step": 128 + }, + { + "epoch": 0.17, + "learning_rate": 4.997840952814484e-05, + "logits/chosen": -2.637038469314575, + "logits/rejected": -2.666442394256592, + "logps/chosen": -151.3613739013672, + "logps/rejected": -160.8176727294922, + "loss": 0.6752, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.4970799684524536, + "rewards/margins": 0.08162279427051544, + "rewards/rejected": -0.5787028074264526, + "step": 129 + }, + { + "epoch": 0.17, + "learning_rate": 4.9976895091056075e-05, + "logits/chosen": -2.7471365928649902, + "logits/rejected": -2.6176395416259766, + "logps/chosen": -204.17808532714844, + "logps/rejected": -224.61956787109375, + "loss": 0.6923, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.6104612946510315, + "rewards/margins": 0.11507527530193329, + "rewards/rejected": -0.725536584854126, + "step": 130 + }, + { + "epoch": 0.17, + "learning_rate": 4.9975329349282826e-05, + "logits/chosen": -2.7353034019470215, + "logits/rejected": -2.743035078048706, + "logps/chosen": -184.143798828125, + "logps/rejected": -197.323974609375, + "loss": 0.6711, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.6166396141052246, + "rewards/margins": 0.08574585616588593, + "rewards/rejected": -0.7023855447769165, + "step": 131 + }, + { + "epoch": 0.17, + "learning_rate": 4.9973712306041256e-05, + "logits/chosen": -2.6548473834991455, + "logits/rejected": -2.6762022972106934, + "logps/chosen": -193.84849548339844, + "logps/rejected": -175.150634765625, + "loss": 0.7597, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.6773683428764343, + "rewards/margins": -0.10929510742425919, + "rewards/rejected": -0.5680732131004333, + "step": 132 + }, + { + "epoch": 0.17, + "learning_rate": 4.997204396465292e-05, + "logits/chosen": -2.787050724029541, + "logits/rejected": -2.777578353881836, + "logps/chosen": -195.51785278320312, + "logps/rejected": -190.51622009277344, + "loss": 0.7073, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.5759019255638123, + "rewards/margins": 0.019099120050668716, + "rewards/rejected": -0.5950011014938354, + "step": 133 + }, + { + "epoch": 0.18, + "learning_rate": 4.997032432854472e-05, + "logits/chosen": -2.6198296546936035, + "logits/rejected": -2.661146640777588, + "logps/chosen": -149.22142028808594, + "logps/rejected": -173.4526824951172, + "loss": 0.6885, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5247339010238647, + "rewards/margins": 0.05187266319990158, + "rewards/rejected": -0.5766065120697021, + "step": 134 + }, + { + "epoch": 0.18, + "learning_rate": 4.996855340124894e-05, + "logits/chosen": -2.637789487838745, + "logits/rejected": -2.6017353534698486, + "logps/chosen": -158.05914306640625, + "logps/rejected": -175.1423797607422, + "loss": 0.6985, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.5851290822029114, + "rewards/margins": 0.03735842555761337, + "rewards/rejected": -0.6224875450134277, + "step": 135 + }, + { + "epoch": 0.18, + "learning_rate": 4.996673118640323e-05, + "logits/chosen": -2.37221360206604, + "logits/rejected": -2.5038788318634033, + "logps/chosen": -143.90518188476562, + "logps/rejected": -222.84115600585938, + "loss": 0.6539, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.36046820878982544, + "rewards/margins": 0.14497990906238556, + "rewards/rejected": -0.5054481029510498, + "step": 136 + }, + { + "epoch": 0.18, + "learning_rate": 4.996485768775055e-05, + "logits/chosen": -2.807823419570923, + "logits/rejected": -2.800899028778076, + "logps/chosen": -169.72129821777344, + "logps/rejected": -181.5267333984375, + "loss": 0.6754, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.543328046798706, + "rewards/margins": 0.11098619550466537, + "rewards/rejected": -0.6543142199516296, + "step": 137 + }, + { + "epoch": 0.18, + "learning_rate": 4.996293290913926e-05, + "logits/chosen": -2.795060634613037, + "logits/rejected": -2.9101216793060303, + "logps/chosen": -136.24945068359375, + "logps/rejected": -162.91119384765625, + "loss": 0.61, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.3900853395462036, + "rewards/margins": 0.19977441430091858, + "rewards/rejected": -0.5898597836494446, + "step": 138 + }, + { + "epoch": 0.18, + "learning_rate": 4.9960956854522986e-05, + "logits/chosen": -2.7642905712127686, + "logits/rejected": -2.7773826122283936, + "logps/chosen": -158.06378173828125, + "logps/rejected": -221.30577087402344, + "loss": 0.625, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.4831012189388275, + "rewards/margins": 0.18627440929412842, + "rewards/rejected": -0.6693755984306335, + "step": 139 + }, + { + "epoch": 0.18, + "learning_rate": 4.995892952796074e-05, + "logits/chosen": -2.7154903411865234, + "logits/rejected": -2.7624685764312744, + "logps/chosen": -187.28146362304688, + "logps/rejected": -189.83071899414062, + "loss": 0.6819, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.6895617842674255, + "rewards/margins": 0.06184637174010277, + "rewards/rejected": -0.7514082193374634, + "step": 140 + }, + { + "epoch": 0.18, + "learning_rate": 4.995685093361682e-05, + "logits/chosen": -2.7003986835479736, + "logits/rejected": -2.754859209060669, + "logps/chosen": -160.55992126464844, + "logps/rejected": -172.67434692382812, + "loss": 0.725, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.6082602739334106, + "rewards/margins": -0.01646682247519493, + "rewards/rejected": -0.5917934775352478, + "step": 141 + }, + { + "epoch": 0.19, + "learning_rate": 4.9954721075760824e-05, + "logits/chosen": -2.7585508823394775, + "logits/rejected": -2.7562849521636963, + "logps/chosen": -186.20509338378906, + "logps/rejected": -190.92132568359375, + "loss": 0.6594, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.5081749558448792, + "rewards/margins": 0.10397283732891083, + "rewards/rejected": -0.6121478080749512, + "step": 142 + }, + { + "epoch": 0.19, + "learning_rate": 4.995253995876767e-05, + "logits/chosen": -2.808187246322632, + "logits/rejected": -2.869479179382324, + "logps/chosen": -172.86203002929688, + "logps/rejected": -175.89739990234375, + "loss": 0.5562, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.31780511140823364, + "rewards/margins": 0.3512324392795563, + "rewards/rejected": -0.6690375804901123, + "step": 143 + }, + { + "epoch": 0.19, + "learning_rate": 4.995030758711756e-05, + "logits/chosen": -2.9907169342041016, + "logits/rejected": -2.968203544616699, + "logps/chosen": -191.64285278320312, + "logps/rejected": -177.73028564453125, + "loss": 0.7513, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.7068686485290527, + "rewards/margins": -0.05182289704680443, + "rewards/rejected": -0.6550456881523132, + "step": 144 + }, + { + "epoch": 0.19, + "learning_rate": 4.994802396539598e-05, + "logits/chosen": -2.8123016357421875, + "logits/rejected": -2.8668291568756104, + "logps/chosen": -172.08924865722656, + "logps/rejected": -195.8844451904297, + "loss": 0.696, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5609222054481506, + "rewards/margins": 0.05493137985467911, + "rewards/rejected": -0.6158535480499268, + "step": 145 + }, + { + "epoch": 0.19, + "learning_rate": 4.994568909829368e-05, + "logits/chosen": -2.892430305480957, + "logits/rejected": -2.762629985809326, + "logps/chosen": -216.95150756835938, + "logps/rejected": -187.0184783935547, + "loss": 0.7119, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.7327225208282471, + "rewards/margins": -0.021204425022006035, + "rewards/rejected": -0.7115181684494019, + "step": 146 + }, + { + "epoch": 0.19, + "learning_rate": 4.9943302990606684e-05, + "logits/chosen": -2.7017452716827393, + "logits/rejected": -2.7307963371276855, + "logps/chosen": -198.5173797607422, + "logps/rejected": -185.40316772460938, + "loss": 0.6751, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.6512372493743896, + "rewards/margins": 0.10363547503948212, + "rewards/rejected": -0.754872739315033, + "step": 147 + }, + { + "epoch": 0.19, + "learning_rate": 4.994086564723626e-05, + "logits/chosen": -2.835409641265869, + "logits/rejected": -2.8388915061950684, + "logps/chosen": -173.46127319335938, + "logps/rejected": -185.3079376220703, + "loss": 0.6937, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.6533925533294678, + "rewards/margins": 0.033594585955142975, + "rewards/rejected": -0.6869871616363525, + "step": 148 + }, + { + "epoch": 0.2, + "learning_rate": 4.9938377073188905e-05, + "logits/chosen": -2.9569547176361084, + "logits/rejected": -2.9091203212738037, + "logps/chosen": -201.7881317138672, + "logps/rejected": -180.14285278320312, + "loss": 0.7056, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.7249323725700378, + "rewards/margins": 0.014494583010673523, + "rewards/rejected": -0.7394269704818726, + "step": 149 + }, + { + "epoch": 0.2, + "learning_rate": 4.993583727357638e-05, + "logits/chosen": -2.6668853759765625, + "logits/rejected": -2.694221258163452, + "logps/chosen": -198.40594482421875, + "logps/rejected": -201.02980041503906, + "loss": 0.7579, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.652782678604126, + "rewards/margins": -0.08612270653247833, + "rewards/rejected": -0.5666600465774536, + "step": 150 + }, + { + "epoch": 0.2, + "learning_rate": 4.993324625361565e-05, + "logits/chosen": -2.757725954055786, + "logits/rejected": -2.7627735137939453, + "logps/chosen": -200.16226196289062, + "logps/rejected": -188.62083435058594, + "loss": 0.6402, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.44451814889907837, + "rewards/margins": 0.1431855410337448, + "rewards/rejected": -0.5877037048339844, + "step": 151 + }, + { + "epoch": 0.2, + "learning_rate": 4.993060401862888e-05, + "logits/chosen": -2.7355546951293945, + "logits/rejected": -2.7701869010925293, + "logps/chosen": -170.37046813964844, + "logps/rejected": -182.28440856933594, + "loss": 0.6787, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5462682247161865, + "rewards/margins": 0.08645598590373993, + "rewards/rejected": -0.6327242255210876, + "step": 152 + }, + { + "epoch": 0.2, + "learning_rate": 4.9927910574043465e-05, + "logits/chosen": -2.893017530441284, + "logits/rejected": -2.9069924354553223, + "logps/chosen": -215.6539764404297, + "logps/rejected": -256.93194580078125, + "loss": 0.642, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.635912299156189, + "rewards/margins": 0.1641692817211151, + "rewards/rejected": -0.8000816702842712, + "step": 153 + }, + { + "epoch": 0.2, + "learning_rate": 4.992516592539196e-05, + "logits/chosen": -2.544395685195923, + "logits/rejected": -2.563568353652954, + "logps/chosen": -144.60275268554688, + "logps/rejected": -150.6583709716797, + "loss": 0.6613, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.40625688433647156, + "rewards/margins": 0.11070521920919418, + "rewards/rejected": -0.5169621706008911, + "step": 154 + }, + { + "epoch": 0.2, + "learning_rate": 4.9922370078312105e-05, + "logits/chosen": -2.6519908905029297, + "logits/rejected": -2.6390066146850586, + "logps/chosen": -187.1649932861328, + "logps/rejected": -163.97708129882812, + "loss": 0.6462, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.4761255383491516, + "rewards/margins": 0.15067748725414276, + "rewards/rejected": -0.6268030405044556, + "step": 155 + }, + { + "epoch": 0.2, + "learning_rate": 4.991952303854682e-05, + "logits/chosen": -2.823216676712036, + "logits/rejected": -2.8248276710510254, + "logps/chosen": -171.64088439941406, + "logps/rejected": -205.40994262695312, + "loss": 0.5695, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.49252718687057495, + "rewards/margins": 0.3139771521091461, + "rewards/rejected": -0.8065043687820435, + "step": 156 + }, + { + "epoch": 0.21, + "learning_rate": 4.9916624811944175e-05, + "logits/chosen": -2.6669604778289795, + "logits/rejected": -2.720827102661133, + "logps/chosen": -145.63650512695312, + "logps/rejected": -143.77822875976562, + "loss": 0.6691, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.4779016673564911, + "rewards/margins": 0.07923712581396103, + "rewards/rejected": -0.5571387410163879, + "step": 157 + }, + { + "epoch": 0.21, + "learning_rate": 4.991367540445735e-05, + "logits/chosen": -2.780358076095581, + "logits/rejected": -2.7683119773864746, + "logps/chosen": -163.51113891601562, + "logps/rejected": -147.0047149658203, + "loss": 0.7295, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.6608467698097229, + "rewards/margins": -0.023487316444516182, + "rewards/rejected": -0.6373594403266907, + "step": 158 + }, + { + "epoch": 0.21, + "learning_rate": 4.991067482214471e-05, + "logits/chosen": -2.660963296890259, + "logits/rejected": -2.6401174068450928, + "logps/chosen": -173.09176635742188, + "logps/rejected": -171.8109130859375, + "loss": 0.7108, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.6182070970535278, + "rewards/margins": -0.01903488114476204, + "rewards/rejected": -0.5991722345352173, + "step": 159 + }, + { + "epoch": 0.21, + "learning_rate": 4.9907623071169686e-05, + "logits/chosen": -2.7048563957214355, + "logits/rejected": -2.5354063510894775, + "logps/chosen": -228.5687255859375, + "logps/rejected": -184.3094482421875, + "loss": 0.7866, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.7742022275924683, + "rewards/margins": -0.1047876849770546, + "rewards/rejected": -0.6694144606590271, + "step": 160 + }, + { + "epoch": 0.21, + "learning_rate": 4.990452015780085e-05, + "logits/chosen": -2.678699016571045, + "logits/rejected": -2.6732139587402344, + "logps/chosen": -217.17123413085938, + "logps/rejected": -211.92489624023438, + "loss": 0.8004, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.7689281702041626, + "rewards/margins": -0.17873500287532806, + "rewards/rejected": -0.5901932120323181, + "step": 161 + }, + { + "epoch": 0.21, + "learning_rate": 4.9901366088411846e-05, + "logits/chosen": -2.650327444076538, + "logits/rejected": -2.6380615234375, + "logps/chosen": -160.45660400390625, + "logps/rejected": -149.87843322753906, + "loss": 0.7334, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.4278574585914612, + "rewards/margins": -0.050989780575037, + "rewards/rejected": -0.3768676817417145, + "step": 162 + }, + { + "epoch": 0.21, + "learning_rate": 4.98981608694814e-05, + "logits/chosen": -2.636261463165283, + "logits/rejected": -2.6120524406433105, + "logps/chosen": -189.56497192382812, + "logps/rejected": -184.8382568359375, + "loss": 0.7106, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.6862293481826782, + "rewards/margins": 0.02107788249850273, + "rewards/rejected": -0.7073072791099548, + "step": 163 + }, + { + "epoch": 0.21, + "learning_rate": 4.9894904507593316e-05, + "logits/chosen": -2.681283712387085, + "logits/rejected": -2.6194002628326416, + "logps/chosen": -158.73968505859375, + "logps/rejected": -180.73330688476562, + "loss": 0.6498, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.4341495633125305, + "rewards/margins": 0.13567671179771423, + "rewards/rejected": -0.5698262453079224, + "step": 164 + }, + { + "epoch": 0.22, + "learning_rate": 4.989159700943643e-05, + "logits/chosen": -2.822274923324585, + "logits/rejected": -2.8219101428985596, + "logps/chosen": -182.0315704345703, + "logps/rejected": -193.2274169921875, + "loss": 0.7126, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.6531980037689209, + "rewards/margins": 0.01201358437538147, + "rewards/rejected": -0.6652116179466248, + "step": 165 + }, + { + "epoch": 0.22, + "learning_rate": 4.988823838180464e-05, + "logits/chosen": -2.804276943206787, + "logits/rejected": -2.8529317378997803, + "logps/chosen": -183.54083251953125, + "logps/rejected": -199.4310302734375, + "loss": 0.6961, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5883633494377136, + "rewards/margins": 0.03549729287624359, + "rewards/rejected": -0.6238605976104736, + "step": 166 + }, + { + "epoch": 0.22, + "learning_rate": 4.988482863159684e-05, + "logits/chosen": -2.7629952430725098, + "logits/rejected": -2.852982759475708, + "logps/chosen": -223.6051483154297, + "logps/rejected": -215.63427734375, + "loss": 0.605, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.5412927865982056, + "rewards/margins": 0.22360366582870483, + "rewards/rejected": -0.7648964524269104, + "step": 167 + }, + { + "epoch": 0.22, + "learning_rate": 4.988136776581696e-05, + "logits/chosen": -2.696824789047241, + "logits/rejected": -2.7131996154785156, + "logps/chosen": -161.2986297607422, + "logps/rejected": -180.24172973632812, + "loss": 0.6756, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.47621679306030273, + "rewards/margins": 0.07477270066738129, + "rewards/rejected": -0.5509894490242004, + "step": 168 + }, + { + "epoch": 0.22, + "learning_rate": 4.9877855791573915e-05, + "logits/chosen": -2.5992307662963867, + "logits/rejected": -2.5558767318725586, + "logps/chosen": -177.31790161132812, + "logps/rejected": -173.00013732910156, + "loss": 0.7614, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.6591533422470093, + "rewards/margins": -0.10336636006832123, + "rewards/rejected": -0.5557870268821716, + "step": 169 + }, + { + "epoch": 0.22, + "learning_rate": 4.9874292716081595e-05, + "logits/chosen": -2.480238914489746, + "logits/rejected": -2.531926155090332, + "logps/chosen": -173.81201171875, + "logps/rejected": -177.76727294921875, + "loss": 0.6506, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.4206813871860504, + "rewards/margins": 0.10889497399330139, + "rewards/rejected": -0.5295763611793518, + "step": 170 + }, + { + "epoch": 0.22, + "learning_rate": 4.9870678546658865e-05, + "logits/chosen": -2.68393611907959, + "logits/rejected": -2.8312528133392334, + "logps/chosen": -238.86911010742188, + "logps/rejected": -268.19732666015625, + "loss": 0.6572, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.5521610975265503, + "rewards/margins": 0.11217445880174637, + "rewards/rejected": -0.6643356084823608, + "step": 171 + }, + { + "epoch": 0.23, + "learning_rate": 4.9867013290729535e-05, + "logits/chosen": -2.580007314682007, + "logits/rejected": -2.5705273151397705, + "logps/chosen": -165.80308532714844, + "logps/rejected": -203.9613800048828, + "loss": 0.7206, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.6492197513580322, + "rewards/margins": 0.02221706137061119, + "rewards/rejected": -0.6714367866516113, + "step": 172 + }, + { + "epoch": 0.23, + "learning_rate": 4.986329695582237e-05, + "logits/chosen": -2.7853593826293945, + "logits/rejected": -2.7307794094085693, + "logps/chosen": -211.93991088867188, + "logps/rejected": -200.86334228515625, + "loss": 0.7051, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5223960876464844, + "rewards/margins": 0.0009156223386526108, + "rewards/rejected": -0.5233116745948792, + "step": 173 + }, + { + "epoch": 0.23, + "learning_rate": 4.985952954957103e-05, + "logits/chosen": -2.6804401874542236, + "logits/rejected": -2.6449456214904785, + "logps/chosen": -187.6370391845703, + "logps/rejected": -193.33245849609375, + "loss": 0.6809, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.513522207736969, + "rewards/margins": 0.04058818519115448, + "rewards/rejected": -0.5541103482246399, + "step": 174 + }, + { + "epoch": 0.23, + "learning_rate": 4.985571107971408e-05, + "logits/chosen": -2.622426986694336, + "logits/rejected": -2.618734836578369, + "logps/chosen": -152.2515869140625, + "logps/rejected": -168.34747314453125, + "loss": 0.7282, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.5195332765579224, + "rewards/margins": -0.02337510883808136, + "rewards/rejected": -0.4961581528186798, + "step": 175 + }, + { + "epoch": 0.23, + "learning_rate": 4.9851841554095e-05, + "logits/chosen": -2.6712746620178223, + "logits/rejected": -2.6609811782836914, + "logps/chosen": -198.9906005859375, + "logps/rejected": -164.44154357910156, + "loss": 0.7105, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.4613041877746582, + "rewards/margins": -0.0029089637100696564, + "rewards/rejected": -0.45839521288871765, + "step": 176 + }, + { + "epoch": 0.23, + "learning_rate": 4.9847920980662134e-05, + "logits/chosen": -2.6356289386749268, + "logits/rejected": -2.6573214530944824, + "logps/chosen": -175.52487182617188, + "logps/rejected": -187.29832458496094, + "loss": 0.6474, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.40463200211524963, + "rewards/margins": 0.12262441217899323, + "rewards/rejected": -0.527256429195404, + "step": 177 + }, + { + "epoch": 0.23, + "learning_rate": 4.984394936746865e-05, + "logits/chosen": -2.357494354248047, + "logits/rejected": -2.411952018737793, + "logps/chosen": -139.59608459472656, + "logps/rejected": -156.5337371826172, + "loss": 0.679, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.4118153154850006, + "rewards/margins": 0.04770222678780556, + "rewards/rejected": -0.45951756834983826, + "step": 178 + }, + { + "epoch": 0.23, + "learning_rate": 4.98399267226726e-05, + "logits/chosen": -2.5636932849884033, + "logits/rejected": -2.6556129455566406, + "logps/chosen": -175.670166015625, + "logps/rejected": -179.19627380371094, + "loss": 0.6572, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.6697508692741394, + "rewards/margins": 0.1040661484003067, + "rewards/rejected": -0.7738169431686401, + "step": 179 + }, + { + "epoch": 0.24, + "learning_rate": 4.9835853054536846e-05, + "logits/chosen": -2.5892560482025146, + "logits/rejected": -2.579235315322876, + "logps/chosen": -168.14564514160156, + "logps/rejected": -163.5547332763672, + "loss": 0.624, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.5034769177436829, + "rewards/margins": 0.1633673459291458, + "rewards/rejected": -0.6668442487716675, + "step": 180 + }, + { + "epoch": 0.24, + "learning_rate": 4.9831728371429046e-05, + "logits/chosen": -2.5526325702667236, + "logits/rejected": -2.602790355682373, + "logps/chosen": -167.693115234375, + "logps/rejected": -191.10301208496094, + "loss": 0.6836, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.5507184267044067, + "rewards/margins": 0.07049673795700073, + "rewards/rejected": -0.6212151646614075, + "step": 181 + }, + { + "epoch": 0.24, + "learning_rate": 4.982755268182164e-05, + "logits/chosen": -2.581120729446411, + "logits/rejected": -2.61881947517395, + "logps/chosen": -176.85264587402344, + "logps/rejected": -202.72483825683594, + "loss": 0.6354, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.6271121501922607, + "rewards/margins": 0.17561408877372742, + "rewards/rejected": -0.8027262687683105, + "step": 182 + }, + { + "epoch": 0.24, + "learning_rate": 4.982332599429187e-05, + "logits/chosen": -2.5083706378936768, + "logits/rejected": -2.5868587493896484, + "logps/chosen": -145.9921112060547, + "logps/rejected": -150.42713928222656, + "loss": 0.6945, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.45637643337249756, + "rewards/margins": 0.06362758576869965, + "rewards/rejected": -0.5200040340423584, + "step": 183 + }, + { + "epoch": 0.24, + "learning_rate": 4.981904831752171e-05, + "logits/chosen": -2.563215970993042, + "logits/rejected": -2.6187920570373535, + "logps/chosen": -147.48265075683594, + "logps/rejected": -155.67405700683594, + "loss": 0.7287, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.5919477939605713, + "rewards/margins": -0.029735613614320755, + "rewards/rejected": -0.5622121691703796, + "step": 184 + }, + { + "epoch": 0.24, + "learning_rate": 4.981471966029787e-05, + "logits/chosen": -2.447539806365967, + "logits/rejected": -2.4295990467071533, + "logps/chosen": -153.93881225585938, + "logps/rejected": -169.62408447265625, + "loss": 0.6502, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.6545721292495728, + "rewards/margins": 0.11100000143051147, + "rewards/rejected": -0.7655720710754395, + "step": 185 + }, + { + "epoch": 0.24, + "learning_rate": 4.981034003151178e-05, + "logits/chosen": -2.4045794010162354, + "logits/rejected": -2.446890354156494, + "logps/chosen": -134.2223358154297, + "logps/rejected": -149.72105407714844, + "loss": 0.6335, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.46206632256507874, + "rewards/margins": 0.19417575001716614, + "rewards/rejected": -0.6562421321868896, + "step": 186 + }, + { + "epoch": 0.24, + "learning_rate": 4.980590944015958e-05, + "logits/chosen": -2.68265700340271, + "logits/rejected": -2.667114496231079, + "logps/chosen": -167.9902801513672, + "logps/rejected": -171.3512420654297, + "loss": 0.6734, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5563911199569702, + "rewards/margins": 0.08469439297914505, + "rewards/rejected": -0.6410855054855347, + "step": 187 + }, + { + "epoch": 0.25, + "learning_rate": 4.98014278953421e-05, + "logits/chosen": -2.5728375911712646, + "logits/rejected": -2.681403875350952, + "logps/chosen": -159.7633056640625, + "logps/rejected": -211.886962890625, + "loss": 0.6439, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.531548023223877, + "rewards/margins": 0.1921529322862625, + "rewards/rejected": -0.7237009406089783, + "step": 188 + }, + { + "epoch": 0.25, + "learning_rate": 4.979689540626479e-05, + "logits/chosen": -2.324286937713623, + "logits/rejected": -2.453277349472046, + "logps/chosen": -168.53738403320312, + "logps/rejected": -180.93719482421875, + "loss": 0.6538, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.4037819802761078, + "rewards/margins": 0.17486746609210968, + "rewards/rejected": -0.5786494016647339, + "step": 189 + }, + { + "epoch": 0.25, + "learning_rate": 4.9792311982237774e-05, + "logits/chosen": -2.773432493209839, + "logits/rejected": -2.74585223197937, + "logps/chosen": -157.39044189453125, + "logps/rejected": -167.32725524902344, + "loss": 0.6255, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.5002456307411194, + "rewards/margins": 0.22034718096256256, + "rewards/rejected": -0.7205928564071655, + "step": 190 + }, + { + "epoch": 0.25, + "learning_rate": 4.9787677632675825e-05, + "logits/chosen": -2.6729888916015625, + "logits/rejected": -2.7148032188415527, + "logps/chosen": -162.77774047851562, + "logps/rejected": -221.85386657714844, + "loss": 0.6881, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.6836182475090027, + "rewards/margins": 0.14588770270347595, + "rewards/rejected": -0.829505980014801, + "step": 191 + }, + { + "epoch": 0.25, + "learning_rate": 4.978299236709826e-05, + "logits/chosen": -2.556713581085205, + "logits/rejected": -2.5743775367736816, + "logps/chosen": -197.9322052001953, + "logps/rejected": -203.74917602539062, + "loss": 0.6874, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.7575306296348572, + "rewards/margins": 0.05150505527853966, + "rewards/rejected": -0.80903559923172, + "step": 192 + }, + { + "epoch": 0.25, + "learning_rate": 4.977825619512904e-05, + "logits/chosen": -2.390803813934326, + "logits/rejected": -2.551340341567993, + "logps/chosen": -152.84097290039062, + "logps/rejected": -192.7227325439453, + "loss": 0.7108, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5909304618835449, + "rewards/margins": -0.009836459532380104, + "rewards/rejected": -0.581093966960907, + "step": 193 + }, + { + "epoch": 0.25, + "learning_rate": 4.977346912649666e-05, + "logits/chosen": -2.451486587524414, + "logits/rejected": -2.423938751220703, + "logps/chosen": -208.7131805419922, + "logps/rejected": -195.0940399169922, + "loss": 0.7075, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.5515446066856384, + "rewards/margins": 0.014907769858837128, + "rewards/rejected": -0.5664523839950562, + "step": 194 + }, + { + "epoch": 0.26, + "learning_rate": 4.9768631171034175e-05, + "logits/chosen": -2.4102437496185303, + "logits/rejected": -2.524508237838745, + "logps/chosen": -171.2530975341797, + "logps/rejected": -197.77008056640625, + "loss": 0.6437, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.6411187648773193, + "rewards/margins": 0.20387138426303864, + "rewards/rejected": -0.8449901342391968, + "step": 195 + }, + { + "epoch": 0.26, + "learning_rate": 4.9763742338679145e-05, + "logits/chosen": -2.6650915145874023, + "logits/rejected": -2.5682055950164795, + "logps/chosen": -280.52142333984375, + "logps/rejected": -250.82424926757812, + "loss": 0.6956, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.7708845138549805, + "rewards/margins": 0.03139163926243782, + "rewards/rejected": -0.8022761344909668, + "step": 196 + }, + { + "epoch": 0.26, + "learning_rate": 4.975880263947367e-05, + "logits/chosen": -2.66872239112854, + "logits/rejected": -2.6272640228271484, + "logps/chosen": -206.67318725585938, + "logps/rejected": -169.04757690429688, + "loss": 0.725, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.7810104489326477, + "rewards/margins": -0.02610369399189949, + "rewards/rejected": -0.7549068331718445, + "step": 197 + }, + { + "epoch": 0.26, + "learning_rate": 4.9753812083564304e-05, + "logits/chosen": -2.4464945793151855, + "logits/rejected": -2.4812588691711426, + "logps/chosen": -153.33660888671875, + "logps/rejected": -176.61399841308594, + "loss": 0.6034, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.44514474272727966, + "rewards/margins": 0.24550259113311768, + "rewards/rejected": -0.6906473636627197, + "step": 198 + }, + { + "epoch": 0.26, + "learning_rate": 4.974877068120208e-05, + "logits/chosen": -2.635669231414795, + "logits/rejected": -2.648510456085205, + "logps/chosen": -182.48388671875, + "logps/rejected": -196.19873046875, + "loss": 0.6649, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.6389026641845703, + "rewards/margins": 0.10669447481632233, + "rewards/rejected": -0.7455971240997314, + "step": 199 + }, + { + "epoch": 0.26, + "learning_rate": 4.974367844274248e-05, + "logits/chosen": -2.5759198665618896, + "logits/rejected": -2.723337173461914, + "logps/chosen": -179.0137939453125, + "logps/rejected": -255.97052001953125, + "loss": 0.6338, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.4965980052947998, + "rewards/margins": 0.18533286452293396, + "rewards/rejected": -0.6819308996200562, + "step": 200 + }, + { + "epoch": 0.26, + "learning_rate": 4.973853537864538e-05, + "logits/chosen": -2.7438008785247803, + "logits/rejected": -2.8121700286865234, + "logps/chosen": -160.9208221435547, + "logps/rejected": -169.40225219726562, + "loss": 0.6689, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.6913070678710938, + "rewards/margins": 0.10465458035469055, + "rewards/rejected": -0.7959617376327515, + "step": 201 + }, + { + "epoch": 0.26, + "learning_rate": 4.973334149947508e-05, + "logits/chosen": -2.70800518989563, + "logits/rejected": -2.6374926567077637, + "logps/chosen": -183.4274444580078, + "logps/rejected": -180.7521209716797, + "loss": 0.6144, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.7033702731132507, + "rewards/margins": 0.2224593162536621, + "rewards/rejected": -0.9258295893669128, + "step": 202 + }, + { + "epoch": 0.27, + "learning_rate": 4.972809681590026e-05, + "logits/chosen": -2.66047739982605, + "logits/rejected": -2.710866928100586, + "logps/chosen": -194.5672607421875, + "logps/rejected": -210.85208129882812, + "loss": 0.655, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.8557997345924377, + "rewards/margins": 0.16004985570907593, + "rewards/rejected": -1.0158495903015137, + "step": 203 + }, + { + "epoch": 0.27, + "learning_rate": 4.972280133869396e-05, + "logits/chosen": -2.433838129043579, + "logits/rejected": -2.564758539199829, + "logps/chosen": -171.2584686279297, + "logps/rejected": -213.25494384765625, + "loss": 0.5993, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.43924885988235474, + "rewards/margins": 0.2442169040441513, + "rewards/rejected": -0.6834657788276672, + "step": 204 + }, + { + "epoch": 0.27, + "learning_rate": 4.971745507873352e-05, + "logits/chosen": -2.681500196456909, + "logits/rejected": -2.6873316764831543, + "logps/chosen": -150.2528839111328, + "logps/rejected": -154.3059844970703, + "loss": 0.627, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.7566659450531006, + "rewards/margins": 0.23124736547470093, + "rewards/rejected": -0.9879133701324463, + "step": 205 + }, + { + "epoch": 0.27, + "learning_rate": 4.971205804700063e-05, + "logits/chosen": -2.464470624923706, + "logits/rejected": -2.3958442211151123, + "logps/chosen": -293.574462890625, + "logps/rejected": -252.32489013671875, + "loss": 0.6749, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5428895354270935, + "rewards/margins": 0.14122185111045837, + "rewards/rejected": -0.6841113567352295, + "step": 206 + }, + { + "epoch": 0.27, + "learning_rate": 4.970661025458125e-05, + "logits/chosen": -2.5775954723358154, + "logits/rejected": -2.6041054725646973, + "logps/chosen": -170.66627502441406, + "logps/rejected": -163.29644775390625, + "loss": 0.721, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.9231572151184082, + "rewards/margins": 0.11258751899003983, + "rewards/rejected": -1.0357446670532227, + "step": 207 + }, + { + "epoch": 0.27, + "learning_rate": 4.9701111712665625e-05, + "logits/chosen": -2.6646294593811035, + "logits/rejected": -2.7594456672668457, + "logps/chosen": -200.36981201171875, + "logps/rejected": -186.59059143066406, + "loss": 0.7514, + "rewards/accuracies": 0.4375, + "rewards/chosen": -1.3253411054611206, + "rewards/margins": -0.0334821492433548, + "rewards/rejected": -1.2918590307235718, + "step": 208 + }, + { + "epoch": 0.27, + "learning_rate": 4.969556243254822e-05, + "logits/chosen": -2.5144646167755127, + "logits/rejected": -2.594902992248535, + "logps/chosen": -147.14231872558594, + "logps/rejected": -176.36328125, + "loss": 0.6548, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.6609407067298889, + "rewards/margins": 0.12413067370653152, + "rewards/rejected": -0.7850713729858398, + "step": 209 + }, + { + "epoch": 0.27, + "learning_rate": 4.968996242562774e-05, + "logits/chosen": -2.6077287197113037, + "logits/rejected": -2.6607818603515625, + "logps/chosen": -199.4670867919922, + "logps/rejected": -201.5868377685547, + "loss": 0.6797, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.8754231333732605, + "rewards/margins": 0.13014619052410126, + "rewards/rejected": -1.005569338798523, + "step": 210 + }, + { + "epoch": 0.28, + "learning_rate": 4.968431170340706e-05, + "logits/chosen": -2.740494966506958, + "logits/rejected": -2.620009660720825, + "logps/chosen": -210.96929931640625, + "logps/rejected": -204.82090759277344, + "loss": 0.7721, + "rewards/accuracies": 0.5, + "rewards/chosen": -1.2561805248260498, + "rewards/margins": -0.05324437841773033, + "rewards/rejected": -1.2029361724853516, + "step": 211 + }, + { + "epoch": 0.28, + "learning_rate": 4.9678610277493275e-05, + "logits/chosen": -2.6105682849884033, + "logits/rejected": -2.5892579555511475, + "logps/chosen": -198.35635375976562, + "logps/rejected": -207.89016723632812, + "loss": 0.6961, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.989006757736206, + "rewards/margins": 0.060812097042798996, + "rewards/rejected": -1.0498188734054565, + "step": 212 + }, + { + "epoch": 0.28, + "learning_rate": 4.967285815959759e-05, + "logits/chosen": -2.714409589767456, + "logits/rejected": -2.7895750999450684, + "logps/chosen": -208.87754821777344, + "logps/rejected": -222.0183563232422, + "loss": 0.57, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.8426048159599304, + "rewards/margins": 0.3045133650302887, + "rewards/rejected": -1.1471182107925415, + "step": 213 + }, + { + "epoch": 0.28, + "learning_rate": 4.9667055361535354e-05, + "logits/chosen": -2.748204231262207, + "logits/rejected": -2.832871675491333, + "logps/chosen": -201.07078552246094, + "logps/rejected": -212.4540252685547, + "loss": 0.6954, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.2435117959976196, + "rewards/margins": 0.14082738757133484, + "rewards/rejected": -1.384339451789856, + "step": 214 + }, + { + "epoch": 0.28, + "learning_rate": 4.9661201895226e-05, + "logits/chosen": -2.7127251625061035, + "logits/rejected": -2.751798629760742, + "logps/chosen": -220.7108154296875, + "logps/rejected": -195.08290100097656, + "loss": 0.6729, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.7872539758682251, + "rewards/margins": 0.12220478057861328, + "rewards/rejected": -0.9094586968421936, + "step": 215 + }, + { + "epoch": 0.28, + "learning_rate": 4.965529777269306e-05, + "logits/chosen": -2.6204776763916016, + "logits/rejected": -2.664301872253418, + "logps/chosen": -166.66172790527344, + "logps/rejected": -205.79847717285156, + "loss": 0.7922, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.9887948036193848, + "rewards/margins": -0.06140782684087753, + "rewards/rejected": -0.927386999130249, + "step": 216 + }, + { + "epoch": 0.28, + "learning_rate": 4.964934300606411e-05, + "logits/chosen": -2.48382830619812, + "logits/rejected": -2.493025541305542, + "logps/chosen": -169.9669189453125, + "logps/rejected": -181.6856689453125, + "loss": 0.7197, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.6650858521461487, + "rewards/margins": 0.017848990857601166, + "rewards/rejected": -0.6829348802566528, + "step": 217 + }, + { + "epoch": 0.29, + "learning_rate": 4.964333760757074e-05, + "logits/chosen": -2.648463726043701, + "logits/rejected": -2.6576759815216064, + "logps/chosen": -178.34747314453125, + "logps/rejected": -188.2498779296875, + "loss": 0.6239, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.95872962474823, + "rewards/margins": 0.26567110419273376, + "rewards/rejected": -1.2244007587432861, + "step": 218 + }, + { + "epoch": 0.29, + "learning_rate": 4.963728158954856e-05, + "logits/chosen": -2.9130921363830566, + "logits/rejected": -2.894216299057007, + "logps/chosen": -221.04931640625, + "logps/rejected": -237.8624725341797, + "loss": 0.6594, + "rewards/accuracies": 0.5625, + "rewards/chosen": -1.084302306175232, + "rewards/margins": 0.13000546395778656, + "rewards/rejected": -1.2143077850341797, + "step": 219 + }, + { + "epoch": 0.29, + "learning_rate": 4.963117496443715e-05, + "logits/chosen": -2.68157958984375, + "logits/rejected": -2.8279314041137695, + "logps/chosen": -165.5657196044922, + "logps/rejected": -202.75865173339844, + "loss": 0.6575, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.7596678733825684, + "rewards/margins": 0.2009599506855011, + "rewards/rejected": -0.9606277942657471, + "step": 220 + }, + { + "epoch": 0.29, + "learning_rate": 4.9625017744780045e-05, + "logits/chosen": -2.614312171936035, + "logits/rejected": -2.6585357189178467, + "logps/chosen": -216.45230102539062, + "logps/rejected": -195.06643676757812, + "loss": 0.7323, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.9276168346405029, + "rewards/margins": -0.02379007264971733, + "rewards/rejected": -0.9038268327713013, + "step": 221 + }, + { + "epoch": 0.29, + "learning_rate": 4.96188099432247e-05, + "logits/chosen": -2.7345945835113525, + "logits/rejected": -2.702479362487793, + "logps/chosen": -227.8175048828125, + "logps/rejected": -238.62513732910156, + "loss": 0.6852, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.9885910749435425, + "rewards/margins": 0.07757923752069473, + "rewards/rejected": -1.0661702156066895, + "step": 222 + }, + { + "epoch": 0.29, + "learning_rate": 4.9612551572522464e-05, + "logits/chosen": -2.6463451385498047, + "logits/rejected": -2.6843183040618896, + "logps/chosen": -150.67254638671875, + "logps/rejected": -155.01316833496094, + "loss": 0.6833, + "rewards/accuracies": 0.6875, + "rewards/chosen": -1.0718083381652832, + "rewards/margins": 0.0895613357424736, + "rewards/rejected": -1.1613696813583374, + "step": 223 + }, + { + "epoch": 0.29, + "learning_rate": 4.960624264552858e-05, + "logits/chosen": -2.6091978549957275, + "logits/rejected": -2.6224098205566406, + "logps/chosen": -134.08544921875, + "logps/rejected": -145.00137329101562, + "loss": 0.617, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.6298830509185791, + "rewards/margins": 0.1894720196723938, + "rewards/rejected": -0.8193551301956177, + "step": 224 + }, + { + "epoch": 0.29, + "learning_rate": 4.9599883175202124e-05, + "logits/chosen": -2.689610004425049, + "logits/rejected": -2.641515016555786, + "logps/chosen": -175.27386474609375, + "logps/rejected": -176.47059631347656, + "loss": 0.6933, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.8815383911132812, + "rewards/margins": 0.05579657852649689, + "rewards/rejected": -0.9373350143432617, + "step": 225 + }, + { + "epoch": 0.3, + "learning_rate": 4.9593473174605974e-05, + "logits/chosen": -2.673809051513672, + "logits/rejected": -2.6921756267547607, + "logps/chosen": -210.48333740234375, + "logps/rejected": -222.2653045654297, + "loss": 0.7605, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.9074676036834717, + "rewards/margins": -0.08400504291057587, + "rewards/rejected": -0.8234625458717346, + "step": 226 + }, + { + "epoch": 0.3, + "learning_rate": 4.958701265690685e-05, + "logits/chosen": -2.608705759048462, + "logits/rejected": -2.622281074523926, + "logps/chosen": -194.5504150390625, + "logps/rejected": -197.58035278320312, + "loss": 0.7456, + "rewards/accuracies": 0.4375, + "rewards/chosen": -1.1094239950180054, + "rewards/margins": -0.014511600136756897, + "rewards/rejected": -1.0949124097824097, + "step": 227 + }, + { + "epoch": 0.3, + "learning_rate": 4.958050163537519e-05, + "logits/chosen": -2.598935127258301, + "logits/rejected": -2.648134231567383, + "logps/chosen": -135.84756469726562, + "logps/rejected": -160.54318237304688, + "loss": 0.7163, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.7799862623214722, + "rewards/margins": 0.07278753817081451, + "rewards/rejected": -0.8527737855911255, + "step": 228 + }, + { + "epoch": 0.3, + "learning_rate": 4.957394012338519e-05, + "logits/chosen": -2.5633938312530518, + "logits/rejected": -2.528005838394165, + "logps/chosen": -179.5242919921875, + "logps/rejected": -175.9528350830078, + "loss": 0.6776, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.6330910325050354, + "rewards/margins": 0.09188088774681091, + "rewards/rejected": -0.7249718904495239, + "step": 229 + }, + { + "epoch": 0.3, + "learning_rate": 4.956732813441477e-05, + "logits/chosen": -2.681288957595825, + "logits/rejected": -2.742069959640503, + "logps/chosen": -157.13189697265625, + "logps/rejected": -170.25006103515625, + "loss": 0.6152, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.7350885272026062, + "rewards/margins": 0.18710875511169434, + "rewards/rejected": -0.9221972227096558, + "step": 230 + }, + { + "epoch": 0.3, + "learning_rate": 4.956066568204552e-05, + "logits/chosen": -2.6132476329803467, + "logits/rejected": -2.6484899520874023, + "logps/chosen": -175.98236083984375, + "logps/rejected": -185.96463012695312, + "loss": 0.5907, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.6415359973907471, + "rewards/margins": 0.3225414752960205, + "rewards/rejected": -0.9640775322914124, + "step": 231 + }, + { + "epoch": 0.3, + "learning_rate": 4.955395277996268e-05, + "logits/chosen": -2.654163122177124, + "logits/rejected": -2.6380820274353027, + "logps/chosen": -191.48721313476562, + "logps/rejected": -160.77561950683594, + "loss": 0.724, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.868155837059021, + "rewards/margins": 0.054594431072473526, + "rewards/rejected": -0.9227503538131714, + "step": 232 + }, + { + "epoch": 0.3, + "learning_rate": 4.954718944195512e-05, + "logits/chosen": -2.6072440147399902, + "logits/rejected": -2.6623966693878174, + "logps/chosen": -184.31289672851562, + "logps/rejected": -173.9798126220703, + "loss": 0.7546, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.8332484364509583, + "rewards/margins": -0.08707739412784576, + "rewards/rejected": -0.7461711168289185, + "step": 233 + }, + { + "epoch": 0.31, + "learning_rate": 4.954037568191534e-05, + "logits/chosen": -2.6110591888427734, + "logits/rejected": -2.568448781967163, + "logps/chosen": -222.3007049560547, + "logps/rejected": -193.83935546875, + "loss": 0.769, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.9347423315048218, + "rewards/margins": -0.09616127610206604, + "rewards/rejected": -0.8385810256004333, + "step": 234 + }, + { + "epoch": 0.31, + "learning_rate": 4.9533511513839384e-05, + "logits/chosen": -2.7532308101654053, + "logits/rejected": -2.7335729598999023, + "logps/chosen": -218.836669921875, + "logps/rejected": -247.48204040527344, + "loss": 0.7134, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.9885715246200562, + "rewards/margins": 0.15268389880657196, + "rewards/rejected": -1.1412553787231445, + "step": 235 + }, + { + "epoch": 0.31, + "learning_rate": 4.9526596951826824e-05, + "logits/chosen": -2.6721489429473877, + "logits/rejected": -2.6597042083740234, + "logps/chosen": -187.82127380371094, + "logps/rejected": -174.8271026611328, + "loss": 0.6244, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.5805238485336304, + "rewards/margins": 0.22618348896503448, + "rewards/rejected": -0.8067073225975037, + "step": 236 + }, + { + "epoch": 0.31, + "learning_rate": 4.951963201008076e-05, + "logits/chosen": -2.855266571044922, + "logits/rejected": -2.8631551265716553, + "logps/chosen": -243.0494384765625, + "logps/rejected": -228.80966186523438, + "loss": 0.7247, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.934760332107544, + "rewards/margins": 0.034967467188835144, + "rewards/rejected": -0.9697277545928955, + "step": 237 + }, + { + "epoch": 0.31, + "learning_rate": 4.951261670290781e-05, + "logits/chosen": -2.664848566055298, + "logits/rejected": -2.730018138885498, + "logps/chosen": -192.22723388671875, + "logps/rejected": -171.22459411621094, + "loss": 0.6765, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.6986839771270752, + "rewards/margins": 0.09998993575572968, + "rewards/rejected": -0.7986739277839661, + "step": 238 + }, + { + "epoch": 0.31, + "learning_rate": 4.950555104471799e-05, + "logits/chosen": -2.6024093627929688, + "logits/rejected": -2.6131458282470703, + "logps/chosen": -164.9868927001953, + "logps/rejected": -153.97813415527344, + "loss": 0.7123, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.7849129438400269, + "rewards/margins": 0.029222920536994934, + "rewards/rejected": -0.8141359090805054, + "step": 239 + }, + { + "epoch": 0.31, + "learning_rate": 4.949843505002477e-05, + "logits/chosen": -2.509110689163208, + "logits/rejected": -2.5836997032165527, + "logps/chosen": -156.37550354003906, + "logps/rejected": -177.84518432617188, + "loss": 0.5906, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.40633201599121094, + "rewards/margins": 0.2858356237411499, + "rewards/rejected": -0.6921676397323608, + "step": 240 + }, + { + "epoch": 0.32, + "learning_rate": 4.9491268733445034e-05, + "logits/chosen": -2.5929789543151855, + "logits/rejected": -2.593715190887451, + "logps/chosen": -159.74212646484375, + "logps/rejected": -177.87425231933594, + "loss": 0.6413, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.4330030679702759, + "rewards/margins": 0.15460006892681122, + "rewards/rejected": -0.5876031517982483, + "step": 241 + }, + { + "epoch": 0.32, + "learning_rate": 4.9484052109698984e-05, + "logits/chosen": -2.581435441970825, + "logits/rejected": -2.598817825317383, + "logps/chosen": -169.76776123046875, + "logps/rejected": -170.72422790527344, + "loss": 0.6379, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.5581039786338806, + "rewards/margins": 0.1779794692993164, + "rewards/rejected": -0.736083447933197, + "step": 242 + }, + { + "epoch": 0.32, + "learning_rate": 4.947678519361021e-05, + "logits/chosen": -2.5178732872009277, + "logits/rejected": -2.5693912506103516, + "logps/chosen": -173.69699096679688, + "logps/rejected": -174.0066680908203, + "loss": 0.6503, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.5450050830841064, + "rewards/margins": 0.11375146359205246, + "rewards/rejected": -0.6587565541267395, + "step": 243 + }, + { + "epoch": 0.32, + "learning_rate": 4.946946800010556e-05, + "logits/chosen": -2.5523407459259033, + "logits/rejected": -2.537888288497925, + "logps/chosen": -175.92843627929688, + "logps/rejected": -160.6066436767578, + "loss": 0.7642, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.7941773533821106, + "rewards/margins": -0.09637541323900223, + "rewards/rejected": -0.697801947593689, + "step": 244 + }, + { + "epoch": 0.32, + "learning_rate": 4.946210054421518e-05, + "logits/chosen": -2.688391923904419, + "logits/rejected": -2.702990770339966, + "logps/chosen": -222.63352966308594, + "logps/rejected": -215.47323608398438, + "loss": 0.6372, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.6142177581787109, + "rewards/margins": 0.2073356807231903, + "rewards/rejected": -0.8215534687042236, + "step": 245 + }, + { + "epoch": 0.32, + "learning_rate": 4.945468284107246e-05, + "logits/chosen": -2.5451714992523193, + "logits/rejected": -2.5484871864318848, + "logps/chosen": -196.5067138671875, + "logps/rejected": -190.59320068359375, + "loss": 0.6548, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.5240797996520996, + "rewards/margins": 0.11533726006746292, + "rewards/rejected": -0.6394170522689819, + "step": 246 + }, + { + "epoch": 0.32, + "learning_rate": 4.944721490591401e-05, + "logits/chosen": -2.472393274307251, + "logits/rejected": -2.587679386138916, + "logps/chosen": -165.8987274169922, + "logps/rejected": -188.1234130859375, + "loss": 0.5868, + "rewards/accuracies": 0.875, + "rewards/chosen": -0.4879373013973236, + "rewards/margins": 0.2605348229408264, + "rewards/rejected": -0.7484720349311829, + "step": 247 + }, + { + "epoch": 0.32, + "learning_rate": 4.9439696754079595e-05, + "logits/chosen": -2.800347089767456, + "logits/rejected": -2.7822184562683105, + "logps/chosen": -248.7989044189453, + "logps/rejected": -214.1162567138672, + "loss": 0.8436, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.8426647782325745, + "rewards/margins": -0.20171405375003815, + "rewards/rejected": -0.6409507393836975, + "step": 248 + }, + { + "epoch": 0.33, + "learning_rate": 4.9432128401012144e-05, + "logits/chosen": -2.5632007122039795, + "logits/rejected": -2.5047194957733154, + "logps/chosen": -185.19070434570312, + "logps/rejected": -204.64364624023438, + "loss": 0.7785, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.7219403386116028, + "rewards/margins": -0.07751601189374924, + "rewards/rejected": -0.6444243788719177, + "step": 249 + }, + { + "epoch": 0.33, + "learning_rate": 4.9424509862257706e-05, + "logits/chosen": -2.529867649078369, + "logits/rejected": -2.5607314109802246, + "logps/chosen": -219.9513397216797, + "logps/rejected": -240.11671447753906, + "loss": 0.6592, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.4247688055038452, + "rewards/margins": 0.10957615077495575, + "rewards/rejected": -0.5343449115753174, + "step": 250 + }, + { + "epoch": 0.33, + "learning_rate": 4.941684115346541e-05, + "logits/chosen": -2.7805376052856445, + "logits/rejected": -2.837836742401123, + "logps/chosen": -177.21543884277344, + "logps/rejected": -213.7078094482422, + "loss": 0.6014, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.5495901107788086, + "rewards/margins": 0.268564373254776, + "rewards/rejected": -0.8181545734405518, + "step": 251 + }, + { + "epoch": 0.33, + "learning_rate": 4.940912229038745e-05, + "logits/chosen": -2.56017804145813, + "logits/rejected": -2.5503697395324707, + "logps/chosen": -170.02735900878906, + "logps/rejected": -161.25509643554688, + "loss": 0.7194, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.5123621225357056, + "rewards/margins": -0.0268712155520916, + "rewards/rejected": -0.4854908883571625, + "step": 252 + }, + { + "epoch": 0.33, + "learning_rate": 4.9401353288879024e-05, + "logits/chosen": -2.5448572635650635, + "logits/rejected": -2.56659197807312, + "logps/chosen": -169.89077758789062, + "logps/rejected": -190.93545532226562, + "loss": 0.6295, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.4579932391643524, + "rewards/margins": 0.1804841011762619, + "rewards/rejected": -0.6384773254394531, + "step": 253 + }, + { + "epoch": 0.33, + "learning_rate": 4.9393534164898335e-05, + "logits/chosen": -2.55391526222229, + "logits/rejected": -2.5868449211120605, + "logps/chosen": -158.18197631835938, + "logps/rejected": -197.00271606445312, + "loss": 0.6476, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.6932105422019958, + "rewards/margins": 0.13315898180007935, + "rewards/rejected": -0.8263695240020752, + "step": 254 + }, + { + "epoch": 0.33, + "learning_rate": 4.9385664934506526e-05, + "logits/chosen": -2.554259777069092, + "logits/rejected": -2.684239625930786, + "logps/chosen": -159.8101348876953, + "logps/rejected": -183.30532836914062, + "loss": 0.6516, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.5770647525787354, + "rewards/margins": 0.14322030544281006, + "rewards/rejected": -0.7202850580215454, + "step": 255 + }, + { + "epoch": 0.34, + "learning_rate": 4.937774561386768e-05, + "logits/chosen": -2.5128979682922363, + "logits/rejected": -2.6388025283813477, + "logps/chosen": -174.58401489257812, + "logps/rejected": -184.96910095214844, + "loss": 0.577, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.5345829129219055, + "rewards/margins": 0.3504784107208252, + "rewards/rejected": -0.8850612640380859, + "step": 256 + }, + { + "epoch": 0.34, + "learning_rate": 4.936977621924875e-05, + "logits/chosen": -2.6937482357025146, + "logits/rejected": -2.650275707244873, + "logps/chosen": -165.79302978515625, + "logps/rejected": -177.26170349121094, + "loss": 0.6382, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.614943265914917, + "rewards/margins": 0.14949731528759003, + "rewards/rejected": -0.7644405961036682, + "step": 257 + }, + { + "epoch": 0.34, + "learning_rate": 4.9361756767019564e-05, + "logits/chosen": -2.5641212463378906, + "logits/rejected": -2.6175105571746826, + "logps/chosen": -182.44837951660156, + "logps/rejected": -225.40805053710938, + "loss": 0.692, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.6991927027702332, + "rewards/margins": 0.0908452570438385, + "rewards/rejected": -0.790037989616394, + "step": 258 + }, + { + "epoch": 0.34, + "learning_rate": 4.935368727365276e-05, + "logits/chosen": -2.6357033252716064, + "logits/rejected": -2.57110857963562, + "logps/chosen": -182.70291137695312, + "logps/rejected": -223.89857482910156, + "loss": 0.6278, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.7485102415084839, + "rewards/margins": 0.1697012186050415, + "rewards/rejected": -0.9182114601135254, + "step": 259 + }, + { + "epoch": 0.34, + "learning_rate": 4.934556775572377e-05, + "logits/chosen": -2.6168265342712402, + "logits/rejected": -2.6644468307495117, + "logps/chosen": -165.17758178710938, + "logps/rejected": -188.50119018554688, + "loss": 0.7554, + "rewards/accuracies": 0.25, + "rewards/chosen": -0.5862306356430054, + "rewards/margins": -0.08211595565080643, + "rewards/rejected": -0.5041146874427795, + "step": 260 + }, + { + "epoch": 0.34, + "learning_rate": 4.9337398229910784e-05, + "logits/chosen": -2.595906972885132, + "logits/rejected": -2.594231128692627, + "logps/chosen": -188.159912109375, + "logps/rejected": -176.15074157714844, + "loss": 0.6936, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.6705946922302246, + "rewards/margins": 0.03840280696749687, + "rewards/rejected": -0.7089974284172058, + "step": 261 + }, + { + "epoch": 0.34, + "learning_rate": 4.932917871299471e-05, + "logits/chosen": -2.672065496444702, + "logits/rejected": -2.6840691566467285, + "logps/chosen": -173.274169921875, + "logps/rejected": -183.5944061279297, + "loss": 0.6154, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.48323020339012146, + "rewards/margins": 0.2292642742395401, + "rewards/rejected": -0.7124944925308228, + "step": 262 + }, + { + "epoch": 0.34, + "learning_rate": 4.9320909221859134e-05, + "logits/chosen": -2.660583972930908, + "logits/rejected": -2.7142574787139893, + "logps/chosen": -192.77145385742188, + "logps/rejected": -185.00450134277344, + "loss": 0.6823, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.7117425799369812, + "rewards/margins": 0.10454593598842621, + "rewards/rejected": -0.8162885308265686, + "step": 263 + }, + { + "epoch": 0.35, + "learning_rate": 4.9312589773490304e-05, + "logits/chosen": -2.5676193237304688, + "logits/rejected": -2.510430335998535, + "logps/chosen": -176.65432739257812, + "logps/rejected": -154.3079833984375, + "loss": 0.7678, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.8498928546905518, + "rewards/margins": -0.05464668944478035, + "rewards/rejected": -0.7952461242675781, + "step": 264 + }, + { + "epoch": 0.35, + "learning_rate": 4.930422038497708e-05, + "logits/chosen": -2.5558042526245117, + "logits/rejected": -2.672940492630005, + "logps/chosen": -173.69500732421875, + "logps/rejected": -180.78118896484375, + "loss": 0.5704, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.5608630776405334, + "rewards/margins": 0.38767609000205994, + "rewards/rejected": -0.948539137840271, + "step": 265 + }, + { + "epoch": 0.35, + "learning_rate": 4.92958010735109e-05, + "logits/chosen": -2.508685350418091, + "logits/rejected": -2.4861252307891846, + "logps/chosen": -196.0287628173828, + "logps/rejected": -212.8455810546875, + "loss": 0.6453, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.6239847540855408, + "rewards/margins": 0.18039949238300323, + "rewards/rejected": -0.8043842315673828, + "step": 266 + }, + { + "epoch": 0.35, + "learning_rate": 4.928733185638575e-05, + "logits/chosen": -2.641526222229004, + "logits/rejected": -2.630765199661255, + "logps/chosen": -205.0409698486328, + "logps/rejected": -201.60269165039062, + "loss": 0.651, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.7301749587059021, + "rewards/margins": 0.1289985626935959, + "rewards/rejected": -0.859173595905304, + "step": 267 + }, + { + "epoch": 0.35, + "learning_rate": 4.927881275099815e-05, + "logits/chosen": -2.5749480724334717, + "logits/rejected": -2.5817878246307373, + "logps/chosen": -187.1511688232422, + "logps/rejected": -236.642822265625, + "loss": 0.6596, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.6212616562843323, + "rewards/margins": 0.13300594687461853, + "rewards/rejected": -0.7542675733566284, + "step": 268 + }, + { + "epoch": 0.35, + "learning_rate": 4.927024377484705e-05, + "logits/chosen": -2.6451222896575928, + "logits/rejected": -2.700425863265991, + "logps/chosen": -176.2154083251953, + "logps/rejected": -211.41744995117188, + "loss": 0.7046, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.8719490766525269, + "rewards/margins": 0.05133984610438347, + "rewards/rejected": -0.9232889413833618, + "step": 269 + }, + { + "epoch": 0.35, + "learning_rate": 4.9261624945533855e-05, + "logits/chosen": -2.5780887603759766, + "logits/rejected": -2.669583320617676, + "logps/chosen": -181.3977813720703, + "logps/rejected": -242.43626403808594, + "loss": 0.6873, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.6802850365638733, + "rewards/margins": 0.029887204989790916, + "rewards/rejected": -0.7101722955703735, + "step": 270 + }, + { + "epoch": 0.35, + "learning_rate": 4.925295628076241e-05, + "logits/chosen": -2.6090972423553467, + "logits/rejected": -2.683279275894165, + "logps/chosen": -192.7779541015625, + "logps/rejected": -241.75607299804688, + "loss": 0.6072, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.7047219276428223, + "rewards/margins": 0.2544190287590027, + "rewards/rejected": -0.9591410160064697, + "step": 271 + }, + { + "epoch": 0.36, + "learning_rate": 4.9244237798338866e-05, + "logits/chosen": -2.7260890007019043, + "logits/rejected": -2.7258718013763428, + "logps/chosen": -206.13421630859375, + "logps/rejected": -211.11663818359375, + "loss": 0.7085, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.855944037437439, + "rewards/margins": 0.07277999073266983, + "rewards/rejected": -0.9287241101264954, + "step": 272 + }, + { + "epoch": 0.36, + "learning_rate": 4.923546951617175e-05, + "logits/chosen": -2.648552417755127, + "logits/rejected": -2.5934343338012695, + "logps/chosen": -170.73721313476562, + "logps/rejected": -186.487548828125, + "loss": 0.6374, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.7491826415061951, + "rewards/margins": 0.1971735656261444, + "rewards/rejected": -0.9463562369346619, + "step": 273 + }, + { + "epoch": 0.36, + "learning_rate": 4.922665145227187e-05, + "logits/chosen": -2.5815610885620117, + "logits/rejected": -2.5234570503234863, + "logps/chosen": -153.86798095703125, + "logps/rejected": -140.2138671875, + "loss": 0.8232, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.9449439644813538, + "rewards/margins": -0.13928692042827606, + "rewards/rejected": -0.8056570291519165, + "step": 274 + }, + { + "epoch": 0.36, + "learning_rate": 4.9217783624752266e-05, + "logits/chosen": -2.4257960319519043, + "logits/rejected": -2.481285333633423, + "logps/chosen": -129.22506713867188, + "logps/rejected": -133.507080078125, + "loss": 0.6863, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.7729750275611877, + "rewards/margins": 0.07025709748268127, + "rewards/rejected": -0.8432320356369019, + "step": 275 + }, + { + "epoch": 0.36, + "learning_rate": 4.920886605182823e-05, + "logits/chosen": -2.8374218940734863, + "logits/rejected": -2.840282440185547, + "logps/chosen": -183.80465698242188, + "logps/rejected": -185.8097381591797, + "loss": 0.6878, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.8535721898078918, + "rewards/margins": 0.04765475541353226, + "rewards/rejected": -0.9012269377708435, + "step": 276 + }, + { + "epoch": 0.36, + "learning_rate": 4.919989875181722e-05, + "logits/chosen": -2.7011358737945557, + "logits/rejected": -2.758044719696045, + "logps/chosen": -176.79345703125, + "logps/rejected": -174.94239807128906, + "loss": 0.6994, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.159913182258606, + "rewards/margins": 0.09292294830083847, + "rewards/rejected": -1.2528361082077026, + "step": 277 + }, + { + "epoch": 0.36, + "learning_rate": 4.919088174313884e-05, + "logits/chosen": -2.614689826965332, + "logits/rejected": -2.645404577255249, + "logps/chosen": -134.0303955078125, + "logps/rejected": -162.16595458984375, + "loss": 0.6113, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.6662919521331787, + "rewards/margins": 0.26484376192092896, + "rewards/rejected": -0.9311355948448181, + "step": 278 + }, + { + "epoch": 0.37, + "learning_rate": 4.91818150443148e-05, + "logits/chosen": -2.6832685470581055, + "logits/rejected": -2.670450448989868, + "logps/chosen": -196.30052185058594, + "logps/rejected": -179.43482971191406, + "loss": 0.6394, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.9087074995040894, + "rewards/margins": 0.1679982841014862, + "rewards/rejected": -1.076705813407898, + "step": 279 + }, + { + "epoch": 0.37, + "learning_rate": 4.917269867396886e-05, + "logits/chosen": -2.8207449913024902, + "logits/rejected": -2.773184061050415, + "logps/chosen": -198.20509338378906, + "logps/rejected": -184.33151245117188, + "loss": 0.7475, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.9387542009353638, + "rewards/margins": 0.003928817808628082, + "rewards/rejected": -0.9426830410957336, + "step": 280 + }, + { + "epoch": 0.37, + "learning_rate": 4.916353265082686e-05, + "logits/chosen": -2.75034236907959, + "logits/rejected": -2.7130117416381836, + "logps/chosen": -187.1002197265625, + "logps/rejected": -193.2103271484375, + "loss": 0.8817, + "rewards/accuracies": 0.25, + "rewards/chosen": -1.0758837461471558, + "rewards/margins": -0.2836476266384125, + "rewards/rejected": -0.7922362089157104, + "step": 281 + }, + { + "epoch": 0.37, + "learning_rate": 4.9154316993716565e-05, + "logits/chosen": -2.7884602546691895, + "logits/rejected": -2.889566421508789, + "logps/chosen": -173.12225341796875, + "logps/rejected": -181.60842895507812, + "loss": 0.6636, + "rewards/accuracies": 0.4375, + "rewards/chosen": -1.0141631364822388, + "rewards/margins": 0.16198423504829407, + "rewards/rejected": -1.1761474609375, + "step": 282 + }, + { + "epoch": 0.37, + "learning_rate": 4.9145051721567734e-05, + "logits/chosen": -2.711033344268799, + "logits/rejected": -2.731414318084717, + "logps/chosen": -190.81759643554688, + "logps/rejected": -214.6211700439453, + "loss": 0.5886, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.9142743945121765, + "rewards/margins": 0.34260398149490356, + "rewards/rejected": -1.25687837600708, + "step": 283 + }, + { + "epoch": 0.37, + "learning_rate": 4.913573685341205e-05, + "logits/chosen": -2.534449338912964, + "logits/rejected": -2.6240017414093018, + "logps/chosen": -158.4725341796875, + "logps/rejected": -144.93075561523438, + "loss": 0.6633, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.8281882405281067, + "rewards/margins": 0.08525969088077545, + "rewards/rejected": -0.9134478569030762, + "step": 284 + }, + { + "epoch": 0.37, + "learning_rate": 4.9126372408383025e-05, + "logits/chosen": -2.8126165866851807, + "logits/rejected": -2.9268059730529785, + "logps/chosen": -165.80160522460938, + "logps/rejected": -195.13014221191406, + "loss": 0.6719, + "rewards/accuracies": 0.6875, + "rewards/chosen": -1.0856614112854004, + "rewards/margins": 0.1156027615070343, + "rewards/rejected": -1.2012642621994019, + "step": 285 + }, + { + "epoch": 0.37, + "learning_rate": 4.911695840571605e-05, + "logits/chosen": -2.8474719524383545, + "logits/rejected": -2.870950698852539, + "logps/chosen": -185.34619140625, + "logps/rejected": -204.24951171875, + "loss": 0.6981, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.9124755859375, + "rewards/margins": 0.032730571925640106, + "rewards/rejected": -0.9452061057090759, + "step": 286 + }, + { + "epoch": 0.38, + "learning_rate": 4.910749486474828e-05, + "logits/chosen": -2.7202816009521484, + "logits/rejected": -2.7738454341888428, + "logps/chosen": -183.0021514892578, + "logps/rejected": -176.14559936523438, + "loss": 0.7423, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.8500471711158752, + "rewards/margins": -0.009690776467323303, + "rewards/rejected": -0.8403564691543579, + "step": 287 + }, + { + "epoch": 0.38, + "learning_rate": 4.909798180491865e-05, + "logits/chosen": -2.8064842224121094, + "logits/rejected": -2.8379313945770264, + "logps/chosen": -181.36424255371094, + "logps/rejected": -186.806884765625, + "loss": 0.7156, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.9925443530082703, + "rewards/margins": 0.02362808585166931, + "rewards/rejected": -1.0161724090576172, + "step": 288 + }, + { + "epoch": 0.38, + "learning_rate": 4.9088419245767803e-05, + "logits/chosen": -2.591299057006836, + "logits/rejected": -2.6505491733551025, + "logps/chosen": -165.84970092773438, + "logps/rejected": -189.2133331298828, + "loss": 0.6588, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.6631417870521545, + "rewards/margins": 0.1258070468902588, + "rewards/rejected": -0.7889488339424133, + "step": 289 + }, + { + "epoch": 0.38, + "learning_rate": 4.907880720693804e-05, + "logits/chosen": -2.9960107803344727, + "logits/rejected": -2.9374802112579346, + "logps/chosen": -213.4750518798828, + "logps/rejected": -237.82672119140625, + "loss": 0.656, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.8233738541603088, + "rewards/margins": 0.1019565686583519, + "rewards/rejected": -0.9253304600715637, + "step": 290 + }, + { + "epoch": 0.38, + "learning_rate": 4.9069145708173324e-05, + "logits/chosen": -2.6092634201049805, + "logits/rejected": -2.6358261108398438, + "logps/chosen": -197.22938537597656, + "logps/rejected": -190.7332000732422, + "loss": 0.613, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.7702199220657349, + "rewards/margins": 0.23700743913650513, + "rewards/rejected": -1.0072274208068848, + "step": 291 + }, + { + "epoch": 0.38, + "learning_rate": 4.9059434769319205e-05, + "logits/chosen": -2.731299638748169, + "logits/rejected": -2.8226795196533203, + "logps/chosen": -202.76451110839844, + "logps/rejected": -244.38844299316406, + "loss": 0.589, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.6375839114189148, + "rewards/margins": 0.2866172790527344, + "rewards/rejected": -0.9242011904716492, + "step": 292 + }, + { + "epoch": 0.38, + "learning_rate": 4.904967441032278e-05, + "logits/chosen": -2.478205680847168, + "logits/rejected": -2.541795253753662, + "logps/chosen": -195.00408935546875, + "logps/rejected": -225.96731567382812, + "loss": 0.6568, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.6428064107894897, + "rewards/margins": 0.17480693757534027, + "rewards/rejected": -0.8176133632659912, + "step": 293 + }, + { + "epoch": 0.38, + "learning_rate": 4.903986465123266e-05, + "logits/chosen": -2.692394733428955, + "logits/rejected": -2.7363274097442627, + "logps/chosen": -167.2650146484375, + "logps/rejected": -220.62228393554688, + "loss": 0.7124, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.8301478624343872, + "rewards/margins": 0.07167816907167435, + "rewards/rejected": -0.901826024055481, + "step": 294 + }, + { + "epoch": 0.39, + "learning_rate": 4.903000551219894e-05, + "logits/chosen": -2.782505989074707, + "logits/rejected": -2.81825590133667, + "logps/chosen": -153.23898315429688, + "logps/rejected": -157.7719268798828, + "loss": 0.7488, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.8630144596099854, + "rewards/margins": -0.03182988613843918, + "rewards/rejected": -0.8311845660209656, + "step": 295 + }, + { + "epoch": 0.39, + "learning_rate": 4.902009701347313e-05, + "logits/chosen": -2.7017691135406494, + "logits/rejected": -2.751739978790283, + "logps/chosen": -199.3386688232422, + "logps/rejected": -189.38465881347656, + "loss": 0.7063, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.7444071769714355, + "rewards/margins": 0.05856693163514137, + "rewards/rejected": -0.8029740452766418, + "step": 296 + }, + { + "epoch": 0.39, + "learning_rate": 4.901013917540814e-05, + "logits/chosen": -2.62956166267395, + "logits/rejected": -2.65138578414917, + "logps/chosen": -209.6748046875, + "logps/rejected": -200.0463409423828, + "loss": 0.6986, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.778890073299408, + "rewards/margins": 0.03998234495520592, + "rewards/rejected": -0.8188724517822266, + "step": 297 + }, + { + "epoch": 0.39, + "learning_rate": 4.900013201845821e-05, + "logits/chosen": -2.604013204574585, + "logits/rejected": -2.6575911045074463, + "logps/chosen": -192.33749389648438, + "logps/rejected": -196.74574279785156, + "loss": 0.7202, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.7086251378059387, + "rewards/margins": 0.026372164487838745, + "rewards/rejected": -0.7349973917007446, + "step": 298 + }, + { + "epoch": 0.39, + "learning_rate": 4.899007556317893e-05, + "logits/chosen": -2.672982931137085, + "logits/rejected": -2.7704596519470215, + "logps/chosen": -232.0260009765625, + "logps/rejected": -230.7442169189453, + "loss": 0.666, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.703597366809845, + "rewards/margins": 0.1020139679312706, + "rewards/rejected": -0.8056113123893738, + "step": 299 + }, + { + "epoch": 0.39, + "learning_rate": 4.8979969830227086e-05, + "logits/chosen": -2.7685508728027344, + "logits/rejected": -2.798677921295166, + "logps/chosen": -170.82254028320312, + "logps/rejected": -211.26219177246094, + "loss": 0.6242, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.6501718759536743, + "rewards/margins": 0.23585857450962067, + "rewards/rejected": -0.8860303163528442, + "step": 300 + }, + { + "epoch": 0.39, + "learning_rate": 4.896981484036074e-05, + "logits/chosen": -2.742246150970459, + "logits/rejected": -2.7133634090423584, + "logps/chosen": -188.86024475097656, + "logps/rejected": -199.79791259765625, + "loss": 0.5836, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.5299299955368042, + "rewards/margins": 0.2819333076477051, + "rewards/rejected": -0.8118634223937988, + "step": 301 + }, + { + "epoch": 0.4, + "learning_rate": 4.895961061443911e-05, + "logits/chosen": -2.726637840270996, + "logits/rejected": -2.728475332260132, + "logps/chosen": -204.32278442382812, + "logps/rejected": -233.36709594726562, + "loss": 0.8344, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.8258933424949646, + "rewards/margins": -0.08616884052753448, + "rewards/rejected": -0.7397244572639465, + "step": 302 + }, + { + "epoch": 0.4, + "learning_rate": 4.894935717342255e-05, + "logits/chosen": -2.757063865661621, + "logits/rejected": -2.7701942920684814, + "logps/chosen": -192.92251586914062, + "logps/rejected": -192.76185607910156, + "loss": 0.7007, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.71357262134552, + "rewards/margins": 0.04112683981657028, + "rewards/rejected": -0.7546994686126709, + "step": 303 + }, + { + "epoch": 0.4, + "learning_rate": 4.8939054538372496e-05, + "logits/chosen": -2.6160852909088135, + "logits/rejected": -2.6638543605804443, + "logps/chosen": -153.6888427734375, + "logps/rejected": -204.19139099121094, + "loss": 0.6179, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.5954157710075378, + "rewards/margins": 0.2527124583721161, + "rewards/rejected": -0.8481282591819763, + "step": 304 + }, + { + "epoch": 0.4, + "learning_rate": 4.8928702730451456e-05, + "logits/chosen": -2.635458469390869, + "logits/rejected": -2.740834951400757, + "logps/chosen": -229.58181762695312, + "logps/rejected": -209.87832641601562, + "loss": 0.7217, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.7451784610748291, + "rewards/margins": 0.019871072843670845, + "rewards/rejected": -0.7650495171546936, + "step": 305 + }, + { + "epoch": 0.4, + "learning_rate": 4.891830177092294e-05, + "logits/chosen": -2.515532970428467, + "logits/rejected": -2.5850772857666016, + "logps/chosen": -173.58749389648438, + "logps/rejected": -197.440185546875, + "loss": 0.6521, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.6022768616676331, + "rewards/margins": 0.11426748335361481, + "rewards/rejected": -0.7165443301200867, + "step": 306 + }, + { + "epoch": 0.4, + "learning_rate": 4.8907851681151396e-05, + "logits/chosen": -2.708148956298828, + "logits/rejected": -2.735069751739502, + "logps/chosen": -146.8754425048828, + "logps/rejected": -182.95477294921875, + "loss": 0.6006, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.35234156250953674, + "rewards/margins": 0.24402308464050293, + "rewards/rejected": -0.5963646173477173, + "step": 307 + }, + { + "epoch": 0.4, + "learning_rate": 4.889735248260221e-05, + "logits/chosen": -2.657132625579834, + "logits/rejected": -2.7396936416625977, + "logps/chosen": -172.76268005371094, + "logps/rejected": -185.73153686523438, + "loss": 0.6844, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.7841547727584839, + "rewards/margins": 0.08935101330280304, + "rewards/rejected": -0.8735058307647705, + "step": 308 + }, + { + "epoch": 0.4, + "learning_rate": 4.8886804196841626e-05, + "logits/chosen": -2.6601943969726562, + "logits/rejected": -2.6764605045318604, + "logps/chosen": -182.19937133789062, + "logps/rejected": -196.57550048828125, + "loss": 0.6327, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.7912774085998535, + "rewards/margins": 0.19412587583065033, + "rewards/rejected": -0.9854032397270203, + "step": 309 + }, + { + "epoch": 0.41, + "learning_rate": 4.887620684553674e-05, + "logits/chosen": -2.5685248374938965, + "logits/rejected": -2.5472164154052734, + "logps/chosen": -161.2342529296875, + "logps/rejected": -201.6751708984375, + "loss": 0.683, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.6504772901535034, + "rewards/margins": 0.089483842253685, + "rewards/rejected": -0.7399611473083496, + "step": 310 + }, + { + "epoch": 0.41, + "learning_rate": 4.886556045045542e-05, + "logits/chosen": -2.7824556827545166, + "logits/rejected": -2.772510528564453, + "logps/chosen": -183.0452880859375, + "logps/rejected": -197.40408325195312, + "loss": 0.8643, + "rewards/accuracies": 0.375, + "rewards/chosen": -1.0302786827087402, + "rewards/margins": -0.2259717881679535, + "rewards/rejected": -0.8043068647384644, + "step": 311 + }, + { + "epoch": 0.41, + "learning_rate": 4.8854865033466275e-05, + "logits/chosen": -2.457235097885132, + "logits/rejected": -2.5225629806518555, + "logps/chosen": -137.17259216308594, + "logps/rejected": -151.13467407226562, + "loss": 0.5958, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.6421889662742615, + "rewards/margins": 0.3034355044364929, + "rewards/rejected": -0.9456245303153992, + "step": 312 + }, + { + "epoch": 0.41, + "learning_rate": 4.88441206165386e-05, + "logits/chosen": -2.7572522163391113, + "logits/rejected": -2.7597944736480713, + "logps/chosen": -194.38626098632812, + "logps/rejected": -208.2905731201172, + "loss": 0.848, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.8563791513442993, + "rewards/margins": -0.21535280346870422, + "rewards/rejected": -0.6410263180732727, + "step": 313 + }, + { + "epoch": 0.41, + "learning_rate": 4.8833327221742356e-05, + "logits/chosen": -2.589763641357422, + "logits/rejected": -2.5469565391540527, + "logps/chosen": -159.60533142089844, + "logps/rejected": -153.5530548095703, + "loss": 0.6174, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.6452800035476685, + "rewards/margins": 0.2377607673406601, + "rewards/rejected": -0.8830407857894897, + "step": 314 + }, + { + "epoch": 0.41, + "learning_rate": 4.88224848712481e-05, + "logits/chosen": -2.6565144062042236, + "logits/rejected": -2.694272994995117, + "logps/chosen": -175.2017059326172, + "logps/rejected": -171.8043975830078, + "loss": 0.6785, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.514467179775238, + "rewards/margins": 0.11190642416477203, + "rewards/rejected": -0.6263736486434937, + "step": 315 + }, + { + "epoch": 0.41, + "learning_rate": 4.881159358732694e-05, + "logits/chosen": -2.5067477226257324, + "logits/rejected": -2.551682710647583, + "logps/chosen": -187.95533752441406, + "logps/rejected": -237.65907287597656, + "loss": 0.6805, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.6460933685302734, + "rewards/margins": 0.15910674631595612, + "rewards/rejected": -0.8052000403404236, + "step": 316 + }, + { + "epoch": 0.41, + "learning_rate": 4.8800653392350526e-05, + "logits/chosen": -2.4550201892852783, + "logits/rejected": -2.519341468811035, + "logps/chosen": -156.3563690185547, + "logps/rejected": -171.7753143310547, + "loss": 0.7012, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.7904012799263, + "rewards/margins": 0.05794687569141388, + "rewards/rejected": -0.8483481407165527, + "step": 317 + }, + { + "epoch": 0.42, + "learning_rate": 4.8789664308790936e-05, + "logits/chosen": -2.7174458503723145, + "logits/rejected": -2.6885735988616943, + "logps/chosen": -172.3370361328125, + "logps/rejected": -164.46519470214844, + "loss": 0.6861, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.6009785532951355, + "rewards/margins": 0.08145736157894135, + "rewards/rejected": -0.682435929775238, + "step": 318 + }, + { + "epoch": 0.42, + "learning_rate": 4.8778626359220715e-05, + "logits/chosen": -2.731224536895752, + "logits/rejected": -2.6857399940490723, + "logps/chosen": -185.449951171875, + "logps/rejected": -206.4256591796875, + "loss": 0.7672, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.7531261444091797, + "rewards/margins": -0.07965384423732758, + "rewards/rejected": -0.6734722852706909, + "step": 319 + }, + { + "epoch": 0.42, + "learning_rate": 4.8767539566312734e-05, + "logits/chosen": -2.5604355335235596, + "logits/rejected": -2.704071044921875, + "logps/chosen": -174.9366912841797, + "logps/rejected": -202.98768615722656, + "loss": 0.6137, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.6721182465553284, + "rewards/margins": 0.21909648180007935, + "rewards/rejected": -0.8912147283554077, + "step": 320 + }, + { + "epoch": 0.42, + "learning_rate": 4.875640395284023e-05, + "logits/chosen": -2.758918046951294, + "logits/rejected": -2.7820916175842285, + "logps/chosen": -195.2515869140625, + "logps/rejected": -231.6067352294922, + "loss": 0.5285, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.38608187437057495, + "rewards/margins": 0.4270917773246765, + "rewards/rejected": -0.8131736516952515, + "step": 321 + }, + { + "epoch": 0.42, + "learning_rate": 4.874521954167671e-05, + "logits/chosen": -2.7890138626098633, + "logits/rejected": -2.8068411350250244, + "logps/chosen": -209.79307556152344, + "logps/rejected": -207.8020477294922, + "loss": 0.6367, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.6039459109306335, + "rewards/margins": 0.1874314397573471, + "rewards/rejected": -0.7913773059844971, + "step": 322 + }, + { + "epoch": 0.42, + "learning_rate": 4.8733986355795905e-05, + "logits/chosen": -2.675286054611206, + "logits/rejected": -2.699337959289551, + "logps/chosen": -233.40780639648438, + "logps/rejected": -208.18724060058594, + "loss": 0.6598, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.6822580695152283, + "rewards/margins": 0.1497945785522461, + "rewards/rejected": -0.8320526480674744, + "step": 323 + }, + { + "epoch": 0.42, + "learning_rate": 4.8722704418271745e-05, + "logits/chosen": -2.5140862464904785, + "logits/rejected": -2.632192850112915, + "logps/chosen": -176.1000213623047, + "logps/rejected": -196.43846130371094, + "loss": 0.7119, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.7464176416397095, + "rewards/margins": 0.01839565485715866, + "rewards/rejected": -0.7648133039474487, + "step": 324 + }, + { + "epoch": 0.43, + "learning_rate": 4.871137375227829e-05, + "logits/chosen": -2.6084189414978027, + "logits/rejected": -2.6140480041503906, + "logps/chosen": -188.7124481201172, + "logps/rejected": -177.5293426513672, + "loss": 0.748, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.618392288684845, + "rewards/margins": 0.016466360539197922, + "rewards/rejected": -0.6348586678504944, + "step": 325 + }, + { + "epoch": 0.43, + "learning_rate": 4.869999438108971e-05, + "logits/chosen": -2.6994099617004395, + "logits/rejected": -2.743457555770874, + "logps/chosen": -182.1778564453125, + "logps/rejected": -194.54884338378906, + "loss": 0.6603, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.5676161050796509, + "rewards/margins": 0.1599595993757248, + "rewards/rejected": -0.7275756597518921, + "step": 326 + }, + { + "epoch": 0.43, + "learning_rate": 4.8688566328080215e-05, + "logits/chosen": -2.5730295181274414, + "logits/rejected": -2.573648452758789, + "logps/chosen": -199.49908447265625, + "logps/rejected": -235.17568969726562, + "loss": 0.5873, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.379108190536499, + "rewards/margins": 0.30178508162498474, + "rewards/rejected": -0.6808933019638062, + "step": 327 + }, + { + "epoch": 0.43, + "learning_rate": 4.867708961672399e-05, + "logits/chosen": -2.5466060638427734, + "logits/rejected": -2.661322593688965, + "logps/chosen": -183.09585571289062, + "logps/rejected": -185.6378936767578, + "loss": 0.666, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.49351951479911804, + "rewards/margins": 0.11638712882995605, + "rewards/rejected": -0.6099066734313965, + "step": 328 + }, + { + "epoch": 0.43, + "learning_rate": 4.866556427059519e-05, + "logits/chosen": -2.7101516723632812, + "logits/rejected": -2.644563913345337, + "logps/chosen": -187.73348999023438, + "logps/rejected": -169.40293884277344, + "loss": 0.7781, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.8404239416122437, + "rewards/margins": -0.026825089007616043, + "rewards/rejected": -0.8135988116264343, + "step": 329 + }, + { + "epoch": 0.43, + "learning_rate": 4.865399031336787e-05, + "logits/chosen": -2.4861361980438232, + "logits/rejected": -2.565809726715088, + "logps/chosen": -146.98423767089844, + "logps/rejected": -163.1490478515625, + "loss": 0.7333, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5940142869949341, + "rewards/margins": 0.03410058468580246, + "rewards/rejected": -0.6281149387359619, + "step": 330 + }, + { + "epoch": 0.43, + "learning_rate": 4.8642367768815936e-05, + "logits/chosen": -2.6412410736083984, + "logits/rejected": -2.747821807861328, + "logps/chosen": -174.8326873779297, + "logps/rejected": -219.83612060546875, + "loss": 0.5947, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.4866630434989929, + "rewards/margins": 0.2575409412384033, + "rewards/rejected": -0.744204044342041, + "step": 331 + }, + { + "epoch": 0.43, + "learning_rate": 4.863069666081307e-05, + "logits/chosen": -2.6463019847869873, + "logits/rejected": -2.796405553817749, + "logps/chosen": -164.44869995117188, + "logps/rejected": -217.47230529785156, + "loss": 0.6092, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.5829634070396423, + "rewards/margins": 0.26363080739974976, + "rewards/rejected": -0.8465942144393921, + "step": 332 + }, + { + "epoch": 0.44, + "learning_rate": 4.861897701333274e-05, + "logits/chosen": -2.6675515174865723, + "logits/rejected": -2.6761202812194824, + "logps/chosen": -187.8043212890625, + "logps/rejected": -175.5827178955078, + "loss": 0.8043, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.8770172595977783, + "rewards/margins": -0.06917458772659302, + "rewards/rejected": -0.8078427314758301, + "step": 333 + }, + { + "epoch": 0.44, + "learning_rate": 4.86072088504481e-05, + "logits/chosen": -2.6193923950195312, + "logits/rejected": -2.652172088623047, + "logps/chosen": -180.9799346923828, + "logps/rejected": -197.08245849609375, + "loss": 0.756, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.8450060486793518, + "rewards/margins": -0.008023982867598534, + "rewards/rejected": -0.8369821310043335, + "step": 334 + }, + { + "epoch": 0.44, + "learning_rate": 4.859539219633199e-05, + "logits/chosen": -2.4201056957244873, + "logits/rejected": -2.547577381134033, + "logps/chosen": -142.2472381591797, + "logps/rejected": -171.77197265625, + "loss": 0.5808, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.25414225459098816, + "rewards/margins": 0.2882387042045593, + "rewards/rejected": -0.5423809289932251, + "step": 335 + }, + { + "epoch": 0.44, + "learning_rate": 4.8583527075256804e-05, + "logits/chosen": -2.6075916290283203, + "logits/rejected": -2.6171302795410156, + "logps/chosen": -185.60699462890625, + "logps/rejected": -199.539306640625, + "loss": 0.6039, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.6799944043159485, + "rewards/margins": 0.21594738960266113, + "rewards/rejected": -0.8959417939186096, + "step": 336 + }, + { + "epoch": 0.44, + "learning_rate": 4.857161351159454e-05, + "logits/chosen": -2.7377195358276367, + "logits/rejected": -2.722515821456909, + "logps/chosen": -225.74755859375, + "logps/rejected": -224.8936004638672, + "loss": 0.6793, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.8174278140068054, + "rewards/margins": 0.10962995886802673, + "rewards/rejected": -0.9270578026771545, + "step": 337 + }, + { + "epoch": 0.44, + "learning_rate": 4.8559651529816664e-05, + "logits/chosen": -2.597797393798828, + "logits/rejected": -2.6779723167419434, + "logps/chosen": -159.58380126953125, + "logps/rejected": -185.880615234375, + "loss": 0.679, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.7312408089637756, + "rewards/margins": 0.1112150177359581, + "rewards/rejected": -0.8424558639526367, + "step": 338 + }, + { + "epoch": 0.44, + "learning_rate": 4.854764115449411e-05, + "logits/chosen": -2.7188682556152344, + "logits/rejected": -2.691462993621826, + "logps/chosen": -143.06845092773438, + "logps/rejected": -142.25157165527344, + "loss": 0.6675, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.675825297832489, + "rewards/margins": 0.130048006772995, + "rewards/rejected": -0.8058732748031616, + "step": 339 + }, + { + "epoch": 0.45, + "learning_rate": 4.853558241029723e-05, + "logits/chosen": -2.629163980484009, + "logits/rejected": -2.5702826976776123, + "logps/chosen": -224.11196899414062, + "logps/rejected": -171.19097900390625, + "loss": 0.718, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.6522189378738403, + "rewards/margins": 0.046554647386074066, + "rewards/rejected": -0.6987735629081726, + "step": 340 + }, + { + "epoch": 0.45, + "learning_rate": 4.8523475321995715e-05, + "logits/chosen": -2.7033562660217285, + "logits/rejected": -2.5383901596069336, + "logps/chosen": -181.98219299316406, + "logps/rejected": -172.13865661621094, + "loss": 0.719, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.5484719276428223, + "rewards/margins": 0.0074146464467048645, + "rewards/rejected": -0.5558865666389465, + "step": 341 + }, + { + "epoch": 0.45, + "learning_rate": 4.8511319914458555e-05, + "logits/chosen": -2.5836706161499023, + "logits/rejected": -2.5528974533081055, + "logps/chosen": -223.9678497314453, + "logps/rejected": -211.67184448242188, + "loss": 0.7904, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.7669371366500854, + "rewards/margins": -0.11547183990478516, + "rewards/rejected": -0.6514652967453003, + "step": 342 + }, + { + "epoch": 0.45, + "learning_rate": 4.849911621265401e-05, + "logits/chosen": -2.652245044708252, + "logits/rejected": -2.6907477378845215, + "logps/chosen": -172.45669555664062, + "logps/rejected": -173.8350067138672, + "loss": 0.7916, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.713798999786377, + "rewards/margins": -0.09893038123846054, + "rewards/rejected": -0.6148686408996582, + "step": 343 + }, + { + "epoch": 0.45, + "learning_rate": 4.848686424164953e-05, + "logits/chosen": -2.703127384185791, + "logits/rejected": -2.704859733581543, + "logps/chosen": -214.36392211914062, + "logps/rejected": -185.55931091308594, + "loss": 0.7281, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.7356032133102417, + "rewards/margins": -0.018105890601873398, + "rewards/rejected": -0.7174972891807556, + "step": 344 + }, + { + "epoch": 0.45, + "learning_rate": 4.84745640266117e-05, + "logits/chosen": -2.690885066986084, + "logits/rejected": -2.740234851837158, + "logps/chosen": -176.29266357421875, + "logps/rejected": -206.47596740722656, + "loss": 0.5979, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.6022865176200867, + "rewards/margins": 0.31845831871032715, + "rewards/rejected": -0.9207448363304138, + "step": 345 + }, + { + "epoch": 0.45, + "learning_rate": 4.846221559280624e-05, + "logits/chosen": -2.6713180541992188, + "logits/rejected": -2.7118892669677734, + "logps/chosen": -144.1902313232422, + "logps/rejected": -170.08709716796875, + "loss": 0.7913, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.737549364566803, + "rewards/margins": -0.0928279459476471, + "rewards/rejected": -0.6447213888168335, + "step": 346 + }, + { + "epoch": 0.45, + "learning_rate": 4.844981896559787e-05, + "logits/chosen": -2.7589731216430664, + "logits/rejected": -2.6973843574523926, + "logps/chosen": -185.05255126953125, + "logps/rejected": -177.13946533203125, + "loss": 0.7678, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.8261479735374451, + "rewards/margins": -0.09880837798118591, + "rewards/rejected": -0.7273396253585815, + "step": 347 + }, + { + "epoch": 0.46, + "learning_rate": 4.8437374170450344e-05, + "logits/chosen": -2.659578323364258, + "logits/rejected": -2.617631196975708, + "logps/chosen": -175.1558074951172, + "logps/rejected": -198.41343688964844, + "loss": 0.6794, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.8928545117378235, + "rewards/margins": 0.10537480562925339, + "rewards/rejected": -0.9982293844223022, + "step": 348 + }, + { + "epoch": 0.46, + "learning_rate": 4.842488123292632e-05, + "logits/chosen": -2.6015379428863525, + "logits/rejected": -2.5835628509521484, + "logps/chosen": -188.88092041015625, + "logps/rejected": -183.85733032226562, + "loss": 0.6908, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.7475460171699524, + "rewards/margins": 0.20796708762645721, + "rewards/rejected": -0.9555131196975708, + "step": 349 + }, + { + "epoch": 0.46, + "learning_rate": 4.8412340178687374e-05, + "logits/chosen": -2.652589797973633, + "logits/rejected": -2.7427022457122803, + "logps/chosen": -198.62112426757812, + "logps/rejected": -206.33192443847656, + "loss": 0.5545, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.6447871923446655, + "rewards/margins": 0.38781607151031494, + "rewards/rejected": -1.0326032638549805, + "step": 350 + }, + { + "epoch": 0.46, + "learning_rate": 4.839975103349391e-05, + "logits/chosen": -2.79940128326416, + "logits/rejected": -2.8857078552246094, + "logps/chosen": -206.7621307373047, + "logps/rejected": -231.99472045898438, + "loss": 0.5811, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.5604078769683838, + "rewards/margins": 0.5052842497825623, + "rewards/rejected": -1.0656920671463013, + "step": 351 + }, + { + "epoch": 0.46, + "learning_rate": 4.8387113823205096e-05, + "logits/chosen": -2.562626361846924, + "logits/rejected": -2.5439820289611816, + "logps/chosen": -198.9171142578125, + "logps/rejected": -184.92816162109375, + "loss": 0.7251, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.6547099351882935, + "rewards/margins": 0.07420587539672852, + "rewards/rejected": -0.7289157509803772, + "step": 352 + }, + { + "epoch": 0.46, + "learning_rate": 4.8374428573778864e-05, + "logits/chosen": -2.6110103130340576, + "logits/rejected": -2.612607479095459, + "logps/chosen": -176.07400512695312, + "logps/rejected": -181.38418579101562, + "loss": 0.8065, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.758232593536377, + "rewards/margins": -0.0620691180229187, + "rewards/rejected": -0.6961634755134583, + "step": 353 + }, + { + "epoch": 0.46, + "learning_rate": 4.8361695311271795e-05, + "logits/chosen": -2.8273181915283203, + "logits/rejected": -2.830404758453369, + "logps/chosen": -205.66763305664062, + "logps/rejected": -222.3781280517578, + "loss": 0.671, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.8250411152839661, + "rewards/margins": 0.13250316679477692, + "rewards/rejected": -0.9575443267822266, + "step": 354 + }, + { + "epoch": 0.46, + "learning_rate": 4.83489140618391e-05, + "logits/chosen": -2.666761875152588, + "logits/rejected": -2.757248878479004, + "logps/chosen": -215.3235626220703, + "logps/rejected": -216.06568908691406, + "loss": 0.6942, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.7060136795043945, + "rewards/margins": 0.16570770740509033, + "rewards/rejected": -0.8717214465141296, + "step": 355 + }, + { + "epoch": 0.47, + "learning_rate": 4.833608485173457e-05, + "logits/chosen": -2.3725719451904297, + "logits/rejected": -2.4071693420410156, + "logps/chosen": -146.78704833984375, + "logps/rejected": -186.87950134277344, + "loss": 0.6066, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.4698937237262726, + "rewards/margins": 0.2167559713125229, + "rewards/rejected": -0.6866496801376343, + "step": 356 + }, + { + "epoch": 0.47, + "learning_rate": 4.8323207707310496e-05, + "logits/chosen": -2.6677677631378174, + "logits/rejected": -2.5784990787506104, + "logps/chosen": -204.05075073242188, + "logps/rejected": -224.5699920654297, + "loss": 0.5351, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.4006405174732208, + "rewards/margins": 0.5226905941963196, + "rewards/rejected": -0.923331081867218, + "step": 357 + }, + { + "epoch": 0.47, + "learning_rate": 4.831028265501764e-05, + "logits/chosen": -2.6300296783447266, + "logits/rejected": -2.810920238494873, + "logps/chosen": -177.45635986328125, + "logps/rejected": -234.4469451904297, + "loss": 0.5034, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.4587719440460205, + "rewards/margins": 0.5265656113624573, + "rewards/rejected": -0.9853376150131226, + "step": 358 + }, + { + "epoch": 0.47, + "learning_rate": 4.829730972140517e-05, + "logits/chosen": -2.5978078842163086, + "logits/rejected": -2.570225954055786, + "logps/chosen": -133.3565673828125, + "logps/rejected": -136.15902709960938, + "loss": 0.632, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.3776158094406128, + "rewards/margins": 0.1921352744102478, + "rewards/rejected": -0.5697510242462158, + "step": 359 + }, + { + "epoch": 0.47, + "learning_rate": 4.8284288933120594e-05, + "logits/chosen": -2.5982682704925537, + "logits/rejected": -2.6651642322540283, + "logps/chosen": -181.2775115966797, + "logps/rejected": -203.33876037597656, + "loss": 0.5832, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.4960383474826813, + "rewards/margins": 0.31738704442977905, + "rewards/rejected": -0.8134254217147827, + "step": 360 + }, + { + "epoch": 0.47, + "learning_rate": 4.8271220316909735e-05, + "logits/chosen": -2.761739730834961, + "logits/rejected": -2.715710401535034, + "logps/chosen": -191.56158447265625, + "logps/rejected": -175.9281463623047, + "loss": 0.7011, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.62202388048172, + "rewards/margins": 0.03054755926132202, + "rewards/rejected": -0.652571439743042, + "step": 361 + }, + { + "epoch": 0.47, + "learning_rate": 4.825810389961666e-05, + "logits/chosen": -2.614866018295288, + "logits/rejected": -2.6899471282958984, + "logps/chosen": -135.42494201660156, + "logps/rejected": -168.2887725830078, + "loss": 0.6542, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.5677900910377502, + "rewards/margins": 0.13706976175308228, + "rewards/rejected": -0.7048598527908325, + "step": 362 + }, + { + "epoch": 0.48, + "learning_rate": 4.8244939708183596e-05, + "logits/chosen": -2.643815040588379, + "logits/rejected": -2.6820266246795654, + "logps/chosen": -178.99908447265625, + "logps/rejected": -212.68692016601562, + "loss": 0.5719, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.5018581748008728, + "rewards/margins": 0.3394116163253784, + "rewards/rejected": -0.841269850730896, + "step": 363 + }, + { + "epoch": 0.48, + "learning_rate": 4.823172776965094e-05, + "logits/chosen": -2.702993869781494, + "logits/rejected": -2.7458367347717285, + "logps/chosen": -239.1004638671875, + "logps/rejected": -227.72450256347656, + "loss": 0.6493, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.4328669309616089, + "rewards/margins": 0.1465311497449875, + "rewards/rejected": -0.5793980360031128, + "step": 364 + }, + { + "epoch": 0.48, + "learning_rate": 4.821846811115713e-05, + "logits/chosen": -2.658914804458618, + "logits/rejected": -2.728074073791504, + "logps/chosen": -207.8568115234375, + "logps/rejected": -194.11788940429688, + "loss": 0.643, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.5129541754722595, + "rewards/margins": 0.19354072213172913, + "rewards/rejected": -0.706494927406311, + "step": 365 + }, + { + "epoch": 0.48, + "learning_rate": 4.820516075993865e-05, + "logits/chosen": -2.741415023803711, + "logits/rejected": -2.6675500869750977, + "logps/chosen": -156.90858459472656, + "logps/rejected": -166.85830688476562, + "loss": 0.8793, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.9281965494155884, + "rewards/margins": -0.2124374508857727, + "rewards/rejected": -0.7157591581344604, + "step": 366 + }, + { + "epoch": 0.48, + "learning_rate": 4.819180574332994e-05, + "logits/chosen": -2.7196927070617676, + "logits/rejected": -2.748945713043213, + "logps/chosen": -202.15493774414062, + "logps/rejected": -194.124755859375, + "loss": 0.7879, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.8033719062805176, + "rewards/margins": -0.02236950770020485, + "rewards/rejected": -0.7810022830963135, + "step": 367 + }, + { + "epoch": 0.48, + "learning_rate": 4.8178403088763355e-05, + "logits/chosen": -2.678246259689331, + "logits/rejected": -2.729729652404785, + "logps/chosen": -160.1791534423828, + "logps/rejected": -185.8133544921875, + "loss": 0.7682, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.5224413871765137, + "rewards/margins": -0.03584878519177437, + "rewards/rejected": -0.4865925908088684, + "step": 368 + }, + { + "epoch": 0.48, + "learning_rate": 4.8164952823769085e-05, + "logits/chosen": -2.649317979812622, + "logits/rejected": -2.655850410461426, + "logps/chosen": -241.83953857421875, + "logps/rejected": -226.36309814453125, + "loss": 0.6643, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.6058496236801147, + "rewards/margins": 0.16187947988510132, + "rewards/rejected": -0.7677291035652161, + "step": 369 + }, + { + "epoch": 0.48, + "learning_rate": 4.815145497597514e-05, + "logits/chosen": -2.711923837661743, + "logits/rejected": -2.7140746116638184, + "logps/chosen": -157.05551147460938, + "logps/rejected": -153.10430908203125, + "loss": 0.7461, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.6182957291603088, + "rewards/margins": -0.003988802433013916, + "rewards/rejected": -0.6143069267272949, + "step": 370 + }, + { + "epoch": 0.49, + "learning_rate": 4.8137909573107246e-05, + "logits/chosen": -2.632993459701538, + "logits/rejected": -2.7051479816436768, + "logps/chosen": -193.94879150390625, + "logps/rejected": -189.07920837402344, + "loss": 0.6392, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.6661685705184937, + "rewards/margins": 0.24933494627475739, + "rewards/rejected": -0.9155035614967346, + "step": 371 + }, + { + "epoch": 0.49, + "learning_rate": 4.812431664298883e-05, + "logits/chosen": -2.8046534061431885, + "logits/rejected": -2.7321465015411377, + "logps/chosen": -205.9781951904297, + "logps/rejected": -212.83636474609375, + "loss": 0.6998, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.49066948890686035, + "rewards/margins": 0.02486901544034481, + "rewards/rejected": -0.5155385136604309, + "step": 372 + }, + { + "epoch": 0.49, + "learning_rate": 4.811067621354094e-05, + "logits/chosen": -2.471921920776367, + "logits/rejected": -2.464104175567627, + "logps/chosen": -146.0660400390625, + "logps/rejected": -147.8424835205078, + "loss": 0.6922, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5911726951599121, + "rewards/margins": 0.050756677985191345, + "rewards/rejected": -0.6419293284416199, + "step": 373 + }, + { + "epoch": 0.49, + "learning_rate": 4.8096988312782174e-05, + "logits/chosen": -2.733851671218872, + "logits/rejected": -2.734400510787964, + "logps/chosen": -201.3523712158203, + "logps/rejected": -213.01361083984375, + "loss": 0.6494, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.4939979016780853, + "rewards/margins": 0.25225183367729187, + "rewards/rejected": -0.7462497353553772, + "step": 374 + }, + { + "epoch": 0.49, + "learning_rate": 4.8083252968828665e-05, + "logits/chosen": -2.5853540897369385, + "logits/rejected": -2.5439412593841553, + "logps/chosen": -240.09124755859375, + "logps/rejected": -200.9342803955078, + "loss": 0.7954, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.7007974982261658, + "rewards/margins": -0.10030128061771393, + "rewards/rejected": -0.6004961133003235, + "step": 375 + }, + { + "epoch": 0.49, + "learning_rate": 4.8069470209893974e-05, + "logits/chosen": -2.4914681911468506, + "logits/rejected": -2.5363824367523193, + "logps/chosen": -176.0529327392578, + "logps/rejected": -212.58383178710938, + "loss": 0.59, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.4798307418823242, + "rewards/margins": 0.2897476851940155, + "rewards/rejected": -0.7695784568786621, + "step": 376 + }, + { + "epoch": 0.49, + "learning_rate": 4.8055640064289086e-05, + "logits/chosen": -2.627121686935425, + "logits/rejected": -2.6512184143066406, + "logps/chosen": -174.460205078125, + "logps/rejected": -205.20346069335938, + "loss": 0.6553, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.5525593757629395, + "rewards/margins": 0.1847618669271469, + "rewards/rejected": -0.7373212575912476, + "step": 377 + }, + { + "epoch": 0.49, + "learning_rate": 4.80417625604223e-05, + "logits/chosen": -2.405069589614868, + "logits/rejected": -2.5229604244232178, + "logps/chosen": -179.3883819580078, + "logps/rejected": -195.5463409423828, + "loss": 0.5946, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.7376325726509094, + "rewards/margins": 0.28595036268234253, + "rewards/rejected": -1.023582935333252, + "step": 378 + }, + { + "epoch": 0.5, + "learning_rate": 4.8027837726799205e-05, + "logits/chosen": -2.6785571575164795, + "logits/rejected": -2.725235939025879, + "logps/chosen": -177.39048767089844, + "logps/rejected": -180.63433837890625, + "loss": 0.7008, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.4321979880332947, + "rewards/margins": 0.021436618641018867, + "rewards/rejected": -0.4536346197128296, + "step": 379 + }, + { + "epoch": 0.5, + "learning_rate": 4.801386559202259e-05, + "logits/chosen": -2.338463544845581, + "logits/rejected": -2.3258635997772217, + "logps/chosen": -136.97865295410156, + "logps/rejected": -175.3179168701172, + "loss": 0.6352, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.3434058427810669, + "rewards/margins": 0.15200775861740112, + "rewards/rejected": -0.495413601398468, + "step": 380 + }, + { + "epoch": 0.5, + "learning_rate": 4.799984618479242e-05, + "logits/chosen": -2.6278395652770996, + "logits/rejected": -2.508350372314453, + "logps/chosen": -154.35214233398438, + "logps/rejected": -188.7537841796875, + "loss": 0.6372, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.4511483311653137, + "rewards/margins": 0.18060770630836487, + "rewards/rejected": -0.6317560076713562, + "step": 381 + }, + { + "epoch": 0.5, + "learning_rate": 4.798577953390577e-05, + "logits/chosen": -2.561044931411743, + "logits/rejected": -2.6394059658050537, + "logps/chosen": -197.1724395751953, + "logps/rejected": -189.59738159179688, + "loss": 0.6657, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5811476111412048, + "rewards/margins": 0.09012848883867264, + "rewards/rejected": -0.6712760329246521, + "step": 382 + }, + { + "epoch": 0.5, + "learning_rate": 4.797166566825675e-05, + "logits/chosen": -2.4525938034057617, + "logits/rejected": -2.509936809539795, + "logps/chosen": -134.09764099121094, + "logps/rejected": -151.4327850341797, + "loss": 0.6024, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.24274428188800812, + "rewards/margins": 0.2151957005262375, + "rewards/rejected": -0.4579399824142456, + "step": 383 + }, + { + "epoch": 0.5, + "learning_rate": 4.795750461683644e-05, + "logits/chosen": -2.5733039379119873, + "logits/rejected": -2.624908924102783, + "logps/chosen": -152.46946716308594, + "logps/rejected": -176.73748779296875, + "loss": 0.7849, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5938777923583984, + "rewards/margins": -0.11171096563339233, + "rewards/rejected": -0.4821667969226837, + "step": 384 + }, + { + "epoch": 0.5, + "learning_rate": 4.794329640873285e-05, + "logits/chosen": -2.552907705307007, + "logits/rejected": -2.564852237701416, + "logps/chosen": -215.24732971191406, + "logps/rejected": -214.79995727539062, + "loss": 0.725, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.522678017616272, + "rewards/margins": 0.005169212818145752, + "rewards/rejected": -0.527847170829773, + "step": 385 + }, + { + "epoch": 0.51, + "learning_rate": 4.7929041073130867e-05, + "logits/chosen": -2.681658983230591, + "logits/rejected": -2.733393669128418, + "logps/chosen": -154.22958374023438, + "logps/rejected": -182.50245666503906, + "loss": 0.6893, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.47931969165802, + "rewards/margins": 0.07224328815937042, + "rewards/rejected": -0.5515629649162292, + "step": 386 + }, + { + "epoch": 0.51, + "learning_rate": 4.7914738639312165e-05, + "logits/chosen": -2.462939739227295, + "logits/rejected": -2.5003418922424316, + "logps/chosen": -152.27236938476562, + "logps/rejected": -162.45211791992188, + "loss": 0.6494, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.47114118933677673, + "rewards/margins": 0.16562382876873016, + "rewards/rejected": -0.6367650628089905, + "step": 387 + }, + { + "epoch": 0.51, + "learning_rate": 4.790038913665519e-05, + "logits/chosen": -2.4999754428863525, + "logits/rejected": -2.5065388679504395, + "logps/chosen": -213.994873046875, + "logps/rejected": -214.79046630859375, + "loss": 0.6382, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.5417348146438599, + "rewards/margins": 0.19294968247413635, + "rewards/rejected": -0.7346844673156738, + "step": 388 + }, + { + "epoch": 0.51, + "learning_rate": 4.788599259463502e-05, + "logits/chosen": -2.5474274158477783, + "logits/rejected": -2.5991437435150146, + "logps/chosen": -180.50039672851562, + "logps/rejected": -225.59510803222656, + "loss": 0.6276, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.7096745371818542, + "rewards/margins": 0.2028300166130066, + "rewards/rejected": -0.9125044941902161, + "step": 389 + }, + { + "epoch": 0.51, + "learning_rate": 4.787154904282341e-05, + "logits/chosen": -2.5795090198516846, + "logits/rejected": -2.577873468399048, + "logps/chosen": -163.51168823242188, + "logps/rejected": -140.55410766601562, + "loss": 0.794, + "rewards/accuracies": 0.25, + "rewards/chosen": -0.5473388433456421, + "rewards/margins": -0.11595198512077332, + "rewards/rejected": -0.4313868582248688, + "step": 390 + }, + { + "epoch": 0.51, + "learning_rate": 4.7857058510888645e-05, + "logits/chosen": -2.516535758972168, + "logits/rejected": -2.5428926944732666, + "logps/chosen": -208.22772216796875, + "logps/rejected": -231.32977294921875, + "loss": 0.6828, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5902161598205566, + "rewards/margins": 0.16028547286987305, + "rewards/rejected": -0.7505015730857849, + "step": 391 + }, + { + "epoch": 0.51, + "learning_rate": 4.7842521028595526e-05, + "logits/chosen": -2.379845380783081, + "logits/rejected": -2.3832809925079346, + "logps/chosen": -200.7274169921875, + "logps/rejected": -164.21742248535156, + "loss": 0.6737, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.5952718257904053, + "rewards/margins": 0.09693758189678192, + "rewards/rejected": -0.6922094225883484, + "step": 392 + }, + { + "epoch": 0.51, + "learning_rate": 4.7827936625805284e-05, + "logits/chosen": -2.6402270793914795, + "logits/rejected": -2.7206149101257324, + "logps/chosen": -166.23167419433594, + "logps/rejected": -214.43209838867188, + "loss": 0.6564, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.5481399893760681, + "rewards/margins": 0.19679725170135498, + "rewards/rejected": -0.7449373006820679, + "step": 393 + }, + { + "epoch": 0.52, + "learning_rate": 4.7813305332475535e-05, + "logits/chosen": -2.5474841594696045, + "logits/rejected": -2.398405075073242, + "logps/chosen": -184.8474884033203, + "logps/rejected": -174.7438507080078, + "loss": 0.814, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.6769263744354248, + "rewards/margins": -0.10798013210296631, + "rewards/rejected": -0.5689462423324585, + "step": 394 + }, + { + "epoch": 0.52, + "learning_rate": 4.77986271786602e-05, + "logits/chosen": -2.4804301261901855, + "logits/rejected": -2.41290545463562, + "logps/chosen": -196.86683654785156, + "logps/rejected": -196.78880310058594, + "loss": 0.7094, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.5920239686965942, + "rewards/margins": 0.030932970345020294, + "rewards/rejected": -0.6229569315910339, + "step": 395 + }, + { + "epoch": 0.52, + "learning_rate": 4.778390219450949e-05, + "logits/chosen": -2.37176775932312, + "logits/rejected": -2.5298891067504883, + "logps/chosen": -161.76962280273438, + "logps/rejected": -179.2218475341797, + "loss": 0.6622, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.6016180515289307, + "rewards/margins": 0.17984583973884583, + "rewards/rejected": -0.7814638614654541, + "step": 396 + }, + { + "epoch": 0.52, + "learning_rate": 4.776913041026976e-05, + "logits/chosen": -2.4908242225646973, + "logits/rejected": -2.6029117107391357, + "logps/chosen": -147.92144775390625, + "logps/rejected": -160.30494689941406, + "loss": 0.7192, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.4882547855377197, + "rewards/margins": 0.03175659850239754, + "rewards/rejected": -0.5200113654136658, + "step": 397 + }, + { + "epoch": 0.52, + "learning_rate": 4.775431185628353e-05, + "logits/chosen": -2.4055118560791016, + "logits/rejected": -2.442111015319824, + "logps/chosen": -168.19540405273438, + "logps/rejected": -166.55137634277344, + "loss": 0.7503, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.6233083605766296, + "rewards/margins": -0.03455538675189018, + "rewards/rejected": -0.5887529850006104, + "step": 398 + }, + { + "epoch": 0.52, + "learning_rate": 4.7739446562989384e-05, + "logits/chosen": -2.4722681045532227, + "logits/rejected": -2.572155237197876, + "logps/chosen": -207.08755493164062, + "logps/rejected": -210.96417236328125, + "loss": 0.6213, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.5305930376052856, + "rewards/margins": 0.21656833589076996, + "rewards/rejected": -0.7471613883972168, + "step": 399 + }, + { + "epoch": 0.52, + "learning_rate": 4.772453456092191e-05, + "logits/chosen": -2.158127546310425, + "logits/rejected": -2.197726249694824, + "logps/chosen": -188.421142578125, + "logps/rejected": -190.73631286621094, + "loss": 0.6757, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.4641679525375366, + "rewards/margins": 0.13141167163848877, + "rewards/rejected": -0.5955795645713806, + "step": 400 + }, + { + "epoch": 0.52, + "learning_rate": 4.7709575880711634e-05, + "logits/chosen": -2.5840139389038086, + "logits/rejected": -2.645707130432129, + "logps/chosen": -188.22262573242188, + "logps/rejected": -196.6393585205078, + "loss": 0.6874, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.7071235775947571, + "rewards/margins": 0.0883268266916275, + "rewards/rejected": -0.7954504489898682, + "step": 401 + }, + { + "epoch": 0.53, + "learning_rate": 4.769457055308497e-05, + "logits/chosen": -2.4420454502105713, + "logits/rejected": -2.4484505653381348, + "logps/chosen": -169.99232482910156, + "logps/rejected": -169.7140350341797, + "loss": 0.7436, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.5594102144241333, + "rewards/margins": -0.01718483492732048, + "rewards/rejected": -0.5422253608703613, + "step": 402 + }, + { + "epoch": 0.53, + "learning_rate": 4.767951860886415e-05, + "logits/chosen": -2.530424118041992, + "logits/rejected": -2.601032257080078, + "logps/chosen": -208.61627197265625, + "logps/rejected": -235.3765411376953, + "loss": 0.671, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.7119523882865906, + "rewards/margins": 0.13131096959114075, + "rewards/rejected": -0.8432632684707642, + "step": 403 + }, + { + "epoch": 0.53, + "learning_rate": 4.766442007896715e-05, + "logits/chosen": -2.662515640258789, + "logits/rejected": -2.574281930923462, + "logps/chosen": -206.23391723632812, + "logps/rejected": -185.04434204101562, + "loss": 0.6698, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.6086325645446777, + "rewards/margins": 0.1881732940673828, + "rewards/rejected": -0.7968058586120605, + "step": 404 + }, + { + "epoch": 0.53, + "learning_rate": 4.764927499440767e-05, + "logits/chosen": -2.4306862354278564, + "logits/rejected": -2.48148512840271, + "logps/chosen": -149.6880340576172, + "logps/rejected": -164.9370574951172, + "loss": 0.6242, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.5055115222930908, + "rewards/margins": 0.18054383993148804, + "rewards/rejected": -0.6860553026199341, + "step": 405 + }, + { + "epoch": 0.53, + "learning_rate": 4.763408338629498e-05, + "logits/chosen": -2.6945927143096924, + "logits/rejected": -2.712038278579712, + "logps/chosen": -195.50482177734375, + "logps/rejected": -207.51431274414062, + "loss": 0.7339, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.7351135015487671, + "rewards/margins": 0.00292108952999115, + "rewards/rejected": -0.7380346059799194, + "step": 406 + }, + { + "epoch": 0.53, + "learning_rate": 4.761884528583396e-05, + "logits/chosen": -2.4739484786987305, + "logits/rejected": -2.5098178386688232, + "logps/chosen": -199.3061065673828, + "logps/rejected": -180.8711395263672, + "loss": 0.7055, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.6267393827438354, + "rewards/margins": 0.034921929240226746, + "rewards/rejected": -0.6616613268852234, + "step": 407 + }, + { + "epoch": 0.53, + "learning_rate": 4.760356072432498e-05, + "logits/chosen": -2.5606906414031982, + "logits/rejected": -2.6224427223205566, + "logps/chosen": -147.893798828125, + "logps/rejected": -156.9450225830078, + "loss": 0.7825, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.8380388021469116, + "rewards/margins": -0.05697247013449669, + "rewards/rejected": -0.7810662388801575, + "step": 408 + }, + { + "epoch": 0.54, + "learning_rate": 4.7588229733163834e-05, + "logits/chosen": -2.479783058166504, + "logits/rejected": -2.5184426307678223, + "logps/chosen": -214.60830688476562, + "logps/rejected": -204.91534423828125, + "loss": 0.6624, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.745360255241394, + "rewards/margins": 0.17358280718326569, + "rewards/rejected": -0.9189431071281433, + "step": 409 + }, + { + "epoch": 0.54, + "learning_rate": 4.757285234384169e-05, + "logits/chosen": -2.5682218074798584, + "logits/rejected": -2.5506396293640137, + "logps/chosen": -158.36090087890625, + "logps/rejected": -163.76829528808594, + "loss": 0.7196, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.7472302913665771, + "rewards/margins": 0.03290612995624542, + "rewards/rejected": -0.7801364064216614, + "step": 410 + }, + { + "epoch": 0.54, + "learning_rate": 4.755742858794503e-05, + "logits/chosen": -2.476301908493042, + "logits/rejected": -2.4283623695373535, + "logps/chosen": -180.48712158203125, + "logps/rejected": -190.4640655517578, + "loss": 0.7285, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.5941387414932251, + "rewards/margins": -0.008222660049796104, + "rewards/rejected": -0.5859161019325256, + "step": 411 + }, + { + "epoch": 0.54, + "learning_rate": 4.754195849715557e-05, + "logits/chosen": -2.4996767044067383, + "logits/rejected": -2.5315232276916504, + "logps/chosen": -211.7996826171875, + "logps/rejected": -221.70632934570312, + "loss": 0.6269, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.6848435997962952, + "rewards/margins": 0.2251822054386139, + "rewards/rejected": -0.9100258350372314, + "step": 412 + }, + { + "epoch": 0.54, + "learning_rate": 4.75264421032502e-05, + "logits/chosen": -2.3915770053863525, + "logits/rejected": -2.443027973175049, + "logps/chosen": -189.36318969726562, + "logps/rejected": -234.4507293701172, + "loss": 0.59, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.6983102560043335, + "rewards/margins": 0.33968544006347656, + "rewards/rejected": -1.03799569606781, + "step": 413 + }, + { + "epoch": 0.54, + "learning_rate": 4.751087943810093e-05, + "logits/chosen": -2.4122979640960693, + "logits/rejected": -2.4623374938964844, + "logps/chosen": -150.90496826171875, + "logps/rejected": -155.33792114257812, + "loss": 0.5973, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.6139358282089233, + "rewards/margins": 0.2593153715133667, + "rewards/rejected": -0.87325119972229, + "step": 414 + }, + { + "epoch": 0.54, + "learning_rate": 4.749527053367481e-05, + "logits/chosen": -2.4751663208007812, + "logits/rejected": -2.5102648735046387, + "logps/chosen": -199.7110595703125, + "logps/rejected": -211.5991973876953, + "loss": 0.6728, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5334303379058838, + "rewards/margins": 0.10008269548416138, + "rewards/rejected": -0.6335129737854004, + "step": 415 + }, + { + "epoch": 0.54, + "learning_rate": 4.747961542203386e-05, + "logits/chosen": -2.458789110183716, + "logits/rejected": -2.4540915489196777, + "logps/chosen": -202.07606506347656, + "logps/rejected": -233.15286254882812, + "loss": 0.7338, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.7183946371078491, + "rewards/margins": -0.020335379987955093, + "rewards/rejected": -0.6980592608451843, + "step": 416 + }, + { + "epoch": 0.55, + "learning_rate": 4.746391413533503e-05, + "logits/chosen": -2.577547788619995, + "logits/rejected": -2.6487934589385986, + "logps/chosen": -200.61962890625, + "logps/rejected": -197.4369354248047, + "loss": 0.6989, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.6610112190246582, + "rewards/margins": 0.05467906594276428, + "rewards/rejected": -0.7156902551651001, + "step": 417 + }, + { + "epoch": 0.55, + "learning_rate": 4.74481667058301e-05, + "logits/chosen": -2.460482120513916, + "logits/rejected": -2.428621768951416, + "logps/chosen": -173.61727905273438, + "logps/rejected": -174.00869750976562, + "loss": 0.6909, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.6227931380271912, + "rewards/margins": 0.061159878969192505, + "rewards/rejected": -0.6839529275894165, + "step": 418 + }, + { + "epoch": 0.55, + "learning_rate": 4.743237316586564e-05, + "logits/chosen": -2.7055728435516357, + "logits/rejected": -2.694801092147827, + "logps/chosen": -233.16864013671875, + "logps/rejected": -219.25518798828125, + "loss": 0.5653, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.7030515670776367, + "rewards/margins": 0.3454846143722534, + "rewards/rejected": -1.0485361814498901, + "step": 419 + }, + { + "epoch": 0.55, + "learning_rate": 4.741653354788295e-05, + "logits/chosen": -2.5690486431121826, + "logits/rejected": -2.5955686569213867, + "logps/chosen": -219.93161010742188, + "logps/rejected": -219.0428466796875, + "loss": 0.6281, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.6727169156074524, + "rewards/margins": 0.190028116106987, + "rewards/rejected": -0.8627450466156006, + "step": 420 + }, + { + "epoch": 0.55, + "learning_rate": 4.7400647884417956e-05, + "logits/chosen": -2.4904708862304688, + "logits/rejected": -2.62727689743042, + "logps/chosen": -178.09906005859375, + "logps/rejected": -193.81495666503906, + "loss": 0.9252, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.8763137459754944, + "rewards/margins": -0.29989194869995117, + "rewards/rejected": -0.5764217376708984, + "step": 421 + }, + { + "epoch": 0.55, + "learning_rate": 4.7384716208101166e-05, + "logits/chosen": -2.443995952606201, + "logits/rejected": -2.4667203426361084, + "logps/chosen": -187.14244079589844, + "logps/rejected": -178.54718017578125, + "loss": 0.6852, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5054149031639099, + "rewards/margins": 0.07299378514289856, + "rewards/rejected": -0.5784087777137756, + "step": 422 + }, + { + "epoch": 0.55, + "learning_rate": 4.736873855165762e-05, + "logits/chosen": -2.535909652709961, + "logits/rejected": -2.5671615600585938, + "logps/chosen": -188.70001220703125, + "logps/rejected": -204.35406494140625, + "loss": 0.5707, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.46517521142959595, + "rewards/margins": 0.3614599108695984, + "rewards/rejected": -0.8266351222991943, + "step": 423 + }, + { + "epoch": 0.55, + "learning_rate": 4.735271494790678e-05, + "logits/chosen": -2.528953790664673, + "logits/rejected": -2.6590449810028076, + "logps/chosen": -187.15150451660156, + "logps/rejected": -224.23590087890625, + "loss": 0.5801, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.5339723229408264, + "rewards/margins": 0.28138864040374756, + "rewards/rejected": -0.8153610229492188, + "step": 424 + }, + { + "epoch": 0.56, + "learning_rate": 4.733664542976253e-05, + "logits/chosen": -2.4930217266082764, + "logits/rejected": -2.5632638931274414, + "logps/chosen": -190.24932861328125, + "logps/rejected": -229.89834594726562, + "loss": 0.6699, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.7542927265167236, + "rewards/margins": 0.1013142317533493, + "rewards/rejected": -0.8556069731712341, + "step": 425 + }, + { + "epoch": 0.56, + "learning_rate": 4.732053003023301e-05, + "logits/chosen": -2.4712095260620117, + "logits/rejected": -2.5324177742004395, + "logps/chosen": -174.3723907470703, + "logps/rejected": -206.7808837890625, + "loss": 0.6939, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.6538185477256775, + "rewards/margins": 0.04219439998269081, + "rewards/rejected": -0.6960129141807556, + "step": 426 + }, + { + "epoch": 0.56, + "learning_rate": 4.730436878242064e-05, + "logits/chosen": -2.5648436546325684, + "logits/rejected": -2.6675539016723633, + "logps/chosen": -201.86000061035156, + "logps/rejected": -222.20242309570312, + "loss": 0.5946, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.752447783946991, + "rewards/margins": 0.23879489302635193, + "rewards/rejected": -0.9912427067756653, + "step": 427 + }, + { + "epoch": 0.56, + "learning_rate": 4.7288161719522016e-05, + "logits/chosen": -2.55771541595459, + "logits/rejected": -2.597944974899292, + "logps/chosen": -168.26791381835938, + "logps/rejected": -158.33251953125, + "loss": 0.647, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.43088653683662415, + "rewards/margins": 0.134020134806633, + "rewards/rejected": -0.5649065971374512, + "step": 428 + }, + { + "epoch": 0.56, + "learning_rate": 4.727190887482783e-05, + "logits/chosen": -2.801307201385498, + "logits/rejected": -2.7668023109436035, + "logps/chosen": -221.6161346435547, + "logps/rejected": -212.2379913330078, + "loss": 0.6348, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.5888996124267578, + "rewards/margins": 0.16605983674526215, + "rewards/rejected": -0.7549594044685364, + "step": 429 + }, + { + "epoch": 0.56, + "learning_rate": 4.725561028172282e-05, + "logits/chosen": -2.459953784942627, + "logits/rejected": -2.4980850219726562, + "logps/chosen": -201.91897583007812, + "logps/rejected": -208.00759887695312, + "loss": 0.6881, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.6136542558670044, + "rewards/margins": 0.06774169206619263, + "rewards/rejected": -0.6813960075378418, + "step": 430 + }, + { + "epoch": 0.56, + "learning_rate": 4.7239265973685696e-05, + "logits/chosen": -2.551776647567749, + "logits/rejected": -2.5754733085632324, + "logps/chosen": -185.41299438476562, + "logps/rejected": -187.89947509765625, + "loss": 0.6601, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5381264090538025, + "rewards/margins": 0.11568471044301987, + "rewards/rejected": -0.6538110971450806, + "step": 431 + }, + { + "epoch": 0.57, + "learning_rate": 4.722287598428907e-05, + "logits/chosen": -2.588721990585327, + "logits/rejected": -2.629152774810791, + "logps/chosen": -197.33180236816406, + "logps/rejected": -211.42718505859375, + "loss": 0.5501, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.6033443808555603, + "rewards/margins": 0.405744731426239, + "rewards/rejected": -1.0090891122817993, + "step": 432 + }, + { + "epoch": 0.57, + "learning_rate": 4.720644034719938e-05, + "logits/chosen": -2.5057499408721924, + "logits/rejected": -2.6138525009155273, + "logps/chosen": -175.42535400390625, + "logps/rejected": -199.4925079345703, + "loss": 0.723, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.8974592685699463, + "rewards/margins": 0.013055291026830673, + "rewards/rejected": -0.9105146527290344, + "step": 433 + }, + { + "epoch": 0.57, + "learning_rate": 4.7189959096176825e-05, + "logits/chosen": -2.4195330142974854, + "logits/rejected": -2.467151165008545, + "logps/chosen": -168.0084686279297, + "logps/rejected": -181.73574829101562, + "loss": 0.8057, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.7599891424179077, + "rewards/margins": -0.08929312229156494, + "rewards/rejected": -0.6706960201263428, + "step": 434 + }, + { + "epoch": 0.57, + "learning_rate": 4.7173432265075334e-05, + "logits/chosen": -2.511300802230835, + "logits/rejected": -2.510061502456665, + "logps/chosen": -195.68515014648438, + "logps/rejected": -173.744140625, + "loss": 0.6779, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.8897146582603455, + "rewards/margins": 0.1359187215566635, + "rewards/rejected": -1.0256333351135254, + "step": 435 + }, + { + "epoch": 0.57, + "learning_rate": 4.7156859887842416e-05, + "logits/chosen": -2.5270705223083496, + "logits/rejected": -2.5860514640808105, + "logps/chosen": -211.92176818847656, + "logps/rejected": -210.06216430664062, + "loss": 0.7143, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.7650929689407349, + "rewards/margins": 0.06967158615589142, + "rewards/rejected": -0.8347646594047546, + "step": 436 + }, + { + "epoch": 0.57, + "learning_rate": 4.714024199851915e-05, + "logits/chosen": -2.6095833778381348, + "logits/rejected": -2.683137893676758, + "logps/chosen": -166.6543426513672, + "logps/rejected": -197.75425720214844, + "loss": 0.7672, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.9517545104026794, + "rewards/margins": -0.0435512438416481, + "rewards/rejected": -0.9082032442092896, + "step": 437 + }, + { + "epoch": 0.57, + "learning_rate": 4.712357863124013e-05, + "logits/chosen": -2.583829641342163, + "logits/rejected": -2.4270806312561035, + "logps/chosen": -184.97288513183594, + "logps/rejected": -151.17689514160156, + "loss": 0.8427, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.9021130800247192, + "rewards/margins": -0.22366446256637573, + "rewards/rejected": -0.6784486770629883, + "step": 438 + }, + { + "epoch": 0.57, + "learning_rate": 4.710686982023332e-05, + "logits/chosen": -2.6779322624206543, + "logits/rejected": -2.6819846630096436, + "logps/chosen": -202.7177734375, + "logps/rejected": -180.97412109375, + "loss": 0.6769, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.7709564566612244, + "rewards/margins": 0.1275014728307724, + "rewards/rejected": -0.8984578251838684, + "step": 439 + }, + { + "epoch": 0.58, + "learning_rate": 4.709011559982006e-05, + "logits/chosen": -2.5081942081451416, + "logits/rejected": -2.457127094268799, + "logps/chosen": -199.16038513183594, + "logps/rejected": -227.66439819335938, + "loss": 0.715, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.8098132014274597, + "rewards/margins": 0.005300614982843399, + "rewards/rejected": -0.8151137232780457, + "step": 440 + }, + { + "epoch": 0.58, + "learning_rate": 4.707331600441495e-05, + "logits/chosen": -2.5350513458251953, + "logits/rejected": -2.498892307281494, + "logps/chosen": -169.93370056152344, + "logps/rejected": -166.59365844726562, + "loss": 0.6809, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.6930581331253052, + "rewards/margins": 0.06999292969703674, + "rewards/rejected": -0.7630510330200195, + "step": 441 + }, + { + "epoch": 0.58, + "learning_rate": 4.705647106852581e-05, + "logits/chosen": -2.5681025981903076, + "logits/rejected": -2.5515594482421875, + "logps/chosen": -232.0432586669922, + "logps/rejected": -224.9477081298828, + "loss": 0.7262, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.9011371731758118, + "rewards/margins": 0.08735474199056625, + "rewards/rejected": -0.988491952419281, + "step": 442 + }, + { + "epoch": 0.58, + "learning_rate": 4.7039580826753564e-05, + "logits/chosen": -2.538968563079834, + "logits/rejected": -2.5133233070373535, + "logps/chosen": -158.14520263671875, + "logps/rejected": -201.77728271484375, + "loss": 0.6074, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.6365475654602051, + "rewards/margins": 0.2535760998725891, + "rewards/rejected": -0.8901236057281494, + "step": 443 + }, + { + "epoch": 0.58, + "learning_rate": 4.7022645313792235e-05, + "logits/chosen": -2.5245423316955566, + "logits/rejected": -2.588785171508789, + "logps/chosen": -211.5054473876953, + "logps/rejected": -183.34361267089844, + "loss": 0.6541, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.7383630275726318, + "rewards/margins": 0.1513664424419403, + "rewards/rejected": -0.8897294998168945, + "step": 444 + }, + { + "epoch": 0.58, + "learning_rate": 4.700566456442882e-05, + "logits/chosen": -2.426429033279419, + "logits/rejected": -2.4127559661865234, + "logps/chosen": -170.16091918945312, + "logps/rejected": -181.26731872558594, + "loss": 0.685, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.555845320224762, + "rewards/margins": 0.09964090585708618, + "rewards/rejected": -0.6554862260818481, + "step": 445 + }, + { + "epoch": 0.58, + "learning_rate": 4.6988638613543216e-05, + "logits/chosen": -2.5047028064727783, + "logits/rejected": -2.6594462394714355, + "logps/chosen": -192.5770263671875, + "logps/rejected": -196.17718505859375, + "loss": 0.6536, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.6389065384864807, + "rewards/margins": 0.1103520542383194, + "rewards/rejected": -0.7492585778236389, + "step": 446 + }, + { + "epoch": 0.59, + "learning_rate": 4.6971567496108206e-05, + "logits/chosen": -2.4204087257385254, + "logits/rejected": -2.4866108894348145, + "logps/chosen": -186.6705780029297, + "logps/rejected": -183.05654907226562, + "loss": 0.6868, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.7241983413696289, + "rewards/margins": 0.062242452055215836, + "rewards/rejected": -0.7864408493041992, + "step": 447 + }, + { + "epoch": 0.59, + "learning_rate": 4.695445124718931e-05, + "logits/chosen": -2.599712610244751, + "logits/rejected": -2.678144693374634, + "logps/chosen": -171.8934326171875, + "logps/rejected": -187.55401611328125, + "loss": 0.7119, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.8643355369567871, + "rewards/margins": 0.07741285860538483, + "rewards/rejected": -0.9417483806610107, + "step": 448 + }, + { + "epoch": 0.59, + "learning_rate": 4.693728990194479e-05, + "logits/chosen": -2.5691065788269043, + "logits/rejected": -2.515324592590332, + "logps/chosen": -175.92041015625, + "logps/rejected": -169.9009552001953, + "loss": 0.7915, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.7048065662384033, + "rewards/margins": -0.13590610027313232, + "rewards/rejected": -0.5689005851745605, + "step": 449 + }, + { + "epoch": 0.59, + "learning_rate": 4.692008349562551e-05, + "logits/chosen": -2.5656256675720215, + "logits/rejected": -2.5296895503997803, + "logps/chosen": -181.8729248046875, + "logps/rejected": -166.05845642089844, + "loss": 0.6528, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.6643526554107666, + "rewards/margins": 0.1753290742635727, + "rewards/rejected": -0.8396817445755005, + "step": 450 + }, + { + "epoch": 0.59, + "learning_rate": 4.690283206357491e-05, + "logits/chosen": -2.6595234870910645, + "logits/rejected": -2.6016488075256348, + "logps/chosen": -227.77488708496094, + "logps/rejected": -197.7751007080078, + "loss": 0.7501, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.9334564208984375, + "rewards/margins": 0.008943833410739899, + "rewards/rejected": -0.942400336265564, + "step": 451 + }, + { + "epoch": 0.59, + "learning_rate": 4.6885535641228904e-05, + "logits/chosen": -2.6872832775115967, + "logits/rejected": -2.6639106273651123, + "logps/chosen": -198.11798095703125, + "logps/rejected": -215.65846252441406, + "loss": 0.6973, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.661568284034729, + "rewards/margins": 0.10554268211126328, + "rewards/rejected": -0.7671110033988953, + "step": 452 + }, + { + "epoch": 0.59, + "learning_rate": 4.6868194264115833e-05, + "logits/chosen": -2.6474506855010986, + "logits/rejected": -2.6153347492218018, + "logps/chosen": -179.79026794433594, + "logps/rejected": -181.140869140625, + "loss": 0.668, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.7167541980743408, + "rewards/margins": 0.10385677218437195, + "rewards/rejected": -0.8206108808517456, + "step": 453 + }, + { + "epoch": 0.59, + "learning_rate": 4.685080796785637e-05, + "logits/chosen": -2.6611721515655518, + "logits/rejected": -2.5946366786956787, + "logps/chosen": -175.3307647705078, + "logps/rejected": -182.19512939453125, + "loss": 0.6937, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5467984080314636, + "rewards/margins": 0.12983733415603638, + "rewards/rejected": -0.6766356825828552, + "step": 454 + }, + { + "epoch": 0.6, + "learning_rate": 4.683337678816345e-05, + "logits/chosen": -2.564751148223877, + "logits/rejected": -2.610701322555542, + "logps/chosen": -160.96405029296875, + "logps/rejected": -211.45936584472656, + "loss": 0.5668, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.6448365449905396, + "rewards/margins": 0.34849783778190613, + "rewards/rejected": -0.9933344125747681, + "step": 455 + }, + { + "epoch": 0.6, + "learning_rate": 4.6815900760842236e-05, + "logits/chosen": -2.5778658390045166, + "logits/rejected": -2.650463104248047, + "logps/chosen": -178.8852081298828, + "logps/rejected": -184.25819396972656, + "loss": 0.6166, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.6312435865402222, + "rewards/margins": 0.21195663511753082, + "rewards/rejected": -0.8432002067565918, + "step": 456 + }, + { + "epoch": 0.6, + "learning_rate": 4.679837992178996e-05, + "logits/chosen": -2.6255109310150146, + "logits/rejected": -2.568089246749878, + "logps/chosen": -186.581787109375, + "logps/rejected": -179.66636657714844, + "loss": 0.7966, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.7909551858901978, + "rewards/margins": -0.12891662120819092, + "rewards/rejected": -0.6620385646820068, + "step": 457 + }, + { + "epoch": 0.6, + "learning_rate": 4.678081430699594e-05, + "logits/chosen": -2.548609972000122, + "logits/rejected": -2.5789599418640137, + "logps/chosen": -149.14419555664062, + "logps/rejected": -175.63099670410156, + "loss": 0.672, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.6500214338302612, + "rewards/margins": 0.1032710000872612, + "rewards/rejected": -0.7532925009727478, + "step": 458 + }, + { + "epoch": 0.6, + "learning_rate": 4.676320395254146e-05, + "logits/chosen": -2.6516225337982178, + "logits/rejected": -2.5997962951660156, + "logps/chosen": -179.6166229248047, + "logps/rejected": -181.91600036621094, + "loss": 0.6797, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.5761557221412659, + "rewards/margins": 0.09436798840761185, + "rewards/rejected": -0.6705237627029419, + "step": 459 + }, + { + "epoch": 0.6, + "learning_rate": 4.674554889459968e-05, + "logits/chosen": -2.547447681427002, + "logits/rejected": -2.531512498855591, + "logps/chosen": -158.489013671875, + "logps/rejected": -158.9821319580078, + "loss": 0.6921, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.551589846611023, + "rewards/margins": 0.06521686166524887, + "rewards/rejected": -0.61680668592453, + "step": 460 + }, + { + "epoch": 0.6, + "learning_rate": 4.672784916943562e-05, + "logits/chosen": -2.6113603115081787, + "logits/rejected": -2.7179596424102783, + "logps/chosen": -161.98703002929688, + "logps/rejected": -183.0166778564453, + "loss": 0.6035, + "rewards/accuracies": 0.8125, + "rewards/chosen": -0.4798649549484253, + "rewards/margins": 0.25054579973220825, + "rewards/rejected": -0.7304107546806335, + "step": 461 + }, + { + "epoch": 0.6, + "learning_rate": 4.6710104813406034e-05, + "logits/chosen": -2.542372703552246, + "logits/rejected": -2.522775650024414, + "logps/chosen": -179.12545776367188, + "logps/rejected": -175.42910766601562, + "loss": 0.6264, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.5295111536979675, + "rewards/margins": 0.1804201453924179, + "rewards/rejected": -0.7099313139915466, + "step": 462 + }, + { + "epoch": 0.61, + "learning_rate": 4.669231586295934e-05, + "logits/chosen": -2.604105234146118, + "logits/rejected": -2.6281673908233643, + "logps/chosen": -201.89901733398438, + "logps/rejected": -226.806396484375, + "loss": 0.7092, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.4338040053844452, + "rewards/margins": 0.028335200622677803, + "rewards/rejected": -0.46213918924331665, + "step": 463 + }, + { + "epoch": 0.61, + "learning_rate": 4.667448235463557e-05, + "logits/chosen": -2.639315605163574, + "logits/rejected": -2.572453022003174, + "logps/chosen": -211.58755493164062, + "logps/rejected": -191.2517547607422, + "loss": 0.5992, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.4935193359851837, + "rewards/margins": 0.28121161460876465, + "rewards/rejected": -0.7747309803962708, + "step": 464 + }, + { + "epoch": 0.61, + "learning_rate": 4.665660432506629e-05, + "logits/chosen": -2.577928304672241, + "logits/rejected": -2.5669662952423096, + "logps/chosen": -200.63458251953125, + "logps/rejected": -192.37644958496094, + "loss": 0.684, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.6348070502281189, + "rewards/margins": 0.1238020658493042, + "rewards/rejected": -0.7586091756820679, + "step": 465 + }, + { + "epoch": 0.61, + "learning_rate": 4.6638681810974496e-05, + "logits/chosen": -2.5102195739746094, + "logits/rejected": -2.4570584297180176, + "logps/chosen": -184.1721954345703, + "logps/rejected": -186.46142578125, + "loss": 0.6753, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.5391435027122498, + "rewards/margins": 0.12175430357456207, + "rewards/rejected": -0.6608977913856506, + "step": 466 + }, + { + "epoch": 0.61, + "learning_rate": 4.6620714849174576e-05, + "logits/chosen": -2.5037519931793213, + "logits/rejected": -2.514277219772339, + "logps/chosen": -221.79428100585938, + "logps/rejected": -226.41201782226562, + "loss": 0.6953, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.4555547833442688, + "rewards/margins": 0.06287840008735657, + "rewards/rejected": -0.5184332132339478, + "step": 467 + }, + { + "epoch": 0.61, + "learning_rate": 4.660270347657219e-05, + "logits/chosen": -2.4214844703674316, + "logits/rejected": -2.451406717300415, + "logps/chosen": -164.8655242919922, + "logps/rejected": -202.42665100097656, + "loss": 0.6557, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.4477751851081848, + "rewards/margins": 0.17700761556625366, + "rewards/rejected": -0.6247828006744385, + "step": 468 + }, + { + "epoch": 0.61, + "learning_rate": 4.658464773016428e-05, + "logits/chosen": -2.6548826694488525, + "logits/rejected": -2.7413270473480225, + "logps/chosen": -203.27706909179688, + "logps/rejected": -195.96438598632812, + "loss": 0.7196, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.6162900924682617, + "rewards/margins": 0.04476276412606239, + "rewards/rejected": -0.6610528230667114, + "step": 469 + }, + { + "epoch": 0.62, + "learning_rate": 4.6566547647038864e-05, + "logits/chosen": -2.6256935596466064, + "logits/rejected": -2.544764757156372, + "logps/chosen": -197.11715698242188, + "logps/rejected": -174.37977600097656, + "loss": 0.5997, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.7993966937065125, + "rewards/margins": 0.2561304569244385, + "rewards/rejected": -1.0555272102355957, + "step": 470 + }, + { + "epoch": 0.62, + "learning_rate": 4.6548403264375074e-05, + "logits/chosen": -2.4777655601501465, + "logits/rejected": -2.466064691543579, + "logps/chosen": -206.86097717285156, + "logps/rejected": -222.15383911132812, + "loss": 0.8362, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.8698564767837524, + "rewards/margins": -0.1482725888490677, + "rewards/rejected": -0.721583902835846, + "step": 471 + }, + { + "epoch": 0.62, + "learning_rate": 4.6530214619443037e-05, + "logits/chosen": -2.5265445709228516, + "logits/rejected": -2.5728039741516113, + "logps/chosen": -205.71484375, + "logps/rejected": -193.68008422851562, + "loss": 0.7739, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.6624203324317932, + "rewards/margins": -0.09644677489995956, + "rewards/rejected": -0.5659735202789307, + "step": 472 + }, + { + "epoch": 0.62, + "learning_rate": 4.6511981749603775e-05, + "logits/chosen": -2.4683680534362793, + "logits/rejected": -2.553187847137451, + "logps/chosen": -188.4405975341797, + "logps/rejected": -206.36973571777344, + "loss": 0.6291, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.44957125186920166, + "rewards/margins": 0.21964800357818604, + "rewards/rejected": -0.6692192554473877, + "step": 473 + }, + { + "epoch": 0.62, + "learning_rate": 4.6493704692309175e-05, + "logits/chosen": -2.4824113845825195, + "logits/rejected": -2.4895737171173096, + "logps/chosen": -173.37237548828125, + "logps/rejected": -182.10430908203125, + "loss": 0.6773, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.5202856063842773, + "rewards/margins": 0.12215665727853775, + "rewards/rejected": -0.6424421668052673, + "step": 474 + }, + { + "epoch": 0.62, + "learning_rate": 4.647538348510189e-05, + "logits/chosen": -2.6831271648406982, + "logits/rejected": -2.7396373748779297, + "logps/chosen": -175.16824340820312, + "logps/rejected": -194.82957458496094, + "loss": 0.653, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.6846888065338135, + "rewards/margins": 0.17653468251228333, + "rewards/rejected": -0.861223578453064, + "step": 475 + }, + { + "epoch": 0.62, + "learning_rate": 4.645701816561523e-05, + "logits/chosen": -2.55379581451416, + "logits/rejected": -2.6678338050842285, + "logps/chosen": -214.75527954101562, + "logps/rejected": -180.83958435058594, + "loss": 0.6331, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.6419520974159241, + "rewards/margins": 0.22850137948989868, + "rewards/rejected": -0.8704534769058228, + "step": 476 + }, + { + "epoch": 0.62, + "learning_rate": 4.643860877157314e-05, + "logits/chosen": -2.560502529144287, + "logits/rejected": -2.6059296131134033, + "logps/chosen": -225.27346801757812, + "logps/rejected": -238.6981658935547, + "loss": 0.7312, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.6071743369102478, + "rewards/margins": 0.0013379361480474472, + "rewards/rejected": -0.6085121631622314, + "step": 477 + }, + { + "epoch": 0.63, + "learning_rate": 4.642015534079012e-05, + "logits/chosen": -2.6502907276153564, + "logits/rejected": -2.4651901721954346, + "logps/chosen": -194.60215759277344, + "logps/rejected": -208.71707153320312, + "loss": 0.6499, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.41506141424179077, + "rewards/margins": 0.14238694310188293, + "rewards/rejected": -0.5574483871459961, + "step": 478 + }, + { + "epoch": 0.63, + "learning_rate": 4.640165791117106e-05, + "logits/chosen": -2.597409248352051, + "logits/rejected": -2.671415090560913, + "logps/chosen": -169.68240356445312, + "logps/rejected": -189.24998474121094, + "loss": 0.6624, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.6197757124900818, + "rewards/margins": 0.10906066745519638, + "rewards/rejected": -0.7288363575935364, + "step": 479 + }, + { + "epoch": 0.63, + "learning_rate": 4.63831165207113e-05, + "logits/chosen": -2.2800724506378174, + "logits/rejected": -2.3521509170532227, + "logps/chosen": -171.45944213867188, + "logps/rejected": -175.93577575683594, + "loss": 0.6243, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.498868465423584, + "rewards/margins": 0.19707906246185303, + "rewards/rejected": -0.6959475874900818, + "step": 480 + }, + { + "epoch": 0.63, + "learning_rate": 4.6364531207496426e-05, + "logits/chosen": -2.549999713897705, + "logits/rejected": -2.5626368522644043, + "logps/chosen": -162.32720947265625, + "logps/rejected": -192.87612915039062, + "loss": 0.7312, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.5747624635696411, + "rewards/margins": 0.024262502789497375, + "rewards/rejected": -0.5990250110626221, + "step": 481 + }, + { + "epoch": 0.63, + "learning_rate": 4.634590200970227e-05, + "logits/chosen": -2.482072591781616, + "logits/rejected": -2.6162638664245605, + "logps/chosen": -176.58897399902344, + "logps/rejected": -207.56654357910156, + "loss": 0.6501, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.7275586128234863, + "rewards/margins": 0.14506082236766815, + "rewards/rejected": -0.8726193904876709, + "step": 482 + }, + { + "epoch": 0.63, + "learning_rate": 4.632722896559481e-05, + "logits/chosen": -2.668024778366089, + "logits/rejected": -2.6363847255706787, + "logps/chosen": -263.9557800292969, + "logps/rejected": -249.15301513671875, + "loss": 0.7493, + "rewards/accuracies": 0.625, + "rewards/chosen": -1.0229195356369019, + "rewards/margins": -0.055171869695186615, + "rewards/rejected": -0.967747688293457, + "step": 483 + }, + { + "epoch": 0.63, + "learning_rate": 4.630851211353007e-05, + "logits/chosen": -2.6272599697113037, + "logits/rejected": -2.644105911254883, + "logps/chosen": -169.70347595214844, + "logps/rejected": -194.94564819335938, + "loss": 0.6384, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.7246192693710327, + "rewards/margins": 0.25421109795570374, + "rewards/rejected": -0.9788303971290588, + "step": 484 + }, + { + "epoch": 0.63, + "learning_rate": 4.628975149195407e-05, + "logits/chosen": -2.530022621154785, + "logits/rejected": -2.5307507514953613, + "logps/chosen": -165.1283721923828, + "logps/rejected": -191.00559997558594, + "loss": 0.6223, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.6375387907028198, + "rewards/margins": 0.26712220907211304, + "rewards/rejected": -0.9046609997749329, + "step": 485 + }, + { + "epoch": 0.64, + "learning_rate": 4.6270947139402744e-05, + "logits/chosen": -2.54582142829895, + "logits/rejected": -2.5507116317749023, + "logps/chosen": -197.43624877929688, + "logps/rejected": -200.06396484375, + "loss": 0.695, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.6543585062026978, + "rewards/margins": 0.08520226180553436, + "rewards/rejected": -0.7395609021186829, + "step": 486 + }, + { + "epoch": 0.64, + "learning_rate": 4.6252099094501834e-05, + "logits/chosen": -2.502086639404297, + "logits/rejected": -2.507930040359497, + "logps/chosen": -192.9934539794922, + "logps/rejected": -207.4755859375, + "loss": 0.5737, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.5259783864021301, + "rewards/margins": 0.3163459002971649, + "rewards/rejected": -0.8423242568969727, + "step": 487 + }, + { + "epoch": 0.64, + "learning_rate": 4.623320739596685e-05, + "logits/chosen": -2.5075860023498535, + "logits/rejected": -2.537548065185547, + "logps/chosen": -180.3726348876953, + "logps/rejected": -195.20867919921875, + "loss": 0.6342, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.6498013138771057, + "rewards/margins": 0.19693784415721893, + "rewards/rejected": -0.8467391729354858, + "step": 488 + }, + { + "epoch": 0.64, + "learning_rate": 4.621427208260296e-05, + "logits/chosen": -2.5124759674072266, + "logits/rejected": -2.5460541248321533, + "logps/chosen": -170.8051300048828, + "logps/rejected": -183.51568603515625, + "loss": 0.6292, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.7085085511207581, + "rewards/margins": 0.24482092261314392, + "rewards/rejected": -0.9533295035362244, + "step": 489 + }, + { + "epoch": 0.64, + "learning_rate": 4.6195293193304915e-05, + "logits/chosen": -2.568577289581299, + "logits/rejected": -2.730349063873291, + "logps/chosen": -157.42147827148438, + "logps/rejected": -204.0237274169922, + "loss": 0.6416, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.7542508244514465, + "rewards/margins": 0.18506285548210144, + "rewards/rejected": -0.9393137693405151, + "step": 490 + }, + { + "epoch": 0.64, + "learning_rate": 4.6176270767056976e-05, + "logits/chosen": -2.658346652984619, + "logits/rejected": -2.697397232055664, + "logps/chosen": -160.69436645507812, + "logps/rejected": -182.4514923095703, + "loss": 0.6066, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.5536147952079773, + "rewards/margins": 0.21064530313014984, + "rewards/rejected": -0.7642600536346436, + "step": 491 + }, + { + "epoch": 0.64, + "learning_rate": 4.615720484293286e-05, + "logits/chosen": -2.621380090713501, + "logits/rejected": -2.5975751876831055, + "logps/chosen": -172.44583129882812, + "logps/rejected": -168.9969482421875, + "loss": 0.7824, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.7546152472496033, + "rewards/margins": -0.09415875375270844, + "rewards/rejected": -0.6604564785957336, + "step": 492 + }, + { + "epoch": 0.65, + "learning_rate": 4.613809546009558e-05, + "logits/chosen": -2.5632882118225098, + "logits/rejected": -2.528463840484619, + "logps/chosen": -174.4139404296875, + "logps/rejected": -178.07061767578125, + "loss": 0.7851, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.783379077911377, + "rewards/margins": -0.13342618942260742, + "rewards/rejected": -0.64995276927948, + "step": 493 + }, + { + "epoch": 0.65, + "learning_rate": 4.611894265779748e-05, + "logits/chosen": -2.6725411415100098, + "logits/rejected": -2.6337125301361084, + "logps/chosen": -158.43734741210938, + "logps/rejected": -149.51394653320312, + "loss": 0.6479, + "rewards/accuracies": 0.625, + "rewards/chosen": -0.7037851810455322, + "rewards/margins": 0.15217718482017517, + "rewards/rejected": -0.8559622764587402, + "step": 494 + }, + { + "epoch": 0.65, + "learning_rate": 4.609974647538003e-05, + "logits/chosen": -2.5839614868164062, + "logits/rejected": -2.5436882972717285, + "logps/chosen": -186.00880432128906, + "logps/rejected": -176.95623779296875, + "loss": 0.6352, + "rewards/accuracies": 0.5625, + "rewards/chosen": -0.7761020660400391, + "rewards/margins": 0.2549971640110016, + "rewards/rejected": -1.0310992002487183, + "step": 495 + }, + { + "epoch": 0.65, + "learning_rate": 4.608050695227385e-05, + "logits/chosen": -2.663245439529419, + "logits/rejected": -2.629906415939331, + "logps/chosen": -187.883056640625, + "logps/rejected": -181.8142852783203, + "loss": 0.727, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.8239506483078003, + "rewards/margins": 0.07172393053770065, + "rewards/rejected": -0.8956745862960815, + "step": 496 + }, + { + "epoch": 0.65, + "learning_rate": 4.606122412799857e-05, + "logits/chosen": -2.864988327026367, + "logits/rejected": -2.8613595962524414, + "logps/chosen": -179.66754150390625, + "logps/rejected": -183.17547607421875, + "loss": 0.7754, + "rewards/accuracies": 0.3125, + "rewards/chosen": -0.7847499847412109, + "rewards/margins": -0.03732617199420929, + "rewards/rejected": -0.7474238872528076, + "step": 497 + }, + { + "epoch": 0.65, + "learning_rate": 4.6041898042162764e-05, + "logits/chosen": -2.67081618309021, + "logits/rejected": -2.630650281906128, + "logps/chosen": -184.00350952148438, + "logps/rejected": -213.1297607421875, + "loss": 0.7759, + "rewards/accuracies": 0.375, + "rewards/chosen": -0.9428163170814514, + "rewards/margins": -0.10311193764209747, + "rewards/rejected": -0.8397043347358704, + "step": 498 + }, + { + "epoch": 0.65, + "learning_rate": 4.602252873446386e-05, + "logits/chosen": -2.5779166221618652, + "logits/rejected": -2.657582998275757, + "logps/chosen": -255.50413513183594, + "logps/rejected": -295.5150146484375, + "loss": 0.6389, + "rewards/accuracies": 0.6875, + "rewards/chosen": -0.6901088356971741, + "rewards/margins": 0.2417818158864975, + "rewards/rejected": -0.9318906664848328, + "step": 499 + }, + { + "epoch": 0.65, + "learning_rate": 4.60031162446881e-05, + "logits/chosen": -2.6082730293273926, + "logits/rejected": -2.7072064876556396, + "logps/chosen": -186.39288330078125, + "logps/rejected": -197.77581787109375, + "loss": 0.7185, + "rewards/accuracies": 0.4375, + "rewards/chosen": -0.8791942596435547, + "rewards/margins": -0.021152300760149956, + "rewards/rejected": -0.8580418825149536, + "step": 500 + } + ], + "logging_steps": 1, + "max_steps": 2292, + "num_input_tokens_seen": 0, + "num_train_epochs": 3, + "save_steps": 500, + "total_flos": 0.0, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +}