diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,4150 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 3.0, + "eval_steps": 100, + "global_step": 2907, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "epoch": 0.0, + "learning_rate": 1.7182130584192438e-09, + "logits/chosen": -2.1327524185180664, + "logits/rejected": -1.609220027923584, + "logps/chosen": -91.77880859375, + "logps/rejected": -81.204345703125, + "loss": 0.6931, + "rewards/accuracies": 0.0, + "rewards/chosen": 0.0, + "rewards/margins": 0.0, + "rewards/rejected": 0.0, + "step": 1 + }, + { + "epoch": 0.01, + "learning_rate": 1.718213058419244e-08, + "logits/chosen": -2.3409039974212646, + "logits/rejected": -2.2914419174194336, + "logps/chosen": -218.2519989013672, + "logps/rejected": -192.96514892578125, + "loss": 0.6949, + "rewards/accuracies": 0.6111111044883728, + "rewards/chosen": 0.008321777917444706, + "rewards/margins": 0.005371565464884043, + "rewards/rejected": 0.0029502129182219505, + "step": 10 + }, + { + "epoch": 0.02, + "learning_rate": 3.436426116838488e-08, + "logits/chosen": -2.259495735168457, + "logits/rejected": -2.44956636428833, + "logps/chosen": -274.8431091308594, + "logps/rejected": -225.97793579101562, + "loss": 0.6933, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.01440061442553997, + "rewards/margins": 0.013972322456538677, + "rewards/rejected": 0.00042829400626942515, + "step": 20 + }, + { + "epoch": 0.03, + "learning_rate": 5.154639175257731e-08, + "logits/chosen": -2.5081024169921875, + "logits/rejected": -2.4530844688415527, + "logps/chosen": -292.6970520019531, + "logps/rejected": -215.64553833007812, + "loss": 0.6933, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.008286448195576668, + "rewards/margins": 0.008681153878569603, + "rewards/rejected": -0.00039470643969252706, + "step": 30 + }, + { + "epoch": 0.04, + "learning_rate": 6.872852233676976e-08, + "logits/chosen": -2.6037235260009766, + "logits/rejected": -2.4282469749450684, + "logps/chosen": -378.5215759277344, + "logps/rejected": -226.55136108398438, + "loss": 0.6933, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": 0.004302692599594593, + "rewards/margins": 0.008366527035832405, + "rewards/rejected": -0.004063835833221674, + "step": 40 + }, + { + "epoch": 0.05, + "learning_rate": 8.59106529209622e-08, + "logits/chosen": -2.3304362297058105, + "logits/rejected": -2.30436372756958, + "logps/chosen": -275.41754150390625, + "logps/rejected": -209.0893096923828, + "loss": 0.6934, + "rewards/accuracies": 0.30000001192092896, + "rewards/chosen": -0.01541445404291153, + "rewards/margins": -0.023714235052466393, + "rewards/rejected": 0.008299780078232288, + "step": 50 + }, + { + "epoch": 0.06, + "learning_rate": 1.0309278350515462e-07, + "logits/chosen": -2.408499240875244, + "logits/rejected": -2.518784284591675, + "logps/chosen": -298.5494689941406, + "logps/rejected": -267.29962158203125, + "loss": 0.6947, + "rewards/accuracies": 0.44999998807907104, + "rewards/chosen": -0.01625905930995941, + "rewards/margins": -0.008457355201244354, + "rewards/rejected": -0.007801705040037632, + "step": 60 + }, + { + "epoch": 0.07, + "learning_rate": 1.202749140893471e-07, + "logits/chosen": -2.4770851135253906, + "logits/rejected": -2.2523791790008545, + "logps/chosen": -296.0211486816406, + "logps/rejected": -145.6873321533203, + "loss": 0.6918, + "rewards/accuracies": 0.30000001192092896, + "rewards/chosen": -0.010021962225437164, + "rewards/margins": -0.006689900998026133, + "rewards/rejected": -0.00333206239156425, + "step": 70 + }, + { + "epoch": 0.08, + "learning_rate": 1.3745704467353952e-07, + "logits/chosen": -2.4601337909698486, + "logits/rejected": -2.488020181655884, + "logps/chosen": -243.9931182861328, + "logps/rejected": -210.5398406982422, + "loss": 0.6928, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.0015067579224705696, + "rewards/margins": 0.0036226934753358364, + "rewards/rejected": -0.002115936018526554, + "step": 80 + }, + { + "epoch": 0.09, + "learning_rate": 1.5463917525773197e-07, + "logits/chosen": -2.45121431350708, + "logits/rejected": -2.3661465644836426, + "logps/chosen": -214.99539184570312, + "logps/rejected": -186.57984924316406, + "loss": 0.6917, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": 0.017697982490062714, + "rewards/margins": 0.007963714189827442, + "rewards/rejected": 0.009734268300235271, + "step": 90 + }, + { + "epoch": 0.1, + "learning_rate": 1.718213058419244e-07, + "logits/chosen": -2.476562261581421, + "logits/rejected": -2.409389019012451, + "logps/chosen": -310.9223327636719, + "logps/rejected": -162.9552764892578, + "loss": 0.6911, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": 0.007375421933829784, + "rewards/margins": 0.01819022186100483, + "rewards/rejected": -0.010814799927175045, + "step": 100 + }, + { + "epoch": 0.11, + "learning_rate": 1.8900343642611682e-07, + "logits/chosen": -2.426025867462158, + "logits/rejected": -2.0674071311950684, + "logps/chosen": -303.84112548828125, + "logps/rejected": -193.99363708496094, + "loss": 0.6914, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.01276397705078125, + "rewards/margins": 0.010143814608454704, + "rewards/rejected": 0.0026201631408184767, + "step": 110 + }, + { + "epoch": 0.12, + "learning_rate": 2.0618556701030925e-07, + "logits/chosen": -2.2423105239868164, + "logits/rejected": -2.5096383094787598, + "logps/chosen": -295.35986328125, + "logps/rejected": -273.8446350097656, + "loss": 0.6914, + "rewards/accuracies": 0.25, + "rewards/chosen": -0.015526200644671917, + "rewards/margins": -0.02725866064429283, + "rewards/rejected": 0.011732463724911213, + "step": 120 + }, + { + "epoch": 0.13, + "learning_rate": 2.2336769759450173e-07, + "logits/chosen": -2.538625955581665, + "logits/rejected": -2.2574923038482666, + "logps/chosen": -289.69061279296875, + "logps/rejected": -183.3281707763672, + "loss": 0.69, + "rewards/accuracies": 0.3499999940395355, + "rewards/chosen": -0.0016457748133689165, + "rewards/margins": -0.00430142879486084, + "rewards/rejected": 0.0026556537486612797, + "step": 130 + }, + { + "epoch": 0.14, + "learning_rate": 2.405498281786942e-07, + "logits/chosen": -2.540769338607788, + "logits/rejected": -2.2134933471679688, + "logps/chosen": -221.8517303466797, + "logps/rejected": -183.58389282226562, + "loss": 0.6897, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.004911613650619984, + "rewards/margins": 0.005466667469590902, + "rewards/rejected": -0.0005550530040636659, + "step": 140 + }, + { + "epoch": 0.15, + "learning_rate": 2.5773195876288655e-07, + "logits/chosen": -2.1684834957122803, + "logits/rejected": -2.418734073638916, + "logps/chosen": -366.19830322265625, + "logps/rejected": -190.3058319091797, + "loss": 0.6865, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 0.025678252801299095, + "rewards/margins": 0.029336413368582726, + "rewards/rejected": -0.0036581619642674923, + "step": 150 + }, + { + "epoch": 0.17, + "learning_rate": 2.7491408934707903e-07, + "logits/chosen": -2.698945999145508, + "logits/rejected": -2.6068835258483887, + "logps/chosen": -441.375, + "logps/rejected": -300.767822265625, + "loss": 0.6841, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.030073052272200584, + "rewards/margins": 0.047654617577791214, + "rewards/rejected": -0.017581569030880928, + "step": 160 + }, + { + "epoch": 0.18, + "learning_rate": 2.9209621993127146e-07, + "logits/chosen": -2.3280515670776367, + "logits/rejected": -2.320730686187744, + "logps/chosen": -271.07452392578125, + "logps/rejected": -224.6231231689453, + "loss": 0.6864, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": 0.02470027096569538, + "rewards/margins": 0.01814831793308258, + "rewards/rejected": 0.006551951169967651, + "step": 170 + }, + { + "epoch": 0.19, + "learning_rate": 3.0927835051546394e-07, + "logits/chosen": -2.5555853843688965, + "logits/rejected": -2.6365621089935303, + "logps/chosen": -271.2387390136719, + "logps/rejected": -235.908935546875, + "loss": 0.6835, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.03413509577512741, + "rewards/margins": 0.046364832669496536, + "rewards/rejected": -0.012229740619659424, + "step": 180 + }, + { + "epoch": 0.2, + "learning_rate": 3.2646048109965636e-07, + "logits/chosen": -2.7220354080200195, + "logits/rejected": -2.5860915184020996, + "logps/chosen": -302.74267578125, + "logps/rejected": -208.666015625, + "loss": 0.6814, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.01786138489842415, + "rewards/margins": 0.0315425843000412, + "rewards/rejected": -0.013681203126907349, + "step": 190 + }, + { + "epoch": 0.21, + "learning_rate": 3.436426116838488e-07, + "logits/chosen": -2.4432010650634766, + "logits/rejected": -2.346902370452881, + "logps/chosen": -259.94525146484375, + "logps/rejected": -188.97792053222656, + "loss": 0.679, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": 0.021963179111480713, + "rewards/margins": 0.03701996058225632, + "rewards/rejected": -0.015056787058711052, + "step": 200 + }, + { + "epoch": 0.22, + "learning_rate": 3.608247422680412e-07, + "logits/chosen": -2.793883800506592, + "logits/rejected": -2.4634127616882324, + "logps/chosen": -378.5939025878906, + "logps/rejected": -226.9193115234375, + "loss": 0.6767, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": 0.013226890936493874, + "rewards/margins": 0.02839084342122078, + "rewards/rejected": -0.015163958072662354, + "step": 210 + }, + { + "epoch": 0.23, + "learning_rate": 3.7800687285223364e-07, + "logits/chosen": -2.3424696922302246, + "logits/rejected": -2.2594411373138428, + "logps/chosen": -166.11044311523438, + "logps/rejected": -209.3076934814453, + "loss": 0.6758, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.006952253170311451, + "rewards/margins": 0.02028687857091427, + "rewards/rejected": -0.027239132672548294, + "step": 220 + }, + { + "epoch": 0.24, + "learning_rate": 3.9518900343642607e-07, + "logits/chosen": -2.5350894927978516, + "logits/rejected": -2.4550373554229736, + "logps/chosen": -335.16162109375, + "logps/rejected": -251.5087890625, + "loss": 0.674, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.0555986650288105, + "rewards/margins": 0.072585329413414, + "rewards/rejected": -0.01698666624724865, + "step": 230 + }, + { + "epoch": 0.25, + "learning_rate": 4.123711340206185e-07, + "logits/chosen": -2.4425277709960938, + "logits/rejected": -2.4560999870300293, + "logps/chosen": -364.9286193847656, + "logps/rejected": -251.4180450439453, + "loss": 0.6683, + "rewards/accuracies": 0.5, + "rewards/chosen": 0.006999121513217688, + "rewards/margins": 0.015280758030712605, + "rewards/rejected": -0.00828163605183363, + "step": 240 + }, + { + "epoch": 0.26, + "learning_rate": 4.2955326460481097e-07, + "logits/chosen": -2.358187675476074, + "logits/rejected": -2.3916361331939697, + "logps/chosen": -278.9718322753906, + "logps/rejected": -262.45458984375, + "loss": 0.6684, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.025192927569150925, + "rewards/margins": 0.08774158358573914, + "rewards/rejected": -0.06254865229129791, + "step": 250 + }, + { + "epoch": 0.27, + "learning_rate": 4.4673539518900345e-07, + "logits/chosen": -2.117164134979248, + "logits/rejected": -2.433020830154419, + "logps/chosen": -203.03009033203125, + "logps/rejected": -217.13827514648438, + "loss": 0.6679, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.003108497243374586, + "rewards/margins": 0.0338444858789444, + "rewards/rejected": -0.030735988169908524, + "step": 260 + }, + { + "epoch": 0.28, + "learning_rate": 4.639175257731959e-07, + "logits/chosen": -2.236057996749878, + "logits/rejected": -2.4115004539489746, + "logps/chosen": -236.7349395751953, + "logps/rejected": -238.6702880859375, + "loss": 0.6612, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": 0.0009882673621177673, + "rewards/margins": 0.07193086296319962, + "rewards/rejected": -0.07094259560108185, + "step": 270 + }, + { + "epoch": 0.29, + "learning_rate": 4.810996563573884e-07, + "logits/chosen": -2.474492311477661, + "logits/rejected": -2.3905348777770996, + "logps/chosen": -330.3529357910156, + "logps/rejected": -231.5391082763672, + "loss": 0.6578, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 0.03507215529680252, + "rewards/margins": 0.10007444769144058, + "rewards/rejected": -0.06500230729579926, + "step": 280 + }, + { + "epoch": 0.3, + "learning_rate": 4.982817869415807e-07, + "logits/chosen": -2.564732074737549, + "logits/rejected": -2.3240487575531006, + "logps/chosen": -277.9839172363281, + "logps/rejected": -197.73281860351562, + "loss": 0.6567, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 0.05555497482419014, + "rewards/margins": 0.15978679060935974, + "rewards/rejected": -0.1042318120598793, + "step": 290 + }, + { + "epoch": 0.31, + "learning_rate": 4.982798165137615e-07, + "logits/chosen": -2.3241677284240723, + "logits/rejected": -2.348520040512085, + "logps/chosen": -209.7371063232422, + "logps/rejected": -238.69595336914062, + "loss": 0.6501, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.041191719472408295, + "rewards/margins": 0.12445306777954102, + "rewards/rejected": -0.08326133340597153, + "step": 300 + }, + { + "epoch": 0.32, + "learning_rate": 4.963685015290519e-07, + "logits/chosen": -2.466608762741089, + "logits/rejected": -2.338318347930908, + "logps/chosen": -360.8646545410156, + "logps/rejected": -246.0187530517578, + "loss": 0.6505, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 0.04523809999227524, + "rewards/margins": 0.1772933453321457, + "rewards/rejected": -0.13205525279045105, + "step": 310 + }, + { + "epoch": 0.33, + "learning_rate": 4.944571865443424e-07, + "logits/chosen": -2.318599224090576, + "logits/rejected": -2.3406758308410645, + "logps/chosen": -217.85122680664062, + "logps/rejected": -143.0299072265625, + "loss": 0.6362, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": 0.05060691758990288, + "rewards/margins": 0.1060304194688797, + "rewards/rejected": -0.05542349815368652, + "step": 320 + }, + { + "epoch": 0.34, + "learning_rate": 4.92545871559633e-07, + "logits/chosen": -2.477591037750244, + "logits/rejected": -2.4527573585510254, + "logps/chosen": -326.4366455078125, + "logps/rejected": -284.9319763183594, + "loss": 0.6424, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 0.0541137270629406, + "rewards/margins": 0.10549300909042358, + "rewards/rejected": -0.051379282027482986, + "step": 330 + }, + { + "epoch": 0.35, + "learning_rate": 4.906345565749235e-07, + "logits/chosen": -2.402550220489502, + "logits/rejected": -2.2470858097076416, + "logps/chosen": -232.6081085205078, + "logps/rejected": -135.73312377929688, + "loss": 0.6441, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 0.018155479803681374, + "rewards/margins": 0.245174378156662, + "rewards/rejected": -0.22701887786388397, + "step": 340 + }, + { + "epoch": 0.36, + "learning_rate": 4.88723241590214e-07, + "logits/chosen": -2.4398138523101807, + "logits/rejected": -2.2956748008728027, + "logps/chosen": -379.9845275878906, + "logps/rejected": -224.34408569335938, + "loss": 0.6282, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 0.09958186000585556, + "rewards/margins": 0.3024463653564453, + "rewards/rejected": -0.20286449790000916, + "step": 350 + }, + { + "epoch": 0.37, + "learning_rate": 4.868119266055046e-07, + "logits/chosen": -2.2719640731811523, + "logits/rejected": -2.3680129051208496, + "logps/chosen": -213.4238739013672, + "logps/rejected": -282.51495361328125, + "loss": 0.6226, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.02809527888894081, + "rewards/margins": 0.08343404531478882, + "rewards/rejected": -0.11152933537960052, + "step": 360 + }, + { + "epoch": 0.38, + "learning_rate": 4.849006116207951e-07, + "logits/chosen": -2.400339365005493, + "logits/rejected": -2.400993824005127, + "logps/chosen": -310.1695861816406, + "logps/rejected": -258.10113525390625, + "loss": 0.6198, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 0.06970132142305374, + "rewards/margins": 0.2196888029575348, + "rewards/rejected": -0.14998750388622284, + "step": 370 + }, + { + "epoch": 0.39, + "learning_rate": 4.829892966360856e-07, + "logits/chosen": -2.42053484916687, + "logits/rejected": -2.3133673667907715, + "logps/chosen": -279.19329833984375, + "logps/rejected": -253.25363159179688, + "loss": 0.6356, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 0.14709529280662537, + "rewards/margins": 0.32831987738609314, + "rewards/rejected": -0.18122461438179016, + "step": 380 + }, + { + "epoch": 0.4, + "learning_rate": 4.810779816513762e-07, + "logits/chosen": -2.3398146629333496, + "logits/rejected": -2.497467041015625, + "logps/chosen": -199.78628540039062, + "logps/rejected": -176.40805053710938, + "loss": 0.6268, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.019132133573293686, + "rewards/margins": 0.16773070394992828, + "rewards/rejected": -0.18686284124851227, + "step": 390 + }, + { + "epoch": 0.41, + "learning_rate": 4.791666666666667e-07, + "logits/chosen": -2.5169873237609863, + "logits/rejected": -2.568854808807373, + "logps/chosen": -289.6071472167969, + "logps/rejected": -184.77845764160156, + "loss": 0.6193, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 0.08133763074874878, + "rewards/margins": 0.25977957248687744, + "rewards/rejected": -0.17844195663928986, + "step": 400 + }, + { + "epoch": 0.42, + "learning_rate": 4.772553516819572e-07, + "logits/chosen": -2.1512584686279297, + "logits/rejected": -2.186790943145752, + "logps/chosen": -119.762451171875, + "logps/rejected": -174.71890258789062, + "loss": 0.6209, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.12059535086154938, + "rewards/margins": 0.047021061182022095, + "rewards/rejected": -0.16761639714241028, + "step": 410 + }, + { + "epoch": 0.43, + "learning_rate": 4.753440366972477e-07, + "logits/chosen": -2.5430076122283936, + "logits/rejected": -2.671963691711426, + "logps/chosen": -280.49481201171875, + "logps/rejected": -277.0762939453125, + "loss": 0.6283, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.05606144666671753, + "rewards/margins": 0.046925198286771774, + "rewards/rejected": -0.10298663377761841, + "step": 420 + }, + { + "epoch": 0.44, + "learning_rate": 4.7343272171253825e-07, + "logits/chosen": -2.290693759918213, + "logits/rejected": -2.2883286476135254, + "logps/chosen": -248.0079345703125, + "logps/rejected": -237.4206085205078, + "loss": 0.6154, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 0.013114357367157936, + "rewards/margins": 0.2169400155544281, + "rewards/rejected": -0.2038256675004959, + "step": 430 + }, + { + "epoch": 0.45, + "learning_rate": 4.715214067278288e-07, + "logits/chosen": -2.404177188873291, + "logits/rejected": -2.43928861618042, + "logps/chosen": -233.565185546875, + "logps/rejected": -212.0560302734375, + "loss": 0.6021, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.052248239517211914, + "rewards/margins": 0.1868211030960083, + "rewards/rejected": -0.23906934261322021, + "step": 440 + }, + { + "epoch": 0.46, + "learning_rate": 4.696100917431192e-07, + "logits/chosen": -2.405780792236328, + "logits/rejected": -2.449907064437866, + "logps/chosen": -254.3538818359375, + "logps/rejected": -255.2841796875, + "loss": 0.6353, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.08873947709798813, + "rewards/margins": 0.23247499763965607, + "rewards/rejected": -0.14373552799224854, + "step": 450 + }, + { + "epoch": 0.47, + "learning_rate": 4.6769877675840974e-07, + "logits/chosen": -2.3749642372131348, + "logits/rejected": -2.4225552082061768, + "logps/chosen": -226.59994506835938, + "logps/rejected": -198.53065490722656, + "loss": 0.5953, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 0.0910528227686882, + "rewards/margins": 0.3178223967552185, + "rewards/rejected": -0.2267695963382721, + "step": 460 + }, + { + "epoch": 0.49, + "learning_rate": 4.6578746177370027e-07, + "logits/chosen": -2.500232696533203, + "logits/rejected": -2.5585453510284424, + "logps/chosen": -290.7154541015625, + "logps/rejected": -141.37062072753906, + "loss": 0.598, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 0.04495004937052727, + "rewards/margins": 0.2956233620643616, + "rewards/rejected": -0.2506733536720276, + "step": 470 + }, + { + "epoch": 0.5, + "learning_rate": 4.638761467889908e-07, + "logits/chosen": -2.41825532913208, + "logits/rejected": -2.539612054824829, + "logps/chosen": -201.26425170898438, + "logps/rejected": -197.85702514648438, + "loss": 0.6148, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.03925345838069916, + "rewards/margins": 0.20073261857032776, + "rewards/rejected": -0.2399860918521881, + "step": 480 + }, + { + "epoch": 0.51, + "learning_rate": 4.6196483180428133e-07, + "logits/chosen": -2.517301559448242, + "logits/rejected": -2.471219539642334, + "logps/chosen": -248.6940460205078, + "logps/rejected": -149.48056030273438, + "loss": 0.5953, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.005128766410052776, + "rewards/margins": 0.24176044762134552, + "rewards/rejected": -0.24688920378684998, + "step": 490 + }, + { + "epoch": 0.52, + "learning_rate": 4.600535168195718e-07, + "logits/chosen": -2.421499729156494, + "logits/rejected": -2.313544511795044, + "logps/chosen": -198.4822540283203, + "logps/rejected": -208.656982421875, + "loss": 0.5988, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.08058812469244003, + "rewards/margins": 0.1655338704586029, + "rewards/rejected": -0.24612200260162354, + "step": 500 + }, + { + "epoch": 0.53, + "learning_rate": 4.5814220183486234e-07, + "logits/chosen": -2.4364054203033447, + "logits/rejected": -2.474491834640503, + "logps/chosen": -357.8948059082031, + "logps/rejected": -313.86407470703125, + "loss": 0.5964, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09066192060709, + "rewards/margins": 0.11807866394519806, + "rewards/rejected": -0.20874059200286865, + "step": 510 + }, + { + "epoch": 0.54, + "learning_rate": 4.562308868501529e-07, + "logits/chosen": -2.4329869747161865, + "logits/rejected": -2.3959157466888428, + "logps/chosen": -252.12997436523438, + "logps/rejected": -272.59930419921875, + "loss": 0.5992, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 0.09437114745378494, + "rewards/margins": 0.3432508707046509, + "rewards/rejected": -0.24887971580028534, + "step": 520 + }, + { + "epoch": 0.55, + "learning_rate": 4.543195718654434e-07, + "logits/chosen": -2.5292797088623047, + "logits/rejected": -2.472195625305176, + "logps/chosen": -311.61810302734375, + "logps/rejected": -204.62269592285156, + "loss": 0.6052, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.022383611649274826, + "rewards/margins": 0.4391644597053528, + "rewards/rejected": -0.4615480303764343, + "step": 530 + }, + { + "epoch": 0.56, + "learning_rate": 4.5240825688073394e-07, + "logits/chosen": -2.4026424884796143, + "logits/rejected": -2.410029649734497, + "logps/chosen": -254.65774536132812, + "logps/rejected": -220.6438751220703, + "loss": 0.6005, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": 0.06496790796518326, + "rewards/margins": 0.32214078307151794, + "rewards/rejected": -0.2571728527545929, + "step": 540 + }, + { + "epoch": 0.57, + "learning_rate": 4.504969418960244e-07, + "logits/chosen": -2.489189624786377, + "logits/rejected": -2.509007215499878, + "logps/chosen": -304.0901184082031, + "logps/rejected": -276.55523681640625, + "loss": 0.6, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.08526446670293808, + "rewards/margins": 0.23875808715820312, + "rewards/rejected": -0.15349361300468445, + "step": 550 + }, + { + "epoch": 0.58, + "learning_rate": 4.4858562691131495e-07, + "logits/chosen": -2.626237392425537, + "logits/rejected": -2.4529411792755127, + "logps/chosen": -250.4290008544922, + "logps/rejected": -236.1934814453125, + "loss": 0.5868, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 0.14224590361118317, + "rewards/margins": 0.5076346397399902, + "rewards/rejected": -0.3653886914253235, + "step": 560 + }, + { + "epoch": 0.59, + "learning_rate": 4.466743119266055e-07, + "logits/chosen": -2.608710527420044, + "logits/rejected": -2.4026219844818115, + "logps/chosen": -322.64434814453125, + "logps/rejected": -315.7530212402344, + "loss": 0.6015, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": 0.017860546708106995, + "rewards/margins": 0.29392343759536743, + "rewards/rejected": -0.27606287598609924, + "step": 570 + }, + { + "epoch": 0.6, + "learning_rate": 4.44762996941896e-07, + "logits/chosen": -2.3832476139068604, + "logits/rejected": -2.451291561126709, + "logps/chosen": -298.6803894042969, + "logps/rejected": -213.0013885498047, + "loss": 0.5851, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 0.08139020204544067, + "rewards/margins": 0.48270922899246216, + "rewards/rejected": -0.4013189375400543, + "step": 580 + }, + { + "epoch": 0.61, + "learning_rate": 4.4285168195718655e-07, + "logits/chosen": -2.4148542881011963, + "logits/rejected": -2.6071524620056152, + "logps/chosen": -235.28567504882812, + "logps/rejected": -229.70938110351562, + "loss": 0.5857, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.05157552286982536, + "rewards/margins": 0.3185363709926605, + "rewards/rejected": -0.3701118528842926, + "step": 590 + }, + { + "epoch": 0.62, + "learning_rate": 4.40940366972477e-07, + "logits/chosen": -2.6201112270355225, + "logits/rejected": -2.5222606658935547, + "logps/chosen": -187.01014709472656, + "logps/rejected": -190.72042846679688, + "loss": 0.591, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.02282007224857807, + "rewards/margins": 0.2915310561656952, + "rewards/rejected": -0.3143511116504669, + "step": 600 + }, + { + "epoch": 0.63, + "learning_rate": 4.3902905198776756e-07, + "logits/chosen": -2.701819896697998, + "logits/rejected": -2.6200711727142334, + "logps/chosen": -400.83453369140625, + "logps/rejected": -312.381591796875, + "loss": 0.5893, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.1581762284040451, + "rewards/margins": 0.5981796979904175, + "rewards/rejected": -0.4400033950805664, + "step": 610 + }, + { + "epoch": 0.64, + "learning_rate": 4.371177370030581e-07, + "logits/chosen": -2.5501413345336914, + "logits/rejected": -2.316926956176758, + "logps/chosen": -223.07455444335938, + "logps/rejected": -188.2316436767578, + "loss": 0.5698, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.12886852025985718, + "rewards/margins": 0.3416977822780609, + "rewards/rejected": -0.47056636214256287, + "step": 620 + }, + { + "epoch": 0.65, + "learning_rate": 4.352064220183486e-07, + "logits/chosen": -2.261867046356201, + "logits/rejected": -2.159829616546631, + "logps/chosen": -219.3220977783203, + "logps/rejected": -205.33786010742188, + "loss": 0.5838, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 0.04584250971674919, + "rewards/margins": 0.3203400671482086, + "rewards/rejected": -0.2744975686073303, + "step": 630 + }, + { + "epoch": 0.66, + "learning_rate": 4.3329510703363915e-07, + "logits/chosen": -2.319148063659668, + "logits/rejected": -2.3123698234558105, + "logps/chosen": -306.0313415527344, + "logps/rejected": -234.1546630859375, + "loss": 0.5699, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.04760587960481644, + "rewards/margins": 0.6112427711486816, + "rewards/rejected": -0.563636839389801, + "step": 640 + }, + { + "epoch": 0.67, + "learning_rate": 4.313837920489297e-07, + "logits/chosen": -2.3105361461639404, + "logits/rejected": -2.3982295989990234, + "logps/chosen": -284.56072998046875, + "logps/rejected": -315.3321838378906, + "loss": 0.58, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 0.0843208059668541, + "rewards/margins": 0.43474799394607544, + "rewards/rejected": -0.35042712092399597, + "step": 650 + }, + { + "epoch": 0.68, + "learning_rate": 4.2947247706422016e-07, + "logits/chosen": -2.4114909172058105, + "logits/rejected": -2.4491775035858154, + "logps/chosen": -219.2405548095703, + "logps/rejected": -253.74575805664062, + "loss": 0.5704, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.01176774688065052, + "rewards/margins": 0.2667906582355499, + "rewards/rejected": -0.2785584330558777, + "step": 660 + }, + { + "epoch": 0.69, + "learning_rate": 4.275611620795107e-07, + "logits/chosen": -2.519127368927002, + "logits/rejected": -2.356675863265991, + "logps/chosen": -315.9088439941406, + "logps/rejected": -185.24163818359375, + "loss": 0.5741, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.10262980312108994, + "rewards/margins": 0.5418930053710938, + "rewards/rejected": -0.6445227861404419, + "step": 670 + }, + { + "epoch": 0.7, + "learning_rate": 4.2564984709480123e-07, + "logits/chosen": -2.520284652709961, + "logits/rejected": -2.5212578773498535, + "logps/chosen": -339.563720703125, + "logps/rejected": -229.5682373046875, + "loss": 0.6126, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.15836027264595032, + "rewards/margins": 0.2690756916999817, + "rewards/rejected": -0.427435964345932, + "step": 680 + }, + { + "epoch": 0.71, + "learning_rate": 4.2373853211009176e-07, + "logits/chosen": -2.461088180541992, + "logits/rejected": -2.431992292404175, + "logps/chosen": -314.1544494628906, + "logps/rejected": -220.8463897705078, + "loss": 0.6011, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.0033186853397637606, + "rewards/margins": 0.4757082462310791, + "rewards/rejected": -0.4790269732475281, + "step": 690 + }, + { + "epoch": 0.72, + "learning_rate": 4.2182721712538224e-07, + "logits/chosen": -2.554452657699585, + "logits/rejected": -2.2479560375213623, + "logps/chosen": -234.7042236328125, + "logps/rejected": -188.83786010742188, + "loss": 0.5883, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.0076250312849879265, + "rewards/margins": 0.5563846230506897, + "rewards/rejected": -0.5487595796585083, + "step": 700 + }, + { + "epoch": 0.73, + "learning_rate": 4.199159021406727e-07, + "logits/chosen": -2.318946361541748, + "logits/rejected": -2.453735828399658, + "logps/chosen": -209.1856231689453, + "logps/rejected": -227.36953735351562, + "loss": 0.5861, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.13343460857868195, + "rewards/margins": 0.22984579205513, + "rewards/rejected": -0.36328038573265076, + "step": 710 + }, + { + "epoch": 0.74, + "learning_rate": 4.1800458715596325e-07, + "logits/chosen": -2.4781854152679443, + "logits/rejected": -2.2781224250793457, + "logps/chosen": -226.40469360351562, + "logps/rejected": -289.3362121582031, + "loss": 0.5966, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.2653411030769348, + "rewards/margins": 0.09751178324222565, + "rewards/rejected": -0.36285287141799927, + "step": 720 + }, + { + "epoch": 0.75, + "learning_rate": 4.160932721712538e-07, + "logits/chosen": -2.4424407482147217, + "logits/rejected": -2.4211478233337402, + "logps/chosen": -273.59869384765625, + "logps/rejected": -269.41717529296875, + "loss": 0.5795, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": 0.003456842852756381, + "rewards/margins": 0.21872258186340332, + "rewards/rejected": -0.21526578068733215, + "step": 730 + }, + { + "epoch": 0.76, + "learning_rate": 4.141819571865443e-07, + "logits/chosen": -2.2280640602111816, + "logits/rejected": -2.307762622833252, + "logps/chosen": -247.0033721923828, + "logps/rejected": -227.2200927734375, + "loss": 0.5638, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.06740056723356247, + "rewards/margins": 0.43576961755752563, + "rewards/rejected": -0.5031701326370239, + "step": 740 + }, + { + "epoch": 0.77, + "learning_rate": 4.1227064220183485e-07, + "logits/chosen": -2.397576332092285, + "logits/rejected": -2.354574680328369, + "logps/chosen": -214.11892700195312, + "logps/rejected": -215.72268676757812, + "loss": 0.5816, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.17300334572792053, + "rewards/margins": 0.3608870208263397, + "rewards/rejected": -0.5338903665542603, + "step": 750 + }, + { + "epoch": 0.78, + "learning_rate": 4.103593272171253e-07, + "logits/chosen": -2.33254337310791, + "logits/rejected": -2.3918721675872803, + "logps/chosen": -340.80072021484375, + "logps/rejected": -347.1974792480469, + "loss": 0.5778, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.130618155002594, + "rewards/margins": 0.32134371995925903, + "rewards/rejected": -0.4519619047641754, + "step": 760 + }, + { + "epoch": 0.79, + "learning_rate": 4.0844801223241586e-07, + "logits/chosen": -2.3820478916168213, + "logits/rejected": -2.4310269355773926, + "logps/chosen": -228.812744140625, + "logps/rejected": -197.6693115234375, + "loss": 0.5567, + "rewards/accuracies": 0.44999998807907104, + "rewards/chosen": -0.0885491669178009, + "rewards/margins": 0.15732769668102264, + "rewards/rejected": -0.24587683379650116, + "step": 770 + }, + { + "epoch": 0.8, + "learning_rate": 4.065366972477064e-07, + "logits/chosen": -2.5541110038757324, + "logits/rejected": -2.3786189556121826, + "logps/chosen": -273.0157165527344, + "logps/rejected": -203.12271118164062, + "loss": 0.5676, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.07696160674095154, + "rewards/margins": 0.5536447763442993, + "rewards/rejected": -0.6306063532829285, + "step": 780 + }, + { + "epoch": 0.82, + "learning_rate": 4.046253822629969e-07, + "logits/chosen": -2.2037081718444824, + "logits/rejected": -2.20261549949646, + "logps/chosen": -158.65432739257812, + "logps/rejected": -185.50302124023438, + "loss": 0.5411, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.19409586489200592, + "rewards/margins": 0.23793645203113556, + "rewards/rejected": -0.4320322871208191, + "step": 790 + }, + { + "epoch": 0.83, + "learning_rate": 4.0271406727828745e-07, + "logits/chosen": -2.465301990509033, + "logits/rejected": -2.5763792991638184, + "logps/chosen": -271.5575256347656, + "logps/rejected": -209.1376953125, + "loss": 0.5711, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.005071232561022043, + "rewards/margins": 0.3356703221797943, + "rewards/rejected": -0.3407415747642517, + "step": 800 + }, + { + "epoch": 0.84, + "learning_rate": 4.00802752293578e-07, + "logits/chosen": -2.3711938858032227, + "logits/rejected": -2.4800353050231934, + "logps/chosen": -240.88314819335938, + "logps/rejected": -233.5401153564453, + "loss": 0.5449, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.08367098867893219, + "rewards/margins": 0.7126539945602417, + "rewards/rejected": -0.7963249683380127, + "step": 810 + }, + { + "epoch": 0.85, + "learning_rate": 3.9889143730886847e-07, + "logits/chosen": -2.42783784866333, + "logits/rejected": -2.4193737506866455, + "logps/chosen": -281.68548583984375, + "logps/rejected": -190.40451049804688, + "loss": 0.5617, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.049332987517118454, + "rewards/margins": 0.5959105491638184, + "rewards/rejected": -0.5465775728225708, + "step": 820 + }, + { + "epoch": 0.86, + "learning_rate": 3.96980122324159e-07, + "logits/chosen": -2.5133376121520996, + "logits/rejected": -2.4929747581481934, + "logps/chosen": -226.41513061523438, + "logps/rejected": -218.5712890625, + "loss": 0.5419, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.004038684070110321, + "rewards/margins": 0.6471161842346191, + "rewards/rejected": -0.643077552318573, + "step": 830 + }, + { + "epoch": 0.87, + "learning_rate": 3.9506880733944953e-07, + "logits/chosen": -2.366255760192871, + "logits/rejected": -2.1581902503967285, + "logps/chosen": -243.42214965820312, + "logps/rejected": -193.34178161621094, + "loss": 0.5987, + "rewards/accuracies": 0.4000000059604645, + "rewards/chosen": -0.29814183712005615, + "rewards/margins": 0.10912153869867325, + "rewards/rejected": -0.4072634279727936, + "step": 840 + }, + { + "epoch": 0.88, + "learning_rate": 3.9315749235474006e-07, + "logits/chosen": -2.4702069759368896, + "logits/rejected": -2.2905399799346924, + "logps/chosen": -202.70205688476562, + "logps/rejected": -229.86154174804688, + "loss": 0.5651, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.14235912263393402, + "rewards/margins": 0.43106895685195923, + "rewards/rejected": -0.5734280347824097, + "step": 850 + }, + { + "epoch": 0.89, + "learning_rate": 3.912461773700306e-07, + "logits/chosen": -2.488694429397583, + "logits/rejected": -2.2790422439575195, + "logps/chosen": -393.3223876953125, + "logps/rejected": -257.35986328125, + "loss": 0.5666, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 0.01125104445964098, + "rewards/margins": 0.7673934698104858, + "rewards/rejected": -0.7561424374580383, + "step": 860 + }, + { + "epoch": 0.9, + "learning_rate": 3.8933486238532107e-07, + "logits/chosen": -2.2432663440704346, + "logits/rejected": -2.2947564125061035, + "logps/chosen": -359.70989990234375, + "logps/rejected": -240.7715301513672, + "loss": 0.5618, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.03136935830116272, + "rewards/margins": 0.8124542236328125, + "rewards/rejected": -0.8438236117362976, + "step": 870 + }, + { + "epoch": 0.91, + "learning_rate": 3.874235474006116e-07, + "logits/chosen": -2.173022747039795, + "logits/rejected": -2.1610169410705566, + "logps/chosen": -201.29830932617188, + "logps/rejected": -236.1139678955078, + "loss": 0.5597, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.15942071378231049, + "rewards/margins": 0.318124383687973, + "rewards/rejected": -0.4775451123714447, + "step": 880 + }, + { + "epoch": 0.92, + "learning_rate": 3.8551223241590214e-07, + "logits/chosen": -2.410968542098999, + "logits/rejected": -2.1644670963287354, + "logps/chosen": -223.5022735595703, + "logps/rejected": -212.260986328125, + "loss": 0.5794, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.12731042504310608, + "rewards/margins": 0.8949457406997681, + "rewards/rejected": -1.0222561359405518, + "step": 890 + }, + { + "epoch": 0.93, + "learning_rate": 3.8360091743119267e-07, + "logits/chosen": -2.505922794342041, + "logits/rejected": -2.4096271991729736, + "logps/chosen": -276.6337890625, + "logps/rejected": -244.76632690429688, + "loss": 0.5492, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 0.0883268266916275, + "rewards/margins": 0.560738205909729, + "rewards/rejected": -0.4724113345146179, + "step": 900 + }, + { + "epoch": 0.94, + "learning_rate": 3.816896024464832e-07, + "logits/chosen": -2.4202027320861816, + "logits/rejected": -2.3245034217834473, + "logps/chosen": -171.32737731933594, + "logps/rejected": -179.85043334960938, + "loss": 0.5671, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.12893405556678772, + "rewards/margins": 0.2722606360912323, + "rewards/rejected": -0.4011947214603424, + "step": 910 + }, + { + "epoch": 0.95, + "learning_rate": 3.797782874617737e-07, + "logits/chosen": -2.489109754562378, + "logits/rejected": -2.5008530616760254, + "logps/chosen": -306.02740478515625, + "logps/rejected": -235.5460662841797, + "loss": 0.5567, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.11618290096521378, + "rewards/margins": 0.5509032011032104, + "rewards/rejected": -0.6670862436294556, + "step": 920 + }, + { + "epoch": 0.96, + "learning_rate": 3.778669724770642e-07, + "logits/chosen": -2.308039426803589, + "logits/rejected": -2.1885459423065186, + "logps/chosen": -318.63189697265625, + "logps/rejected": -247.6229248046875, + "loss": 0.566, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.18256303668022156, + "rewards/margins": 0.7064282298088074, + "rewards/rejected": -0.8889913558959961, + "step": 930 + }, + { + "epoch": 0.97, + "learning_rate": 3.7595565749235474e-07, + "logits/chosen": -2.522726535797119, + "logits/rejected": -2.366619110107422, + "logps/chosen": -268.7926025390625, + "logps/rejected": -197.31088256835938, + "loss": 0.5705, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.130192369222641, + "rewards/margins": 0.7135027647018433, + "rewards/rejected": -0.8436950445175171, + "step": 940 + }, + { + "epoch": 0.98, + "learning_rate": 3.740443425076452e-07, + "logits/chosen": -2.3446176052093506, + "logits/rejected": -2.3508477210998535, + "logps/chosen": -267.4695739746094, + "logps/rejected": -273.3948974609375, + "loss": 0.57, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": 0.022189509123563766, + "rewards/margins": 0.3890397548675537, + "rewards/rejected": -0.3668502867221832, + "step": 950 + }, + { + "epoch": 0.99, + "learning_rate": 3.7213302752293575e-07, + "logits/chosen": -2.5760252475738525, + "logits/rejected": -2.461642026901245, + "logps/chosen": -264.9154052734375, + "logps/rejected": -198.0699005126953, + "loss": 0.5491, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.11176663637161255, + "rewards/margins": 0.19738611578941345, + "rewards/rejected": -0.309152752161026, + "step": 960 + }, + { + "epoch": 1.0, + "eval_logits/chosen": -2.1374547481536865, + "eval_logits/rejected": -1.9755022525787354, + "eval_logps/chosen": -248.40455627441406, + "eval_logps/rejected": -195.833251953125, + "eval_loss": 0.5562577247619629, + "eval_rewards/accuracies": 0.78125, + "eval_rewards/chosen": -0.09624822437763214, + "eval_rewards/margins": 0.6263414025306702, + "eval_rewards/rejected": -0.7225896120071411, + "eval_runtime": 49.7794, + "eval_samples_per_second": 40.177, + "eval_steps_per_second": 0.321, + "step": 969 + }, + { + "epoch": 1.0, + "learning_rate": 3.702217125382263e-07, + "logits/chosen": -2.383577823638916, + "logits/rejected": -2.2432830333709717, + "logps/chosen": -269.8841247558594, + "logps/rejected": -266.1859436035156, + "loss": 0.5489, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 0.028007108718156815, + "rewards/margins": 0.6903571486473083, + "rewards/rejected": -0.662350058555603, + "step": 970 + }, + { + "epoch": 1.01, + "learning_rate": 3.6831039755351677e-07, + "logits/chosen": -2.481004238128662, + "logits/rejected": -2.3184635639190674, + "logps/chosen": -239.337890625, + "logps/rejected": -234.6234893798828, + "loss": 0.5453, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.06412211060523987, + "rewards/margins": 0.32830190658569336, + "rewards/rejected": -0.3924240171909332, + "step": 980 + }, + { + "epoch": 1.02, + "learning_rate": 3.663990825688073e-07, + "logits/chosen": -2.437516689300537, + "logits/rejected": -2.390587091445923, + "logps/chosen": -252.07528686523438, + "logps/rejected": -195.83224487304688, + "loss": 0.6027, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.1875399798154831, + "rewards/margins": 0.6106002330780029, + "rewards/rejected": -0.7981401681900024, + "step": 990 + }, + { + "epoch": 1.03, + "learning_rate": 3.6448776758409783e-07, + "logits/chosen": -2.3859527111053467, + "logits/rejected": -2.4744107723236084, + "logps/chosen": -246.77200317382812, + "logps/rejected": -183.67501831054688, + "loss": 0.5812, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.18464355170726776, + "rewards/margins": 0.5135353803634644, + "rewards/rejected": -0.6981789469718933, + "step": 1000 + }, + { + "epoch": 1.04, + "learning_rate": 3.6257645259938836e-07, + "logits/chosen": -2.596139669418335, + "logits/rejected": -2.5789005756378174, + "logps/chosen": -280.490966796875, + "logps/rejected": -216.73489379882812, + "loss": 0.5545, + "rewards/accuracies": 1.0, + "rewards/chosen": 0.08334894478321075, + "rewards/margins": 0.8474000692367554, + "rewards/rejected": -0.7640511393547058, + "step": 1010 + }, + { + "epoch": 1.05, + "learning_rate": 3.606651376146789e-07, + "logits/chosen": -2.3291637897491455, + "logits/rejected": -2.3714513778686523, + "logps/chosen": -224.9653778076172, + "logps/rejected": -184.2610321044922, + "loss": 0.5464, + "rewards/accuracies": 0.5, + "rewards/chosen": -0.2414972335100174, + "rewards/margins": 0.3471061587333679, + "rewards/rejected": -0.5886033773422241, + "step": 1020 + }, + { + "epoch": 1.06, + "learning_rate": 3.5875382262996937e-07, + "logits/chosen": -2.275418758392334, + "logits/rejected": -2.099208354949951, + "logps/chosen": -197.90513610839844, + "logps/rejected": -201.25003051757812, + "loss": 0.5503, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.23300638794898987, + "rewards/margins": 0.42851170897483826, + "rewards/rejected": -0.6615180969238281, + "step": 1030 + }, + { + "epoch": 1.07, + "learning_rate": 3.568425076452599e-07, + "logits/chosen": -2.285912036895752, + "logits/rejected": -2.2500832080841064, + "logps/chosen": -213.13992309570312, + "logps/rejected": -196.5510711669922, + "loss": 0.551, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.3874604105949402, + "rewards/margins": 0.24448566138744354, + "rewards/rejected": -0.6319460868835449, + "step": 1040 + }, + { + "epoch": 1.08, + "learning_rate": 3.5493119266055044e-07, + "logits/chosen": -2.257537364959717, + "logits/rejected": -2.1868162155151367, + "logps/chosen": -262.30859375, + "logps/rejected": -174.8310546875, + "loss": 0.5329, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09565829485654831, + "rewards/margins": 0.5677075982093811, + "rewards/rejected": -0.6633658409118652, + "step": 1050 + }, + { + "epoch": 1.09, + "learning_rate": 3.5301987767584097e-07, + "logits/chosen": -2.5957443714141846, + "logits/rejected": -2.3026747703552246, + "logps/chosen": -248.4452667236328, + "logps/rejected": -291.78692626953125, + "loss": 0.5419, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 0.01751767471432686, + "rewards/margins": 1.0502655506134033, + "rewards/rejected": -1.0327479839324951, + "step": 1060 + }, + { + "epoch": 1.1, + "learning_rate": 3.511085626911315e-07, + "logits/chosen": -2.495492458343506, + "logits/rejected": -2.360352039337158, + "logps/chosen": -379.7645568847656, + "logps/rejected": -288.1813049316406, + "loss": 0.5355, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.29672688245773315, + "rewards/margins": 0.2822774350643158, + "rewards/rejected": -0.5790044069290161, + "step": 1070 + }, + { + "epoch": 1.11, + "learning_rate": 3.49197247706422e-07, + "logits/chosen": -2.5327889919281006, + "logits/rejected": -2.4286463260650635, + "logps/chosen": -283.114501953125, + "logps/rejected": -184.7639923095703, + "loss": 0.5419, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 0.13121911883354187, + "rewards/margins": 0.8478299975395203, + "rewards/rejected": -0.716610848903656, + "step": 1080 + }, + { + "epoch": 1.12, + "learning_rate": 3.472859327217125e-07, + "logits/chosen": -2.3253655433654785, + "logits/rejected": -2.4310760498046875, + "logps/chosen": -363.16827392578125, + "logps/rejected": -301.8953552246094, + "loss": 0.5571, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.21858613193035126, + "rewards/margins": 0.8502481579780579, + "rewards/rejected": -0.631662130355835, + "step": 1090 + }, + { + "epoch": 1.14, + "learning_rate": 3.4537461773700304e-07, + "logits/chosen": -2.407773494720459, + "logits/rejected": -2.4023048877716064, + "logps/chosen": -173.5346221923828, + "logps/rejected": -123.51615905761719, + "loss": 0.563, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.0015801489353179932, + "rewards/margins": 0.45527520775794983, + "rewards/rejected": -0.4568553566932678, + "step": 1100 + }, + { + "epoch": 1.15, + "learning_rate": 3.434633027522936e-07, + "logits/chosen": -2.2176413536071777, + "logits/rejected": -2.1938424110412598, + "logps/chosen": -279.9801330566406, + "logps/rejected": -209.1507568359375, + "loss": 0.5494, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.01898578554391861, + "rewards/margins": 0.47637319564819336, + "rewards/rejected": -0.45738738775253296, + "step": 1110 + }, + { + "epoch": 1.16, + "learning_rate": 3.415519877675841e-07, + "logits/chosen": -2.5352399349212646, + "logits/rejected": -2.4146199226379395, + "logps/chosen": -213.72891235351562, + "logps/rejected": -270.6646728515625, + "loss": 0.5422, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.11530435085296631, + "rewards/margins": 0.4574407935142517, + "rewards/rejected": -0.5727452039718628, + "step": 1120 + }, + { + "epoch": 1.17, + "learning_rate": 3.3964067278287464e-07, + "logits/chosen": -2.37849760055542, + "logits/rejected": -2.2588951587677, + "logps/chosen": -306.17840576171875, + "logps/rejected": -276.8265075683594, + "loss": 0.5349, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 0.07973052561283112, + "rewards/margins": 0.7714017033576965, + "rewards/rejected": -0.6916711926460266, + "step": 1130 + }, + { + "epoch": 1.18, + "learning_rate": 3.377293577981651e-07, + "logits/chosen": -2.130993366241455, + "logits/rejected": -2.4299304485321045, + "logps/chosen": -229.61618041992188, + "logps/rejected": -189.0056915283203, + "loss": 0.508, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.2313126027584076, + "rewards/margins": 0.4152100682258606, + "rewards/rejected": -0.6465227007865906, + "step": 1140 + }, + { + "epoch": 1.19, + "learning_rate": 3.3581804281345565e-07, + "logits/chosen": -2.485884428024292, + "logits/rejected": -2.3729846477508545, + "logps/chosen": -346.53497314453125, + "logps/rejected": -233.9870147705078, + "loss": 0.5674, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.24053406715393066, + "rewards/margins": 0.43819236755371094, + "rewards/rejected": -0.6787264943122864, + "step": 1150 + }, + { + "epoch": 1.2, + "learning_rate": 3.339067278287462e-07, + "logits/chosen": -2.3505210876464844, + "logits/rejected": -2.153390645980835, + "logps/chosen": -230.1878204345703, + "logps/rejected": -220.8087158203125, + "loss": 0.532, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.11666283756494522, + "rewards/margins": 0.6117745637893677, + "rewards/rejected": -0.7284374237060547, + "step": 1160 + }, + { + "epoch": 1.21, + "learning_rate": 3.319954128440367e-07, + "logits/chosen": -2.525207281112671, + "logits/rejected": -2.4221832752227783, + "logps/chosen": -354.8018493652344, + "logps/rejected": -194.69482421875, + "loss": 0.5507, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.03960106521844864, + "rewards/margins": 0.9505462646484375, + "rewards/rejected": -0.9901474118232727, + "step": 1170 + }, + { + "epoch": 1.22, + "learning_rate": 3.3008409785932725e-07, + "logits/chosen": -2.316929340362549, + "logits/rejected": -2.223125696182251, + "logps/chosen": -189.4244384765625, + "logps/rejected": -190.21742248535156, + "loss": 0.5381, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.4602813720703125, + "rewards/margins": 0.2248728722333908, + "rewards/rejected": -0.6851542592048645, + "step": 1180 + }, + { + "epoch": 1.23, + "learning_rate": 3.2817278287461773e-07, + "logits/chosen": -2.5135419368743896, + "logits/rejected": -2.398132801055908, + "logps/chosen": -205.8257293701172, + "logps/rejected": -299.1632385253906, + "loss": 0.5313, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.21163006126880646, + "rewards/margins": 0.4841296672821045, + "rewards/rejected": -0.6957597732543945, + "step": 1190 + }, + { + "epoch": 1.24, + "learning_rate": 3.262614678899082e-07, + "logits/chosen": -2.211836338043213, + "logits/rejected": -2.2863128185272217, + "logps/chosen": -228.488525390625, + "logps/rejected": -252.99923706054688, + "loss": 0.522, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.0907645896077156, + "rewards/margins": 0.7704148292541504, + "rewards/rejected": -0.8611793518066406, + "step": 1200 + }, + { + "epoch": 1.25, + "learning_rate": 3.2435015290519874e-07, + "logits/chosen": -2.300359010696411, + "logits/rejected": -2.210038185119629, + "logps/chosen": -193.10565185546875, + "logps/rejected": -223.6150665283203, + "loss": 0.5302, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.04345179349184036, + "rewards/margins": 0.6841613054275513, + "rewards/rejected": -0.727613091468811, + "step": 1210 + }, + { + "epoch": 1.26, + "learning_rate": 3.2243883792048927e-07, + "logits/chosen": -2.5476744174957275, + "logits/rejected": -2.430663585662842, + "logps/chosen": -229.6248321533203, + "logps/rejected": -247.6537322998047, + "loss": 0.5573, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.20007988810539246, + "rewards/margins": 0.406427800655365, + "rewards/rejected": -0.6065077185630798, + "step": 1220 + }, + { + "epoch": 1.27, + "learning_rate": 3.205275229357798e-07, + "logits/chosen": -2.5259690284729004, + "logits/rejected": -2.2766363620758057, + "logps/chosen": -265.3452453613281, + "logps/rejected": -221.7385711669922, + "loss": 0.5653, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.11124776303768158, + "rewards/margins": 0.5815609693527222, + "rewards/rejected": -0.6928088068962097, + "step": 1230 + }, + { + "epoch": 1.28, + "learning_rate": 3.186162079510703e-07, + "logits/chosen": -2.3755877017974854, + "logits/rejected": -2.4818127155303955, + "logps/chosen": -253.17007446289062, + "logps/rejected": -265.7911071777344, + "loss": 0.5397, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.14208722114562988, + "rewards/margins": 0.624567985534668, + "rewards/rejected": -0.7666550874710083, + "step": 1240 + }, + { + "epoch": 1.29, + "learning_rate": 3.167048929663608e-07, + "logits/chosen": -2.2667508125305176, + "logits/rejected": -2.1175003051757812, + "logps/chosen": -214.2106475830078, + "logps/rejected": -216.2061767578125, + "loss": 0.5402, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.12058589607477188, + "rewards/margins": 0.762450098991394, + "rewards/rejected": -0.8830360174179077, + "step": 1250 + }, + { + "epoch": 1.3, + "learning_rate": 3.1479357798165134e-07, + "logits/chosen": -2.475630283355713, + "logits/rejected": -2.5030765533447266, + "logps/chosen": -229.7735595703125, + "logps/rejected": -204.53561401367188, + "loss": 0.545, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.023252641782164574, + "rewards/margins": 0.7630415558815002, + "rewards/rejected": -0.7397890090942383, + "step": 1260 + }, + { + "epoch": 1.31, + "learning_rate": 3.128822629969419e-07, + "logits/chosen": -2.7179012298583984, + "logits/rejected": -2.6036970615386963, + "logps/chosen": -276.84918212890625, + "logps/rejected": -254.32498168945312, + "loss": 0.5474, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.22881421446800232, + "rewards/margins": 0.6891492009162903, + "rewards/rejected": -0.9179633259773254, + "step": 1270 + }, + { + "epoch": 1.32, + "learning_rate": 3.109709480122324e-07, + "logits/chosen": -2.344045877456665, + "logits/rejected": -2.5483055114746094, + "logps/chosen": -286.34063720703125, + "logps/rejected": -220.1922607421875, + "loss": 0.5663, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.27246418595314026, + "rewards/margins": 0.20061273872852325, + "rewards/rejected": -0.4730769991874695, + "step": 1280 + }, + { + "epoch": 1.33, + "learning_rate": 3.0905963302752294e-07, + "logits/chosen": -2.1131272315979004, + "logits/rejected": -2.2049593925476074, + "logps/chosen": -215.07150268554688, + "logps/rejected": -210.6974639892578, + "loss": 0.5606, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.24951574206352234, + "rewards/margins": 0.28748106956481934, + "rewards/rejected": -0.5369968414306641, + "step": 1290 + }, + { + "epoch": 1.34, + "learning_rate": 3.071483180428134e-07, + "logits/chosen": -2.578439950942993, + "logits/rejected": -2.587803602218628, + "logps/chosen": -395.8404846191406, + "logps/rejected": -238.2346954345703, + "loss": 0.5476, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 0.028068387880921364, + "rewards/margins": 0.7790915966033936, + "rewards/rejected": -0.7510231733322144, + "step": 1300 + }, + { + "epoch": 1.35, + "learning_rate": 3.0523700305810395e-07, + "logits/chosen": -2.45281720161438, + "logits/rejected": -2.3570828437805176, + "logps/chosen": -279.39410400390625, + "logps/rejected": -279.7311096191406, + "loss": 0.5405, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.1633271872997284, + "rewards/margins": 0.6046531200408936, + "rewards/rejected": -0.7679802775382996, + "step": 1310 + }, + { + "epoch": 1.36, + "learning_rate": 3.033256880733945e-07, + "logits/chosen": -2.2366137504577637, + "logits/rejected": -2.1086764335632324, + "logps/chosen": -169.0766143798828, + "logps/rejected": -210.2600555419922, + "loss": 0.5082, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.2424238622188568, + "rewards/margins": 0.16653873026371002, + "rewards/rejected": -0.4089626371860504, + "step": 1320 + }, + { + "epoch": 1.37, + "learning_rate": 3.01414373088685e-07, + "logits/chosen": -2.2935876846313477, + "logits/rejected": -2.4133498668670654, + "logps/chosen": -272.6535949707031, + "logps/rejected": -256.1263122558594, + "loss": 0.5408, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 0.06631962209939957, + "rewards/margins": 0.8952635526657104, + "rewards/rejected": -0.8289439082145691, + "step": 1330 + }, + { + "epoch": 1.38, + "learning_rate": 2.9950305810397555e-07, + "logits/chosen": -2.4236862659454346, + "logits/rejected": -2.5300862789154053, + "logps/chosen": -264.49200439453125, + "logps/rejected": -250.42605590820312, + "loss": 0.5558, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.4723237156867981, + "rewards/margins": 0.4657681882381439, + "rewards/rejected": -0.9380919337272644, + "step": 1340 + }, + { + "epoch": 1.39, + "learning_rate": 2.9759174311926603e-07, + "logits/chosen": -2.388587236404419, + "logits/rejected": -2.268752336502075, + "logps/chosen": -311.207763671875, + "logps/rejected": -154.0712127685547, + "loss": 0.5471, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.054443489760160446, + "rewards/margins": 0.8568767309188843, + "rewards/rejected": -0.9113203883171082, + "step": 1350 + }, + { + "epoch": 1.4, + "learning_rate": 2.9568042813455656e-07, + "logits/chosen": -2.134000301361084, + "logits/rejected": -2.2762789726257324, + "logps/chosen": -236.97683715820312, + "logps/rejected": -211.66000366210938, + "loss": 0.5532, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.23458454012870789, + "rewards/margins": 0.4157086908817291, + "rewards/rejected": -0.650293231010437, + "step": 1360 + }, + { + "epoch": 1.41, + "learning_rate": 2.937691131498471e-07, + "logits/chosen": -2.4096202850341797, + "logits/rejected": -2.525867462158203, + "logps/chosen": -311.18206787109375, + "logps/rejected": -282.1830749511719, + "loss": 0.5697, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.14300379157066345, + "rewards/margins": 0.38485369086265564, + "rewards/rejected": -0.5278575420379639, + "step": 1370 + }, + { + "epoch": 1.42, + "learning_rate": 2.918577981651376e-07, + "logits/chosen": -2.3166534900665283, + "logits/rejected": -2.1045656204223633, + "logps/chosen": -318.7552490234375, + "logps/rejected": -169.04383850097656, + "loss": 0.5451, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.16076600551605225, + "rewards/margins": 0.43872982263565063, + "rewards/rejected": -0.5994957685470581, + "step": 1380 + }, + { + "epoch": 1.43, + "learning_rate": 2.8994648318042816e-07, + "logits/chosen": -2.477952718734741, + "logits/rejected": -2.3459110260009766, + "logps/chosen": -246.3095245361328, + "logps/rejected": -196.61399841308594, + "loss": 0.5566, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.06157383322715759, + "rewards/margins": 0.7134921550750732, + "rewards/rejected": -0.6519182920455933, + "step": 1390 + }, + { + "epoch": 1.44, + "learning_rate": 2.8803516819571863e-07, + "logits/chosen": -2.489950180053711, + "logits/rejected": -2.50527286529541, + "logps/chosen": -308.19573974609375, + "logps/rejected": -222.0939178466797, + "loss": 0.5669, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.28509682416915894, + "rewards/margins": 0.6225865483283997, + "rewards/rejected": -0.9076833724975586, + "step": 1400 + }, + { + "epoch": 1.46, + "learning_rate": 2.8612385321100917e-07, + "logits/chosen": -2.2142746448516846, + "logits/rejected": -2.1834685802459717, + "logps/chosen": -199.5335235595703, + "logps/rejected": -232.9855499267578, + "loss": 0.5256, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.20894494652748108, + "rewards/margins": 0.24761509895324707, + "rewards/rejected": -0.45656007528305054, + "step": 1410 + }, + { + "epoch": 1.47, + "learning_rate": 2.842125382262997e-07, + "logits/chosen": -2.416759490966797, + "logits/rejected": -2.349916934967041, + "logps/chosen": -277.214599609375, + "logps/rejected": -271.69854736328125, + "loss": 0.5399, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 0.003354682121425867, + "rewards/margins": 0.93927001953125, + "rewards/rejected": -0.9359153509140015, + "step": 1420 + }, + { + "epoch": 1.48, + "learning_rate": 2.8230122324159023e-07, + "logits/chosen": -2.2614686489105225, + "logits/rejected": -2.343787670135498, + "logps/chosen": -205.03573608398438, + "logps/rejected": -297.828369140625, + "loss": 0.5376, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.3863915205001831, + "rewards/margins": 0.32738059759140015, + "rewards/rejected": -0.7137721180915833, + "step": 1430 + }, + { + "epoch": 1.49, + "learning_rate": 2.8038990825688076e-07, + "logits/chosen": -2.5332446098327637, + "logits/rejected": -2.485593795776367, + "logps/chosen": -204.37960815429688, + "logps/rejected": -191.53567504882812, + "loss": 0.547, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.4227626323699951, + "rewards/margins": 0.1967758685350418, + "rewards/rejected": -0.619538426399231, + "step": 1440 + }, + { + "epoch": 1.5, + "learning_rate": 2.784785932721712e-07, + "logits/chosen": -2.4867165088653564, + "logits/rejected": -2.367835521697998, + "logps/chosen": -264.09783935546875, + "logps/rejected": -226.6724395751953, + "loss": 0.5612, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.03983074054121971, + "rewards/margins": 1.0130517482757568, + "rewards/rejected": -1.052882432937622, + "step": 1450 + }, + { + "epoch": 1.51, + "learning_rate": 2.765672782874617e-07, + "logits/chosen": -2.376364231109619, + "logits/rejected": -2.221336841583252, + "logps/chosen": -327.6852111816406, + "logps/rejected": -228.00344848632812, + "loss": 0.555, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.287401020526886, + "rewards/margins": 0.42843931913375854, + "rewards/rejected": -0.7158403396606445, + "step": 1460 + }, + { + "epoch": 1.52, + "learning_rate": 2.7465596330275225e-07, + "logits/chosen": -2.417738437652588, + "logits/rejected": -2.3728537559509277, + "logps/chosen": -222.4464111328125, + "logps/rejected": -159.91334533691406, + "loss": 0.5632, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.3034583330154419, + "rewards/margins": 0.43387946486473083, + "rewards/rejected": -0.7373377680778503, + "step": 1470 + }, + { + "epoch": 1.53, + "learning_rate": 2.727446483180428e-07, + "logits/chosen": -2.5264039039611816, + "logits/rejected": -2.549905776977539, + "logps/chosen": -311.6380615234375, + "logps/rejected": -241.33175659179688, + "loss": 0.5474, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.31216129660606384, + "rewards/margins": 0.6543269157409668, + "rewards/rejected": -0.966488242149353, + "step": 1480 + }, + { + "epoch": 1.54, + "learning_rate": 2.708333333333333e-07, + "logits/chosen": -2.4928135871887207, + "logits/rejected": -2.5519299507141113, + "logps/chosen": -278.2198181152344, + "logps/rejected": -230.333984375, + "loss": 0.5281, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.3294491767883301, + "rewards/margins": 0.6884486079216003, + "rewards/rejected": -1.0178978443145752, + "step": 1490 + }, + { + "epoch": 1.55, + "learning_rate": 2.6892201834862385e-07, + "logits/chosen": -2.3910984992980957, + "logits/rejected": -2.5310654640197754, + "logps/chosen": -318.42059326171875, + "logps/rejected": -227.05142211914062, + "loss": 0.5449, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.03567828983068466, + "rewards/margins": 1.0869849920272827, + "rewards/rejected": -1.1226632595062256, + "step": 1500 + }, + { + "epoch": 1.56, + "learning_rate": 2.6701070336391433e-07, + "logits/chosen": -2.120537042617798, + "logits/rejected": -1.9794002771377563, + "logps/chosen": -287.77618408203125, + "logps/rejected": -180.72586059570312, + "loss": 0.5251, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.17248104512691498, + "rewards/margins": 0.49385708570480347, + "rewards/rejected": -0.6663382053375244, + "step": 1510 + }, + { + "epoch": 1.57, + "learning_rate": 2.6509938837920486e-07, + "logits/chosen": -2.598473310470581, + "logits/rejected": -2.6752312183380127, + "logps/chosen": -303.84912109375, + "logps/rejected": -220.67562866210938, + "loss": 0.5453, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.2263271063566208, + "rewards/margins": 0.805108904838562, + "rewards/rejected": -1.0314362049102783, + "step": 1520 + }, + { + "epoch": 1.58, + "learning_rate": 2.631880733944954e-07, + "logits/chosen": -2.563636541366577, + "logits/rejected": -2.4898366928100586, + "logps/chosen": -317.47259521484375, + "logps/rejected": -213.1184844970703, + "loss": 0.5185, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.07109083235263824, + "rewards/margins": 0.6358667612075806, + "rewards/rejected": -0.7069576382637024, + "step": 1530 + }, + { + "epoch": 1.59, + "learning_rate": 2.612767584097859e-07, + "logits/chosen": -2.4102084636688232, + "logits/rejected": -2.329968214035034, + "logps/chosen": -262.5071716308594, + "logps/rejected": -234.21939086914062, + "loss": 0.5491, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.028744656592607498, + "rewards/margins": 0.5971611738204956, + "rewards/rejected": -0.6259058713912964, + "step": 1540 + }, + { + "epoch": 1.6, + "learning_rate": 2.5936544342507646e-07, + "logits/chosen": -2.52365779876709, + "logits/rejected": -2.2771174907684326, + "logps/chosen": -292.43560791015625, + "logps/rejected": -262.5408020019531, + "loss": 0.5316, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.04525692015886307, + "rewards/margins": 0.5967898368835449, + "rewards/rejected": -0.5515329241752625, + "step": 1550 + }, + { + "epoch": 1.61, + "learning_rate": 2.5745412844036693e-07, + "logits/chosen": -2.4652435779571533, + "logits/rejected": -2.3399059772491455, + "logps/chosen": -295.6546325683594, + "logps/rejected": -239.66946411132812, + "loss": 0.547, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.045461829751729965, + "rewards/margins": 0.6490108370780945, + "rewards/rejected": -0.6944726705551147, + "step": 1560 + }, + { + "epoch": 1.62, + "learning_rate": 2.5554281345565747e-07, + "logits/chosen": -2.425602674484253, + "logits/rejected": -2.3701093196868896, + "logps/chosen": -333.5090637207031, + "logps/rejected": -249.1916961669922, + "loss": 0.5225, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.054739952087402344, + "rewards/margins": 1.113675832748413, + "rewards/rejected": -1.0589358806610107, + "step": 1570 + }, + { + "epoch": 1.63, + "learning_rate": 2.53631498470948e-07, + "logits/chosen": -2.476578712463379, + "logits/rejected": -2.3098702430725098, + "logps/chosen": -277.14373779296875, + "logps/rejected": -208.9362335205078, + "loss": 0.5154, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.12385344505310059, + "rewards/margins": 0.5760471820831299, + "rewards/rejected": -0.6999006867408752, + "step": 1580 + }, + { + "epoch": 1.64, + "learning_rate": 2.5172018348623853e-07, + "logits/chosen": -2.3377368450164795, + "logits/rejected": -2.5816705226898193, + "logps/chosen": -333.68310546875, + "logps/rejected": -167.23974609375, + "loss": 0.564, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 0.043125562369823456, + "rewards/margins": 0.8181279897689819, + "rewards/rejected": -0.7750024795532227, + "step": 1590 + }, + { + "epoch": 1.65, + "learning_rate": 2.4980886850152906e-07, + "logits/chosen": -2.3544070720672607, + "logits/rejected": -2.522584915161133, + "logps/chosen": -372.16339111328125, + "logps/rejected": -314.8898620605469, + "loss": 0.5526, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 0.10558883845806122, + "rewards/margins": 1.0972599983215332, + "rewards/rejected": -0.9916712045669556, + "step": 1600 + }, + { + "epoch": 1.66, + "learning_rate": 2.478975535168196e-07, + "logits/chosen": -2.4156651496887207, + "logits/rejected": -2.321596622467041, + "logps/chosen": -234.2825164794922, + "logps/rejected": -187.53053283691406, + "loss": 0.5361, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.13712987303733826, + "rewards/margins": 0.6930993795394897, + "rewards/rejected": -0.8302291631698608, + "step": 1610 + }, + { + "epoch": 1.67, + "learning_rate": 2.459862385321101e-07, + "logits/chosen": -2.536264657974243, + "logits/rejected": -2.373924732208252, + "logps/chosen": -377.7065734863281, + "logps/rejected": -278.75494384765625, + "loss": 0.5685, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.011585181578993797, + "rewards/margins": 0.7624789476394653, + "rewards/rejected": -0.750893771648407, + "step": 1620 + }, + { + "epoch": 1.68, + "learning_rate": 2.440749235474006e-07, + "logits/chosen": -2.322601795196533, + "logits/rejected": -2.2549948692321777, + "logps/chosen": -292.2939147949219, + "logps/rejected": -266.98040771484375, + "loss": 0.535, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 0.1699330061674118, + "rewards/margins": 0.8751620054244995, + "rewards/rejected": -0.7052290439605713, + "step": 1630 + }, + { + "epoch": 1.69, + "learning_rate": 2.421636085626911e-07, + "logits/chosen": -2.3065285682678223, + "logits/rejected": -2.3722896575927734, + "logps/chosen": -338.43115234375, + "logps/rejected": -282.4959411621094, + "loss": 0.555, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.2029719352722168, + "rewards/margins": 0.37522464990615845, + "rewards/rejected": -0.57819664478302, + "step": 1640 + }, + { + "epoch": 1.7, + "learning_rate": 2.402522935779816e-07, + "logits/chosen": -2.526841163635254, + "logits/rejected": -2.2794435024261475, + "logps/chosen": -308.1125183105469, + "logps/rejected": -253.7852020263672, + "loss": 0.5306, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.15976569056510925, + "rewards/margins": 0.9058405160903931, + "rewards/rejected": -1.0656063556671143, + "step": 1650 + }, + { + "epoch": 1.71, + "learning_rate": 2.3834097859327215e-07, + "logits/chosen": -2.308931350708008, + "logits/rejected": -2.195425033569336, + "logps/chosen": -227.3255615234375, + "logps/rejected": -257.3186950683594, + "loss": 0.5538, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.19937238097190857, + "rewards/margins": 0.6596201658248901, + "rewards/rejected": -0.8589925765991211, + "step": 1660 + }, + { + "epoch": 1.72, + "learning_rate": 2.3642966360856268e-07, + "logits/chosen": -2.505361795425415, + "logits/rejected": -2.2748889923095703, + "logps/chosen": -275.67230224609375, + "logps/rejected": -213.0087890625, + "loss": 0.502, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.023041725158691406, + "rewards/margins": 0.6609295010566711, + "rewards/rejected": -0.6839712262153625, + "step": 1670 + }, + { + "epoch": 1.73, + "learning_rate": 2.345183486238532e-07, + "logits/chosen": -2.4382357597351074, + "logits/rejected": -2.337517499923706, + "logps/chosen": -257.69146728515625, + "logps/rejected": -238.8438262939453, + "loss": 0.5522, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.17831094563007355, + "rewards/margins": 0.3667464554309845, + "rewards/rejected": -0.5450573563575745, + "step": 1680 + }, + { + "epoch": 1.74, + "learning_rate": 2.3260703363914372e-07, + "logits/chosen": -2.54907488822937, + "logits/rejected": -2.3391122817993164, + "logps/chosen": -345.74346923828125, + "logps/rejected": -252.90414428710938, + "loss": 0.5239, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.07564310729503632, + "rewards/margins": 0.590453028678894, + "rewards/rejected": -0.6660962104797363, + "step": 1690 + }, + { + "epoch": 1.75, + "learning_rate": 2.3069571865443425e-07, + "logits/chosen": -2.417306900024414, + "logits/rejected": -2.506610155105591, + "logps/chosen": -223.8877716064453, + "logps/rejected": -169.25540161132812, + "loss": 0.5207, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.12120304256677628, + "rewards/margins": 0.6357380151748657, + "rewards/rejected": -0.7569410800933838, + "step": 1700 + }, + { + "epoch": 1.76, + "learning_rate": 2.2878440366972476e-07, + "logits/chosen": -2.6201210021972656, + "logits/rejected": -2.4392848014831543, + "logps/chosen": -270.24615478515625, + "logps/rejected": -343.59423828125, + "loss": 0.533, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.28634923696517944, + "rewards/margins": 0.4523712992668152, + "rewards/rejected": -0.7387205958366394, + "step": 1710 + }, + { + "epoch": 1.78, + "learning_rate": 2.268730886850153e-07, + "logits/chosen": -2.069031000137329, + "logits/rejected": -1.9285329580307007, + "logps/chosen": -210.6043243408203, + "logps/rejected": -177.20159912109375, + "loss": 0.5444, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 0.03490941971540451, + "rewards/margins": 0.9221685528755188, + "rewards/rejected": -0.8872591853141785, + "step": 1720 + }, + { + "epoch": 1.79, + "learning_rate": 2.249617737003058e-07, + "logits/chosen": -2.2896201610565186, + "logits/rejected": -2.356297254562378, + "logps/chosen": -292.9852294921875, + "logps/rejected": -277.8864440917969, + "loss": 0.5467, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.21859049797058105, + "rewards/margins": 0.5577865839004517, + "rewards/rejected": -0.7763770818710327, + "step": 1730 + }, + { + "epoch": 1.8, + "learning_rate": 2.2305045871559633e-07, + "logits/chosen": -2.3246283531188965, + "logits/rejected": -2.1089653968811035, + "logps/chosen": -273.15118408203125, + "logps/rejected": -292.574951171875, + "loss": 0.5207, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.37160351872444153, + "rewards/margins": 0.06535868346691132, + "rewards/rejected": -0.43696221709251404, + "step": 1740 + }, + { + "epoch": 1.81, + "learning_rate": 2.2113914373088686e-07, + "logits/chosen": -2.4427847862243652, + "logits/rejected": -2.560529947280884, + "logps/chosen": -153.76699829101562, + "logps/rejected": -248.28408813476562, + "loss": 0.5375, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.2880316376686096, + "rewards/margins": 0.30386772751808167, + "rewards/rejected": -0.5918993949890137, + "step": 1750 + }, + { + "epoch": 1.82, + "learning_rate": 2.1922782874617736e-07, + "logits/chosen": -2.6262247562408447, + "logits/rejected": -2.416951894760132, + "logps/chosen": -257.9002990722656, + "logps/rejected": -275.48553466796875, + "loss": 0.5315, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.2771896719932556, + "rewards/margins": 0.2873901128768921, + "rewards/rejected": -0.5645797848701477, + "step": 1760 + }, + { + "epoch": 1.83, + "learning_rate": 2.1731651376146787e-07, + "logits/chosen": -2.1698267459869385, + "logits/rejected": -2.3672404289245605, + "logps/chosen": -256.00689697265625, + "logps/rejected": -232.8870849609375, + "loss": 0.5626, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.13111671805381775, + "rewards/margins": 0.5583267211914062, + "rewards/rejected": -0.6894434690475464, + "step": 1770 + }, + { + "epoch": 1.84, + "learning_rate": 2.154051987767584e-07, + "logits/chosen": -2.2943813800811768, + "logits/rejected": -2.3611464500427246, + "logps/chosen": -289.64422607421875, + "logps/rejected": -286.1085205078125, + "loss": 0.5383, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.17674392461776733, + "rewards/margins": 0.36700400710105896, + "rewards/rejected": -0.5437479615211487, + "step": 1780 + }, + { + "epoch": 1.85, + "learning_rate": 2.134938837920489e-07, + "logits/chosen": -2.2364659309387207, + "logits/rejected": -2.2148382663726807, + "logps/chosen": -202.55038452148438, + "logps/rejected": -206.96762084960938, + "loss": 0.5432, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.26416245102882385, + "rewards/margins": 0.6946617960929871, + "rewards/rejected": -0.9588242769241333, + "step": 1790 + }, + { + "epoch": 1.86, + "learning_rate": 2.1158256880733944e-07, + "logits/chosen": -2.305535316467285, + "logits/rejected": -2.4840779304504395, + "logps/chosen": -338.34271240234375, + "logps/rejected": -185.28201293945312, + "loss": 0.5132, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.16436448693275452, + "rewards/margins": 0.42132654786109924, + "rewards/rejected": -0.5856910943984985, + "step": 1800 + }, + { + "epoch": 1.87, + "learning_rate": 2.0967125382262994e-07, + "logits/chosen": -2.7777082920074463, + "logits/rejected": -2.5519793033599854, + "logps/chosen": -233.9488067626953, + "logps/rejected": -237.7256622314453, + "loss": 0.5536, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.48247796297073364, + "rewards/margins": 0.04958399385213852, + "rewards/rejected": -0.5320619344711304, + "step": 1810 + }, + { + "epoch": 1.88, + "learning_rate": 2.0775993883792048e-07, + "logits/chosen": -2.5612552165985107, + "logits/rejected": -2.420253276824951, + "logps/chosen": -339.1038818359375, + "logps/rejected": -225.16360473632812, + "loss": 0.5151, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.03660213202238083, + "rewards/margins": 1.2038971185684204, + "rewards/rejected": -1.2404992580413818, + "step": 1820 + }, + { + "epoch": 1.89, + "learning_rate": 2.05848623853211e-07, + "logits/chosen": -2.586907386779785, + "logits/rejected": -2.5101799964904785, + "logps/chosen": -310.6505432128906, + "logps/rejected": -244.5113983154297, + "loss": 0.5221, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.411550909280777, + "rewards/margins": 0.3417314291000366, + "rewards/rejected": -0.7532823085784912, + "step": 1830 + }, + { + "epoch": 1.9, + "learning_rate": 2.0393730886850151e-07, + "logits/chosen": -2.2709717750549316, + "logits/rejected": -2.39669132232666, + "logps/chosen": -226.12057495117188, + "logps/rejected": -200.52218627929688, + "loss": 0.5242, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 0.046432264149188995, + "rewards/margins": 1.110090732574463, + "rewards/rejected": -1.0636584758758545, + "step": 1840 + }, + { + "epoch": 1.91, + "learning_rate": 2.0202599388379205e-07, + "logits/chosen": -2.3261749744415283, + "logits/rejected": -2.5248923301696777, + "logps/chosen": -222.13772583007812, + "logps/rejected": -425.5648498535156, + "loss": 0.5189, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.3807342052459717, + "rewards/margins": 0.5107225775718689, + "rewards/rejected": -0.8914567232131958, + "step": 1850 + }, + { + "epoch": 1.92, + "learning_rate": 2.0011467889908258e-07, + "logits/chosen": -2.398069381713867, + "logits/rejected": -2.2855401039123535, + "logps/chosen": -314.5137939453125, + "logps/rejected": -217.12588500976562, + "loss": 0.5264, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 0.029106635600328445, + "rewards/margins": 1.0325161218643188, + "rewards/rejected": -1.003409504890442, + "step": 1860 + }, + { + "epoch": 1.93, + "learning_rate": 1.9820336391437308e-07, + "logits/chosen": -2.290491819381714, + "logits/rejected": -2.378493070602417, + "logps/chosen": -286.273681640625, + "logps/rejected": -229.33828735351562, + "loss": 0.5646, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.08369754254817963, + "rewards/margins": 0.795507550239563, + "rewards/rejected": -0.879205048084259, + "step": 1870 + }, + { + "epoch": 1.94, + "learning_rate": 1.9629204892966362e-07, + "logits/chosen": -2.601147413253784, + "logits/rejected": -2.4275565147399902, + "logps/chosen": -390.74151611328125, + "logps/rejected": -276.0337829589844, + "loss": 0.53, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 0.009929287247359753, + "rewards/margins": 0.7174602746963501, + "rewards/rejected": -0.7075310945510864, + "step": 1880 + }, + { + "epoch": 1.95, + "learning_rate": 1.943807339449541e-07, + "logits/chosen": -2.18162202835083, + "logits/rejected": -1.9625205993652344, + "logps/chosen": -205.7775421142578, + "logps/rejected": -185.28477478027344, + "loss": 0.5423, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.2230360060930252, + "rewards/margins": 0.8223746418952942, + "rewards/rejected": -1.0454107522964478, + "step": 1890 + }, + { + "epoch": 1.96, + "learning_rate": 1.9246941896024463e-07, + "logits/chosen": -2.2929797172546387, + "logits/rejected": -2.081751823425293, + "logps/chosen": -176.2759246826172, + "logps/rejected": -127.28376770019531, + "loss": 0.5754, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.16625508666038513, + "rewards/margins": 0.5883089900016785, + "rewards/rejected": -0.754564106464386, + "step": 1900 + }, + { + "epoch": 1.97, + "learning_rate": 1.9055810397553516e-07, + "logits/chosen": -2.3370862007141113, + "logits/rejected": -2.5263562202453613, + "logps/chosen": -305.7495422363281, + "logps/rejected": -238.85726928710938, + "loss": 0.5508, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.1391083151102066, + "rewards/margins": 0.8672167062759399, + "rewards/rejected": -1.0063250064849854, + "step": 1910 + }, + { + "epoch": 1.98, + "learning_rate": 1.8864678899082566e-07, + "logits/chosen": -2.6837048530578613, + "logits/rejected": -2.4654018878936768, + "logps/chosen": -278.2798767089844, + "logps/rejected": -200.07553100585938, + "loss": 0.5311, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.21458300948143005, + "rewards/margins": 0.6255909204483032, + "rewards/rejected": -0.8401739001274109, + "step": 1920 + }, + { + "epoch": 1.99, + "learning_rate": 1.867354740061162e-07, + "logits/chosen": -2.3841135501861572, + "logits/rejected": -2.4274752140045166, + "logps/chosen": -241.267333984375, + "logps/rejected": -294.39300537109375, + "loss": 0.5454, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.15124601125717163, + "rewards/margins": 1.2222580909729004, + "rewards/rejected": -1.3735041618347168, + "step": 1930 + }, + { + "epoch": 2.0, + "eval_logits/chosen": -2.0971014499664307, + "eval_logits/rejected": -1.931637167930603, + "eval_logps/chosen": -248.69100952148438, + "eval_logps/rejected": -198.20773315429688, + "eval_loss": 0.531248927116394, + "eval_rewards/accuracies": 0.796875, + "eval_rewards/chosen": -0.12489113211631775, + "eval_rewards/margins": 0.8351471424102783, + "eval_rewards/rejected": -0.9600383639335632, + "eval_runtime": 49.837, + "eval_samples_per_second": 40.131, + "eval_steps_per_second": 0.321, + "step": 1938 + }, + { + "epoch": 2.0, + "learning_rate": 1.8482415902140673e-07, + "logits/chosen": -2.248683452606201, + "logits/rejected": -2.3425230979919434, + "logps/chosen": -222.09530639648438, + "logps/rejected": -244.0989227294922, + "loss": 0.5393, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.25359851121902466, + "rewards/margins": 0.6999626159667969, + "rewards/rejected": -0.9535611271858215, + "step": 1940 + }, + { + "epoch": 2.01, + "learning_rate": 1.8291284403669723e-07, + "logits/chosen": -2.3472368717193604, + "logits/rejected": -2.4190945625305176, + "logps/chosen": -202.47897338867188, + "logps/rejected": -223.9008331298828, + "loss": 0.5342, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.06839686632156372, + "rewards/margins": 0.7683246731758118, + "rewards/rejected": -0.8367214202880859, + "step": 1950 + }, + { + "epoch": 2.02, + "learning_rate": 1.8100152905198777e-07, + "logits/chosen": -2.1655468940734863, + "logits/rejected": -2.06404185295105, + "logps/chosen": -259.51666259765625, + "logps/rejected": -317.1017150878906, + "loss": 0.5335, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 0.24679192900657654, + "rewards/margins": 1.218878984451294, + "rewards/rejected": -0.9720870852470398, + "step": 1960 + }, + { + "epoch": 2.03, + "learning_rate": 1.7909021406727827e-07, + "logits/chosen": -2.5240676403045654, + "logits/rejected": -2.360107660293579, + "logps/chosen": -175.0189971923828, + "logps/rejected": -180.5626983642578, + "loss": 0.5445, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.4591894745826721, + "rewards/margins": 0.5135048627853394, + "rewards/rejected": -0.9726942777633667, + "step": 1970 + }, + { + "epoch": 2.04, + "learning_rate": 1.771788990825688e-07, + "logits/chosen": -2.3183510303497314, + "logits/rejected": -2.134965658187866, + "logps/chosen": -302.5520324707031, + "logps/rejected": -210.1514434814453, + "loss": 0.5426, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.04827744886279106, + "rewards/margins": 1.1514790058135986, + "rewards/rejected": -1.1032016277313232, + "step": 1980 + }, + { + "epoch": 2.05, + "learning_rate": 1.7526758409785934e-07, + "logits/chosen": -2.512765884399414, + "logits/rejected": -2.167898416519165, + "logps/chosen": -355.231201171875, + "logps/rejected": -272.0666809082031, + "loss": 0.5404, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.12298504263162613, + "rewards/margins": 1.1620323657989502, + "rewards/rejected": -1.2850173711776733, + "step": 1990 + }, + { + "epoch": 2.06, + "learning_rate": 1.7335626911314984e-07, + "logits/chosen": -2.495887517929077, + "logits/rejected": -2.2603235244750977, + "logps/chosen": -236.11611938476562, + "logps/rejected": -186.68002319335938, + "loss": 0.5362, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.14387288689613342, + "rewards/margins": 0.7451364398002625, + "rewards/rejected": -0.8890093564987183, + "step": 2000 + }, + { + "epoch": 2.07, + "learning_rate": 1.7144495412844037e-07, + "logits/chosen": -2.6951799392700195, + "logits/rejected": -2.600787878036499, + "logps/chosen": -388.97418212890625, + "logps/rejected": -281.6431884765625, + "loss": 0.5252, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.054711710661649704, + "rewards/margins": 1.36897611618042, + "rewards/rejected": -1.4236878156661987, + "step": 2010 + }, + { + "epoch": 2.08, + "learning_rate": 1.6953363914373088e-07, + "logits/chosen": -2.5666658878326416, + "logits/rejected": -2.512883186340332, + "logps/chosen": -324.7785949707031, + "logps/rejected": -228.8123016357422, + "loss": 0.5496, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.09128103405237198, + "rewards/margins": 1.144072413444519, + "rewards/rejected": -1.2353535890579224, + "step": 2020 + }, + { + "epoch": 2.09, + "learning_rate": 1.6762232415902138e-07, + "logits/chosen": -2.499027729034424, + "logits/rejected": -2.298178195953369, + "logps/chosen": -235.67837524414062, + "logps/rejected": -202.81117248535156, + "loss": 0.5436, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.24017953872680664, + "rewards/margins": 0.8434773683547974, + "rewards/rejected": -1.083656907081604, + "step": 2030 + }, + { + "epoch": 2.11, + "learning_rate": 1.6571100917431192e-07, + "logits/chosen": -2.5607171058654785, + "logits/rejected": -2.353574514389038, + "logps/chosen": -240.97384643554688, + "logps/rejected": -170.1258544921875, + "loss": 0.5185, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.20230352878570557, + "rewards/margins": 0.6808130741119385, + "rewards/rejected": -0.883116602897644, + "step": 2040 + }, + { + "epoch": 2.12, + "learning_rate": 1.6379969418960242e-07, + "logits/chosen": -2.5361034870147705, + "logits/rejected": -2.513338565826416, + "logps/chosen": -381.1001892089844, + "logps/rejected": -299.3647155761719, + "loss": 0.5277, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.016599655151367188, + "rewards/margins": 0.782288134098053, + "rewards/rejected": -0.7988878488540649, + "step": 2050 + }, + { + "epoch": 2.13, + "learning_rate": 1.6188837920489295e-07, + "logits/chosen": -2.3150038719177246, + "logits/rejected": -2.09513783454895, + "logps/chosen": -262.87286376953125, + "logps/rejected": -209.9288330078125, + "loss": 0.5178, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.19605524837970734, + "rewards/margins": 0.7449124455451965, + "rewards/rejected": -0.9409675598144531, + "step": 2060 + }, + { + "epoch": 2.14, + "learning_rate": 1.5997706422018349e-07, + "logits/chosen": -2.9330039024353027, + "logits/rejected": -2.6582858562469482, + "logps/chosen": -341.77850341796875, + "logps/rejected": -298.8485412597656, + "loss": 0.5397, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.009139073081314564, + "rewards/margins": 0.28165051341056824, + "rewards/rejected": -0.27251148223876953, + "step": 2070 + }, + { + "epoch": 2.15, + "learning_rate": 1.58065749235474e-07, + "logits/chosen": -2.320667266845703, + "logits/rejected": -2.201702356338501, + "logps/chosen": -286.5201416015625, + "logps/rejected": -217.8232421875, + "loss": 0.5418, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 0.11670849472284317, + "rewards/margins": 0.8405148386955261, + "rewards/rejected": -0.7238063216209412, + "step": 2080 + }, + { + "epoch": 2.16, + "learning_rate": 1.5615443425076452e-07, + "logits/chosen": -2.363339900970459, + "logits/rejected": -2.3478615283966064, + "logps/chosen": -314.27325439453125, + "logps/rejected": -253.44631958007812, + "loss": 0.5247, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 0.10530459880828857, + "rewards/margins": 1.0353710651397705, + "rewards/rejected": -0.9300664067268372, + "step": 2090 + }, + { + "epoch": 2.17, + "learning_rate": 1.5424311926605506e-07, + "logits/chosen": -2.2379679679870605, + "logits/rejected": -2.2152559757232666, + "logps/chosen": -170.92782592773438, + "logps/rejected": -166.93072509765625, + "loss": 0.5114, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -0.21574795246124268, + "rewards/margins": 1.1248795986175537, + "rewards/rejected": -1.3406277894973755, + "step": 2100 + }, + { + "epoch": 2.18, + "learning_rate": 1.5233180428134556e-07, + "logits/chosen": -2.4279611110687256, + "logits/rejected": -2.4034111499786377, + "logps/chosen": -271.7957763671875, + "logps/rejected": -260.822265625, + "loss": 0.5237, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.07596953958272934, + "rewards/margins": 0.9050176739692688, + "rewards/rejected": -0.9809872508049011, + "step": 2110 + }, + { + "epoch": 2.19, + "learning_rate": 1.504204892966361e-07, + "logits/chosen": -2.213738441467285, + "logits/rejected": -2.2372987270355225, + "logps/chosen": -295.6419677734375, + "logps/rejected": -206.6561279296875, + "loss": 0.5314, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.19369874894618988, + "rewards/margins": 0.6303919553756714, + "rewards/rejected": -0.8240906596183777, + "step": 2120 + }, + { + "epoch": 2.2, + "learning_rate": 1.485091743119266e-07, + "logits/chosen": -2.4710030555725098, + "logits/rejected": -2.3606488704681396, + "logps/chosen": -242.4974822998047, + "logps/rejected": -218.1492156982422, + "loss": 0.5367, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.16626985371112823, + "rewards/margins": 0.7947496771812439, + "rewards/rejected": -0.9610195159912109, + "step": 2130 + }, + { + "epoch": 2.21, + "learning_rate": 1.465978593272171e-07, + "logits/chosen": -2.255868434906006, + "logits/rejected": -2.3592605590820312, + "logps/chosen": -228.03219604492188, + "logps/rejected": -184.50611877441406, + "loss": 0.5165, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.3162696957588196, + "rewards/margins": 0.7572251558303833, + "rewards/rejected": -1.0734949111938477, + "step": 2140 + }, + { + "epoch": 2.22, + "learning_rate": 1.4468654434250764e-07, + "logits/chosen": -2.2551803588867188, + "logits/rejected": -2.189657688140869, + "logps/chosen": -370.01104736328125, + "logps/rejected": -314.4168395996094, + "loss": 0.5523, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 0.027963241562247276, + "rewards/margins": 0.7318282127380371, + "rewards/rejected": -0.7038649320602417, + "step": 2150 + }, + { + "epoch": 2.23, + "learning_rate": 1.4277522935779814e-07, + "logits/chosen": -2.4878127574920654, + "logits/rejected": -2.42626953125, + "logps/chosen": -275.42144775390625, + "logps/rejected": -220.00137329101562, + "loss": 0.5275, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 0.14348439872264862, + "rewards/margins": 1.2011865377426147, + "rewards/rejected": -1.0577021837234497, + "step": 2160 + }, + { + "epoch": 2.24, + "learning_rate": 1.4086391437308867e-07, + "logits/chosen": -2.3342137336730957, + "logits/rejected": -2.4700827598571777, + "logps/chosen": -279.27032470703125, + "logps/rejected": -260.35107421875, + "loss": 0.5171, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.21079519391059875, + "rewards/margins": 0.983725368976593, + "rewards/rejected": -1.1945207118988037, + "step": 2170 + }, + { + "epoch": 2.25, + "learning_rate": 1.389525993883792e-07, + "logits/chosen": -2.543966770172119, + "logits/rejected": -2.178934097290039, + "logps/chosen": -354.38348388671875, + "logps/rejected": -278.5998840332031, + "loss": 0.5621, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.17348089814186096, + "rewards/margins": 1.0600968599319458, + "rewards/rejected": -1.2335779666900635, + "step": 2180 + }, + { + "epoch": 2.26, + "learning_rate": 1.370412844036697e-07, + "logits/chosen": -2.370507001876831, + "logits/rejected": -2.4885435104370117, + "logps/chosen": -282.58270263671875, + "logps/rejected": -245.8155517578125, + "loss": 0.5296, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": 0.010824834927916527, + "rewards/margins": 0.7261137962341309, + "rewards/rejected": -0.7152889966964722, + "step": 2190 + }, + { + "epoch": 2.27, + "learning_rate": 1.3512996941896024e-07, + "logits/chosen": -2.373438835144043, + "logits/rejected": -2.3270111083984375, + "logps/chosen": -255.15109252929688, + "logps/rejected": -247.8589324951172, + "loss": 0.5333, + "rewards/accuracies": 0.44999998807907104, + "rewards/chosen": -0.5083593130111694, + "rewards/margins": 0.12943392992019653, + "rewards/rejected": -0.6377933025360107, + "step": 2200 + }, + { + "epoch": 2.28, + "learning_rate": 1.3321865443425075e-07, + "logits/chosen": -2.239222526550293, + "logits/rejected": -2.1923537254333496, + "logps/chosen": -299.39422607421875, + "logps/rejected": -174.023681640625, + "loss": 0.5336, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.11805693060159683, + "rewards/margins": 1.0056555271148682, + "rewards/rejected": -1.1237125396728516, + "step": 2210 + }, + { + "epoch": 2.29, + "learning_rate": 1.3130733944954128e-07, + "logits/chosen": -2.110752820968628, + "logits/rejected": -1.9507097005844116, + "logps/chosen": -329.23724365234375, + "logps/rejected": -225.7882080078125, + "loss": 0.4955, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.32635951042175293, + "rewards/margins": 0.7226699590682983, + "rewards/rejected": -1.0490295886993408, + "step": 2220 + }, + { + "epoch": 2.3, + "learning_rate": 1.293960244648318e-07, + "logits/chosen": -2.5377070903778076, + "logits/rejected": -2.426079750061035, + "logps/chosen": -373.67645263671875, + "logps/rejected": -226.9227294921875, + "loss": 0.5225, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.15080983936786652, + "rewards/margins": 1.0611189603805542, + "rewards/rejected": -1.2119289636611938, + "step": 2230 + }, + { + "epoch": 2.31, + "learning_rate": 1.2748470948012232e-07, + "logits/chosen": -2.362394332885742, + "logits/rejected": -2.197788715362549, + "logps/chosen": -327.0963134765625, + "logps/rejected": -272.4471130371094, + "loss": 0.5111, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.272097110748291, + "rewards/margins": 0.6045467257499695, + "rewards/rejected": -0.8766437768936157, + "step": 2240 + }, + { + "epoch": 2.32, + "learning_rate": 1.2557339449541285e-07, + "logits/chosen": -2.2135844230651855, + "logits/rejected": -2.296194076538086, + "logps/chosen": -314.9928894042969, + "logps/rejected": -249.1458282470703, + "loss": 0.5293, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.059797704219818115, + "rewards/margins": 1.026885747909546, + "rewards/rejected": -1.0866836309432983, + "step": 2250 + }, + { + "epoch": 2.33, + "learning_rate": 1.2366207951070336e-07, + "logits/chosen": -2.468040943145752, + "logits/rejected": -2.395946741104126, + "logps/chosen": -259.4168395996094, + "logps/rejected": -228.13162231445312, + "loss": 0.5639, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.3209591507911682, + "rewards/margins": 0.7097271680831909, + "rewards/rejected": -1.0306862592697144, + "step": 2260 + }, + { + "epoch": 2.34, + "learning_rate": 1.217507645259939e-07, + "logits/chosen": -2.540865182876587, + "logits/rejected": -2.305960178375244, + "logps/chosen": -327.03668212890625, + "logps/rejected": -316.6974792480469, + "loss": 0.4995, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.10364139080047607, + "rewards/margins": 0.5791498422622681, + "rewards/rejected": -0.6827912330627441, + "step": 2270 + }, + { + "epoch": 2.35, + "learning_rate": 1.198394495412844e-07, + "logits/chosen": -2.5025086402893066, + "logits/rejected": -2.4841408729553223, + "logps/chosen": -346.7644958496094, + "logps/rejected": -261.13916015625, + "loss": 0.5339, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.09549558907747269, + "rewards/margins": 0.9585358500480652, + "rewards/rejected": -1.054031491279602, + "step": 2280 + }, + { + "epoch": 2.36, + "learning_rate": 1.1792813455657493e-07, + "logits/chosen": -2.3317127227783203, + "logits/rejected": -2.3077309131622314, + "logps/chosen": -263.4445495605469, + "logps/rejected": -303.0923156738281, + "loss": 0.5097, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.48188915848731995, + "rewards/margins": 0.5629307627677917, + "rewards/rejected": -1.0448198318481445, + "step": 2290 + }, + { + "epoch": 2.37, + "learning_rate": 1.1601681957186543e-07, + "logits/chosen": -2.5383365154266357, + "logits/rejected": -2.3553810119628906, + "logps/chosen": -303.4425964355469, + "logps/rejected": -202.1547088623047, + "loss": 0.516, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 0.03705354407429695, + "rewards/margins": 1.2515580654144287, + "rewards/rejected": -1.2145044803619385, + "step": 2300 + }, + { + "epoch": 2.38, + "learning_rate": 1.1410550458715595e-07, + "logits/chosen": -2.290337085723877, + "logits/rejected": -2.136826753616333, + "logps/chosen": -309.16717529296875, + "logps/rejected": -254.88674926757812, + "loss": 0.5478, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 0.010205010883510113, + "rewards/margins": 0.6125646829605103, + "rewards/rejected": -0.6023596525192261, + "step": 2310 + }, + { + "epoch": 2.39, + "learning_rate": 1.1219418960244648e-07, + "logits/chosen": -2.3295750617980957, + "logits/rejected": -2.385953426361084, + "logps/chosen": -215.65237426757812, + "logps/rejected": -212.346435546875, + "loss": 0.518, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.3336523175239563, + "rewards/margins": 0.6867604851722717, + "rewards/rejected": -1.0204129219055176, + "step": 2320 + }, + { + "epoch": 2.4, + "learning_rate": 1.10282874617737e-07, + "logits/chosen": -2.4023945331573486, + "logits/rejected": -2.4153189659118652, + "logps/chosen": -240.5900115966797, + "logps/rejected": -228.1215362548828, + "loss": 0.4989, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.16529302299022675, + "rewards/margins": 0.8604542016983032, + "rewards/rejected": -1.0257470607757568, + "step": 2330 + }, + { + "epoch": 2.41, + "learning_rate": 1.0837155963302752e-07, + "logits/chosen": -2.2563438415527344, + "logits/rejected": -2.125026226043701, + "logps/chosen": -267.7855224609375, + "logps/rejected": -219.22030639648438, + "loss": 0.5213, + "rewards/accuracies": 0.949999988079071, + "rewards/chosen": -0.08529398590326309, + "rewards/margins": 0.9822967648506165, + "rewards/rejected": -1.0675907135009766, + "step": 2340 + }, + { + "epoch": 2.43, + "learning_rate": 1.0646024464831804e-07, + "logits/chosen": -2.2471909523010254, + "logits/rejected": -2.319463014602661, + "logps/chosen": -255.18911743164062, + "logps/rejected": -244.0147247314453, + "loss": 0.5512, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.24005265533924103, + "rewards/margins": 0.49893826246261597, + "rewards/rejected": -0.7389909029006958, + "step": 2350 + }, + { + "epoch": 2.44, + "learning_rate": 1.0454892966360856e-07, + "logits/chosen": -2.348158597946167, + "logits/rejected": -2.28670334815979, + "logps/chosen": -271.4364013671875, + "logps/rejected": -196.4866943359375, + "loss": 0.5465, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.04201481118798256, + "rewards/margins": 1.2664215564727783, + "rewards/rejected": -1.3084365129470825, + "step": 2360 + }, + { + "epoch": 2.45, + "learning_rate": 1.0263761467889908e-07, + "logits/chosen": -2.466719150543213, + "logits/rejected": -2.4625251293182373, + "logps/chosen": -284.7028503417969, + "logps/rejected": -224.3902587890625, + "loss": 0.5451, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.19498039782047272, + "rewards/margins": 0.3427335023880005, + "rewards/rejected": -0.537713885307312, + "step": 2370 + }, + { + "epoch": 2.46, + "learning_rate": 1.007262996941896e-07, + "logits/chosen": -2.122545003890991, + "logits/rejected": -2.2347145080566406, + "logps/chosen": -289.37249755859375, + "logps/rejected": -206.8185577392578, + "loss": 0.5535, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.25089073181152344, + "rewards/margins": 0.7263849973678589, + "rewards/rejected": -0.9772756695747375, + "step": 2380 + }, + { + "epoch": 2.47, + "learning_rate": 9.881498470948011e-08, + "logits/chosen": -2.279235363006592, + "logits/rejected": -2.394592523574829, + "logps/chosen": -239.59011840820312, + "logps/rejected": -206.2091827392578, + "loss": 0.5505, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.09624668210744858, + "rewards/margins": 0.9607799649238586, + "rewards/rejected": -1.0570266246795654, + "step": 2390 + }, + { + "epoch": 2.48, + "learning_rate": 9.690366972477065e-08, + "logits/chosen": -2.580003499984741, + "logits/rejected": -2.5049805641174316, + "logps/chosen": -193.94044494628906, + "logps/rejected": -129.92938232421875, + "loss": 0.5501, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.29306405782699585, + "rewards/margins": 0.3221284747123718, + "rewards/rejected": -0.6151925325393677, + "step": 2400 + }, + { + "epoch": 2.49, + "learning_rate": 9.499235474006116e-08, + "logits/chosen": -2.2965879440307617, + "logits/rejected": -2.5327494144439697, + "logps/chosen": -437.3583068847656, + "logps/rejected": -228.34335327148438, + "loss": 0.5281, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.08578985184431076, + "rewards/margins": 0.9632189869880676, + "rewards/rejected": -1.0490089654922485, + "step": 2410 + }, + { + "epoch": 2.5, + "learning_rate": 9.308103975535168e-08, + "logits/chosen": -2.5039114952087402, + "logits/rejected": -2.533884048461914, + "logps/chosen": -328.80194091796875, + "logps/rejected": -245.8604736328125, + "loss": 0.5309, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.1082562655210495, + "rewards/margins": 0.7982146143913269, + "rewards/rejected": -0.9064709544181824, + "step": 2420 + }, + { + "epoch": 2.51, + "learning_rate": 9.116972477064219e-08, + "logits/chosen": -2.3511176109313965, + "logits/rejected": -2.471139430999756, + "logps/chosen": -351.3453063964844, + "logps/rejected": -363.6473693847656, + "loss": 0.5501, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.06536252051591873, + "rewards/margins": 0.8679245710372925, + "rewards/rejected": -0.9332871437072754, + "step": 2430 + }, + { + "epoch": 2.52, + "learning_rate": 8.925840978593272e-08, + "logits/chosen": -2.4342617988586426, + "logits/rejected": -2.3651440143585205, + "logps/chosen": -185.2665252685547, + "logps/rejected": -184.4652099609375, + "loss": 0.5244, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 0.01564156636595726, + "rewards/margins": 0.836595892906189, + "rewards/rejected": -0.820954442024231, + "step": 2440 + }, + { + "epoch": 2.53, + "learning_rate": 8.734709480122324e-08, + "logits/chosen": -2.180828809738159, + "logits/rejected": -2.3217592239379883, + "logps/chosen": -162.02694702148438, + "logps/rejected": -214.3283233642578, + "loss": 0.5294, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.3506285548210144, + "rewards/margins": 0.32000571489334106, + "rewards/rejected": -0.6706342697143555, + "step": 2450 + }, + { + "epoch": 2.54, + "learning_rate": 8.543577981651376e-08, + "logits/chosen": -2.336467742919922, + "logits/rejected": -2.384082317352295, + "logps/chosen": -436.14337158203125, + "logps/rejected": -267.2066955566406, + "loss": 0.5303, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": -0.0021473728120326996, + "rewards/margins": 1.047534465789795, + "rewards/rejected": -1.0496817827224731, + "step": 2460 + }, + { + "epoch": 2.55, + "learning_rate": 8.352446483180428e-08, + "logits/chosen": -2.4472270011901855, + "logits/rejected": -2.302725315093994, + "logps/chosen": -265.781005859375, + "logps/rejected": -185.17855834960938, + "loss": 0.5353, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.04335325211286545, + "rewards/margins": 0.785746157169342, + "rewards/rejected": -0.8290994763374329, + "step": 2470 + }, + { + "epoch": 2.56, + "learning_rate": 8.161314984709481e-08, + "logits/chosen": -2.546318292617798, + "logits/rejected": -2.435561180114746, + "logps/chosen": -341.5940246582031, + "logps/rejected": -224.41000366210938, + "loss": 0.5387, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.08983229100704193, + "rewards/margins": 1.180863380432129, + "rewards/rejected": -1.2706955671310425, + "step": 2480 + }, + { + "epoch": 2.57, + "learning_rate": 7.970183486238531e-08, + "logits/chosen": -2.327275276184082, + "logits/rejected": -2.229645252227783, + "logps/chosen": -198.76148986816406, + "logps/rejected": -200.72247314453125, + "loss": 0.5137, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.06232021003961563, + "rewards/margins": 0.635228157043457, + "rewards/rejected": -0.6975484490394592, + "step": 2490 + }, + { + "epoch": 2.58, + "learning_rate": 7.779051987767583e-08, + "logits/chosen": -2.560883045196533, + "logits/rejected": -2.422776699066162, + "logps/chosen": -403.97991943359375, + "logps/rejected": -216.1969451904297, + "loss": 0.4976, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.12103588879108429, + "rewards/margins": 1.1826056241989136, + "rewards/rejected": -1.0615696907043457, + "step": 2500 + }, + { + "epoch": 2.59, + "learning_rate": 7.587920489296635e-08, + "logits/chosen": -2.3078255653381348, + "logits/rejected": -2.3434040546417236, + "logps/chosen": -200.40487670898438, + "logps/rejected": -198.0846405029297, + "loss": 0.5296, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.07855480164289474, + "rewards/margins": 0.8178641200065613, + "rewards/rejected": -0.8964190483093262, + "step": 2510 + }, + { + "epoch": 2.6, + "learning_rate": 7.396788990825688e-08, + "logits/chosen": -2.438007354736328, + "logits/rejected": -2.413921594619751, + "logps/chosen": -205.8711395263672, + "logps/rejected": -196.82382202148438, + "loss": 0.5469, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.14088700711727142, + "rewards/margins": 0.9455137252807617, + "rewards/rejected": -1.0864007472991943, + "step": 2520 + }, + { + "epoch": 2.61, + "learning_rate": 7.20565749235474e-08, + "logits/chosen": -2.546194553375244, + "logits/rejected": -2.4628357887268066, + "logps/chosen": -181.03305053710938, + "logps/rejected": -232.1865234375, + "loss": 0.5298, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.23946937918663025, + "rewards/margins": 0.2353799045085907, + "rewards/rejected": -0.47484928369522095, + "step": 2530 + }, + { + "epoch": 2.62, + "learning_rate": 7.014525993883792e-08, + "logits/chosen": -2.6189305782318115, + "logits/rejected": -2.1566860675811768, + "logps/chosen": -291.4128112792969, + "logps/rejected": -229.62069702148438, + "loss": 0.5305, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.10792186111211777, + "rewards/margins": 0.8227887153625488, + "rewards/rejected": -0.7148668169975281, + "step": 2540 + }, + { + "epoch": 2.63, + "learning_rate": 6.823394495412843e-08, + "logits/chosen": -2.3303613662719727, + "logits/rejected": -2.243708610534668, + "logps/chosen": -240.57321166992188, + "logps/rejected": -237.77621459960938, + "loss": 0.53, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.18570654094219208, + "rewards/margins": 0.8395042419433594, + "rewards/rejected": -1.0252107381820679, + "step": 2550 + }, + { + "epoch": 2.64, + "learning_rate": 6.632262996941895e-08, + "logits/chosen": -2.184171438217163, + "logits/rejected": -2.440727949142456, + "logps/chosen": -293.82208251953125, + "logps/rejected": -232.8399658203125, + "loss": 0.5551, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.14512939751148224, + "rewards/margins": 0.7270814180374146, + "rewards/rejected": -0.8722109794616699, + "step": 2560 + }, + { + "epoch": 2.65, + "learning_rate": 6.441131498470948e-08, + "logits/chosen": -2.308842182159424, + "logits/rejected": -2.357908010482788, + "logps/chosen": -276.0211181640625, + "logps/rejected": -183.45028686523438, + "loss": 0.5318, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.06593456864356995, + "rewards/margins": 1.2839319705963135, + "rewards/rejected": -1.3498667478561401, + "step": 2570 + }, + { + "epoch": 2.66, + "learning_rate": 6.25e-08, + "logits/chosen": -2.4197561740875244, + "logits/rejected": -2.3011603355407715, + "logps/chosen": -332.51519775390625, + "logps/rejected": -231.3236846923828, + "loss": 0.5394, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.08808217942714691, + "rewards/margins": 0.9846493005752563, + "rewards/rejected": -0.896567165851593, + "step": 2580 + }, + { + "epoch": 2.67, + "learning_rate": 6.058868501529052e-08, + "logits/chosen": -2.518052577972412, + "logits/rejected": -2.333383083343506, + "logps/chosen": -253.10482788085938, + "logps/rejected": -198.84860229492188, + "loss": 0.516, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.3139076828956604, + "rewards/margins": 0.5393510460853577, + "rewards/rejected": -0.8532587289810181, + "step": 2590 + }, + { + "epoch": 2.68, + "learning_rate": 5.8677370030581035e-08, + "logits/chosen": -2.1512084007263184, + "logits/rejected": -2.312035083770752, + "logps/chosen": -288.53680419921875, + "logps/rejected": -222.78762817382812, + "loss": 0.5122, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 0.13815611600875854, + "rewards/margins": 1.1774489879608154, + "rewards/rejected": -1.0392926931381226, + "step": 2600 + }, + { + "epoch": 2.69, + "learning_rate": 5.6766055045871554e-08, + "logits/chosen": -2.2546794414520264, + "logits/rejected": -2.3793716430664062, + "logps/chosen": -343.5367736816406, + "logps/rejected": -309.8175354003906, + "loss": 0.5171, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.32955241203308105, + "rewards/margins": 0.6453115940093994, + "rewards/rejected": -0.9748640060424805, + "step": 2610 + }, + { + "epoch": 2.7, + "learning_rate": 5.485474006116208e-08, + "logits/chosen": -2.549440622329712, + "logits/rejected": -2.3844194412231445, + "logps/chosen": -258.51177978515625, + "logps/rejected": -215.14132690429688, + "loss": 0.5318, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 0.36421215534210205, + "rewards/margins": 1.2265236377716064, + "rewards/rejected": -0.8623113632202148, + "step": 2620 + }, + { + "epoch": 2.71, + "learning_rate": 5.294342507645259e-08, + "logits/chosen": -2.530996561050415, + "logits/rejected": -2.3681914806365967, + "logps/chosen": -335.9801330566406, + "logps/rejected": -263.645263671875, + "loss": 0.5112, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": 0.1152672991156578, + "rewards/margins": 0.8920395970344543, + "rewards/rejected": -0.7767722606658936, + "step": 2630 + }, + { + "epoch": 2.72, + "learning_rate": 5.1032110091743117e-08, + "logits/chosen": -2.185145378112793, + "logits/rejected": -2.3489866256713867, + "logps/chosen": -260.3304138183594, + "logps/rejected": -180.7406768798828, + "loss": 0.5443, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.4966302514076233, + "rewards/margins": 0.4965400695800781, + "rewards/rejected": -0.9931701421737671, + "step": 2640 + }, + { + "epoch": 2.73, + "learning_rate": 4.9120795107033635e-08, + "logits/chosen": -2.3882431983947754, + "logits/rejected": -2.4234635829925537, + "logps/chosen": -267.7854919433594, + "logps/rejected": -259.3836669921875, + "loss": 0.5344, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.1018451452255249, + "rewards/margins": 1.0978671312332153, + "rewards/rejected": -1.1997122764587402, + "step": 2650 + }, + { + "epoch": 2.75, + "learning_rate": 4.7209480122324154e-08, + "logits/chosen": -2.515261650085449, + "logits/rejected": -2.3157753944396973, + "logps/chosen": -299.7487487792969, + "logps/rejected": -317.60516357421875, + "loss": 0.5231, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 0.024449225515127182, + "rewards/margins": 1.3952572345733643, + "rewards/rejected": -1.3708080053329468, + "step": 2660 + }, + { + "epoch": 2.76, + "learning_rate": 4.529816513761467e-08, + "logits/chosen": -2.3622994422912598, + "logits/rejected": -2.125053644180298, + "logps/chosen": -248.30117797851562, + "logps/rejected": -275.47894287109375, + "loss": 0.4947, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.29935550689697266, + "rewards/margins": 0.4452272355556488, + "rewards/rejected": -0.7445827722549438, + "step": 2670 + }, + { + "epoch": 2.77, + "learning_rate": 4.33868501529052e-08, + "logits/chosen": -2.584453821182251, + "logits/rejected": -2.18440580368042, + "logps/chosen": -242.4678955078125, + "logps/rejected": -205.87344360351562, + "loss": 0.541, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.28912895917892456, + "rewards/margins": 0.5633198618888855, + "rewards/rejected": -0.8524488210678101, + "step": 2680 + }, + { + "epoch": 2.78, + "learning_rate": 4.147553516819572e-08, + "logits/chosen": -2.475851058959961, + "logits/rejected": -2.3260536193847656, + "logps/chosen": -214.3905029296875, + "logps/rejected": -229.3086700439453, + "loss": 0.544, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.2511672079563141, + "rewards/margins": 0.7018988132476807, + "rewards/rejected": -0.9530660510063171, + "step": 2690 + }, + { + "epoch": 2.79, + "learning_rate": 3.9564220183486236e-08, + "logits/chosen": -2.319629192352295, + "logits/rejected": -2.3051748275756836, + "logps/chosen": -242.98965454101562, + "logps/rejected": -185.3482666015625, + "loss": 0.5039, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.17325511574745178, + "rewards/margins": 0.6108760833740234, + "rewards/rejected": -0.7841311693191528, + "step": 2700 + }, + { + "epoch": 2.8, + "learning_rate": 3.7652905198776755e-08, + "logits/chosen": -2.633847713470459, + "logits/rejected": -2.576089382171631, + "logps/chosen": -300.53460693359375, + "logps/rejected": -229.891357421875, + "loss": 0.5421, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.30726686120033264, + "rewards/margins": 0.44638171792030334, + "rewards/rejected": -0.753648579120636, + "step": 2710 + }, + { + "epoch": 2.81, + "learning_rate": 3.574159021406728e-08, + "logits/chosen": -2.354677438735962, + "logits/rejected": -2.4129960536956787, + "logps/chosen": -258.06610107421875, + "logps/rejected": -236.42153930664062, + "loss": 0.549, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.44077181816101074, + "rewards/margins": 0.5614089369773865, + "rewards/rejected": -1.002180814743042, + "step": 2720 + }, + { + "epoch": 2.82, + "learning_rate": 3.383027522935779e-08, + "logits/chosen": -2.444614887237549, + "logits/rejected": -2.2034354209899902, + "logps/chosen": -259.88946533203125, + "logps/rejected": -274.34490966796875, + "loss": 0.5441, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 0.042647577822208405, + "rewards/margins": 1.146150827407837, + "rewards/rejected": -1.1035032272338867, + "step": 2730 + }, + { + "epoch": 2.83, + "learning_rate": 3.191896024464832e-08, + "logits/chosen": -2.2883150577545166, + "logits/rejected": -2.3496975898742676, + "logps/chosen": -292.2751770019531, + "logps/rejected": -200.29855346679688, + "loss": 0.5666, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 0.0850500836968422, + "rewards/margins": 1.3057624101638794, + "rewards/rejected": -1.2207123041152954, + "step": 2740 + }, + { + "epoch": 2.84, + "learning_rate": 3.0007645259938836e-08, + "logits/chosen": -2.456864356994629, + "logits/rejected": -2.5583481788635254, + "logps/chosen": -241.4561767578125, + "logps/rejected": -275.2192077636719, + "loss": 0.5165, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.22843988239765167, + "rewards/margins": 0.4716885983943939, + "rewards/rejected": -0.7001284956932068, + "step": 2750 + }, + { + "epoch": 2.85, + "learning_rate": 2.809633027522936e-08, + "logits/chosen": -2.2733702659606934, + "logits/rejected": -2.3403806686401367, + "logps/chosen": -196.03524780273438, + "logps/rejected": -215.77767944335938, + "loss": 0.5142, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.21540656685829163, + "rewards/margins": 0.6115992665290833, + "rewards/rejected": -0.8270059823989868, + "step": 2760 + }, + { + "epoch": 2.86, + "learning_rate": 2.6185015290519877e-08, + "logits/chosen": -2.3207640647888184, + "logits/rejected": -2.3754372596740723, + "logps/chosen": -177.1396026611328, + "logps/rejected": -185.77505493164062, + "loss": 0.5179, + "rewards/accuracies": 0.8500000238418579, + "rewards/chosen": 0.019303400069475174, + "rewards/margins": 0.6052744388580322, + "rewards/rejected": -0.5859710574150085, + "step": 2770 + }, + { + "epoch": 2.87, + "learning_rate": 2.4273700305810396e-08, + "logits/chosen": -2.4921469688415527, + "logits/rejected": -2.460731029510498, + "logps/chosen": -291.66229248046875, + "logps/rejected": -241.90072631835938, + "loss": 0.5378, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": 0.022645365446805954, + "rewards/margins": 1.2275021076202393, + "rewards/rejected": -1.2048569917678833, + "step": 2780 + }, + { + "epoch": 2.88, + "learning_rate": 2.2362385321100918e-08, + "logits/chosen": -2.3739137649536133, + "logits/rejected": -2.2125802040100098, + "logps/chosen": -286.7039794921875, + "logps/rejected": -244.8232421875, + "loss": 0.5323, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.10285166651010513, + "rewards/margins": 0.4872564375400543, + "rewards/rejected": -0.5901080965995789, + "step": 2790 + }, + { + "epoch": 2.89, + "learning_rate": 2.0451070336391437e-08, + "logits/chosen": -1.8592112064361572, + "logits/rejected": -1.7996069192886353, + "logps/chosen": -194.89865112304688, + "logps/rejected": -183.8830108642578, + "loss": 0.5462, + "rewards/accuracies": 0.6000000238418579, + "rewards/chosen": -0.18555238842964172, + "rewards/margins": 0.03723742812871933, + "rewards/rejected": -0.22278980910778046, + "step": 2800 + }, + { + "epoch": 2.9, + "learning_rate": 1.8539755351681956e-08, + "logits/chosen": -2.2919793128967285, + "logits/rejected": -2.5266997814178467, + "logps/chosen": -407.080810546875, + "logps/rejected": -314.6758117675781, + "loss": 0.5106, + "rewards/accuracies": 0.75, + "rewards/chosen": 0.05720346048474312, + "rewards/margins": 0.6664168238639832, + "rewards/rejected": -0.6092134714126587, + "step": 2810 + }, + { + "epoch": 2.91, + "learning_rate": 1.6628440366972478e-08, + "logits/chosen": -2.0835564136505127, + "logits/rejected": -2.263801097869873, + "logps/chosen": -319.478271484375, + "logps/rejected": -161.7843017578125, + "loss": 0.5252, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": -0.046999458223581314, + "rewards/margins": 1.0421249866485596, + "rewards/rejected": -1.0891244411468506, + "step": 2820 + }, + { + "epoch": 2.92, + "learning_rate": 1.4717125382262997e-08, + "logits/chosen": -2.5174002647399902, + "logits/rejected": -2.6593270301818848, + "logps/chosen": -317.7415771484375, + "logps/rejected": -299.45513916015625, + "loss": 0.5221, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.22343751788139343, + "rewards/margins": 0.47214531898498535, + "rewards/rejected": -0.695582926273346, + "step": 2830 + }, + { + "epoch": 2.93, + "learning_rate": 1.2805810397553517e-08, + "logits/chosen": -2.3480212688446045, + "logits/rejected": -2.281808376312256, + "logps/chosen": -368.12774658203125, + "logps/rejected": -359.60064697265625, + "loss": 0.5712, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.30808025598526, + "rewards/margins": 0.5734547972679138, + "rewards/rejected": -0.8815349340438843, + "step": 2840 + }, + { + "epoch": 2.94, + "learning_rate": 1.0894495412844038e-08, + "logits/chosen": -2.2814998626708984, + "logits/rejected": -2.3192315101623535, + "logps/chosen": -251.41836547851562, + "logps/rejected": -233.8671417236328, + "loss": 0.5412, + "rewards/accuracies": 0.550000011920929, + "rewards/chosen": -0.1413925290107727, + "rewards/margins": 0.6429818272590637, + "rewards/rejected": -0.7843743562698364, + "step": 2850 + }, + { + "epoch": 2.95, + "learning_rate": 8.983180428134555e-09, + "logits/chosen": -2.0733284950256348, + "logits/rejected": -2.226485252380371, + "logps/chosen": -212.7808380126953, + "logps/rejected": -141.1074676513672, + "loss": 0.5101, + "rewards/accuracies": 0.800000011920929, + "rewards/chosen": -0.39890027046203613, + "rewards/margins": 0.5048045516014099, + "rewards/rejected": -0.903704822063446, + "step": 2860 + }, + { + "epoch": 2.96, + "learning_rate": 7.071865443425076e-09, + "logits/chosen": -2.551570415496826, + "logits/rejected": -2.3837685585021973, + "logps/chosen": -274.71490478515625, + "logps/rejected": -316.5244140625, + "loss": 0.5186, + "rewards/accuracies": 0.6499999761581421, + "rewards/chosen": -0.2874767482280731, + "rewards/margins": 0.5797588229179382, + "rewards/rejected": -0.867235541343689, + "step": 2870 + }, + { + "epoch": 2.97, + "learning_rate": 5.1605504587155965e-09, + "logits/chosen": -2.5565171241760254, + "logits/rejected": -2.4579882621765137, + "logps/chosen": -253.5585479736328, + "logps/rejected": -178.70553588867188, + "loss": 0.5493, + "rewards/accuracies": 0.699999988079071, + "rewards/chosen": -0.20143918693065643, + "rewards/margins": 0.6308044195175171, + "rewards/rejected": -0.8322436213493347, + "step": 2880 + }, + { + "epoch": 2.98, + "learning_rate": 3.249235474006116e-09, + "logits/chosen": -2.354980707168579, + "logits/rejected": -2.389235496520996, + "logps/chosen": -283.6640319824219, + "logps/rejected": -232.73165893554688, + "loss": 0.532, + "rewards/accuracies": 0.75, + "rewards/chosen": -0.4438861906528473, + "rewards/margins": 0.5923529267311096, + "rewards/rejected": -1.0362392663955688, + "step": 2890 + }, + { + "epoch": 2.99, + "learning_rate": 1.3379204892966359e-09, + "logits/chosen": -2.1370351314544678, + "logits/rejected": -2.2694876194000244, + "logps/chosen": -225.5801239013672, + "logps/rejected": -227.0513916015625, + "loss": 0.5242, + "rewards/accuracies": 0.8999999761581421, + "rewards/chosen": 0.11508389562368393, + "rewards/margins": 1.1479651927947998, + "rewards/rejected": -1.032881259918213, + "step": 2900 + }, + { + "epoch": 3.0, + "eval_logits/chosen": -2.085988998413086, + "eval_logits/rejected": -1.9190013408660889, + "eval_logps/chosen": -248.65191650390625, + "eval_logps/rejected": -198.58494567871094, + "eval_loss": 0.5269633531570435, + "eval_rewards/accuracies": 0.78125, + "eval_rewards/chosen": -0.12098389863967896, + "eval_rewards/margins": 0.8767741918563843, + "eval_rewards/rejected": -0.9977580308914185, + "eval_runtime": 50.0854, + "eval_samples_per_second": 39.932, + "eval_steps_per_second": 0.319, + "step": 2907 + }, + { + "epoch": 3.0, + "step": 2907, + "total_flos": 0.0, + "train_loss": 0.5643668570057567, + "train_runtime": 8096.9375, + "train_samples_per_second": 22.959, + "train_steps_per_second": 0.359 + } + ], + "logging_steps": 10, + "max_steps": 2907, + "num_train_epochs": 3, + "save_steps": 500, + "total_flos": 0.0, + "trial_name": null, + "trial_params": null +}