| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.0, | |
| "eval_steps": 100, | |
| "global_step": 1563, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.0, | |
| "grad_norm": 42.15456008911133, | |
| "kl": 0.017818570137023926, | |
| "learning_rate": 1.592356687898089e-08, | |
| "logps/chosen": -285.75128173828125, | |
| "logps/rejected": -254.7062530517578, | |
| "loss": 0.4999, | |
| "rewards/chosen": 0.004669209010899067, | |
| "rewards/margins": 0.0025329389609396458, | |
| "rewards/rejected": 0.0021362705156207085, | |
| "step": 5 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 44.65619659423828, | |
| "kl": 0.05461766570806503, | |
| "learning_rate": 3.184713375796178e-08, | |
| "logps/chosen": -286.11944580078125, | |
| "logps/rejected": -276.1832580566406, | |
| "loss": 0.496, | |
| "rewards/chosen": 0.03117586299777031, | |
| "rewards/margins": 0.033454541116952896, | |
| "rewards/rejected": -0.00227867579087615, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 39.61723327636719, | |
| "kl": 0.44255906343460083, | |
| "learning_rate": 4.777070063694268e-08, | |
| "logps/chosen": -269.203125, | |
| "logps/rejected": -260.71966552734375, | |
| "loss": 0.4849, | |
| "rewards/chosen": 0.13460782170295715, | |
| "rewards/margins": 0.11511580646038055, | |
| "rewards/rejected": 0.0194920115172863, | |
| "step": 15 | |
| }, | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 36.42387771606445, | |
| "kl": 0.871076226234436, | |
| "learning_rate": 6.369426751592356e-08, | |
| "logps/chosen": -244.55203247070312, | |
| "logps/rejected": -259.15496826171875, | |
| "loss": 0.4573, | |
| "rewards/chosen": 0.3324822187423706, | |
| "rewards/margins": 0.3418889045715332, | |
| "rewards/rejected": -0.00940666627138853, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 35.794044494628906, | |
| "kl": 0.3753414750099182, | |
| "learning_rate": 7.961783439490445e-08, | |
| "logps/chosen": -261.01800537109375, | |
| "logps/rejected": -271.74786376953125, | |
| "loss": 0.4164, | |
| "rewards/chosen": 0.48356980085372925, | |
| "rewards/margins": 0.7098164558410645, | |
| "rewards/rejected": -0.22624659538269043, | |
| "step": 25 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 34.50333786010742, | |
| "kl": 0.0, | |
| "learning_rate": 9.554140127388536e-08, | |
| "logps/chosen": -306.0127868652344, | |
| "logps/rejected": -244.1356658935547, | |
| "loss": 0.3649, | |
| "rewards/chosen": 0.746610164642334, | |
| "rewards/margins": 1.2027219533920288, | |
| "rewards/rejected": -0.4561118483543396, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.02, | |
| "grad_norm": 30.50051498413086, | |
| "kl": 0.0, | |
| "learning_rate": 1.1146496815286624e-07, | |
| "logps/chosen": -280.65692138671875, | |
| "logps/rejected": -262.2502136230469, | |
| "loss": 0.3171, | |
| "rewards/chosen": 0.774597704410553, | |
| "rewards/margins": 1.728833794593811, | |
| "rewards/rejected": -0.9542360305786133, | |
| "step": 35 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 29.806577682495117, | |
| "kl": 0.0, | |
| "learning_rate": 1.2738853503184713e-07, | |
| "logps/chosen": -276.9497375488281, | |
| "logps/rejected": -259.6419982910156, | |
| "loss": 0.2884, | |
| "rewards/chosen": 0.6780191659927368, | |
| "rewards/margins": 2.165379047393799, | |
| "rewards/rejected": -1.4873597621917725, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 22.508939743041992, | |
| "kl": 0.0, | |
| "learning_rate": 1.43312101910828e-07, | |
| "logps/chosen": -253.7576141357422, | |
| "logps/rejected": -268.8052062988281, | |
| "loss": 0.2596, | |
| "rewards/chosen": 1.205094575881958, | |
| "rewards/margins": 2.685385227203369, | |
| "rewards/rejected": -1.4802907705307007, | |
| "step": 45 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 29.29058074951172, | |
| "kl": 0.0, | |
| "learning_rate": 1.592356687898089e-07, | |
| "logps/chosen": -284.4584045410156, | |
| "logps/rejected": -254.226806640625, | |
| "loss": 0.2614, | |
| "rewards/chosen": 1.0118796825408936, | |
| "rewards/margins": 2.8269615173339844, | |
| "rewards/rejected": -1.8150818347930908, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 29.25771141052246, | |
| "kl": 0.0, | |
| "learning_rate": 1.7515923566878978e-07, | |
| "logps/chosen": -266.92254638671875, | |
| "logps/rejected": -260.49261474609375, | |
| "loss": 0.233, | |
| "rewards/chosen": 1.16305673122406, | |
| "rewards/margins": 3.4210219383239746, | |
| "rewards/rejected": -2.257965326309204, | |
| "step": 55 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 20.642614364624023, | |
| "kl": 0.0, | |
| "learning_rate": 1.9108280254777072e-07, | |
| "logps/chosen": -231.07168579101562, | |
| "logps/rejected": -258.5835876464844, | |
| "loss": 0.241, | |
| "rewards/chosen": 1.3277983665466309, | |
| "rewards/margins": 3.532343626022339, | |
| "rewards/rejected": -2.204545497894287, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 23.786853790283203, | |
| "kl": 0.0, | |
| "learning_rate": 2.070063694267516e-07, | |
| "logps/chosen": -251.5561981201172, | |
| "logps/rejected": -267.85986328125, | |
| "loss": 0.2376, | |
| "rewards/chosen": 1.2313438653945923, | |
| "rewards/margins": 3.837125062942505, | |
| "rewards/rejected": -2.605781316757202, | |
| "step": 65 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 24.48412322998047, | |
| "kl": 0.0, | |
| "learning_rate": 2.2292993630573247e-07, | |
| "logps/chosen": -244.3961639404297, | |
| "logps/rejected": -248.7190704345703, | |
| "loss": 0.2304, | |
| "rewards/chosen": 1.338683843612671, | |
| "rewards/margins": 3.6262855529785156, | |
| "rewards/rejected": -2.287601947784424, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 16.654287338256836, | |
| "kl": 0.0, | |
| "learning_rate": 2.388535031847134e-07, | |
| "logps/chosen": -261.95819091796875, | |
| "logps/rejected": -291.40020751953125, | |
| "loss": 0.2003, | |
| "rewards/chosen": 1.3882195949554443, | |
| "rewards/margins": 4.626662254333496, | |
| "rewards/rejected": -3.238443374633789, | |
| "step": 75 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 30.89764976501465, | |
| "kl": 0.0, | |
| "learning_rate": 2.5477707006369425e-07, | |
| "logps/chosen": -267.4027099609375, | |
| "logps/rejected": -271.3486328125, | |
| "loss": 0.1985, | |
| "rewards/chosen": 1.4253151416778564, | |
| "rewards/margins": 4.816943645477295, | |
| "rewards/rejected": -3.3916287422180176, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 55.57185363769531, | |
| "kl": 0.0, | |
| "learning_rate": 2.7070063694267513e-07, | |
| "logps/chosen": -289.416259765625, | |
| "logps/rejected": -282.303955078125, | |
| "loss": 0.2075, | |
| "rewards/chosen": 1.308215856552124, | |
| "rewards/margins": 5.0450119972229, | |
| "rewards/rejected": -3.7367959022521973, | |
| "step": 85 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 34.31930923461914, | |
| "kl": 0.0, | |
| "learning_rate": 2.86624203821656e-07, | |
| "logps/chosen": -255.4053192138672, | |
| "logps/rejected": -293.8216247558594, | |
| "loss": 0.1877, | |
| "rewards/chosen": 1.560572862625122, | |
| "rewards/margins": 5.238338470458984, | |
| "rewards/rejected": -3.6777656078338623, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 22.21489143371582, | |
| "kl": 0.0, | |
| "learning_rate": 3.0254777070063694e-07, | |
| "logps/chosen": -286.2869873046875, | |
| "logps/rejected": -271.2530212402344, | |
| "loss": 0.1985, | |
| "rewards/chosen": 1.478058099746704, | |
| "rewards/margins": 5.051484107971191, | |
| "rewards/rejected": -3.5734260082244873, | |
| "step": 95 | |
| }, | |
| { | |
| "epoch": 0.06, | |
| "grad_norm": 29.9346981048584, | |
| "kl": 0.0, | |
| "learning_rate": 3.184713375796178e-07, | |
| "logps/chosen": -245.080810546875, | |
| "logps/rejected": -287.15435791015625, | |
| "loss": 0.1656, | |
| "rewards/chosen": 1.4543142318725586, | |
| "rewards/margins": 5.874560356140137, | |
| "rewards/rejected": -4.420246124267578, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 22.431930541992188, | |
| "kl": 0.0, | |
| "learning_rate": 3.343949044585987e-07, | |
| "logps/chosen": -259.4337463378906, | |
| "logps/rejected": -309.25164794921875, | |
| "loss": 0.1767, | |
| "rewards/chosen": 1.4657680988311768, | |
| "rewards/margins": 5.859877109527588, | |
| "rewards/rejected": -4.39410924911499, | |
| "step": 105 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 36.517799377441406, | |
| "kl": 0.0, | |
| "learning_rate": 3.5031847133757957e-07, | |
| "logps/chosen": -264.08306884765625, | |
| "logps/rejected": -295.64788818359375, | |
| "loss": 0.1599, | |
| "rewards/chosen": 1.291338562965393, | |
| "rewards/margins": 5.8188042640686035, | |
| "rewards/rejected": -4.5274658203125, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 25.18623924255371, | |
| "kl": 0.0, | |
| "learning_rate": 3.6624203821656045e-07, | |
| "logps/chosen": -268.96527099609375, | |
| "logps/rejected": -278.4827575683594, | |
| "loss": 0.1735, | |
| "rewards/chosen": 1.4270073175430298, | |
| "rewards/margins": 5.516595363616943, | |
| "rewards/rejected": -4.0895891189575195, | |
| "step": 115 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 24.899450302124023, | |
| "kl": 0.0, | |
| "learning_rate": 3.8216560509554143e-07, | |
| "logps/chosen": -257.46807861328125, | |
| "logps/rejected": -306.6875305175781, | |
| "loss": 0.1675, | |
| "rewards/chosen": 1.363029956817627, | |
| "rewards/margins": 5.9437384605407715, | |
| "rewards/rejected": -4.580708980560303, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 18.1636962890625, | |
| "kl": 0.0, | |
| "learning_rate": 3.980891719745223e-07, | |
| "logps/chosen": -274.8599548339844, | |
| "logps/rejected": -316.78057861328125, | |
| "loss": 0.1509, | |
| "rewards/chosen": 1.4044160842895508, | |
| "rewards/margins": 6.668715000152588, | |
| "rewards/rejected": -5.264299392700195, | |
| "step": 125 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 53.83689498901367, | |
| "kl": 0.0, | |
| "learning_rate": 4.140127388535032e-07, | |
| "logps/chosen": -316.65460205078125, | |
| "logps/rejected": -318.2517395019531, | |
| "loss": 0.1667, | |
| "rewards/chosen": 1.331627368927002, | |
| "rewards/margins": 6.6092848777771, | |
| "rewards/rejected": -5.277657508850098, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 16.44365692138672, | |
| "kl": 0.0, | |
| "learning_rate": 4.2993630573248406e-07, | |
| "logps/chosen": -242.02651977539062, | |
| "logps/rejected": -283.90106201171875, | |
| "loss": 0.1762, | |
| "rewards/chosen": 1.6127967834472656, | |
| "rewards/margins": 6.091832160949707, | |
| "rewards/rejected": -4.479035377502441, | |
| "step": 135 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 19.764270782470703, | |
| "kl": 0.0, | |
| "learning_rate": 4.4585987261146494e-07, | |
| "logps/chosen": -252.82180786132812, | |
| "logps/rejected": -319.73199462890625, | |
| "loss": 0.1468, | |
| "rewards/chosen": 1.5330384969711304, | |
| "rewards/margins": 6.702307224273682, | |
| "rewards/rejected": -5.169268608093262, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.09, | |
| "grad_norm": 21.560876846313477, | |
| "kl": 0.0, | |
| "learning_rate": 4.6178343949044587e-07, | |
| "logps/chosen": -247.28427124023438, | |
| "logps/rejected": -282.38848876953125, | |
| "loss": 0.1868, | |
| "rewards/chosen": 1.350294828414917, | |
| "rewards/margins": 5.6418867111206055, | |
| "rewards/rejected": -4.291592597961426, | |
| "step": 145 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 21.553871154785156, | |
| "kl": 0.0, | |
| "learning_rate": 4.777070063694267e-07, | |
| "logps/chosen": -286.14312744140625, | |
| "logps/rejected": -281.7044372558594, | |
| "loss": 0.168, | |
| "rewards/chosen": 1.5479779243469238, | |
| "rewards/margins": 6.192603588104248, | |
| "rewards/rejected": -4.644625663757324, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 24.488561630249023, | |
| "kl": 0.0, | |
| "learning_rate": 4.936305732484076e-07, | |
| "logps/chosen": -272.4622497558594, | |
| "logps/rejected": -323.91082763671875, | |
| "loss": 0.1648, | |
| "rewards/chosen": 1.3805065155029297, | |
| "rewards/margins": 6.456129550933838, | |
| "rewards/rejected": -5.07562255859375, | |
| "step": 155 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 21.614229202270508, | |
| "kl": 0.0, | |
| "learning_rate": 4.989331436699858e-07, | |
| "logps/chosen": -207.8739776611328, | |
| "logps/rejected": -324.73565673828125, | |
| "loss": 0.1435, | |
| "rewards/chosen": 1.465820074081421, | |
| "rewards/margins": 6.953179836273193, | |
| "rewards/rejected": -5.487359046936035, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 20.425729751586914, | |
| "kl": 0.0, | |
| "learning_rate": 4.971550497866287e-07, | |
| "logps/chosen": -287.088623046875, | |
| "logps/rejected": -310.1844787597656, | |
| "loss": 0.1568, | |
| "rewards/chosen": 1.5799640417099, | |
| "rewards/margins": 6.89548397064209, | |
| "rewards/rejected": -5.315520286560059, | |
| "step": 165 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 21.86062240600586, | |
| "kl": 0.0, | |
| "learning_rate": 4.953769559032717e-07, | |
| "logps/chosen": -282.91741943359375, | |
| "logps/rejected": -315.76470947265625, | |
| "loss": 0.1516, | |
| "rewards/chosen": 1.442077875137329, | |
| "rewards/margins": 7.074441432952881, | |
| "rewards/rejected": -5.632363796234131, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 20.31084632873535, | |
| "kl": 0.0, | |
| "learning_rate": 4.935988620199146e-07, | |
| "logps/chosen": -274.0014953613281, | |
| "logps/rejected": -310.8323059082031, | |
| "loss": 0.1477, | |
| "rewards/chosen": 1.4551314115524292, | |
| "rewards/margins": 6.99865198135376, | |
| "rewards/rejected": -5.543520450592041, | |
| "step": 175 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 20.067346572875977, | |
| "kl": 0.0, | |
| "learning_rate": 4.918207681365576e-07, | |
| "logps/chosen": -274.91705322265625, | |
| "logps/rejected": -330.29315185546875, | |
| "loss": 0.1491, | |
| "rewards/chosen": 1.5545661449432373, | |
| "rewards/margins": 7.674098014831543, | |
| "rewards/rejected": -6.119531631469727, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 26.703292846679688, | |
| "kl": 0.0, | |
| "learning_rate": 4.900426742532006e-07, | |
| "logps/chosen": -267.2247619628906, | |
| "logps/rejected": -308.6265563964844, | |
| "loss": 0.142, | |
| "rewards/chosen": 1.4495770931243896, | |
| "rewards/margins": 7.4614386558532715, | |
| "rewards/rejected": -6.011861324310303, | |
| "step": 185 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 22.81442642211914, | |
| "kl": 0.0, | |
| "learning_rate": 4.882645803698435e-07, | |
| "logps/chosen": -243.8881072998047, | |
| "logps/rejected": -318.4960632324219, | |
| "loss": 0.1679, | |
| "rewards/chosen": 1.2335823774337769, | |
| "rewards/margins": 7.564291954040527, | |
| "rewards/rejected": -6.330709934234619, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 17.875883102416992, | |
| "kl": 0.0, | |
| "learning_rate": 4.864864864864865e-07, | |
| "logps/chosen": -276.0986328125, | |
| "logps/rejected": -317.36163330078125, | |
| "loss": 0.1442, | |
| "rewards/chosen": 1.5666391849517822, | |
| "rewards/margins": 7.757128715515137, | |
| "rewards/rejected": -6.190489768981934, | |
| "step": 195 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 22.31620216369629, | |
| "kl": 0.0, | |
| "learning_rate": 4.847083926031294e-07, | |
| "logps/chosen": -273.843994140625, | |
| "logps/rejected": -329.43475341796875, | |
| "loss": 0.154, | |
| "rewards/chosen": 1.6012461185455322, | |
| "rewards/margins": 7.519808292388916, | |
| "rewards/rejected": -5.918562412261963, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 19.976451873779297, | |
| "kl": 0.0, | |
| "learning_rate": 4.829302987197724e-07, | |
| "logps/chosen": -284.374755859375, | |
| "logps/rejected": -326.24371337890625, | |
| "loss": 0.1378, | |
| "rewards/chosen": 1.5799314975738525, | |
| "rewards/margins": 7.78595495223999, | |
| "rewards/rejected": -6.206023216247559, | |
| "step": 205 | |
| }, | |
| { | |
| "epoch": 0.13, | |
| "grad_norm": 23.919170379638672, | |
| "kl": 0.0, | |
| "learning_rate": 4.811522048364154e-07, | |
| "logps/chosen": -236.8705291748047, | |
| "logps/rejected": -316.30413818359375, | |
| "loss": 0.1523, | |
| "rewards/chosen": 1.5169861316680908, | |
| "rewards/margins": 7.513286590576172, | |
| "rewards/rejected": -5.99630069732666, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 25.041824340820312, | |
| "kl": 0.0, | |
| "learning_rate": 4.793741109530583e-07, | |
| "logps/chosen": -219.86117553710938, | |
| "logps/rejected": -331.60418701171875, | |
| "loss": 0.1516, | |
| "rewards/chosen": 1.5909864902496338, | |
| "rewards/margins": 7.9639410972595215, | |
| "rewards/rejected": -6.372954845428467, | |
| "step": 215 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 25.45568084716797, | |
| "kl": 0.0, | |
| "learning_rate": 4.775960170697012e-07, | |
| "logps/chosen": -271.017333984375, | |
| "logps/rejected": -333.2148132324219, | |
| "loss": 0.1628, | |
| "rewards/chosen": 1.5948388576507568, | |
| "rewards/margins": 7.8658246994018555, | |
| "rewards/rejected": -6.2709856033325195, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 22.259382247924805, | |
| "kl": 0.0, | |
| "learning_rate": 4.7581792318634425e-07, | |
| "logps/chosen": -253.55789184570312, | |
| "logps/rejected": -298.8450622558594, | |
| "loss": 0.1452, | |
| "rewards/chosen": 1.640355110168457, | |
| "rewards/margins": 8.043792724609375, | |
| "rewards/rejected": -6.403438568115234, | |
| "step": 225 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 26.135997772216797, | |
| "kl": 0.0, | |
| "learning_rate": 4.7403982930298717e-07, | |
| "logps/chosen": -235.11349487304688, | |
| "logps/rejected": -344.72369384765625, | |
| "loss": 0.1472, | |
| "rewards/chosen": 1.559660792350769, | |
| "rewards/margins": 7.8217010498046875, | |
| "rewards/rejected": -6.262040615081787, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 20.143089294433594, | |
| "kl": 0.0, | |
| "learning_rate": 4.7226173541963014e-07, | |
| "logps/chosen": -249.42196655273438, | |
| "logps/rejected": -277.12554931640625, | |
| "loss": 0.1529, | |
| "rewards/chosen": 1.6460959911346436, | |
| "rewards/margins": 7.348568916320801, | |
| "rewards/rejected": -5.702473163604736, | |
| "step": 235 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 23.44059181213379, | |
| "kl": 0.0, | |
| "learning_rate": 4.7048364153627306e-07, | |
| "logps/chosen": -281.3484802246094, | |
| "logps/rejected": -307.25457763671875, | |
| "loss": 0.1387, | |
| "rewards/chosen": 1.7489140033721924, | |
| "rewards/margins": 7.860198974609375, | |
| "rewards/rejected": -6.111284255981445, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 23.201915740966797, | |
| "kl": 0.0, | |
| "learning_rate": 4.6870554765291604e-07, | |
| "logps/chosen": -272.64031982421875, | |
| "logps/rejected": -310.50909423828125, | |
| "loss": 0.1356, | |
| "rewards/chosen": 1.6725397109985352, | |
| "rewards/margins": 8.012718200683594, | |
| "rewards/rejected": -6.340178489685059, | |
| "step": 245 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 19.45538902282715, | |
| "kl": 0.0, | |
| "learning_rate": 4.66927453769559e-07, | |
| "logps/chosen": -272.3523254394531, | |
| "logps/rejected": -338.20867919921875, | |
| "loss": 0.1246, | |
| "rewards/chosen": 1.727900743484497, | |
| "rewards/margins": 8.646397590637207, | |
| "rewards/rejected": -6.918497562408447, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 24.896251678466797, | |
| "kl": 0.0, | |
| "learning_rate": 4.65149359886202e-07, | |
| "logps/chosen": -269.20794677734375, | |
| "logps/rejected": -328.73486328125, | |
| "loss": 0.1281, | |
| "rewards/chosen": 1.6262487173080444, | |
| "rewards/margins": 9.223333358764648, | |
| "rewards/rejected": -7.597084999084473, | |
| "step": 255 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "grad_norm": 20.800025939941406, | |
| "kl": 0.0, | |
| "learning_rate": 4.633712660028449e-07, | |
| "logps/chosen": -270.3649597167969, | |
| "logps/rejected": -330.872314453125, | |
| "loss": 0.1279, | |
| "rewards/chosen": 1.5803369283676147, | |
| "rewards/margins": 9.708114624023438, | |
| "rewards/rejected": -8.127778053283691, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "grad_norm": 20.315540313720703, | |
| "kl": 0.0, | |
| "learning_rate": 4.615931721194879e-07, | |
| "logps/chosen": -277.21807861328125, | |
| "logps/rejected": -313.4653625488281, | |
| "loss": 0.1461, | |
| "rewards/chosen": 1.6411361694335938, | |
| "rewards/margins": 7.905195713043213, | |
| "rewards/rejected": -6.264059543609619, | |
| "step": 265 | |
| }, | |
| { | |
| "epoch": 0.17, | |
| "grad_norm": 19.68859100341797, | |
| "kl": 0.0, | |
| "learning_rate": 4.5981507823613085e-07, | |
| "logps/chosen": -275.69696044921875, | |
| "logps/rejected": -295.79400634765625, | |
| "loss": 0.1547, | |
| "rewards/chosen": 1.6596572399139404, | |
| "rewards/margins": 7.9982404708862305, | |
| "rewards/rejected": -6.338583469390869, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "grad_norm": 27.070371627807617, | |
| "kl": 0.0, | |
| "learning_rate": 4.580369843527738e-07, | |
| "logps/chosen": -272.72735595703125, | |
| "logps/rejected": -310.302734375, | |
| "loss": 0.1673, | |
| "rewards/chosen": 1.2246049642562866, | |
| "rewards/margins": 8.44649600982666, | |
| "rewards/rejected": -7.221890449523926, | |
| "step": 275 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "grad_norm": 15.166617393493652, | |
| "kl": 0.0, | |
| "learning_rate": 4.562588904694168e-07, | |
| "logps/chosen": -267.9317932128906, | |
| "logps/rejected": -316.60479736328125, | |
| "loss": 0.1304, | |
| "rewards/chosen": 1.7944562435150146, | |
| "rewards/margins": 8.752721786499023, | |
| "rewards/rejected": -6.958265781402588, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "grad_norm": 27.143291473388672, | |
| "kl": 0.0, | |
| "learning_rate": 4.544807965860597e-07, | |
| "logps/chosen": -257.7388916015625, | |
| "logps/rejected": -308.5174560546875, | |
| "loss": 0.1421, | |
| "rewards/chosen": 1.6366589069366455, | |
| "rewards/margins": 8.678224563598633, | |
| "rewards/rejected": -7.04156494140625, | |
| "step": 285 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "grad_norm": 22.1202392578125, | |
| "kl": 0.0, | |
| "learning_rate": 4.5270270270270264e-07, | |
| "logps/chosen": -266.8901062011719, | |
| "logps/rejected": -322.6683654785156, | |
| "loss": 0.1447, | |
| "rewards/chosen": 1.787255883216858, | |
| "rewards/margins": 9.206243515014648, | |
| "rewards/rejected": -7.418987274169922, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "grad_norm": 24.7782039642334, | |
| "kl": 0.0, | |
| "learning_rate": 4.509246088193456e-07, | |
| "logps/chosen": -274.30438232421875, | |
| "logps/rejected": -342.98541259765625, | |
| "loss": 0.1421, | |
| "rewards/chosen": 1.776602029800415, | |
| "rewards/margins": 9.053030967712402, | |
| "rewards/rejected": -7.276429176330566, | |
| "step": 295 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "grad_norm": 23.54313087463379, | |
| "kl": 0.0, | |
| "learning_rate": 4.491465149359886e-07, | |
| "logps/chosen": -215.9270477294922, | |
| "logps/rejected": -303.5273742675781, | |
| "loss": 0.1438, | |
| "rewards/chosen": 1.868506669998169, | |
| "rewards/margins": 9.129117965698242, | |
| "rewards/rejected": -7.260611534118652, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "grad_norm": 17.70969009399414, | |
| "kl": 0.0, | |
| "learning_rate": 4.4736842105263156e-07, | |
| "logps/chosen": -240.4123992919922, | |
| "logps/rejected": -336.1444396972656, | |
| "loss": 0.1403, | |
| "rewards/chosen": 1.7039588689804077, | |
| "rewards/margins": 9.00100326538086, | |
| "rewards/rejected": -7.297044277191162, | |
| "step": 305 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "grad_norm": 19.908315658569336, | |
| "kl": 0.0, | |
| "learning_rate": 4.4559032716927454e-07, | |
| "logps/chosen": -212.1704864501953, | |
| "logps/rejected": -340.3092041015625, | |
| "loss": 0.1338, | |
| "rewards/chosen": 1.6462271213531494, | |
| "rewards/margins": 8.726274490356445, | |
| "rewards/rejected": -7.0800461769104, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "grad_norm": 22.928499221801758, | |
| "kl": 0.0, | |
| "learning_rate": 4.438122332859175e-07, | |
| "logps/chosen": -243.7815704345703, | |
| "logps/rejected": -324.60772705078125, | |
| "loss": 0.1323, | |
| "rewards/chosen": 1.6998401880264282, | |
| "rewards/margins": 8.809240341186523, | |
| "rewards/rejected": -7.109400749206543, | |
| "step": 315 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "grad_norm": 20.08190155029297, | |
| "kl": 0.0, | |
| "learning_rate": 4.420341394025605e-07, | |
| "logps/chosen": -263.37725830078125, | |
| "logps/rejected": -317.4496154785156, | |
| "loss": 0.1443, | |
| "rewards/chosen": 1.706053376197815, | |
| "rewards/margins": 8.708757400512695, | |
| "rewards/rejected": -7.002703666687012, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "grad_norm": 25.728233337402344, | |
| "kl": 0.0, | |
| "learning_rate": 4.4025604551920335e-07, | |
| "logps/chosen": -282.44989013671875, | |
| "logps/rejected": -325.9697265625, | |
| "loss": 0.1351, | |
| "rewards/chosen": 1.6976518630981445, | |
| "rewards/margins": 9.016552925109863, | |
| "rewards/rejected": -7.318901062011719, | |
| "step": 325 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "grad_norm": 22.311450958251953, | |
| "kl": 0.0, | |
| "learning_rate": 4.384779516358463e-07, | |
| "logps/chosen": -260.39447021484375, | |
| "logps/rejected": -309.620849609375, | |
| "loss": 0.1311, | |
| "rewards/chosen": 1.8077850341796875, | |
| "rewards/margins": 9.406209945678711, | |
| "rewards/rejected": -7.598425388336182, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "grad_norm": 23.20633316040039, | |
| "kl": 0.0, | |
| "learning_rate": 4.366998577524893e-07, | |
| "logps/chosen": -302.47509765625, | |
| "logps/rejected": -318.767578125, | |
| "loss": 0.128, | |
| "rewards/chosen": 1.7027689218521118, | |
| "rewards/margins": 10.0870943069458, | |
| "rewards/rejected": -8.38432502746582, | |
| "step": 335 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "grad_norm": 25.405933380126953, | |
| "kl": 0.0, | |
| "learning_rate": 4.3492176386913227e-07, | |
| "logps/chosen": -238.94827270507812, | |
| "logps/rejected": -326.12786865234375, | |
| "loss": 0.1505, | |
| "rewards/chosen": 1.7603483200073242, | |
| "rewards/margins": 9.71554183959961, | |
| "rewards/rejected": -7.955193519592285, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "grad_norm": 22.41493034362793, | |
| "kl": 0.0, | |
| "learning_rate": 4.3314366998577524e-07, | |
| "logps/chosen": -253.826416015625, | |
| "logps/rejected": -330.7270812988281, | |
| "loss": 0.1413, | |
| "rewards/chosen": 1.8580402135849, | |
| "rewards/margins": 9.722744941711426, | |
| "rewards/rejected": -7.8647050857543945, | |
| "step": 345 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "grad_norm": 26.315628051757812, | |
| "kl": 0.0, | |
| "learning_rate": 4.313655761024182e-07, | |
| "logps/chosen": -286.4155578613281, | |
| "logps/rejected": -342.62652587890625, | |
| "loss": 0.1422, | |
| "rewards/chosen": 1.7754218578338623, | |
| "rewards/margins": 10.7272367477417, | |
| "rewards/rejected": -8.951814651489258, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "grad_norm": 22.07290267944336, | |
| "kl": 0.0, | |
| "learning_rate": 4.2958748221906114e-07, | |
| "logps/chosen": -239.4450225830078, | |
| "logps/rejected": -310.7589111328125, | |
| "loss": 0.1108, | |
| "rewards/chosen": 1.9980005025863647, | |
| "rewards/margins": 10.0792818069458, | |
| "rewards/rejected": -8.081281661987305, | |
| "step": 355 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "grad_norm": 22.56456756591797, | |
| "kl": 0.0, | |
| "learning_rate": 4.278093883357041e-07, | |
| "logps/chosen": -269.43560791015625, | |
| "logps/rejected": -328.5326843261719, | |
| "loss": 0.1524, | |
| "rewards/chosen": 1.747078537940979, | |
| "rewards/margins": 9.907144546508789, | |
| "rewards/rejected": -8.160065650939941, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "grad_norm": 24.159225463867188, | |
| "kl": 0.0, | |
| "learning_rate": 4.260312944523471e-07, | |
| "logps/chosen": -241.72286987304688, | |
| "logps/rejected": -353.99493408203125, | |
| "loss": 0.1185, | |
| "rewards/chosen": 2.0929551124572754, | |
| "rewards/margins": 10.861230850219727, | |
| "rewards/rejected": -8.768275260925293, | |
| "step": 365 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "grad_norm": 21.119632720947266, | |
| "kl": 0.0, | |
| "learning_rate": 4.2425320056899e-07, | |
| "logps/chosen": -263.73370361328125, | |
| "logps/rejected": -343.69866943359375, | |
| "loss": 0.1164, | |
| "rewards/chosen": 1.9714298248291016, | |
| "rewards/margins": 10.647806167602539, | |
| "rewards/rejected": -8.676377296447754, | |
| "step": 370 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "grad_norm": 23.59329605102539, | |
| "kl": 0.0, | |
| "learning_rate": 4.22475106685633e-07, | |
| "logps/chosen": -244.0771026611328, | |
| "logps/rejected": -348.35693359375, | |
| "loss": 0.1218, | |
| "rewards/chosen": 1.8124616146087646, | |
| "rewards/margins": 10.569868087768555, | |
| "rewards/rejected": -8.757406234741211, | |
| "step": 375 | |
| }, | |
| { | |
| "epoch": 0.24, | |
| "grad_norm": 23.84940528869629, | |
| "kl": 0.0, | |
| "learning_rate": 4.2069701280227595e-07, | |
| "logps/chosen": -241.4117889404297, | |
| "logps/rejected": -336.4566345214844, | |
| "loss": 0.1294, | |
| "rewards/chosen": 1.8767503499984741, | |
| "rewards/margins": 10.489054679870605, | |
| "rewards/rejected": -8.612303733825684, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "grad_norm": 16.005657196044922, | |
| "kl": 0.0, | |
| "learning_rate": 4.189189189189189e-07, | |
| "logps/chosen": -307.21600341796875, | |
| "logps/rejected": -341.0569152832031, | |
| "loss": 0.1154, | |
| "rewards/chosen": 1.901063323020935, | |
| "rewards/margins": 10.400434494018555, | |
| "rewards/rejected": -8.499369621276855, | |
| "step": 385 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "grad_norm": 16.274534225463867, | |
| "kl": 0.0, | |
| "learning_rate": 4.1714082503556185e-07, | |
| "logps/chosen": -253.8114471435547, | |
| "logps/rejected": -333.0570983886719, | |
| "loss": 0.1327, | |
| "rewards/chosen": 1.955413818359375, | |
| "rewards/margins": 10.895428657531738, | |
| "rewards/rejected": -8.940014839172363, | |
| "step": 390 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "grad_norm": 20.72968101501465, | |
| "kl": 0.0, | |
| "learning_rate": 4.153627311522048e-07, | |
| "logps/chosen": -261.703857421875, | |
| "logps/rejected": -339.4162902832031, | |
| "loss": 0.1308, | |
| "rewards/chosen": 1.7927815914154053, | |
| "rewards/margins": 10.807384490966797, | |
| "rewards/rejected": -9.014602661132812, | |
| "step": 395 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "grad_norm": 21.173845291137695, | |
| "kl": 0.0, | |
| "learning_rate": 4.135846372688478e-07, | |
| "logps/chosen": -260.5010681152344, | |
| "logps/rejected": -359.41851806640625, | |
| "loss": 0.1131, | |
| "rewards/chosen": 1.8654544353485107, | |
| "rewards/margins": 11.833639144897461, | |
| "rewards/rejected": -9.968184471130371, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "grad_norm": 17.008028030395508, | |
| "kl": 0.0, | |
| "learning_rate": 4.1180654338549077e-07, | |
| "logps/chosen": -234.20315551757812, | |
| "logps/rejected": -342.5408020019531, | |
| "loss": 0.1224, | |
| "rewards/chosen": 2.086550235748291, | |
| "rewards/margins": 10.627596855163574, | |
| "rewards/rejected": -8.541048049926758, | |
| "step": 405 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "grad_norm": 19.24855613708496, | |
| "kl": 0.0, | |
| "learning_rate": 4.100284495021337e-07, | |
| "logps/chosen": -282.03106689453125, | |
| "logps/rejected": -347.0184631347656, | |
| "loss": 0.1212, | |
| "rewards/chosen": 1.9099693298339844, | |
| "rewards/margins": 10.274763107299805, | |
| "rewards/rejected": -8.36479377746582, | |
| "step": 410 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "grad_norm": 19.914691925048828, | |
| "kl": 0.0, | |
| "learning_rate": 4.082503556187766e-07, | |
| "logps/chosen": -278.811767578125, | |
| "logps/rejected": -348.43096923828125, | |
| "loss": 0.1209, | |
| "rewards/chosen": 1.7562357187271118, | |
| "rewards/margins": 11.304061889648438, | |
| "rewards/rejected": -9.547826766967773, | |
| "step": 415 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "grad_norm": 32.35190963745117, | |
| "kl": 0.0, | |
| "learning_rate": 4.064722617354196e-07, | |
| "logps/chosen": -272.9192199707031, | |
| "logps/rejected": -339.4073181152344, | |
| "loss": 0.1005, | |
| "rewards/chosen": 2.1069161891937256, | |
| "rewards/margins": 11.689797401428223, | |
| "rewards/rejected": -9.582880973815918, | |
| "step": 420 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "grad_norm": 16.27385902404785, | |
| "kl": 0.0, | |
| "learning_rate": 4.0469416785206256e-07, | |
| "logps/chosen": -233.26083374023438, | |
| "logps/rejected": -329.8385314941406, | |
| "loss": 0.1135, | |
| "rewards/chosen": 2.1503446102142334, | |
| "rewards/margins": 10.886509895324707, | |
| "rewards/rejected": -8.736165046691895, | |
| "step": 425 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "grad_norm": 22.10724449157715, | |
| "kl": 0.0, | |
| "learning_rate": 4.0291607396870553e-07, | |
| "logps/chosen": -267.9533996582031, | |
| "logps/rejected": -359.1152038574219, | |
| "loss": 0.1197, | |
| "rewards/chosen": 1.8767788410186768, | |
| "rewards/margins": 11.441519737243652, | |
| "rewards/rejected": -9.564741134643555, | |
| "step": 430 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "grad_norm": 29.721330642700195, | |
| "kl": 0.0, | |
| "learning_rate": 4.011379800853485e-07, | |
| "logps/chosen": -268.9942932128906, | |
| "logps/rejected": -377.97308349609375, | |
| "loss": 0.1061, | |
| "rewards/chosen": 2.016845703125, | |
| "rewards/margins": 11.474628448486328, | |
| "rewards/rejected": -9.457781791687012, | |
| "step": 435 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "grad_norm": 24.95069122314453, | |
| "kl": 0.0, | |
| "learning_rate": 3.993598862019915e-07, | |
| "logps/chosen": -279.73944091796875, | |
| "logps/rejected": -370.55474853515625, | |
| "loss": 0.1045, | |
| "rewards/chosen": 2.340444803237915, | |
| "rewards/margins": 11.733491897583008, | |
| "rewards/rejected": -9.393046379089355, | |
| "step": 440 | |
| }, | |
| { | |
| "epoch": 0.28, | |
| "grad_norm": 26.99854278564453, | |
| "kl": 0.0, | |
| "learning_rate": 3.975817923186344e-07, | |
| "logps/chosen": -245.24417114257812, | |
| "logps/rejected": -336.7696533203125, | |
| "loss": 0.1234, | |
| "rewards/chosen": 2.2276217937469482, | |
| "rewards/margins": 10.63569164276123, | |
| "rewards/rejected": -8.40807056427002, | |
| "step": 445 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "grad_norm": 22.343507766723633, | |
| "kl": 0.0, | |
| "learning_rate": 3.9580369843527737e-07, | |
| "logps/chosen": -250.68197631835938, | |
| "logps/rejected": -364.1970520019531, | |
| "loss": 0.1342, | |
| "rewards/chosen": 2.1794769763946533, | |
| "rewards/margins": 10.827537536621094, | |
| "rewards/rejected": -8.64806079864502, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "grad_norm": 16.532583236694336, | |
| "kl": 0.0, | |
| "learning_rate": 3.940256045519203e-07, | |
| "logps/chosen": -246.8504638671875, | |
| "logps/rejected": -336.2493896484375, | |
| "loss": 0.1151, | |
| "rewards/chosen": 1.9471546411514282, | |
| "rewards/margins": 11.059728622436523, | |
| "rewards/rejected": -9.112573623657227, | |
| "step": 455 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "grad_norm": 19.48154067993164, | |
| "kl": 0.0, | |
| "learning_rate": 3.9224751066856327e-07, | |
| "logps/chosen": -246.47671508789062, | |
| "logps/rejected": -328.21038818359375, | |
| "loss": 0.1156, | |
| "rewards/chosen": 2.022249221801758, | |
| "rewards/margins": 11.125974655151367, | |
| "rewards/rejected": -9.103724479675293, | |
| "step": 460 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "grad_norm": 17.954296112060547, | |
| "kl": 0.0, | |
| "learning_rate": 3.9046941678520624e-07, | |
| "logps/chosen": -254.69808959960938, | |
| "logps/rejected": -347.45758056640625, | |
| "loss": 0.1254, | |
| "rewards/chosen": 2.0911641120910645, | |
| "rewards/margins": 11.680787086486816, | |
| "rewards/rejected": -9.58962345123291, | |
| "step": 465 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "grad_norm": 21.16987419128418, | |
| "kl": 0.0, | |
| "learning_rate": 3.886913229018492e-07, | |
| "logps/chosen": -226.58358764648438, | |
| "logps/rejected": -334.48138427734375, | |
| "loss": 0.1437, | |
| "rewards/chosen": 1.8157398700714111, | |
| "rewards/margins": 10.399962425231934, | |
| "rewards/rejected": -8.584221839904785, | |
| "step": 470 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "grad_norm": 18.404508590698242, | |
| "kl": 0.0, | |
| "learning_rate": 3.8691322901849213e-07, | |
| "logps/chosen": -263.36395263671875, | |
| "logps/rejected": -337.2552795410156, | |
| "loss": 0.1078, | |
| "rewards/chosen": 2.1132290363311768, | |
| "rewards/margins": 11.090356826782227, | |
| "rewards/rejected": -8.977127075195312, | |
| "step": 475 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "grad_norm": 18.355857849121094, | |
| "kl": 0.0, | |
| "learning_rate": 3.851351351351351e-07, | |
| "logps/chosen": -256.04412841796875, | |
| "logps/rejected": -356.82073974609375, | |
| "loss": 0.1145, | |
| "rewards/chosen": 1.9654195308685303, | |
| "rewards/margins": 11.159137725830078, | |
| "rewards/rejected": -9.193717956542969, | |
| "step": 480 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "grad_norm": 16.297401428222656, | |
| "kl": 0.0, | |
| "learning_rate": 3.833570412517781e-07, | |
| "logps/chosen": -330.457275390625, | |
| "logps/rejected": -367.7776794433594, | |
| "loss": 0.113, | |
| "rewards/chosen": 1.8794314861297607, | |
| "rewards/margins": 11.851155281066895, | |
| "rewards/rejected": -9.971722602844238, | |
| "step": 485 | |
| }, | |
| { | |
| "epoch": 0.31, | |
| "grad_norm": 22.856266021728516, | |
| "kl": 0.0, | |
| "learning_rate": 3.8157894736842105e-07, | |
| "logps/chosen": -254.3659210205078, | |
| "logps/rejected": -343.2192687988281, | |
| "loss": 0.1138, | |
| "rewards/chosen": 2.034006118774414, | |
| "rewards/margins": 11.903759002685547, | |
| "rewards/rejected": -9.869752883911133, | |
| "step": 490 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "grad_norm": 21.443723678588867, | |
| "kl": 0.0, | |
| "learning_rate": 3.7980085348506403e-07, | |
| "logps/chosen": -278.9993591308594, | |
| "logps/rejected": -333.3291320800781, | |
| "loss": 0.1287, | |
| "rewards/chosen": 1.906224012374878, | |
| "rewards/margins": 10.766765594482422, | |
| "rewards/rejected": -8.860541343688965, | |
| "step": 495 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "grad_norm": 24.97509002685547, | |
| "kl": 0.0, | |
| "learning_rate": 3.7802275960170695e-07, | |
| "logps/chosen": -238.44290161132812, | |
| "logps/rejected": -338.4499206542969, | |
| "loss": 0.1142, | |
| "rewards/chosen": 2.103523015975952, | |
| "rewards/margins": 11.162189483642578, | |
| "rewards/rejected": -9.058666229248047, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "grad_norm": 22.563114166259766, | |
| "kl": 0.0, | |
| "learning_rate": 3.7624466571834987e-07, | |
| "logps/chosen": -238.7085723876953, | |
| "logps/rejected": -352.23663330078125, | |
| "loss": 0.1396, | |
| "rewards/chosen": 1.8689464330673218, | |
| "rewards/margins": 10.945247650146484, | |
| "rewards/rejected": -9.076301574707031, | |
| "step": 505 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "grad_norm": 17.017030715942383, | |
| "kl": 0.0, | |
| "learning_rate": 3.7446657183499284e-07, | |
| "logps/chosen": -241.0718231201172, | |
| "logps/rejected": -340.98968505859375, | |
| "loss": 0.1197, | |
| "rewards/chosen": 1.9388542175292969, | |
| "rewards/margins": 11.241914749145508, | |
| "rewards/rejected": -9.303060531616211, | |
| "step": 510 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "grad_norm": 16.066030502319336, | |
| "kl": 0.0, | |
| "learning_rate": 3.726884779516358e-07, | |
| "logps/chosen": -224.85086059570312, | |
| "logps/rejected": -349.51800537109375, | |
| "loss": 0.1094, | |
| "rewards/chosen": 2.1027920246124268, | |
| "rewards/margins": 11.039840698242188, | |
| "rewards/rejected": -8.937047958374023, | |
| "step": 515 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "grad_norm": 17.983535766601562, | |
| "kl": 0.0, | |
| "learning_rate": 3.709103840682788e-07, | |
| "logps/chosen": -269.41693115234375, | |
| "logps/rejected": -347.45477294921875, | |
| "loss": 0.1234, | |
| "rewards/chosen": 2.2909629344940186, | |
| "rewards/margins": 11.061802864074707, | |
| "rewards/rejected": -8.770838737487793, | |
| "step": 520 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "grad_norm": 32.42300796508789, | |
| "kl": 0.0, | |
| "learning_rate": 3.6913229018492176e-07, | |
| "logps/chosen": -274.3470764160156, | |
| "logps/rejected": -335.10345458984375, | |
| "loss": 0.1186, | |
| "rewards/chosen": 2.1239845752716064, | |
| "rewards/margins": 11.3440523147583, | |
| "rewards/rejected": -9.220067977905273, | |
| "step": 525 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "grad_norm": 16.601402282714844, | |
| "kl": 0.0, | |
| "learning_rate": 3.6735419630156474e-07, | |
| "logps/chosen": -232.43392944335938, | |
| "logps/rejected": -354.2206726074219, | |
| "loss": 0.1218, | |
| "rewards/chosen": 2.117833137512207, | |
| "rewards/margins": 11.192573547363281, | |
| "rewards/rejected": -9.07474136352539, | |
| "step": 530 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "grad_norm": 24.04266357421875, | |
| "kl": 0.0, | |
| "learning_rate": 3.655761024182077e-07, | |
| "logps/chosen": -277.41241455078125, | |
| "logps/rejected": -334.04815673828125, | |
| "loss": 0.1025, | |
| "rewards/chosen": 2.3604273796081543, | |
| "rewards/margins": 11.987722396850586, | |
| "rewards/rejected": -9.627294540405273, | |
| "step": 535 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "grad_norm": 15.607158660888672, | |
| "kl": 0.0, | |
| "learning_rate": 3.637980085348506e-07, | |
| "logps/chosen": -286.7027282714844, | |
| "logps/rejected": -319.68896484375, | |
| "loss": 0.1124, | |
| "rewards/chosen": 2.4556756019592285, | |
| "rewards/margins": 11.582249641418457, | |
| "rewards/rejected": -9.126574516296387, | |
| "step": 540 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "grad_norm": 22.26024055480957, | |
| "kl": 0.0, | |
| "learning_rate": 3.6201991465149355e-07, | |
| "logps/chosen": -239.4503173828125, | |
| "logps/rejected": -345.2344970703125, | |
| "loss": 0.112, | |
| "rewards/chosen": 1.9915698766708374, | |
| "rewards/margins": 11.853796005249023, | |
| "rewards/rejected": -9.862226486206055, | |
| "step": 545 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "grad_norm": 29.981088638305664, | |
| "kl": 0.0, | |
| "learning_rate": 3.602418207681365e-07, | |
| "logps/chosen": -241.8583984375, | |
| "logps/rejected": -365.6163024902344, | |
| "loss": 0.1207, | |
| "rewards/chosen": 1.9029747247695923, | |
| "rewards/margins": 11.538789749145508, | |
| "rewards/rejected": -9.635814666748047, | |
| "step": 550 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "grad_norm": 17.248044967651367, | |
| "kl": 0.0, | |
| "learning_rate": 3.584637268847795e-07, | |
| "logps/chosen": -255.35635375976562, | |
| "logps/rejected": -348.6636962890625, | |
| "loss": 0.1267, | |
| "rewards/chosen": 2.0199761390686035, | |
| "rewards/margins": 11.458740234375, | |
| "rewards/rejected": -9.438763618469238, | |
| "step": 555 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "grad_norm": 20.46364974975586, | |
| "kl": 0.0, | |
| "learning_rate": 3.5668563300142247e-07, | |
| "logps/chosen": -271.98846435546875, | |
| "logps/rejected": -319.4902038574219, | |
| "loss": 0.1209, | |
| "rewards/chosen": 2.140854597091675, | |
| "rewards/margins": 11.506881713867188, | |
| "rewards/rejected": -9.36602783203125, | |
| "step": 560 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "grad_norm": 13.751622200012207, | |
| "kl": 0.0, | |
| "learning_rate": 3.5490753911806545e-07, | |
| "logps/chosen": -249.31838989257812, | |
| "logps/rejected": -363.65313720703125, | |
| "loss": 0.1066, | |
| "rewards/chosen": 2.302562713623047, | |
| "rewards/margins": 12.324169158935547, | |
| "rewards/rejected": -10.0216064453125, | |
| "step": 565 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "grad_norm": 23.35749053955078, | |
| "kl": 0.0, | |
| "learning_rate": 3.5312944523470837e-07, | |
| "logps/chosen": -242.62533569335938, | |
| "logps/rejected": -326.2989807128906, | |
| "loss": 0.1213, | |
| "rewards/chosen": 2.164386749267578, | |
| "rewards/margins": 11.125585556030273, | |
| "rewards/rejected": -8.961198806762695, | |
| "step": 570 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "grad_norm": 22.92085075378418, | |
| "kl": 0.0, | |
| "learning_rate": 3.5135135135135134e-07, | |
| "logps/chosen": -215.1080322265625, | |
| "logps/rejected": -343.92315673828125, | |
| "loss": 0.1096, | |
| "rewards/chosen": 2.084261417388916, | |
| "rewards/margins": 12.25261402130127, | |
| "rewards/rejected": -10.168352127075195, | |
| "step": 575 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "grad_norm": 19.450267791748047, | |
| "kl": 0.0, | |
| "learning_rate": 3.495732574679943e-07, | |
| "logps/chosen": -217.7822723388672, | |
| "logps/rejected": -329.91375732421875, | |
| "loss": 0.1294, | |
| "rewards/chosen": 2.3254265785217285, | |
| "rewards/margins": 11.483414649963379, | |
| "rewards/rejected": -9.157987594604492, | |
| "step": 580 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "grad_norm": 18.078920364379883, | |
| "kl": 0.0, | |
| "learning_rate": 3.4779516358463724e-07, | |
| "logps/chosen": -253.4072723388672, | |
| "logps/rejected": -344.491455078125, | |
| "loss": 0.0927, | |
| "rewards/chosen": 2.3925743103027344, | |
| "rewards/margins": 12.241676330566406, | |
| "rewards/rejected": -9.849101066589355, | |
| "step": 585 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "grad_norm": 17.867704391479492, | |
| "kl": 0.0, | |
| "learning_rate": 3.460170697012802e-07, | |
| "logps/chosen": -253.86178588867188, | |
| "logps/rejected": -367.8619689941406, | |
| "loss": 0.1152, | |
| "rewards/chosen": 2.281512975692749, | |
| "rewards/margins": 12.078498840332031, | |
| "rewards/rejected": -9.79698657989502, | |
| "step": 590 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "grad_norm": 19.863170623779297, | |
| "kl": 0.0, | |
| "learning_rate": 3.442389758179232e-07, | |
| "logps/chosen": -289.83990478515625, | |
| "logps/rejected": -346.80987548828125, | |
| "loss": 0.1098, | |
| "rewards/chosen": 2.1612110137939453, | |
| "rewards/margins": 12.505953788757324, | |
| "rewards/rejected": -10.344742774963379, | |
| "step": 595 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "grad_norm": 14.594950675964355, | |
| "kl": 0.0, | |
| "learning_rate": 3.424608819345661e-07, | |
| "logps/chosen": -199.16708374023438, | |
| "logps/rejected": -352.14447021484375, | |
| "loss": 0.0929, | |
| "rewards/chosen": 1.9940685033798218, | |
| "rewards/margins": 12.916508674621582, | |
| "rewards/rejected": -10.922439575195312, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "grad_norm": 20.205915451049805, | |
| "kl": 0.0, | |
| "learning_rate": 3.406827880512091e-07, | |
| "logps/chosen": -251.85546875, | |
| "logps/rejected": -347.35113525390625, | |
| "loss": 0.1137, | |
| "rewards/chosen": 2.3019704818725586, | |
| "rewards/margins": 12.349814414978027, | |
| "rewards/rejected": -10.047843933105469, | |
| "step": 605 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "grad_norm": 21.62274742126465, | |
| "kl": 0.0, | |
| "learning_rate": 3.3890469416785205e-07, | |
| "logps/chosen": -241.4209442138672, | |
| "logps/rejected": -366.311767578125, | |
| "loss": 0.104, | |
| "rewards/chosen": 2.3162879943847656, | |
| "rewards/margins": 11.884176254272461, | |
| "rewards/rejected": -9.567889213562012, | |
| "step": 610 | |
| }, | |
| { | |
| "epoch": 0.39, | |
| "grad_norm": 18.622377395629883, | |
| "kl": 0.0, | |
| "learning_rate": 3.37126600284495e-07, | |
| "logps/chosen": -270.2699279785156, | |
| "logps/rejected": -338.7982482910156, | |
| "loss": 0.1127, | |
| "rewards/chosen": 2.2165682315826416, | |
| "rewards/margins": 11.826861381530762, | |
| "rewards/rejected": -9.6102933883667, | |
| "step": 615 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "grad_norm": 19.930374145507812, | |
| "kl": 0.0, | |
| "learning_rate": 3.35348506401138e-07, | |
| "logps/chosen": -265.74432373046875, | |
| "logps/rejected": -366.4137268066406, | |
| "loss": 0.1235, | |
| "rewards/chosen": 2.140533924102783, | |
| "rewards/margins": 12.077839851379395, | |
| "rewards/rejected": -9.937305450439453, | |
| "step": 620 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "grad_norm": 20.78235626220703, | |
| "kl": 0.0, | |
| "learning_rate": 3.335704125177809e-07, | |
| "logps/chosen": -251.783935546875, | |
| "logps/rejected": -321.7081298828125, | |
| "loss": 0.1251, | |
| "rewards/chosen": 2.4168410301208496, | |
| "rewards/margins": 11.567142486572266, | |
| "rewards/rejected": -9.150300979614258, | |
| "step": 625 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "grad_norm": 18.487886428833008, | |
| "kl": 0.0, | |
| "learning_rate": 3.3179231863442384e-07, | |
| "logps/chosen": -244.60250854492188, | |
| "logps/rejected": -386.168212890625, | |
| "loss": 0.0922, | |
| "rewards/chosen": 2.371992588043213, | |
| "rewards/margins": 13.357122421264648, | |
| "rewards/rejected": -10.985128402709961, | |
| "step": 630 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "grad_norm": 16.354122161865234, | |
| "kl": 0.0, | |
| "learning_rate": 3.300142247510668e-07, | |
| "logps/chosen": -252.8701171875, | |
| "logps/rejected": -334.31744384765625, | |
| "loss": 0.1003, | |
| "rewards/chosen": 2.335937023162842, | |
| "rewards/margins": 12.5576753616333, | |
| "rewards/rejected": -10.2217378616333, | |
| "step": 635 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "grad_norm": 16.126724243164062, | |
| "kl": 0.0, | |
| "learning_rate": 3.282361308677098e-07, | |
| "logps/chosen": -240.2421417236328, | |
| "logps/rejected": -346.31414794921875, | |
| "loss": 0.1069, | |
| "rewards/chosen": 2.2422432899475098, | |
| "rewards/margins": 11.99687671661377, | |
| "rewards/rejected": -9.754633903503418, | |
| "step": 640 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "grad_norm": 21.066267013549805, | |
| "kl": 0.0, | |
| "learning_rate": 3.2645803698435276e-07, | |
| "logps/chosen": -264.81964111328125, | |
| "logps/rejected": -338.95379638671875, | |
| "loss": 0.1126, | |
| "rewards/chosen": 2.3289685249328613, | |
| "rewards/margins": 11.437704086303711, | |
| "rewards/rejected": -9.108736038208008, | |
| "step": 645 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "grad_norm": 14.892672538757324, | |
| "kl": 0.0, | |
| "learning_rate": 3.2467994310099573e-07, | |
| "logps/chosen": -240.17236328125, | |
| "logps/rejected": -355.543212890625, | |
| "loss": 0.0884, | |
| "rewards/chosen": 2.1047472953796387, | |
| "rewards/margins": 12.319875717163086, | |
| "rewards/rejected": -10.215127944946289, | |
| "step": 650 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "grad_norm": 22.722187042236328, | |
| "kl": 0.0, | |
| "learning_rate": 3.229018492176387e-07, | |
| "logps/chosen": -235.84341430664062, | |
| "logps/rejected": -363.74041748046875, | |
| "loss": 0.0958, | |
| "rewards/chosen": 2.262721300125122, | |
| "rewards/margins": 12.947868347167969, | |
| "rewards/rejected": -10.685147285461426, | |
| "step": 655 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "grad_norm": 16.061914443969727, | |
| "kl": 0.0, | |
| "learning_rate": 3.211237553342817e-07, | |
| "logps/chosen": -232.480224609375, | |
| "logps/rejected": -363.93792724609375, | |
| "loss": 0.0947, | |
| "rewards/chosen": 2.245720863342285, | |
| "rewards/margins": 13.019895553588867, | |
| "rewards/rejected": -10.774174690246582, | |
| "step": 660 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "grad_norm": 17.95708656311035, | |
| "kl": 0.0, | |
| "learning_rate": 3.193456614509246e-07, | |
| "logps/chosen": -266.2756652832031, | |
| "logps/rejected": -349.5681457519531, | |
| "loss": 0.0981, | |
| "rewards/chosen": 2.0552451610565186, | |
| "rewards/margins": 12.47476863861084, | |
| "rewards/rejected": -10.419523239135742, | |
| "step": 665 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "grad_norm": 22.32909393310547, | |
| "kl": 0.0, | |
| "learning_rate": 3.175675675675675e-07, | |
| "logps/chosen": -262.5782165527344, | |
| "logps/rejected": -350.5921936035156, | |
| "loss": 0.1155, | |
| "rewards/chosen": 2.324476718902588, | |
| "rewards/margins": 12.327436447143555, | |
| "rewards/rejected": -10.002958297729492, | |
| "step": 670 | |
| }, | |
| { | |
| "epoch": 0.43, | |
| "grad_norm": 14.884044647216797, | |
| "kl": 0.0, | |
| "learning_rate": 3.157894736842105e-07, | |
| "logps/chosen": -257.66094970703125, | |
| "logps/rejected": -344.09869384765625, | |
| "loss": 0.096, | |
| "rewards/chosen": 2.2759206295013428, | |
| "rewards/margins": 13.264117240905762, | |
| "rewards/rejected": -10.988197326660156, | |
| "step": 675 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "grad_norm": 18.041053771972656, | |
| "kl": 0.0, | |
| "learning_rate": 3.1401137980085347e-07, | |
| "logps/chosen": -270.93646240234375, | |
| "logps/rejected": -360.5249938964844, | |
| "loss": 0.0958, | |
| "rewards/chosen": 2.366556167602539, | |
| "rewards/margins": 13.1105318069458, | |
| "rewards/rejected": -10.743974685668945, | |
| "step": 680 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "grad_norm": 17.0706729888916, | |
| "kl": 0.0, | |
| "learning_rate": 3.1223328591749644e-07, | |
| "logps/chosen": -214.73690795898438, | |
| "logps/rejected": -342.7330017089844, | |
| "loss": 0.1107, | |
| "rewards/chosen": 2.152832508087158, | |
| "rewards/margins": 12.126806259155273, | |
| "rewards/rejected": -9.973973274230957, | |
| "step": 685 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "grad_norm": 25.26626968383789, | |
| "kl": 0.0, | |
| "learning_rate": 3.104551920341394e-07, | |
| "logps/chosen": -293.3183898925781, | |
| "logps/rejected": -356.9164123535156, | |
| "loss": 0.0973, | |
| "rewards/chosen": 2.3354926109313965, | |
| "rewards/margins": 13.199078559875488, | |
| "rewards/rejected": -10.863585472106934, | |
| "step": 690 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "grad_norm": 28.42453384399414, | |
| "kl": 0.0, | |
| "learning_rate": 3.0867709815078234e-07, | |
| "logps/chosen": -285.2272644042969, | |
| "logps/rejected": -367.3414001464844, | |
| "loss": 0.1091, | |
| "rewards/chosen": 2.12477707862854, | |
| "rewards/margins": 13.724832534790039, | |
| "rewards/rejected": -11.600054740905762, | |
| "step": 695 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "grad_norm": 17.899433135986328, | |
| "kl": 0.0, | |
| "learning_rate": 3.068990042674253e-07, | |
| "logps/chosen": -262.5505065917969, | |
| "logps/rejected": -371.45025634765625, | |
| "loss": 0.1083, | |
| "rewards/chosen": 2.060147762298584, | |
| "rewards/margins": 12.760570526123047, | |
| "rewards/rejected": -10.700422286987305, | |
| "step": 700 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "grad_norm": 19.646583557128906, | |
| "kl": 0.0, | |
| "learning_rate": 3.051209103840683e-07, | |
| "logps/chosen": -274.9856262207031, | |
| "logps/rejected": -360.3489685058594, | |
| "loss": 0.1105, | |
| "rewards/chosen": 2.4847919940948486, | |
| "rewards/margins": 13.168169975280762, | |
| "rewards/rejected": -10.683378219604492, | |
| "step": 705 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "grad_norm": 14.362700462341309, | |
| "kl": 0.0, | |
| "learning_rate": 3.033428165007112e-07, | |
| "logps/chosen": -253.5523681640625, | |
| "logps/rejected": -336.803466796875, | |
| "loss": 0.1024, | |
| "rewards/chosen": 2.3810598850250244, | |
| "rewards/margins": 12.508206367492676, | |
| "rewards/rejected": -10.127145767211914, | |
| "step": 710 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "grad_norm": 22.93763542175293, | |
| "kl": 0.0, | |
| "learning_rate": 3.015647226173542e-07, | |
| "logps/chosen": -229.1265869140625, | |
| "logps/rejected": -354.0566101074219, | |
| "loss": 0.1135, | |
| "rewards/chosen": 2.1574866771698, | |
| "rewards/margins": 12.193730354309082, | |
| "rewards/rejected": -10.036243438720703, | |
| "step": 715 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "grad_norm": 22.314695358276367, | |
| "kl": 0.0, | |
| "learning_rate": 2.9978662873399715e-07, | |
| "logps/chosen": -274.2774353027344, | |
| "logps/rejected": -405.2040100097656, | |
| "loss": 0.0953, | |
| "rewards/chosen": 2.191157817840576, | |
| "rewards/margins": 13.628583908081055, | |
| "rewards/rejected": -11.43742561340332, | |
| "step": 720 | |
| }, | |
| { | |
| "epoch": 0.46, | |
| "grad_norm": 20.03668975830078, | |
| "kl": 0.0, | |
| "learning_rate": 2.9800853485064007e-07, | |
| "logps/chosen": -250.0018768310547, | |
| "logps/rejected": -340.919677734375, | |
| "loss": 0.1022, | |
| "rewards/chosen": 2.4375367164611816, | |
| "rewards/margins": 12.8548583984375, | |
| "rewards/rejected": -10.417322158813477, | |
| "step": 725 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "grad_norm": 20.21332359313965, | |
| "kl": 0.0, | |
| "learning_rate": 2.9623044096728305e-07, | |
| "logps/chosen": -262.5707092285156, | |
| "logps/rejected": -341.51556396484375, | |
| "loss": 0.113, | |
| "rewards/chosen": 2.2380969524383545, | |
| "rewards/margins": 12.677546501159668, | |
| "rewards/rejected": -10.439449310302734, | |
| "step": 730 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "grad_norm": 18.686241149902344, | |
| "kl": 0.0, | |
| "learning_rate": 2.94452347083926e-07, | |
| "logps/chosen": -259.5292053222656, | |
| "logps/rejected": -340.3607482910156, | |
| "loss": 0.1188, | |
| "rewards/chosen": 2.1912286281585693, | |
| "rewards/margins": 12.15953540802002, | |
| "rewards/rejected": -9.968307495117188, | |
| "step": 735 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "grad_norm": 19.728899002075195, | |
| "kl": 0.0, | |
| "learning_rate": 2.92674253200569e-07, | |
| "logps/chosen": -249.8451385498047, | |
| "logps/rejected": -381.3360290527344, | |
| "loss": 0.0967, | |
| "rewards/chosen": 2.2508890628814697, | |
| "rewards/margins": 13.840009689331055, | |
| "rewards/rejected": -11.589120864868164, | |
| "step": 740 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "grad_norm": 17.338115692138672, | |
| "kl": 0.0, | |
| "learning_rate": 2.9089615931721197e-07, | |
| "logps/chosen": -268.9515380859375, | |
| "logps/rejected": -368.5078125, | |
| "loss": 0.1015, | |
| "rewards/chosen": 2.28879976272583, | |
| "rewards/margins": 13.396432876586914, | |
| "rewards/rejected": -11.107633590698242, | |
| "step": 745 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "grad_norm": 19.87563133239746, | |
| "kl": 0.0, | |
| "learning_rate": 2.8911806543385494e-07, | |
| "logps/chosen": -293.7141418457031, | |
| "logps/rejected": -355.55535888671875, | |
| "loss": 0.1071, | |
| "rewards/chosen": 2.4517719745635986, | |
| "rewards/margins": 13.449313163757324, | |
| "rewards/rejected": -10.997541427612305, | |
| "step": 750 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "grad_norm": 19.807565689086914, | |
| "kl": 0.0, | |
| "learning_rate": 2.873399715504978e-07, | |
| "logps/chosen": -248.6322784423828, | |
| "logps/rejected": -353.8228759765625, | |
| "loss": 0.1088, | |
| "rewards/chosen": 2.2985501289367676, | |
| "rewards/margins": 12.722993850708008, | |
| "rewards/rejected": -10.424444198608398, | |
| "step": 755 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "grad_norm": 14.098715782165527, | |
| "kl": 0.0, | |
| "learning_rate": 2.855618776671408e-07, | |
| "logps/chosen": -240.1453857421875, | |
| "logps/rejected": -330.6163635253906, | |
| "loss": 0.0891, | |
| "rewards/chosen": 2.550762414932251, | |
| "rewards/margins": 12.813929557800293, | |
| "rewards/rejected": -10.263166427612305, | |
| "step": 760 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "grad_norm": 16.266538619995117, | |
| "kl": 0.0, | |
| "learning_rate": 2.8378378378378376e-07, | |
| "logps/chosen": -236.1558380126953, | |
| "logps/rejected": -360.686767578125, | |
| "loss": 0.0973, | |
| "rewards/chosen": 2.2286760807037354, | |
| "rewards/margins": 13.117490768432617, | |
| "rewards/rejected": -10.888814926147461, | |
| "step": 765 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "grad_norm": 16.414714813232422, | |
| "kl": 0.0, | |
| "learning_rate": 2.8200568990042673e-07, | |
| "logps/chosen": -283.36944580078125, | |
| "logps/rejected": -368.7194519042969, | |
| "loss": 0.0967, | |
| "rewards/chosen": 2.4076812267303467, | |
| "rewards/margins": 12.956690788269043, | |
| "rewards/rejected": -10.549009323120117, | |
| "step": 770 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "grad_norm": 18.088735580444336, | |
| "kl": 0.0, | |
| "learning_rate": 2.802275960170697e-07, | |
| "logps/chosen": -302.5070495605469, | |
| "logps/rejected": -348.53289794921875, | |
| "loss": 0.1047, | |
| "rewards/chosen": 2.4432990550994873, | |
| "rewards/margins": 12.935566902160645, | |
| "rewards/rejected": -10.492268562316895, | |
| "step": 775 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "grad_norm": 16.778833389282227, | |
| "kl": 0.0, | |
| "learning_rate": 2.784495021337127e-07, | |
| "logps/chosen": -243.6858367919922, | |
| "logps/rejected": -366.0279235839844, | |
| "loss": 0.0852, | |
| "rewards/chosen": 2.5511088371276855, | |
| "rewards/margins": 13.161443710327148, | |
| "rewards/rejected": -10.610334396362305, | |
| "step": 780 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "grad_norm": 17.379335403442383, | |
| "kl": 0.0, | |
| "learning_rate": 2.766714082503556e-07, | |
| "logps/chosen": -279.8948669433594, | |
| "logps/rejected": -352.0287170410156, | |
| "loss": 0.0957, | |
| "rewards/chosen": 2.464888334274292, | |
| "rewards/margins": 13.103485107421875, | |
| "rewards/rejected": -10.638595581054688, | |
| "step": 785 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "grad_norm": 21.737380981445312, | |
| "kl": 0.0, | |
| "learning_rate": 2.7489331436699857e-07, | |
| "logps/chosen": -258.93865966796875, | |
| "logps/rejected": -345.5055236816406, | |
| "loss": 0.1177, | |
| "rewards/chosen": 2.079983711242676, | |
| "rewards/margins": 12.90088176727295, | |
| "rewards/rejected": -10.820898056030273, | |
| "step": 790 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "grad_norm": 15.227828025817871, | |
| "kl": 0.0, | |
| "learning_rate": 2.7311522048364154e-07, | |
| "logps/chosen": -233.96194458007812, | |
| "logps/rejected": -370.0096130371094, | |
| "loss": 0.1054, | |
| "rewards/chosen": 2.696063280105591, | |
| "rewards/margins": 13.426877975463867, | |
| "rewards/rejected": -10.730813980102539, | |
| "step": 795 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "grad_norm": 13.938480377197266, | |
| "kl": 0.0, | |
| "learning_rate": 2.7133712660028446e-07, | |
| "logps/chosen": -243.730224609375, | |
| "logps/rejected": -350.44183349609375, | |
| "loss": 0.0976, | |
| "rewards/chosen": 2.3282103538513184, | |
| "rewards/margins": 13.082319259643555, | |
| "rewards/rejected": -10.754108428955078, | |
| "step": 800 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "grad_norm": 29.015178680419922, | |
| "kl": 0.0, | |
| "learning_rate": 2.6955903271692744e-07, | |
| "logps/chosen": -216.66488647460938, | |
| "logps/rejected": -368.9279479980469, | |
| "loss": 0.1045, | |
| "rewards/chosen": 2.2219786643981934, | |
| "rewards/margins": 13.255975723266602, | |
| "rewards/rejected": -11.033994674682617, | |
| "step": 805 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "grad_norm": 12.759780883789062, | |
| "kl": 0.0, | |
| "learning_rate": 2.677809388335704e-07, | |
| "logps/chosen": -232.0436248779297, | |
| "logps/rejected": -358.2416076660156, | |
| "loss": 0.0943, | |
| "rewards/chosen": 2.235215425491333, | |
| "rewards/margins": 12.901985168457031, | |
| "rewards/rejected": -10.666769027709961, | |
| "step": 810 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "grad_norm": 19.0715389251709, | |
| "kl": 0.0, | |
| "learning_rate": 2.6600284495021333e-07, | |
| "logps/chosen": -245.0729217529297, | |
| "logps/rejected": -379.9324645996094, | |
| "loss": 0.1173, | |
| "rewards/chosen": 2.423140287399292, | |
| "rewards/margins": 13.03075885772705, | |
| "rewards/rejected": -10.60761833190918, | |
| "step": 815 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "grad_norm": 15.53042984008789, | |
| "kl": 0.0, | |
| "learning_rate": 2.642247510668563e-07, | |
| "logps/chosen": -261.9339294433594, | |
| "logps/rejected": -327.780517578125, | |
| "loss": 0.0903, | |
| "rewards/chosen": 2.4564433097839355, | |
| "rewards/margins": 13.401847839355469, | |
| "rewards/rejected": -10.945404052734375, | |
| "step": 820 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "grad_norm": 26.67053985595703, | |
| "kl": 0.0, | |
| "learning_rate": 2.624466571834993e-07, | |
| "logps/chosen": -248.34487915039062, | |
| "logps/rejected": -345.79083251953125, | |
| "loss": 0.1071, | |
| "rewards/chosen": 2.359476089477539, | |
| "rewards/margins": 13.113537788391113, | |
| "rewards/rejected": -10.754061698913574, | |
| "step": 825 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "grad_norm": 22.24759292602539, | |
| "kl": 0.0, | |
| "learning_rate": 2.6066856330014225e-07, | |
| "logps/chosen": -262.4079895019531, | |
| "logps/rejected": -364.2686462402344, | |
| "loss": 0.1076, | |
| "rewards/chosen": 2.37235689163208, | |
| "rewards/margins": 13.127764701843262, | |
| "rewards/rejected": -10.75540828704834, | |
| "step": 830 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "grad_norm": 17.638994216918945, | |
| "kl": 0.0, | |
| "learning_rate": 2.5889046941678523e-07, | |
| "logps/chosen": -273.74432373046875, | |
| "logps/rejected": -383.0593566894531, | |
| "loss": 0.0984, | |
| "rewards/chosen": 2.275177478790283, | |
| "rewards/margins": 14.033803939819336, | |
| "rewards/rejected": -11.758626937866211, | |
| "step": 835 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "grad_norm": 15.510226249694824, | |
| "kl": 0.0, | |
| "learning_rate": 2.5711237553342815e-07, | |
| "logps/chosen": -268.3088073730469, | |
| "logps/rejected": -372.16876220703125, | |
| "loss": 0.075, | |
| "rewards/chosen": 2.7448737621307373, | |
| "rewards/margins": 14.005487442016602, | |
| "rewards/rejected": -11.260614395141602, | |
| "step": 840 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "grad_norm": 18.002506256103516, | |
| "kl": 0.0, | |
| "learning_rate": 2.5533428165007107e-07, | |
| "logps/chosen": -214.6236572265625, | |
| "logps/rejected": -379.697265625, | |
| "loss": 0.1156, | |
| "rewards/chosen": 2.090531349182129, | |
| "rewards/margins": 14.037800788879395, | |
| "rewards/rejected": -11.947270393371582, | |
| "step": 845 | |
| }, | |
| { | |
| "epoch": 0.54, | |
| "grad_norm": 18.8648681640625, | |
| "kl": 0.0, | |
| "learning_rate": 2.5355618776671404e-07, | |
| "logps/chosen": -230.46810913085938, | |
| "logps/rejected": -345.8827209472656, | |
| "loss": 0.1028, | |
| "rewards/chosen": 2.3623595237731934, | |
| "rewards/margins": 13.34724235534668, | |
| "rewards/rejected": -10.984883308410645, | |
| "step": 850 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "grad_norm": 15.514137268066406, | |
| "kl": 0.0, | |
| "learning_rate": 2.51778093883357e-07, | |
| "logps/chosen": -224.1614227294922, | |
| "logps/rejected": -365.08477783203125, | |
| "loss": 0.0985, | |
| "rewards/chosen": 2.2946102619171143, | |
| "rewards/margins": 14.019018173217773, | |
| "rewards/rejected": -11.724408149719238, | |
| "step": 855 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "grad_norm": 17.679288864135742, | |
| "kl": 0.0, | |
| "learning_rate": 2.5e-07, | |
| "logps/chosen": -231.95565795898438, | |
| "logps/rejected": -349.056640625, | |
| "loss": 0.0966, | |
| "rewards/chosen": 2.3342556953430176, | |
| "rewards/margins": 13.233372688293457, | |
| "rewards/rejected": -10.899115562438965, | |
| "step": 860 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "grad_norm": 19.04916000366211, | |
| "kl": 0.0, | |
| "learning_rate": 2.4822190611664296e-07, | |
| "logps/chosen": -233.5281982421875, | |
| "logps/rejected": -374.4831848144531, | |
| "loss": 0.0922, | |
| "rewards/chosen": 2.58280873298645, | |
| "rewards/margins": 13.661170959472656, | |
| "rewards/rejected": -11.078360557556152, | |
| "step": 865 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "grad_norm": 16.834497451782227, | |
| "kl": 0.0, | |
| "learning_rate": 2.4644381223328594e-07, | |
| "logps/chosen": -236.75106811523438, | |
| "logps/rejected": -353.3365478515625, | |
| "loss": 0.096, | |
| "rewards/chosen": 2.7401375770568848, | |
| "rewards/margins": 13.776086807250977, | |
| "rewards/rejected": -11.035948753356934, | |
| "step": 870 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "grad_norm": 24.789772033691406, | |
| "kl": 0.0, | |
| "learning_rate": 2.4466571834992886e-07, | |
| "logps/chosen": -267.3653869628906, | |
| "logps/rejected": -370.1556091308594, | |
| "loss": 0.0949, | |
| "rewards/chosen": 2.2669379711151123, | |
| "rewards/margins": 13.626495361328125, | |
| "rewards/rejected": -11.35955810546875, | |
| "step": 875 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "grad_norm": 12.911144256591797, | |
| "kl": 0.0, | |
| "learning_rate": 2.4288762446657183e-07, | |
| "logps/chosen": -250.23556518554688, | |
| "logps/rejected": -363.0490417480469, | |
| "loss": 0.0717, | |
| "rewards/chosen": 2.6297707557678223, | |
| "rewards/margins": 15.319549560546875, | |
| "rewards/rejected": -12.689778327941895, | |
| "step": 880 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "grad_norm": 18.159881591796875, | |
| "kl": 0.0, | |
| "learning_rate": 2.411095305832148e-07, | |
| "logps/chosen": -276.918701171875, | |
| "logps/rejected": -351.54132080078125, | |
| "loss": 0.1062, | |
| "rewards/chosen": 2.5637714862823486, | |
| "rewards/margins": 13.2720365524292, | |
| "rewards/rejected": -10.708267211914062, | |
| "step": 885 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "grad_norm": 16.24278450012207, | |
| "kl": 0.0, | |
| "learning_rate": 2.393314366998578e-07, | |
| "logps/chosen": -264.356689453125, | |
| "logps/rejected": -345.72064208984375, | |
| "loss": 0.1061, | |
| "rewards/chosen": 2.57415509223938, | |
| "rewards/margins": 13.096124649047852, | |
| "rewards/rejected": -10.521968841552734, | |
| "step": 890 | |
| }, | |
| { | |
| "epoch": 0.57, | |
| "grad_norm": 15.353395462036133, | |
| "kl": 0.0, | |
| "learning_rate": 2.375533428165007e-07, | |
| "logps/chosen": -245.558349609375, | |
| "logps/rejected": -346.11151123046875, | |
| "loss": 0.1098, | |
| "rewards/chosen": 2.0979220867156982, | |
| "rewards/margins": 13.061151504516602, | |
| "rewards/rejected": -10.963228225708008, | |
| "step": 895 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "grad_norm": 15.291166305541992, | |
| "kl": 0.0, | |
| "learning_rate": 2.3577524893314365e-07, | |
| "logps/chosen": -268.0149841308594, | |
| "logps/rejected": -359.99676513671875, | |
| "loss": 0.0935, | |
| "rewards/chosen": 2.660017251968384, | |
| "rewards/margins": 13.785112380981445, | |
| "rewards/rejected": -11.125094413757324, | |
| "step": 900 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "grad_norm": 27.338308334350586, | |
| "kl": 0.0, | |
| "learning_rate": 2.3399715504978662e-07, | |
| "logps/chosen": -223.98062133789062, | |
| "logps/rejected": -382.53924560546875, | |
| "loss": 0.0733, | |
| "rewards/chosen": 2.573464870452881, | |
| "rewards/margins": 14.070528030395508, | |
| "rewards/rejected": -11.497062683105469, | |
| "step": 905 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "grad_norm": 28.126638412475586, | |
| "kl": 0.0, | |
| "learning_rate": 2.322190611664296e-07, | |
| "logps/chosen": -225.90432739257812, | |
| "logps/rejected": -369.765380859375, | |
| "loss": 0.1068, | |
| "rewards/chosen": 2.4907584190368652, | |
| "rewards/margins": 13.410183906555176, | |
| "rewards/rejected": -10.919425964355469, | |
| "step": 910 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "grad_norm": 23.488555908203125, | |
| "kl": 0.0, | |
| "learning_rate": 2.304409672830725e-07, | |
| "logps/chosen": -264.053466796875, | |
| "logps/rejected": -347.4461364746094, | |
| "loss": 0.0978, | |
| "rewards/chosen": 2.3625271320343018, | |
| "rewards/margins": 13.296697616577148, | |
| "rewards/rejected": -10.934170722961426, | |
| "step": 915 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "grad_norm": 16.538496017456055, | |
| "kl": 0.0, | |
| "learning_rate": 2.2866287339971549e-07, | |
| "logps/chosen": -305.2149353027344, | |
| "logps/rejected": -387.3183898925781, | |
| "loss": 0.0895, | |
| "rewards/chosen": 2.51884126663208, | |
| "rewards/margins": 14.519546508789062, | |
| "rewards/rejected": -12.000704765319824, | |
| "step": 920 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "grad_norm": 14.570239067077637, | |
| "kl": 0.0, | |
| "learning_rate": 2.2688477951635846e-07, | |
| "logps/chosen": -288.55859375, | |
| "logps/rejected": -360.0328063964844, | |
| "loss": 0.0891, | |
| "rewards/chosen": 2.6060962677001953, | |
| "rewards/margins": 13.63697338104248, | |
| "rewards/rejected": -11.030878067016602, | |
| "step": 925 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "grad_norm": 14.252922058105469, | |
| "kl": 0.0, | |
| "learning_rate": 2.251066856330014e-07, | |
| "logps/chosen": -237.12985229492188, | |
| "logps/rejected": -375.7847900390625, | |
| "loss": 0.0968, | |
| "rewards/chosen": 2.277179718017578, | |
| "rewards/margins": 13.618906021118164, | |
| "rewards/rejected": -11.341727256774902, | |
| "step": 930 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "grad_norm": 14.224340438842773, | |
| "kl": 0.0, | |
| "learning_rate": 2.2332859174964438e-07, | |
| "logps/chosen": -247.994384765625, | |
| "logps/rejected": -384.1309509277344, | |
| "loss": 0.0921, | |
| "rewards/chosen": 2.5065815448760986, | |
| "rewards/margins": 14.67822551727295, | |
| "rewards/rejected": -12.17164421081543, | |
| "step": 935 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "grad_norm": 15.533319473266602, | |
| "kl": 0.0, | |
| "learning_rate": 2.2155049786628733e-07, | |
| "logps/chosen": -217.72982788085938, | |
| "logps/rejected": -364.0068054199219, | |
| "loss": 0.1044, | |
| "rewards/chosen": 2.4342358112335205, | |
| "rewards/margins": 13.825854301452637, | |
| "rewards/rejected": -11.391618728637695, | |
| "step": 940 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "grad_norm": 12.951379776000977, | |
| "kl": 0.0, | |
| "learning_rate": 2.1977240398293027e-07, | |
| "logps/chosen": -216.9298858642578, | |
| "logps/rejected": -391.88916015625, | |
| "loss": 0.0959, | |
| "rewards/chosen": 2.2248167991638184, | |
| "rewards/margins": 14.502099990844727, | |
| "rewards/rejected": -12.27728271484375, | |
| "step": 945 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "grad_norm": 12.726017951965332, | |
| "kl": 0.0, | |
| "learning_rate": 2.1799431009957325e-07, | |
| "logps/chosen": -250.4014129638672, | |
| "logps/rejected": -367.2666931152344, | |
| "loss": 0.099, | |
| "rewards/chosen": 2.6326889991760254, | |
| "rewards/margins": 14.203544616699219, | |
| "rewards/rejected": -11.570856094360352, | |
| "step": 950 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "grad_norm": 24.116321563720703, | |
| "kl": 0.0, | |
| "learning_rate": 2.1621621621621622e-07, | |
| "logps/chosen": -256.0219421386719, | |
| "logps/rejected": -359.89410400390625, | |
| "loss": 0.1114, | |
| "rewards/chosen": 2.291504383087158, | |
| "rewards/margins": 13.204824447631836, | |
| "rewards/rejected": -10.913320541381836, | |
| "step": 955 | |
| }, | |
| { | |
| "epoch": 0.61, | |
| "grad_norm": 21.19695472717285, | |
| "kl": 0.0, | |
| "learning_rate": 2.1443812233285914e-07, | |
| "logps/chosen": -253.9010009765625, | |
| "logps/rejected": -348.592041015625, | |
| "loss": 0.0887, | |
| "rewards/chosen": 2.41105580329895, | |
| "rewards/margins": 13.58563232421875, | |
| "rewards/rejected": -11.174577713012695, | |
| "step": 960 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "grad_norm": 19.317626953125, | |
| "kl": 0.0, | |
| "learning_rate": 2.1266002844950212e-07, | |
| "logps/chosen": -245.3134765625, | |
| "logps/rejected": -380.17327880859375, | |
| "loss": 0.0998, | |
| "rewards/chosen": 2.4610061645507812, | |
| "rewards/margins": 13.953335762023926, | |
| "rewards/rejected": -11.492330551147461, | |
| "step": 965 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "grad_norm": 22.688295364379883, | |
| "kl": 0.0, | |
| "learning_rate": 2.108819345661451e-07, | |
| "logps/chosen": -278.2052917480469, | |
| "logps/rejected": -357.1881408691406, | |
| "loss": 0.1063, | |
| "rewards/chosen": 2.404897928237915, | |
| "rewards/margins": 13.51880931854248, | |
| "rewards/rejected": -11.113912582397461, | |
| "step": 970 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "grad_norm": 14.956331253051758, | |
| "kl": 0.0, | |
| "learning_rate": 2.0910384068278806e-07, | |
| "logps/chosen": -274.7865295410156, | |
| "logps/rejected": -348.69024658203125, | |
| "loss": 0.1178, | |
| "rewards/chosen": 2.560181140899658, | |
| "rewards/margins": 13.09942626953125, | |
| "rewards/rejected": -10.539244651794434, | |
| "step": 975 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "grad_norm": 14.130335807800293, | |
| "kl": 0.0, | |
| "learning_rate": 2.0732574679943098e-07, | |
| "logps/chosen": -241.42257690429688, | |
| "logps/rejected": -374.1187438964844, | |
| "loss": 0.0728, | |
| "rewards/chosen": 2.970536470413208, | |
| "rewards/margins": 14.709304809570312, | |
| "rewards/rejected": -11.738768577575684, | |
| "step": 980 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "grad_norm": 20.794334411621094, | |
| "kl": 0.0, | |
| "learning_rate": 2.0554765291607396e-07, | |
| "logps/chosen": -230.2430419921875, | |
| "logps/rejected": -380.54608154296875, | |
| "loss": 0.09, | |
| "rewards/chosen": 2.673881769180298, | |
| "rewards/margins": 14.4403076171875, | |
| "rewards/rejected": -11.766425132751465, | |
| "step": 985 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "grad_norm": 16.297340393066406, | |
| "kl": 0.0, | |
| "learning_rate": 2.0376955903271693e-07, | |
| "logps/chosen": -264.10137939453125, | |
| "logps/rejected": -372.11346435546875, | |
| "loss": 0.0896, | |
| "rewards/chosen": 2.7595813274383545, | |
| "rewards/margins": 13.790555953979492, | |
| "rewards/rejected": -11.030974388122559, | |
| "step": 990 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "grad_norm": 20.304162979125977, | |
| "kl": 0.0, | |
| "learning_rate": 2.0199146514935988e-07, | |
| "logps/chosen": -243.2149200439453, | |
| "logps/rejected": -342.080322265625, | |
| "loss": 0.1091, | |
| "rewards/chosen": 2.3006443977355957, | |
| "rewards/margins": 13.069559097290039, | |
| "rewards/rejected": -10.768914222717285, | |
| "step": 995 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "grad_norm": 19.80646514892578, | |
| "kl": 0.0, | |
| "learning_rate": 2.0021337126600283e-07, | |
| "logps/chosen": -251.78524780273438, | |
| "logps/rejected": -360.4002380371094, | |
| "loss": 0.088, | |
| "rewards/chosen": 2.7110652923583984, | |
| "rewards/margins": 13.96354866027832, | |
| "rewards/rejected": -11.252483367919922, | |
| "step": 1000 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "grad_norm": 19.04472541809082, | |
| "kl": 0.0, | |
| "learning_rate": 1.984352773826458e-07, | |
| "logps/chosen": -227.35873413085938, | |
| "logps/rejected": -363.24713134765625, | |
| "loss": 0.0952, | |
| "rewards/chosen": 2.7294869422912598, | |
| "rewards/margins": 14.287317276000977, | |
| "rewards/rejected": -11.557830810546875, | |
| "step": 1005 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "grad_norm": 14.0511474609375, | |
| "kl": 0.0, | |
| "learning_rate": 1.9665718349928875e-07, | |
| "logps/chosen": -248.2740478515625, | |
| "logps/rejected": -359.29534912109375, | |
| "loss": 0.0783, | |
| "rewards/chosen": 2.7295162677764893, | |
| "rewards/margins": 14.418438911437988, | |
| "rewards/rejected": -11.688921928405762, | |
| "step": 1010 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "grad_norm": 21.88315773010254, | |
| "kl": 0.0, | |
| "learning_rate": 1.9487908961593172e-07, | |
| "logps/chosen": -228.99447631835938, | |
| "logps/rejected": -387.5201721191406, | |
| "loss": 0.0771, | |
| "rewards/chosen": 2.3790504932403564, | |
| "rewards/margins": 13.84051513671875, | |
| "rewards/rejected": -11.461464881896973, | |
| "step": 1015 | |
| }, | |
| { | |
| "epoch": 0.65, | |
| "grad_norm": 15.932575225830078, | |
| "kl": 0.0, | |
| "learning_rate": 1.931009957325747e-07, | |
| "logps/chosen": -278.6766357421875, | |
| "logps/rejected": -372.94793701171875, | |
| "loss": 0.0826, | |
| "rewards/chosen": 2.363306760787964, | |
| "rewards/margins": 14.281936645507812, | |
| "rewards/rejected": -11.918628692626953, | |
| "step": 1020 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "grad_norm": 11.954800605773926, | |
| "kl": 0.0, | |
| "learning_rate": 1.9132290184921761e-07, | |
| "logps/chosen": -223.6442108154297, | |
| "logps/rejected": -371.5616149902344, | |
| "loss": 0.0961, | |
| "rewards/chosen": 2.332761526107788, | |
| "rewards/margins": 13.71965217590332, | |
| "rewards/rejected": -11.386890411376953, | |
| "step": 1025 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "grad_norm": 14.154836654663086, | |
| "kl": 0.0, | |
| "learning_rate": 1.895448079658606e-07, | |
| "logps/chosen": -223.6517791748047, | |
| "logps/rejected": -386.1749267578125, | |
| "loss": 0.0908, | |
| "rewards/chosen": 2.7226128578186035, | |
| "rewards/margins": 14.438260078430176, | |
| "rewards/rejected": -11.71564769744873, | |
| "step": 1030 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "grad_norm": 18.134370803833008, | |
| "kl": 0.0, | |
| "learning_rate": 1.8776671408250356e-07, | |
| "logps/chosen": -260.0550842285156, | |
| "logps/rejected": -346.24127197265625, | |
| "loss": 0.1007, | |
| "rewards/chosen": 2.5339343547821045, | |
| "rewards/margins": 13.723528861999512, | |
| "rewards/rejected": -11.189595222473145, | |
| "step": 1035 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "grad_norm": 15.416353225708008, | |
| "kl": 0.0, | |
| "learning_rate": 1.859886201991465e-07, | |
| "logps/chosen": -247.7535400390625, | |
| "logps/rejected": -371.42034912109375, | |
| "loss": 0.0912, | |
| "rewards/chosen": 2.3352103233337402, | |
| "rewards/margins": 13.56823444366455, | |
| "rewards/rejected": -11.233022689819336, | |
| "step": 1040 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "grad_norm": 18.23054313659668, | |
| "kl": 0.0, | |
| "learning_rate": 1.8421052631578946e-07, | |
| "logps/chosen": -213.8344268798828, | |
| "logps/rejected": -369.88702392578125, | |
| "loss": 0.0758, | |
| "rewards/chosen": 2.6615304946899414, | |
| "rewards/margins": 14.028742790222168, | |
| "rewards/rejected": -11.367212295532227, | |
| "step": 1045 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "grad_norm": 18.42432975769043, | |
| "kl": 0.0, | |
| "learning_rate": 1.8243243243243243e-07, | |
| "logps/chosen": -238.5988006591797, | |
| "logps/rejected": -392.305419921875, | |
| "loss": 0.0965, | |
| "rewards/chosen": 2.5309016704559326, | |
| "rewards/margins": 14.56842041015625, | |
| "rewards/rejected": -12.037518501281738, | |
| "step": 1050 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "grad_norm": 15.395295143127441, | |
| "kl": 0.0, | |
| "learning_rate": 1.8065433854907538e-07, | |
| "logps/chosen": -242.9210968017578, | |
| "logps/rejected": -375.423828125, | |
| "loss": 0.0985, | |
| "rewards/chosen": 2.644166946411133, | |
| "rewards/margins": 13.905471801757812, | |
| "rewards/rejected": -11.26130485534668, | |
| "step": 1055 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "grad_norm": 16.343006134033203, | |
| "kl": 0.0, | |
| "learning_rate": 1.7887624466571835e-07, | |
| "logps/chosen": -198.0528106689453, | |
| "logps/rejected": -354.1863098144531, | |
| "loss": 0.092, | |
| "rewards/chosen": 2.4538371562957764, | |
| "rewards/margins": 12.678072929382324, | |
| "rewards/rejected": -10.224235534667969, | |
| "step": 1060 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "grad_norm": 19.673669815063477, | |
| "kl": 0.0, | |
| "learning_rate": 1.770981507823613e-07, | |
| "logps/chosen": -241.1757049560547, | |
| "logps/rejected": -339.1280822753906, | |
| "loss": 0.088, | |
| "rewards/chosen": 2.550891876220703, | |
| "rewards/margins": 13.831698417663574, | |
| "rewards/rejected": -11.280807495117188, | |
| "step": 1065 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "grad_norm": 15.624777793884277, | |
| "kl": 0.0, | |
| "learning_rate": 1.7532005689900424e-07, | |
| "logps/chosen": -247.8914031982422, | |
| "logps/rejected": -364.5996398925781, | |
| "loss": 0.1038, | |
| "rewards/chosen": 2.5451152324676514, | |
| "rewards/margins": 13.162847518920898, | |
| "rewards/rejected": -10.617732048034668, | |
| "step": 1070 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "grad_norm": 16.0270938873291, | |
| "kl": 0.0, | |
| "learning_rate": 1.7354196301564722e-07, | |
| "logps/chosen": -271.2415466308594, | |
| "logps/rejected": -348.6362609863281, | |
| "loss": 0.0832, | |
| "rewards/chosen": 2.6901183128356934, | |
| "rewards/margins": 13.692281723022461, | |
| "rewards/rejected": -11.002164840698242, | |
| "step": 1075 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "grad_norm": 16.198070526123047, | |
| "kl": 0.0, | |
| "learning_rate": 1.717638691322902e-07, | |
| "logps/chosen": -228.4770965576172, | |
| "logps/rejected": -371.384521484375, | |
| "loss": 0.0879, | |
| "rewards/chosen": 2.317864179611206, | |
| "rewards/margins": 14.147501945495605, | |
| "rewards/rejected": -11.82963752746582, | |
| "step": 1080 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "grad_norm": 24.463367462158203, | |
| "kl": 0.0, | |
| "learning_rate": 1.6998577524893314e-07, | |
| "logps/chosen": -271.0051574707031, | |
| "logps/rejected": -403.8445129394531, | |
| "loss": 0.0817, | |
| "rewards/chosen": 2.82243275642395, | |
| "rewards/margins": 14.68195629119873, | |
| "rewards/rejected": -11.859524726867676, | |
| "step": 1085 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "grad_norm": 21.81173324584961, | |
| "kl": 0.0, | |
| "learning_rate": 1.6820768136557609e-07, | |
| "logps/chosen": -209.64572143554688, | |
| "logps/rejected": -360.5612487792969, | |
| "loss": 0.0849, | |
| "rewards/chosen": 2.4915404319763184, | |
| "rewards/margins": 13.439886093139648, | |
| "rewards/rejected": -10.948348045349121, | |
| "step": 1090 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "grad_norm": 20.325061798095703, | |
| "kl": 0.0, | |
| "learning_rate": 1.6642958748221906e-07, | |
| "logps/chosen": -244.2580108642578, | |
| "logps/rejected": -348.1650390625, | |
| "loss": 0.0804, | |
| "rewards/chosen": 2.744297742843628, | |
| "rewards/margins": 13.572134017944336, | |
| "rewards/rejected": -10.827836990356445, | |
| "step": 1095 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "grad_norm": 24.380229949951172, | |
| "kl": 0.0, | |
| "learning_rate": 1.64651493598862e-07, | |
| "logps/chosen": -253.1858673095703, | |
| "logps/rejected": -348.04144287109375, | |
| "loss": 0.0846, | |
| "rewards/chosen": 2.656752824783325, | |
| "rewards/margins": 13.392297744750977, | |
| "rewards/rejected": -10.735544204711914, | |
| "step": 1100 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "grad_norm": 12.965998649597168, | |
| "kl": 0.0, | |
| "learning_rate": 1.6287339971550498e-07, | |
| "logps/chosen": -262.8606262207031, | |
| "logps/rejected": -356.92242431640625, | |
| "loss": 0.0783, | |
| "rewards/chosen": 2.8926401138305664, | |
| "rewards/margins": 14.1354341506958, | |
| "rewards/rejected": -11.242793083190918, | |
| "step": 1105 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "grad_norm": 16.383338928222656, | |
| "kl": 0.0, | |
| "learning_rate": 1.6109530583214793e-07, | |
| "logps/chosen": -300.94384765625, | |
| "logps/rejected": -358.71588134765625, | |
| "loss": 0.0951, | |
| "rewards/chosen": 2.539355993270874, | |
| "rewards/margins": 13.336338996887207, | |
| "rewards/rejected": -10.79698371887207, | |
| "step": 1110 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "grad_norm": 15.756513595581055, | |
| "kl": 0.0, | |
| "learning_rate": 1.5931721194879087e-07, | |
| "logps/chosen": -261.84637451171875, | |
| "logps/rejected": -374.3198547363281, | |
| "loss": 0.0791, | |
| "rewards/chosen": 2.755915880203247, | |
| "rewards/margins": 14.211552619934082, | |
| "rewards/rejected": -11.455635070800781, | |
| "step": 1115 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "grad_norm": 18.087120056152344, | |
| "kl": 0.0, | |
| "learning_rate": 1.5753911806543385e-07, | |
| "logps/chosen": -274.73370361328125, | |
| "logps/rejected": -326.76123046875, | |
| "loss": 0.0935, | |
| "rewards/chosen": 2.786874294281006, | |
| "rewards/margins": 12.87488079071045, | |
| "rewards/rejected": -10.088006973266602, | |
| "step": 1120 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "grad_norm": 15.65069580078125, | |
| "kl": 0.0, | |
| "learning_rate": 1.5576102418207682e-07, | |
| "logps/chosen": -235.80062866210938, | |
| "logps/rejected": -340.6771545410156, | |
| "loss": 0.1005, | |
| "rewards/chosen": 2.691729784011841, | |
| "rewards/margins": 13.200857162475586, | |
| "rewards/rejected": -10.509127616882324, | |
| "step": 1125 | |
| }, | |
| { | |
| "epoch": 0.72, | |
| "grad_norm": 25.325843811035156, | |
| "kl": 0.0, | |
| "learning_rate": 1.5398293029871974e-07, | |
| "logps/chosen": -244.3973388671875, | |
| "logps/rejected": -343.04412841796875, | |
| "loss": 0.0883, | |
| "rewards/chosen": 2.485719680786133, | |
| "rewards/margins": 13.161664962768555, | |
| "rewards/rejected": -10.675946235656738, | |
| "step": 1130 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "grad_norm": 17.044322967529297, | |
| "kl": 0.0, | |
| "learning_rate": 1.5220483641536272e-07, | |
| "logps/chosen": -244.70559692382812, | |
| "logps/rejected": -358.86920166015625, | |
| "loss": 0.0987, | |
| "rewards/chosen": 2.5949511528015137, | |
| "rewards/margins": 13.485272407531738, | |
| "rewards/rejected": -10.890320777893066, | |
| "step": 1135 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "grad_norm": 27.13024139404297, | |
| "kl": 0.0, | |
| "learning_rate": 1.504267425320057e-07, | |
| "logps/chosen": -247.91641235351562, | |
| "logps/rejected": -343.4930419921875, | |
| "loss": 0.0899, | |
| "rewards/chosen": 2.4175989627838135, | |
| "rewards/margins": 13.832735061645508, | |
| "rewards/rejected": -11.415135383605957, | |
| "step": 1140 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "grad_norm": 14.320551872253418, | |
| "kl": 0.0, | |
| "learning_rate": 1.4864864864864866e-07, | |
| "logps/chosen": -246.96670532226562, | |
| "logps/rejected": -370.0877990722656, | |
| "loss": 0.0758, | |
| "rewards/chosen": 2.691598653793335, | |
| "rewards/margins": 13.812780380249023, | |
| "rewards/rejected": -11.121182441711426, | |
| "step": 1145 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "grad_norm": 16.582712173461914, | |
| "kl": 0.0, | |
| "learning_rate": 1.4687055476529158e-07, | |
| "logps/chosen": -224.07785034179688, | |
| "logps/rejected": -349.31927490234375, | |
| "loss": 0.082, | |
| "rewards/chosen": 2.6934258937835693, | |
| "rewards/margins": 13.236696243286133, | |
| "rewards/rejected": -10.543269157409668, | |
| "step": 1150 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "grad_norm": 17.989919662475586, | |
| "kl": 0.0, | |
| "learning_rate": 1.4509246088193456e-07, | |
| "logps/chosen": -243.8923797607422, | |
| "logps/rejected": -367.2593688964844, | |
| "loss": 0.0846, | |
| "rewards/chosen": 2.7338271141052246, | |
| "rewards/margins": 14.203822135925293, | |
| "rewards/rejected": -11.469995498657227, | |
| "step": 1155 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "grad_norm": 17.582975387573242, | |
| "kl": 0.0, | |
| "learning_rate": 1.4331436699857753e-07, | |
| "logps/chosen": -227.1583709716797, | |
| "logps/rejected": -378.5196228027344, | |
| "loss": 0.1011, | |
| "rewards/chosen": 2.531038284301758, | |
| "rewards/margins": 13.578967094421387, | |
| "rewards/rejected": -11.047929763793945, | |
| "step": 1160 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "grad_norm": 8.821680068969727, | |
| "kl": 0.0, | |
| "learning_rate": 1.4153627311522048e-07, | |
| "logps/chosen": -277.82684326171875, | |
| "logps/rejected": -364.6512145996094, | |
| "loss": 0.0742, | |
| "rewards/chosen": 2.9555764198303223, | |
| "rewards/margins": 14.14686393737793, | |
| "rewards/rejected": -11.19128704071045, | |
| "step": 1165 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "grad_norm": 15.817394256591797, | |
| "kl": 0.0, | |
| "learning_rate": 1.3975817923186345e-07, | |
| "logps/chosen": -271.718017578125, | |
| "logps/rejected": -352.4165954589844, | |
| "loss": 0.0848, | |
| "rewards/chosen": 2.5184245109558105, | |
| "rewards/margins": 13.12096881866455, | |
| "rewards/rejected": -10.602544784545898, | |
| "step": 1170 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "grad_norm": 18.2441463470459, | |
| "kl": 0.0, | |
| "learning_rate": 1.379800853485064e-07, | |
| "logps/chosen": -232.19467163085938, | |
| "logps/rejected": -379.3485412597656, | |
| "loss": 0.0909, | |
| "rewards/chosen": 2.5335030555725098, | |
| "rewards/margins": 14.489652633666992, | |
| "rewards/rejected": -11.956149101257324, | |
| "step": 1175 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "grad_norm": 30.594867706298828, | |
| "kl": 0.0, | |
| "learning_rate": 1.3620199146514935e-07, | |
| "logps/chosen": -265.6886291503906, | |
| "logps/rejected": -361.285400390625, | |
| "loss": 0.0964, | |
| "rewards/chosen": 2.450854778289795, | |
| "rewards/margins": 13.442533493041992, | |
| "rewards/rejected": -10.991681098937988, | |
| "step": 1180 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "grad_norm": 12.528332710266113, | |
| "kl": 0.0, | |
| "learning_rate": 1.3442389758179232e-07, | |
| "logps/chosen": -288.8167724609375, | |
| "logps/rejected": -352.4346618652344, | |
| "loss": 0.0741, | |
| "rewards/chosen": 2.7889742851257324, | |
| "rewards/margins": 14.034700393676758, | |
| "rewards/rejected": -11.245725631713867, | |
| "step": 1185 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "grad_norm": 13.565973281860352, | |
| "kl": 0.0, | |
| "learning_rate": 1.326458036984353e-07, | |
| "logps/chosen": -263.8364562988281, | |
| "logps/rejected": -346.26019287109375, | |
| "loss": 0.0843, | |
| "rewards/chosen": 2.638777256011963, | |
| "rewards/margins": 13.351663589477539, | |
| "rewards/rejected": -10.712886810302734, | |
| "step": 1190 | |
| }, | |
| { | |
| "epoch": 0.76, | |
| "grad_norm": 21.124759674072266, | |
| "kl": 0.0, | |
| "learning_rate": 1.3086770981507821e-07, | |
| "logps/chosen": -245.15576171875, | |
| "logps/rejected": -372.0450134277344, | |
| "loss": 0.0898, | |
| "rewards/chosen": 2.657459259033203, | |
| "rewards/margins": 14.276753425598145, | |
| "rewards/rejected": -11.619293212890625, | |
| "step": 1195 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "grad_norm": 15.396940231323242, | |
| "kl": 0.0, | |
| "learning_rate": 1.290896159317212e-07, | |
| "logps/chosen": -247.9556121826172, | |
| "logps/rejected": -371.3470764160156, | |
| "loss": 0.0828, | |
| "rewards/chosen": 2.4249672889709473, | |
| "rewards/margins": 14.25316333770752, | |
| "rewards/rejected": -11.828195571899414, | |
| "step": 1200 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "grad_norm": 14.409557342529297, | |
| "kl": 0.0, | |
| "learning_rate": 1.2731152204836416e-07, | |
| "logps/chosen": -252.07241821289062, | |
| "logps/rejected": -361.1800842285156, | |
| "loss": 0.085, | |
| "rewards/chosen": 2.680985689163208, | |
| "rewards/margins": 14.30175495147705, | |
| "rewards/rejected": -11.620769500732422, | |
| "step": 1205 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "grad_norm": 13.202485084533691, | |
| "kl": 0.0, | |
| "learning_rate": 1.255334281650071e-07, | |
| "logps/chosen": -237.88565063476562, | |
| "logps/rejected": -350.7021179199219, | |
| "loss": 0.0874, | |
| "rewards/chosen": 2.6545567512512207, | |
| "rewards/margins": 13.95142936706543, | |
| "rewards/rejected": -11.29687213897705, | |
| "step": 1210 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "grad_norm": 28.25654411315918, | |
| "kl": 0.0, | |
| "learning_rate": 1.2375533428165005e-07, | |
| "logps/chosen": -282.26031494140625, | |
| "logps/rejected": -377.5342712402344, | |
| "loss": 0.0771, | |
| "rewards/chosen": 2.6503818035125732, | |
| "rewards/margins": 14.911274909973145, | |
| "rewards/rejected": -12.260892868041992, | |
| "step": 1215 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "grad_norm": 20.115049362182617, | |
| "kl": 0.0, | |
| "learning_rate": 1.2197724039829303e-07, | |
| "logps/chosen": -234.886962890625, | |
| "logps/rejected": -347.6160583496094, | |
| "loss": 0.0879, | |
| "rewards/chosen": 2.644916296005249, | |
| "rewards/margins": 14.264989852905273, | |
| "rewards/rejected": -11.620074272155762, | |
| "step": 1220 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "grad_norm": 17.797800064086914, | |
| "kl": 0.0, | |
| "learning_rate": 1.2019914651493598e-07, | |
| "logps/chosen": -246.7455291748047, | |
| "logps/rejected": -392.910400390625, | |
| "loss": 0.0791, | |
| "rewards/chosen": 2.706545352935791, | |
| "rewards/margins": 14.843210220336914, | |
| "rewards/rejected": -12.136663436889648, | |
| "step": 1225 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "grad_norm": 12.662943840026855, | |
| "kl": 0.0, | |
| "learning_rate": 1.1842105263157894e-07, | |
| "logps/chosen": -225.7235870361328, | |
| "logps/rejected": -402.0716247558594, | |
| "loss": 0.0925, | |
| "rewards/chosen": 2.4518020153045654, | |
| "rewards/margins": 14.207118034362793, | |
| "rewards/rejected": -11.755315780639648, | |
| "step": 1230 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "grad_norm": 16.056779861450195, | |
| "kl": 0.0, | |
| "learning_rate": 1.166429587482219e-07, | |
| "logps/chosen": -215.66708374023438, | |
| "logps/rejected": -375.41717529296875, | |
| "loss": 0.0844, | |
| "rewards/chosen": 2.730950355529785, | |
| "rewards/margins": 14.23332691192627, | |
| "rewards/rejected": -11.502375602722168, | |
| "step": 1235 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "grad_norm": 16.644628524780273, | |
| "kl": 0.0, | |
| "learning_rate": 1.1486486486486487e-07, | |
| "logps/chosen": -266.43878173828125, | |
| "logps/rejected": -358.8226318359375, | |
| "loss": 0.0835, | |
| "rewards/chosen": 2.870779514312744, | |
| "rewards/margins": 14.435707092285156, | |
| "rewards/rejected": -11.56492805480957, | |
| "step": 1240 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "grad_norm": 8.560157775878906, | |
| "kl": 0.0, | |
| "learning_rate": 1.1308677098150782e-07, | |
| "logps/chosen": -238.55319213867188, | |
| "logps/rejected": -390.98358154296875, | |
| "loss": 0.0804, | |
| "rewards/chosen": 2.759446620941162, | |
| "rewards/margins": 14.843562126159668, | |
| "rewards/rejected": -12.084115028381348, | |
| "step": 1245 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "grad_norm": 16.91299819946289, | |
| "kl": 0.0, | |
| "learning_rate": 1.1130867709815078e-07, | |
| "logps/chosen": -234.5216064453125, | |
| "logps/rejected": -338.272705078125, | |
| "loss": 0.093, | |
| "rewards/chosen": 2.591742992401123, | |
| "rewards/margins": 14.04884147644043, | |
| "rewards/rejected": -11.457098007202148, | |
| "step": 1250 | |
| }, | |
| { | |
| "epoch": 0.8, | |
| "grad_norm": 13.1578369140625, | |
| "kl": 0.0, | |
| "learning_rate": 1.0953058321479374e-07, | |
| "logps/chosen": -275.4759826660156, | |
| "logps/rejected": -379.9882507324219, | |
| "loss": 0.0692, | |
| "rewards/chosen": 2.984293222427368, | |
| "rewards/margins": 15.574743270874023, | |
| "rewards/rejected": -12.59045124053955, | |
| "step": 1255 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "grad_norm": 17.815027236938477, | |
| "kl": 0.0, | |
| "learning_rate": 1.077524893314367e-07, | |
| "logps/chosen": -267.3926696777344, | |
| "logps/rejected": -353.8580017089844, | |
| "loss": 0.0725, | |
| "rewards/chosen": 2.5867929458618164, | |
| "rewards/margins": 14.201835632324219, | |
| "rewards/rejected": -11.615041732788086, | |
| "step": 1260 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "grad_norm": 11.758225440979004, | |
| "kl": 0.0, | |
| "learning_rate": 1.0597439544807964e-07, | |
| "logps/chosen": -249.6155242919922, | |
| "logps/rejected": -364.9371337890625, | |
| "loss": 0.0769, | |
| "rewards/chosen": 2.7065796852111816, | |
| "rewards/margins": 14.05157470703125, | |
| "rewards/rejected": -11.344995498657227, | |
| "step": 1265 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "grad_norm": 16.8466739654541, | |
| "kl": 0.0, | |
| "learning_rate": 1.0419630156472262e-07, | |
| "logps/chosen": -238.05673217773438, | |
| "logps/rejected": -373.03546142578125, | |
| "loss": 0.0748, | |
| "rewards/chosen": 2.9012789726257324, | |
| "rewards/margins": 14.46104621887207, | |
| "rewards/rejected": -11.55976676940918, | |
| "step": 1270 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "grad_norm": 14.22836685180664, | |
| "kl": 0.0, | |
| "learning_rate": 1.0241820768136557e-07, | |
| "logps/chosen": -224.74887084960938, | |
| "logps/rejected": -332.44415283203125, | |
| "loss": 0.084, | |
| "rewards/chosen": 2.65814208984375, | |
| "rewards/margins": 13.293182373046875, | |
| "rewards/rejected": -10.635040283203125, | |
| "step": 1275 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "grad_norm": 11.591891288757324, | |
| "kl": 0.0, | |
| "learning_rate": 1.0064011379800854e-07, | |
| "logps/chosen": -280.44427490234375, | |
| "logps/rejected": -365.83392333984375, | |
| "loss": 0.0801, | |
| "rewards/chosen": 2.5924744606018066, | |
| "rewards/margins": 14.31025505065918, | |
| "rewards/rejected": -11.717779159545898, | |
| "step": 1280 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "grad_norm": 14.504942893981934, | |
| "kl": 0.0, | |
| "learning_rate": 9.886201991465149e-08, | |
| "logps/chosen": -277.6451416015625, | |
| "logps/rejected": -350.4718322753906, | |
| "loss": 0.0661, | |
| "rewards/chosen": 2.9364330768585205, | |
| "rewards/margins": 14.681096076965332, | |
| "rewards/rejected": -11.744662284851074, | |
| "step": 1285 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "grad_norm": 16.065649032592773, | |
| "kl": 0.0, | |
| "learning_rate": 9.708392603129445e-08, | |
| "logps/chosen": -262.8613586425781, | |
| "logps/rejected": -391.71649169921875, | |
| "loss": 0.0723, | |
| "rewards/chosen": 2.7925260066986084, | |
| "rewards/margins": 15.433810234069824, | |
| "rewards/rejected": -12.641283988952637, | |
| "step": 1290 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "grad_norm": 21.038543701171875, | |
| "kl": 0.0, | |
| "learning_rate": 9.530583214793741e-08, | |
| "logps/chosen": -262.80584716796875, | |
| "logps/rejected": -380.09625244140625, | |
| "loss": 0.0801, | |
| "rewards/chosen": 2.761460781097412, | |
| "rewards/margins": 15.412538528442383, | |
| "rewards/rejected": -12.651077270507812, | |
| "step": 1295 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "grad_norm": 18.872102737426758, | |
| "kl": 0.0, | |
| "learning_rate": 9.352773826458037e-08, | |
| "logps/chosen": -285.86712646484375, | |
| "logps/rejected": -342.8421325683594, | |
| "loss": 0.0819, | |
| "rewards/chosen": 2.6993701457977295, | |
| "rewards/margins": 14.52336311340332, | |
| "rewards/rejected": -11.823991775512695, | |
| "step": 1300 | |
| }, | |
| { | |
| "epoch": 0.83, | |
| "grad_norm": 15.029848098754883, | |
| "kl": 0.0, | |
| "learning_rate": 9.174964438122331e-08, | |
| "logps/chosen": -245.3412322998047, | |
| "logps/rejected": -356.58172607421875, | |
| "loss": 0.094, | |
| "rewards/chosen": 2.818472385406494, | |
| "rewards/margins": 14.51873779296875, | |
| "rewards/rejected": -11.700265884399414, | |
| "step": 1305 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "grad_norm": 17.082958221435547, | |
| "kl": 0.0, | |
| "learning_rate": 8.997155049786629e-08, | |
| "logps/chosen": -286.61383056640625, | |
| "logps/rejected": -395.6304626464844, | |
| "loss": 0.0716, | |
| "rewards/chosen": 2.8008289337158203, | |
| "rewards/margins": 15.597787857055664, | |
| "rewards/rejected": -12.796960830688477, | |
| "step": 1310 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "grad_norm": 22.768911361694336, | |
| "kl": 0.0, | |
| "learning_rate": 8.819345661450925e-08, | |
| "logps/chosen": -250.6887664794922, | |
| "logps/rejected": -358.8474426269531, | |
| "loss": 0.0929, | |
| "rewards/chosen": 2.7452361583709717, | |
| "rewards/margins": 14.326945304870605, | |
| "rewards/rejected": -11.581708908081055, | |
| "step": 1315 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "grad_norm": 21.478580474853516, | |
| "kl": 0.0, | |
| "learning_rate": 8.64153627311522e-08, | |
| "logps/chosen": -267.79974365234375, | |
| "logps/rejected": -369.04815673828125, | |
| "loss": 0.0838, | |
| "rewards/chosen": 2.744828224182129, | |
| "rewards/margins": 14.584070205688477, | |
| "rewards/rejected": -11.839241027832031, | |
| "step": 1320 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "grad_norm": 12.477672576904297, | |
| "kl": 0.0, | |
| "learning_rate": 8.463726884779517e-08, | |
| "logps/chosen": -239.75399780273438, | |
| "logps/rejected": -383.63812255859375, | |
| "loss": 0.0833, | |
| "rewards/chosen": 2.6762642860412598, | |
| "rewards/margins": 14.272786140441895, | |
| "rewards/rejected": -11.596521377563477, | |
| "step": 1325 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "grad_norm": 19.415245056152344, | |
| "kl": 0.0, | |
| "learning_rate": 8.285917496443812e-08, | |
| "logps/chosen": -258.97406005859375, | |
| "logps/rejected": -372.96173095703125, | |
| "loss": 0.0874, | |
| "rewards/chosen": 2.7613649368286133, | |
| "rewards/margins": 14.08574390411377, | |
| "rewards/rejected": -11.324378967285156, | |
| "step": 1330 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "grad_norm": 13.463936805725098, | |
| "kl": 0.0, | |
| "learning_rate": 8.108108108108108e-08, | |
| "logps/chosen": -228.6225128173828, | |
| "logps/rejected": -344.24688720703125, | |
| "loss": 0.0777, | |
| "rewards/chosen": 2.942438840866089, | |
| "rewards/margins": 14.587237358093262, | |
| "rewards/rejected": -11.644798278808594, | |
| "step": 1335 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "grad_norm": 13.836956977844238, | |
| "kl": 0.0, | |
| "learning_rate": 7.930298719772404e-08, | |
| "logps/chosen": -247.45742797851562, | |
| "logps/rejected": -371.2969055175781, | |
| "loss": 0.0721, | |
| "rewards/chosen": 3.017321825027466, | |
| "rewards/margins": 14.967196464538574, | |
| "rewards/rejected": -11.949874877929688, | |
| "step": 1340 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "grad_norm": 16.551321029663086, | |
| "kl": 0.0, | |
| "learning_rate": 7.7524893314367e-08, | |
| "logps/chosen": -230.99227905273438, | |
| "logps/rejected": -384.6443786621094, | |
| "loss": 0.0871, | |
| "rewards/chosen": 2.6670827865600586, | |
| "rewards/margins": 14.841870307922363, | |
| "rewards/rejected": -12.174787521362305, | |
| "step": 1345 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "grad_norm": 15.87748908996582, | |
| "kl": 0.0, | |
| "learning_rate": 7.574679943100994e-08, | |
| "logps/chosen": -266.7972717285156, | |
| "logps/rejected": -335.5400390625, | |
| "loss": 0.0976, | |
| "rewards/chosen": 2.702658176422119, | |
| "rewards/margins": 13.561399459838867, | |
| "rewards/rejected": -10.858741760253906, | |
| "step": 1350 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "grad_norm": 17.662395477294922, | |
| "kl": 0.0, | |
| "learning_rate": 7.396870554765292e-08, | |
| "logps/chosen": -269.71026611328125, | |
| "logps/rejected": -383.00628662109375, | |
| "loss": 0.0921, | |
| "rewards/chosen": 2.5814826488494873, | |
| "rewards/margins": 14.34886646270752, | |
| "rewards/rejected": -11.76738452911377, | |
| "step": 1355 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "grad_norm": 16.09382438659668, | |
| "kl": 0.0, | |
| "learning_rate": 7.219061166429587e-08, | |
| "logps/chosen": -269.3471984863281, | |
| "logps/rejected": -375.32208251953125, | |
| "loss": 0.0886, | |
| "rewards/chosen": 2.8077282905578613, | |
| "rewards/margins": 14.510396003723145, | |
| "rewards/rejected": -11.702667236328125, | |
| "step": 1360 | |
| }, | |
| { | |
| "epoch": 0.87, | |
| "grad_norm": 13.27104377746582, | |
| "kl": 0.0, | |
| "learning_rate": 7.041251778093883e-08, | |
| "logps/chosen": -273.42950439453125, | |
| "logps/rejected": -371.14837646484375, | |
| "loss": 0.0685, | |
| "rewards/chosen": 2.944091558456421, | |
| "rewards/margins": 15.448992729187012, | |
| "rewards/rejected": -12.504900932312012, | |
| "step": 1365 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "grad_norm": 12.18476390838623, | |
| "kl": 0.0, | |
| "learning_rate": 6.863442389758179e-08, | |
| "logps/chosen": -256.47161865234375, | |
| "logps/rejected": -384.1866760253906, | |
| "loss": 0.0742, | |
| "rewards/chosen": 2.7861623764038086, | |
| "rewards/margins": 15.34550952911377, | |
| "rewards/rejected": -12.559347152709961, | |
| "step": 1370 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "grad_norm": 23.00941276550293, | |
| "kl": 0.0, | |
| "learning_rate": 6.685633001422475e-08, | |
| "logps/chosen": -242.3636474609375, | |
| "logps/rejected": -404.00848388671875, | |
| "loss": 0.0747, | |
| "rewards/chosen": 2.5074126720428467, | |
| "rewards/margins": 14.919418334960938, | |
| "rewards/rejected": -12.412006378173828, | |
| "step": 1375 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "grad_norm": 10.992572784423828, | |
| "kl": 0.0, | |
| "learning_rate": 6.507823613086771e-08, | |
| "logps/chosen": -248.75967407226562, | |
| "logps/rejected": -368.59063720703125, | |
| "loss": 0.0867, | |
| "rewards/chosen": 2.401717185974121, | |
| "rewards/margins": 14.726252555847168, | |
| "rewards/rejected": -12.32453441619873, | |
| "step": 1380 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "grad_norm": 11.035951614379883, | |
| "kl": 0.0, | |
| "learning_rate": 6.330014224751067e-08, | |
| "logps/chosen": -240.3590850830078, | |
| "logps/rejected": -347.6131896972656, | |
| "loss": 0.0921, | |
| "rewards/chosen": 2.680974245071411, | |
| "rewards/margins": 14.051698684692383, | |
| "rewards/rejected": -11.370722770690918, | |
| "step": 1385 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "grad_norm": 21.619415283203125, | |
| "kl": 0.0, | |
| "learning_rate": 6.152204836415363e-08, | |
| "logps/chosen": -240.7654266357422, | |
| "logps/rejected": -381.09307861328125, | |
| "loss": 0.087, | |
| "rewards/chosen": 2.646531105041504, | |
| "rewards/margins": 14.633687019348145, | |
| "rewards/rejected": -11.987154960632324, | |
| "step": 1390 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "grad_norm": 20.996952056884766, | |
| "kl": 0.0, | |
| "learning_rate": 5.974395448079659e-08, | |
| "logps/chosen": -228.12826538085938, | |
| "logps/rejected": -373.8050231933594, | |
| "loss": 0.0903, | |
| "rewards/chosen": 2.5932085514068604, | |
| "rewards/margins": 14.422918319702148, | |
| "rewards/rejected": -11.829710960388184, | |
| "step": 1395 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "grad_norm": 12.941699981689453, | |
| "kl": 0.0, | |
| "learning_rate": 5.796586059743954e-08, | |
| "logps/chosen": -266.34759521484375, | |
| "logps/rejected": -377.67572021484375, | |
| "loss": 0.0733, | |
| "rewards/chosen": 2.6969380378723145, | |
| "rewards/margins": 14.922724723815918, | |
| "rewards/rejected": -12.225786209106445, | |
| "step": 1400 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "grad_norm": 18.563539505004883, | |
| "kl": 0.0, | |
| "learning_rate": 5.61877667140825e-08, | |
| "logps/chosen": -233.05337524414062, | |
| "logps/rejected": -351.65478515625, | |
| "loss": 0.0819, | |
| "rewards/chosen": 2.4380977153778076, | |
| "rewards/margins": 14.38970947265625, | |
| "rewards/rejected": -11.951611518859863, | |
| "step": 1405 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "grad_norm": 14.77774715423584, | |
| "kl": 0.0, | |
| "learning_rate": 5.4409672830725456e-08, | |
| "logps/chosen": -272.6517028808594, | |
| "logps/rejected": -359.89013671875, | |
| "loss": 0.0762, | |
| "rewards/chosen": 2.888676404953003, | |
| "rewards/margins": 14.460576057434082, | |
| "rewards/rejected": -11.571900367736816, | |
| "step": 1410 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "grad_norm": 15.590229988098145, | |
| "kl": 0.0, | |
| "learning_rate": 5.2631578947368416e-08, | |
| "logps/chosen": -256.3451843261719, | |
| "logps/rejected": -375.3144226074219, | |
| "loss": 0.0762, | |
| "rewards/chosen": 2.667102336883545, | |
| "rewards/margins": 15.10230827331543, | |
| "rewards/rejected": -12.435206413269043, | |
| "step": 1415 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "grad_norm": 10.8274507522583, | |
| "kl": 0.0, | |
| "learning_rate": 5.0853485064011376e-08, | |
| "logps/chosen": -268.83636474609375, | |
| "logps/rejected": -372.27838134765625, | |
| "loss": 0.0648, | |
| "rewards/chosen": 2.9176650047302246, | |
| "rewards/margins": 14.840046882629395, | |
| "rewards/rejected": -11.922381401062012, | |
| "step": 1420 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "grad_norm": 15.774340629577637, | |
| "kl": 0.0, | |
| "learning_rate": 4.9075391180654337e-08, | |
| "logps/chosen": -260.6099853515625, | |
| "logps/rejected": -358.04241943359375, | |
| "loss": 0.0585, | |
| "rewards/chosen": 2.9233345985412598, | |
| "rewards/margins": 14.653867721557617, | |
| "rewards/rejected": -11.730535507202148, | |
| "step": 1425 | |
| }, | |
| { | |
| "epoch": 0.91, | |
| "grad_norm": 15.554561614990234, | |
| "kl": 0.0, | |
| "learning_rate": 4.72972972972973e-08, | |
| "logps/chosen": -238.23739624023438, | |
| "logps/rejected": -386.2373962402344, | |
| "loss": 0.0833, | |
| "rewards/chosen": 2.658268451690674, | |
| "rewards/margins": 14.15376091003418, | |
| "rewards/rejected": -11.49549388885498, | |
| "step": 1430 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "grad_norm": 17.29914093017578, | |
| "kl": 0.0, | |
| "learning_rate": 4.551920341394026e-08, | |
| "logps/chosen": -248.63204956054688, | |
| "logps/rejected": -371.215087890625, | |
| "loss": 0.0872, | |
| "rewards/chosen": 2.9267373085021973, | |
| "rewards/margins": 14.192481994628906, | |
| "rewards/rejected": -11.265745162963867, | |
| "step": 1435 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "grad_norm": 14.301885604858398, | |
| "kl": 0.0, | |
| "learning_rate": 4.374110953058322e-08, | |
| "logps/chosen": -240.0107421875, | |
| "logps/rejected": -355.2042541503906, | |
| "loss": 0.0846, | |
| "rewards/chosen": 2.4140007495880127, | |
| "rewards/margins": 13.864062309265137, | |
| "rewards/rejected": -11.450060844421387, | |
| "step": 1440 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "grad_norm": 13.32541275024414, | |
| "kl": 0.0, | |
| "learning_rate": 4.196301564722617e-08, | |
| "logps/chosen": -247.22152709960938, | |
| "logps/rejected": -360.07110595703125, | |
| "loss": 0.0659, | |
| "rewards/chosen": 2.7749335765838623, | |
| "rewards/margins": 14.867749214172363, | |
| "rewards/rejected": -12.092815399169922, | |
| "step": 1445 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "grad_norm": 12.356550216674805, | |
| "kl": 0.0, | |
| "learning_rate": 4.018492176386913e-08, | |
| "logps/chosen": -256.69683837890625, | |
| "logps/rejected": -363.08489990234375, | |
| "loss": 0.0799, | |
| "rewards/chosen": 2.8861594200134277, | |
| "rewards/margins": 14.505203247070312, | |
| "rewards/rejected": -11.619044303894043, | |
| "step": 1450 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "grad_norm": 14.969185829162598, | |
| "kl": 0.0, | |
| "learning_rate": 3.840682788051209e-08, | |
| "logps/chosen": -254.7665252685547, | |
| "logps/rejected": -354.30841064453125, | |
| "loss": 0.0757, | |
| "rewards/chosen": 2.8421378135681152, | |
| "rewards/margins": 14.373845100402832, | |
| "rewards/rejected": -11.531707763671875, | |
| "step": 1455 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "grad_norm": 17.030691146850586, | |
| "kl": 0.0, | |
| "learning_rate": 3.6628733997155046e-08, | |
| "logps/chosen": -263.8084411621094, | |
| "logps/rejected": -395.4825439453125, | |
| "loss": 0.0759, | |
| "rewards/chosen": 2.9121432304382324, | |
| "rewards/margins": 15.505640983581543, | |
| "rewards/rejected": -12.593496322631836, | |
| "step": 1460 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "grad_norm": 12.211559295654297, | |
| "kl": 0.0, | |
| "learning_rate": 3.4850640113798006e-08, | |
| "logps/chosen": -236.26431274414062, | |
| "logps/rejected": -357.1573791503906, | |
| "loss": 0.0755, | |
| "rewards/chosen": 2.843881130218506, | |
| "rewards/margins": 14.507527351379395, | |
| "rewards/rejected": -11.66364574432373, | |
| "step": 1465 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "grad_norm": 17.574806213378906, | |
| "kl": 0.0, | |
| "learning_rate": 3.3072546230440967e-08, | |
| "logps/chosen": -285.59942626953125, | |
| "logps/rejected": -368.08245849609375, | |
| "loss": 0.0676, | |
| "rewards/chosen": 2.877531051635742, | |
| "rewards/margins": 14.671765327453613, | |
| "rewards/rejected": -11.794233322143555, | |
| "step": 1470 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "grad_norm": 13.167614936828613, | |
| "kl": 0.0, | |
| "learning_rate": 3.129445234708392e-08, | |
| "logps/chosen": -240.7215576171875, | |
| "logps/rejected": -382.0995788574219, | |
| "loss": 0.0651, | |
| "rewards/chosen": 2.778978109359741, | |
| "rewards/margins": 14.833264350891113, | |
| "rewards/rejected": -12.054286003112793, | |
| "step": 1475 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "grad_norm": 19.462112426757812, | |
| "kl": 0.0, | |
| "learning_rate": 2.9516358463726884e-08, | |
| "logps/chosen": -243.0684356689453, | |
| "logps/rejected": -366.4331359863281, | |
| "loss": 0.0802, | |
| "rewards/chosen": 2.8256452083587646, | |
| "rewards/margins": 15.236946105957031, | |
| "rewards/rejected": -12.411300659179688, | |
| "step": 1480 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "grad_norm": 20.093833923339844, | |
| "kl": 0.0, | |
| "learning_rate": 2.7738264580369844e-08, | |
| "logps/chosen": -209.17770385742188, | |
| "logps/rejected": -359.9656677246094, | |
| "loss": 0.0839, | |
| "rewards/chosen": 2.901711940765381, | |
| "rewards/margins": 14.679159164428711, | |
| "rewards/rejected": -11.777449607849121, | |
| "step": 1485 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "grad_norm": 13.270203590393066, | |
| "kl": 0.0, | |
| "learning_rate": 2.59601706970128e-08, | |
| "logps/chosen": -272.44610595703125, | |
| "logps/rejected": -357.05975341796875, | |
| "loss": 0.0868, | |
| "rewards/chosen": 2.6972391605377197, | |
| "rewards/margins": 15.144004821777344, | |
| "rewards/rejected": -12.446764945983887, | |
| "step": 1490 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "grad_norm": 14.922286987304688, | |
| "kl": 0.0, | |
| "learning_rate": 2.418207681365576e-08, | |
| "logps/chosen": -238.18191528320312, | |
| "logps/rejected": -367.4979553222656, | |
| "loss": 0.0824, | |
| "rewards/chosen": 2.5710983276367188, | |
| "rewards/margins": 14.130340576171875, | |
| "rewards/rejected": -11.559242248535156, | |
| "step": 1495 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "grad_norm": 21.186229705810547, | |
| "kl": 0.0, | |
| "learning_rate": 2.240398293029872e-08, | |
| "logps/chosen": -240.6565399169922, | |
| "logps/rejected": -387.65936279296875, | |
| "loss": 0.0783, | |
| "rewards/chosen": 2.6365768909454346, | |
| "rewards/margins": 15.131543159484863, | |
| "rewards/rejected": -12.494966506958008, | |
| "step": 1500 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "grad_norm": 14.05397891998291, | |
| "kl": 0.0, | |
| "learning_rate": 2.0625889046941676e-08, | |
| "logps/chosen": -307.04583740234375, | |
| "logps/rejected": -361.1058654785156, | |
| "loss": 0.0903, | |
| "rewards/chosen": 2.804368257522583, | |
| "rewards/margins": 14.683168411254883, | |
| "rewards/rejected": -11.878799438476562, | |
| "step": 1505 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "grad_norm": 10.290091514587402, | |
| "kl": 0.0, | |
| "learning_rate": 1.8847795163584636e-08, | |
| "logps/chosen": -280.72406005859375, | |
| "logps/rejected": -393.7200622558594, | |
| "loss": 0.0643, | |
| "rewards/chosen": 2.76891827583313, | |
| "rewards/margins": 15.010998725891113, | |
| "rewards/rejected": -12.24207878112793, | |
| "step": 1510 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "grad_norm": 13.187542915344238, | |
| "kl": 0.0, | |
| "learning_rate": 1.7069701280227596e-08, | |
| "logps/chosen": -250.7726287841797, | |
| "logps/rejected": -378.99993896484375, | |
| "loss": 0.0697, | |
| "rewards/chosen": 2.8748562335968018, | |
| "rewards/margins": 15.209932327270508, | |
| "rewards/rejected": -12.335077285766602, | |
| "step": 1515 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "grad_norm": 13.64420223236084, | |
| "kl": 0.0, | |
| "learning_rate": 1.5291607396870554e-08, | |
| "logps/chosen": -253.7127227783203, | |
| "logps/rejected": -367.41558837890625, | |
| "loss": 0.0824, | |
| "rewards/chosen": 3.031802177429199, | |
| "rewards/margins": 14.762832641601562, | |
| "rewards/rejected": -11.73102855682373, | |
| "step": 1520 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "grad_norm": 10.866116523742676, | |
| "kl": 0.0, | |
| "learning_rate": 1.3513513513513514e-08, | |
| "logps/chosen": -250.94400024414062, | |
| "logps/rejected": -381.5174865722656, | |
| "loss": 0.0793, | |
| "rewards/chosen": 2.8337435722351074, | |
| "rewards/margins": 15.124834060668945, | |
| "rewards/rejected": -12.29109001159668, | |
| "step": 1525 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "grad_norm": 11.604872703552246, | |
| "kl": 0.0, | |
| "learning_rate": 1.1735419630156473e-08, | |
| "logps/chosen": -233.5682830810547, | |
| "logps/rejected": -377.261962890625, | |
| "loss": 0.0687, | |
| "rewards/chosen": 3.119915008544922, | |
| "rewards/margins": 15.134057998657227, | |
| "rewards/rejected": -12.014142990112305, | |
| "step": 1530 | |
| }, | |
| { | |
| "epoch": 0.98, | |
| "grad_norm": 20.006731033325195, | |
| "kl": 0.0, | |
| "learning_rate": 9.95732574679943e-09, | |
| "logps/chosen": -257.07525634765625, | |
| "logps/rejected": -346.5194396972656, | |
| "loss": 0.0907, | |
| "rewards/chosen": 2.676340103149414, | |
| "rewards/margins": 14.137725830078125, | |
| "rewards/rejected": -11.461384773254395, | |
| "step": 1535 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "grad_norm": 17.203874588012695, | |
| "kl": 0.0, | |
| "learning_rate": 8.179231863442388e-09, | |
| "logps/chosen": -226.61843872070312, | |
| "logps/rejected": -381.11029052734375, | |
| "loss": 0.0683, | |
| "rewards/chosen": 2.940342903137207, | |
| "rewards/margins": 15.23707103729248, | |
| "rewards/rejected": -12.296728134155273, | |
| "step": 1540 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "grad_norm": 14.477093696594238, | |
| "kl": 0.0, | |
| "learning_rate": 6.401137980085348e-09, | |
| "logps/chosen": -240.14157104492188, | |
| "logps/rejected": -388.9137268066406, | |
| "loss": 0.0658, | |
| "rewards/chosen": 2.875831127166748, | |
| "rewards/margins": 15.294588088989258, | |
| "rewards/rejected": -12.418756484985352, | |
| "step": 1545 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "grad_norm": 10.444808959960938, | |
| "kl": 0.0, | |
| "learning_rate": 4.623044096728307e-09, | |
| "logps/chosen": -273.6938781738281, | |
| "logps/rejected": -361.58782958984375, | |
| "loss": 0.0592, | |
| "rewards/chosen": 3.0903258323669434, | |
| "rewards/margins": 15.089597702026367, | |
| "rewards/rejected": -11.999273300170898, | |
| "step": 1550 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "grad_norm": 13.01125717163086, | |
| "kl": 0.0, | |
| "learning_rate": 2.844950213371266e-09, | |
| "logps/chosen": -263.2748107910156, | |
| "logps/rejected": -382.49481201171875, | |
| "loss": 0.0724, | |
| "rewards/chosen": 2.801457166671753, | |
| "rewards/margins": 14.89887809753418, | |
| "rewards/rejected": -12.097421646118164, | |
| "step": 1555 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "grad_norm": 13.397442817687988, | |
| "kl": 0.0, | |
| "learning_rate": 1.0668563300142248e-09, | |
| "logps/chosen": -225.07894897460938, | |
| "logps/rejected": -353.08856201171875, | |
| "loss": 0.0824, | |
| "rewards/chosen": 2.47590970993042, | |
| "rewards/margins": 14.8174409866333, | |
| "rewards/rejected": -12.341531753540039, | |
| "step": 1560 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "step": 1563, | |
| "total_flos": 0.0, | |
| "train_loss": 0.11715243643320149, | |
| "train_runtime": 11089.6574, | |
| "train_samples_per_second": 9.017, | |
| "train_steps_per_second": 0.141 | |
| } | |
| ], | |
| "logging_steps": 5, | |
| "max_steps": 1563, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 100, | |
| "total_flos": 0.0, | |
| "train_batch_size": 8, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |