{ "best_metric": null, "best_model_checkpoint": null, "epoch": 1.0, "eval_steps": 500, "global_step": 1875, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "epoch": 0.0, "learning_rate": 2.6595744680851065e-08, "logits/chosen": 0.4583740830421448, "logits/rejected": 0.45381295680999756, "logps/chosen": -403.16717529296875, "logps/rejected": -354.3865051269531, "loss": 0.1853, "rewards/accuracies": 0.0, "rewards/chosen": 0.0, "rewards/margins": 0.0, "rewards/rejected": 0.0, "step": 1 }, { "epoch": 0.01, "learning_rate": 2.6595744680851066e-07, "logits/chosen": 0.18826468288898468, "logits/rejected": 0.16825373470783234, "logps/chosen": -402.1011962890625, "logps/rejected": -396.2879638671875, "loss": 0.2347, "rewards/accuracies": 0.25, "rewards/chosen": -0.09647607803344727, "rewards/margins": -0.014048744924366474, "rewards/rejected": -0.08242733776569366, "step": 10 }, { "epoch": 0.01, "learning_rate": 5.319148936170213e-07, "logits/chosen": 0.16707709431648254, "logits/rejected": 0.2701728343963623, "logps/chosen": -453.8223571777344, "logps/rejected": -442.9689025878906, "loss": 0.2389, "rewards/accuracies": 0.3812499940395355, "rewards/chosen": -0.06698164343833923, "rewards/margins": -0.012189853005111217, "rewards/rejected": -0.05479179695248604, "step": 20 }, { "epoch": 0.02, "learning_rate": 7.97872340425532e-07, "logits/chosen": 0.17515605688095093, "logits/rejected": 0.2583461403846741, "logps/chosen": -361.522705078125, "logps/rejected": -338.58416748046875, "loss": 0.2243, "rewards/accuracies": 0.3375000059604645, "rewards/chosen": -0.07758971303701401, "rewards/margins": 0.0016961194342002273, "rewards/rejected": -0.07928583025932312, "step": 30 }, { "epoch": 0.02, "learning_rate": 1.0638297872340427e-06, "logits/chosen": 0.17781862616539001, "logits/rejected": 0.23920877277851105, "logps/chosen": -419.521728515625, "logps/rejected": -420.13299560546875, "loss": 0.216, "rewards/accuracies": 0.3499999940395355, "rewards/chosen": -0.14323017001152039, "rewards/margins": -0.003557709977030754, "rewards/rejected": -0.13967247307300568, "step": 40 }, { "epoch": 0.03, "learning_rate": 1.3297872340425533e-06, "logits/chosen": 0.20945556461811066, "logits/rejected": 0.301052451133728, "logps/chosen": -369.31622314453125, "logps/rejected": -369.2641906738281, "loss": 0.2338, "rewards/accuracies": 0.34375, "rewards/chosen": -0.10775469243526459, "rewards/margins": 0.00019715214148163795, "rewards/rejected": -0.10795185714960098, "step": 50 }, { "epoch": 0.03, "learning_rate": 1.595744680851064e-06, "logits/chosen": 0.21086159348487854, "logits/rejected": 0.2350298911333084, "logps/chosen": -401.9427490234375, "logps/rejected": -401.28765869140625, "loss": 0.2272, "rewards/accuracies": 0.3375000059604645, "rewards/chosen": -0.13143998384475708, "rewards/margins": -0.003936966881155968, "rewards/rejected": -0.12750300765037537, "step": 60 }, { "epoch": 0.04, "learning_rate": 1.8617021276595745e-06, "logits/chosen": 0.22450070083141327, "logits/rejected": 0.24035272002220154, "logps/chosen": -421.1205139160156, "logps/rejected": -449.4365234375, "loss": 0.2225, "rewards/accuracies": 0.4000000059604645, "rewards/chosen": -0.07039676606655121, "rewards/margins": 0.013278981670737267, "rewards/rejected": -0.08367574214935303, "step": 70 }, { "epoch": 0.04, "learning_rate": 2.1276595744680853e-06, "logits/chosen": 0.2308211624622345, "logits/rejected": 0.2585153579711914, "logps/chosen": -418.4386291503906, "logps/rejected": -397.7836608886719, "loss": 0.2146, "rewards/accuracies": 0.38749998807907104, "rewards/chosen": -0.13351131975650787, "rewards/margins": 0.0027887646574527025, "rewards/rejected": -0.13630008697509766, "step": 80 }, { "epoch": 0.05, "learning_rate": 2.393617021276596e-06, "logits/chosen": 0.17430104315280914, "logits/rejected": 0.24182358384132385, "logps/chosen": -393.2335510253906, "logps/rejected": -386.56982421875, "loss": 0.2165, "rewards/accuracies": 0.39375001192092896, "rewards/chosen": -0.12810206413269043, "rewards/margins": 0.02029189094901085, "rewards/rejected": -0.14839394390583038, "step": 90 }, { "epoch": 0.05, "learning_rate": 2.6595744680851065e-06, "logits/chosen": 0.2197931557893753, "logits/rejected": 0.213352769613266, "logps/chosen": -412.4981384277344, "logps/rejected": -393.7420349121094, "loss": 0.2092, "rewards/accuracies": 0.41874998807907104, "rewards/chosen": -0.13062937557697296, "rewards/margins": 0.031539879739284515, "rewards/rejected": -0.16216926276683807, "step": 100 }, { "epoch": 0.06, "learning_rate": 2.9255319148936174e-06, "logits/chosen": 0.1948339343070984, "logits/rejected": 0.3156844973564148, "logps/chosen": -412.1725158691406, "logps/rejected": -388.34942626953125, "loss": 0.1994, "rewards/accuracies": 0.42500001192092896, "rewards/chosen": -0.22077541053295135, "rewards/margins": 0.02942301705479622, "rewards/rejected": -0.2501984238624573, "step": 110 }, { "epoch": 0.06, "learning_rate": 3.191489361702128e-06, "logits/chosen": 0.15572600066661835, "logits/rejected": 0.27886322140693665, "logps/chosen": -405.5447998046875, "logps/rejected": -378.16302490234375, "loss": 0.2062, "rewards/accuracies": 0.45625001192092896, "rewards/chosen": -0.6035518646240234, "rewards/margins": 0.06773124635219574, "rewards/rejected": -0.6712831258773804, "step": 120 }, { "epoch": 0.07, "learning_rate": 3.457446808510639e-06, "logits/chosen": 0.21951737999916077, "logits/rejected": 0.21863070130348206, "logps/chosen": -439.3607482910156, "logps/rejected": -423.91033935546875, "loss": 0.2045, "rewards/accuracies": 0.5249999761581421, "rewards/chosen": -1.0598005056381226, "rewards/margins": 0.10918780416250229, "rewards/rejected": -1.1689883470535278, "step": 130 }, { "epoch": 0.07, "learning_rate": 3.723404255319149e-06, "logits/chosen": 0.12865397334098816, "logits/rejected": 0.22429411113262177, "logps/chosen": -425.0618591308594, "logps/rejected": -411.78656005859375, "loss": 0.2059, "rewards/accuracies": 0.518750011920929, "rewards/chosen": -1.486228346824646, "rewards/margins": 0.10093434154987335, "rewards/rejected": -1.5871626138687134, "step": 140 }, { "epoch": 0.08, "learning_rate": 3.98936170212766e-06, "logits/chosen": 0.1495121419429779, "logits/rejected": 0.2696373760700226, "logps/chosen": -455.36224365234375, "logps/rejected": -423.36993408203125, "loss": 0.2499, "rewards/accuracies": 0.44999998807907104, "rewards/chosen": -2.2363038063049316, "rewards/margins": 0.07195089757442474, "rewards/rejected": -2.3082547187805176, "step": 150 }, { "epoch": 0.09, "learning_rate": 4.255319148936171e-06, "logits/chosen": 0.17146244645118713, "logits/rejected": 0.33353060483932495, "logps/chosen": -477.47491455078125, "logps/rejected": -447.4056701660156, "loss": 0.2363, "rewards/accuracies": 0.46875, "rewards/chosen": -2.126039981842041, "rewards/margins": 0.09269236773252487, "rewards/rejected": -2.2187325954437256, "step": 160 }, { "epoch": 0.09, "learning_rate": 4.521276595744681e-06, "logits/chosen": 0.25921598076820374, "logits/rejected": 0.21789881587028503, "logps/chosen": -444.46038818359375, "logps/rejected": -439.04541015625, "loss": 0.2294, "rewards/accuracies": 0.45625001192092896, "rewards/chosen": -1.5025267601013184, "rewards/margins": 0.05006232112646103, "rewards/rejected": -1.5525890588760376, "step": 170 }, { "epoch": 0.1, "learning_rate": 4.787234042553192e-06, "logits/chosen": 0.16481132805347443, "logits/rejected": 0.3523365259170532, "logps/chosen": -417.71234130859375, "logps/rejected": -409.2852478027344, "loss": 0.2052, "rewards/accuracies": 0.48124998807907104, "rewards/chosen": -1.2100210189819336, "rewards/margins": 0.08468395471572876, "rewards/rejected": -1.2947049140930176, "step": 180 }, { "epoch": 0.1, "learning_rate": 4.999982660399688e-06, "logits/chosen": 0.21249070763587952, "logits/rejected": 0.1487680971622467, "logps/chosen": -381.04010009765625, "logps/rejected": -370.06121826171875, "loss": 0.2126, "rewards/accuracies": 0.44999998807907104, "rewards/chosen": -0.23919078707695007, "rewards/margins": 0.046520624309778214, "rewards/rejected": -0.285711407661438, "step": 190 }, { "epoch": 0.11, "learning_rate": 4.99937579964398e-06, "logits/chosen": 0.14420565962791443, "logits/rejected": 0.2520049512386322, "logps/chosen": -407.8086242675781, "logps/rejected": -404.40960693359375, "loss": 0.2139, "rewards/accuracies": 0.4749999940395355, "rewards/chosen": 0.31316059827804565, "rewards/margins": 0.07112900167703629, "rewards/rejected": 0.24203161895275116, "step": 200 }, { "epoch": 0.11, "learning_rate": 4.9979021993870645e-06, "logits/chosen": 0.20850297808647156, "logits/rejected": 0.15908761322498322, "logps/chosen": -393.0495300292969, "logps/rejected": -385.91131591796875, "loss": 0.1919, "rewards/accuracies": 0.4625000059604645, "rewards/chosen": 0.2354341298341751, "rewards/margins": 0.09232009202241898, "rewards/rejected": 0.14311401546001434, "step": 210 }, { "epoch": 0.12, "learning_rate": 4.995562370647553e-06, "logits/chosen": 0.18888719379901886, "logits/rejected": 0.2506316602230072, "logps/chosen": -400.5332946777344, "logps/rejected": -396.8497009277344, "loss": 0.213, "rewards/accuracies": 0.45625001192092896, "rewards/chosen": 0.3825472295284271, "rewards/margins": 0.1113465204834938, "rewards/rejected": 0.27120068669319153, "step": 220 }, { "epoch": 0.12, "learning_rate": 4.992357124836838e-06, "logits/chosen": 0.18391093611717224, "logits/rejected": 0.2248292863368988, "logps/chosen": -414.39495849609375, "logps/rejected": -410.48773193359375, "loss": 0.1955, "rewards/accuracies": 0.48750001192092896, "rewards/chosen": 0.43902724981307983, "rewards/margins": 0.07995086908340454, "rewards/rejected": 0.3590763509273529, "step": 230 }, { "epoch": 0.13, "learning_rate": 4.9882875734777044e-06, "logits/chosen": 0.17369148135185242, "logits/rejected": 0.2072429358959198, "logps/chosen": -379.5187683105469, "logps/rejected": -356.64935302734375, "loss": 0.2006, "rewards/accuracies": 0.4749999940395355, "rewards/chosen": 0.48646441102027893, "rewards/margins": 0.07142899930477142, "rewards/rejected": 0.4150354266166687, "step": 240 }, { "epoch": 0.13, "learning_rate": 4.983355127818882e-06, "logits/chosen": 0.22464045882225037, "logits/rejected": 0.21831247210502625, "logps/chosen": -360.67718505859375, "logps/rejected": -375.7092590332031, "loss": 0.2103, "rewards/accuracies": 0.42500001192092896, "rewards/chosen": 0.6559606790542603, "rewards/margins": 0.08051694184541702, "rewards/rejected": 0.5754436254501343, "step": 250 }, { "epoch": 0.14, "learning_rate": 4.977561498345639e-06, "logits/chosen": 0.27521076798439026, "logits/rejected": 0.2506524324417114, "logps/chosen": -416.90625, "logps/rejected": -403.52020263671875, "loss": 0.2265, "rewards/accuracies": 0.4000000059604645, "rewards/chosen": 0.5058245062828064, "rewards/margins": 0.06458650529384613, "rewards/rejected": 0.44123801589012146, "step": 260 }, { "epoch": 0.14, "learning_rate": 4.970908694186624e-06, "logits/chosen": 0.15694484114646912, "logits/rejected": 0.22934310138225555, "logps/chosen": -393.33367919921875, "logps/rejected": -378.1922912597656, "loss": 0.2514, "rewards/accuracies": 0.48124998807907104, "rewards/chosen": 0.3451576232910156, "rewards/margins": 0.09955648332834244, "rewards/rejected": 0.24560114741325378, "step": 270 }, { "epoch": 0.15, "learning_rate": 4.9633990224171305e-06, "logits/chosen": 0.22818629443645477, "logits/rejected": 0.3253491222858429, "logps/chosen": -439.2140197753906, "logps/rejected": -449.018798828125, "loss": 0.2199, "rewards/accuracies": 0.574999988079071, "rewards/chosen": 0.3162192404270172, "rewards/margins": 0.17989328503608704, "rewards/rejected": 0.13632595539093018, "step": 280 }, { "epoch": 0.15, "learning_rate": 4.955035087259046e-06, "logits/chosen": 0.13608984649181366, "logits/rejected": 0.22955039143562317, "logps/chosen": -382.9739074707031, "logps/rejected": -382.9454040527344, "loss": 0.2369, "rewards/accuracies": 0.4749999940395355, "rewards/chosen": -0.2025183141231537, "rewards/margins": 0.10606689751148224, "rewards/rejected": -0.3085852265357971, "step": 290 }, { "epoch": 0.16, "learning_rate": 4.945819789177756e-06, "logits/chosen": 0.15429742634296417, "logits/rejected": 0.22013196349143982, "logps/chosen": -417.97894287109375, "logps/rejected": -397.3367614746094, "loss": 0.222, "rewards/accuracies": 0.5, "rewards/chosen": -0.940466582775116, "rewards/margins": 0.11995697021484375, "rewards/rejected": -1.060423493385315, "step": 300 }, { "epoch": 0.17, "learning_rate": 4.935756323876306e-06, "logits/chosen": 0.19988210499286652, "logits/rejected": 0.19450345635414124, "logps/chosen": -406.30523681640625, "logps/rejected": -398.8627014160156, "loss": 0.2481, "rewards/accuracies": 0.48124998807907104, "rewards/chosen": -1.3156739473342896, "rewards/margins": 0.10400591045618057, "rewards/rejected": -1.4196797609329224, "step": 310 }, { "epoch": 0.17, "learning_rate": 4.924848181187199e-06, "logits/chosen": 0.2409110963344574, "logits/rejected": 0.19854632019996643, "logps/chosen": -412.65631103515625, "logps/rejected": -411.14892578125, "loss": 0.2431, "rewards/accuracies": 0.48124998807907104, "rewards/chosen": -1.3011319637298584, "rewards/margins": 0.14340858161449432, "rewards/rejected": -1.4445403814315796, "step": 320 }, { "epoch": 0.18, "learning_rate": 4.913099143862173e-06, "logits/chosen": 0.22796742618083954, "logits/rejected": 0.1370314657688141, "logps/chosen": -410.01593017578125, "logps/rejected": -423.0164489746094, "loss": 0.2662, "rewards/accuracies": 0.550000011920929, "rewards/chosen": -1.125431776046753, "rewards/margins": 0.15198691189289093, "rewards/rejected": -1.2774187326431274, "step": 330 }, { "epoch": 0.18, "learning_rate": 4.900513286260416e-06, "logits/chosen": 0.23252587020397186, "logits/rejected": 0.22327935695648193, "logps/chosen": -422.98388671875, "logps/rejected": -409.4496765136719, "loss": 0.4587, "rewards/accuracies": 0.4437499940395355, "rewards/chosen": -1.244390845298767, "rewards/margins": 0.12163744866847992, "rewards/rejected": -1.366028070449829, "step": 340 }, { "epoch": 0.19, "learning_rate": 4.887094972935645e-06, "logits/chosen": 0.19463476538658142, "logits/rejected": 0.26694804430007935, "logps/chosen": -458.14923095703125, "logps/rejected": -462.924560546875, "loss": 0.5137, "rewards/accuracies": 0.5, "rewards/chosen": -2.0200841426849365, "rewards/margins": 0.041815903037786484, "rewards/rejected": -2.0619003772735596, "step": 350 }, { "epoch": 0.19, "learning_rate": 4.87284885712256e-06, "logits/chosen": 0.18824467062950134, "logits/rejected": 0.28755128383636475, "logps/chosen": -443.279052734375, "logps/rejected": -453.8575134277344, "loss": 0.3629, "rewards/accuracies": 0.46875, "rewards/chosen": -2.572296619415283, "rewards/margins": 0.1011347621679306, "rewards/rejected": -2.673431396484375, "step": 360 }, { "epoch": 0.2, "learning_rate": 4.857779879123181e-06, "logits/chosen": 0.24380216002464294, "logits/rejected": 0.185393288731575, "logps/chosen": -400.14642333984375, "logps/rejected": -419.3863220214844, "loss": 0.3068, "rewards/accuracies": 0.4375, "rewards/chosen": -2.2855589389801025, "rewards/margins": 0.08926688134670258, "rewards/rejected": -2.3748257160186768, "step": 370 }, { "epoch": 0.2, "learning_rate": 4.841893264593643e-06, "logits/chosen": 0.15558484196662903, "logits/rejected": 0.2775138020515442, "logps/chosen": -436.8177795410156, "logps/rejected": -401.4188232421875, "loss": 0.2526, "rewards/accuracies": 0.4937500059604645, "rewards/chosen": -1.875754714012146, "rewards/margins": 0.08787569403648376, "rewards/rejected": -1.9636303186416626, "step": 380 }, { "epoch": 0.21, "learning_rate": 4.825194522732023e-06, "logits/chosen": 0.1963922530412674, "logits/rejected": 0.20690886676311493, "logps/chosen": -378.81109619140625, "logps/rejected": -389.9710998535156, "loss": 0.2406, "rewards/accuracies": 0.4937500059604645, "rewards/chosen": -1.7464733123779297, "rewards/margins": 0.09245175123214722, "rewards/rejected": -1.8389251232147217, "step": 390 }, { "epoch": 0.21, "learning_rate": 4.807689444367853e-06, "logits/chosen": 0.2619530260562897, "logits/rejected": 0.25581902265548706, "logps/chosen": -466.8785705566406, "logps/rejected": -460.354248046875, "loss": 0.3071, "rewards/accuracies": 0.4124999940395355, "rewards/chosen": -2.3323142528533936, "rewards/margins": 0.07366655021905899, "rewards/rejected": -2.4059810638427734, "step": 400 }, { "epoch": 0.22, "learning_rate": 4.78938409995396e-06, "logits/chosen": 0.1963699460029602, "logits/rejected": 0.32746168971061707, "logps/chosen": -433.58453369140625, "logps/rejected": -417.2007751464844, "loss": 0.2297, "rewards/accuracies": 0.518750011920929, "rewards/chosen": -2.2858786582946777, "rewards/margins": 0.13152393698692322, "rewards/rejected": -2.417402505874634, "step": 410 }, { "epoch": 0.22, "learning_rate": 4.770284837461342e-06, "logits/chosen": 0.2083076685667038, "logits/rejected": 0.3537079393863678, "logps/chosen": -425.3860778808594, "logps/rejected": -437.66357421875, "loss": 0.2828, "rewards/accuracies": 0.4312500059604645, "rewards/chosen": -1.9256458282470703, "rewards/margins": 0.05349540710449219, "rewards/rejected": -1.9791409969329834, "step": 420 }, { "epoch": 0.23, "learning_rate": 4.7503982801778015e-06, "logits/chosen": 0.26357799768447876, "logits/rejected": 0.24686399102210999, "logps/chosen": -413.22430419921875, "logps/rejected": -388.62860107421875, "loss": 0.2328, "rewards/accuracies": 0.4375, "rewards/chosen": -1.4066696166992188, "rewards/margins": 0.0737091675400734, "rewards/rejected": -1.4803787469863892, "step": 430 }, { "epoch": 0.23, "learning_rate": 4.729731324411104e-06, "logits/chosen": 0.20708036422729492, "logits/rejected": 0.25638309121131897, "logps/chosen": -433.58782958984375, "logps/rejected": -406.76568603515625, "loss": 0.2482, "rewards/accuracies": 0.4749999940395355, "rewards/chosen": -1.2260335683822632, "rewards/margins": 0.10625864565372467, "rewards/rejected": -1.3322923183441162, "step": 440 }, { "epoch": 0.24, "learning_rate": 4.7082911370974645e-06, "logits/chosen": 0.1832914650440216, "logits/rejected": 0.19458040595054626, "logps/chosen": -370.4656066894531, "logps/rejected": -370.5750732421875, "loss": 0.2305, "rewards/accuracies": 0.41874998807907104, "rewards/chosen": -1.1384856700897217, "rewards/margins": 0.10250405967235565, "rewards/rejected": -1.2409899234771729, "step": 450 }, { "epoch": 0.25, "learning_rate": 4.68608515331618e-06, "logits/chosen": 0.1749318540096283, "logits/rejected": 0.20707306265830994, "logps/chosen": -378.585693359375, "logps/rejected": -360.33709716796875, "loss": 0.2466, "rewards/accuracies": 0.4437499940395355, "rewards/chosen": -1.2511470317840576, "rewards/margins": 0.10291622579097748, "rewards/rejected": -1.3540633916854858, "step": 460 }, { "epoch": 0.25, "learning_rate": 4.663121073711269e-06, "logits/chosen": 0.2571523189544678, "logits/rejected": 0.18344727158546448, "logps/chosen": -420.41204833984375, "logps/rejected": -417.1241149902344, "loss": 0.2738, "rewards/accuracies": 0.39375001192092896, "rewards/chosen": -1.400221586227417, "rewards/margins": 0.04325573518872261, "rewards/rejected": -1.4434772729873657, "step": 470 }, { "epoch": 0.26, "learning_rate": 4.63940686182103e-06, "logits/chosen": 0.22877541184425354, "logits/rejected": 0.330875039100647, "logps/chosen": -389.0229797363281, "logps/rejected": -399.605712890625, "loss": 0.2411, "rewards/accuracies": 0.42500001192092896, "rewards/chosen": -1.2312822341918945, "rewards/margins": 0.11593781411647797, "rewards/rejected": -1.3472201824188232, "step": 480 }, { "epoch": 0.26, "learning_rate": 4.614950741316425e-06, "logits/chosen": 0.1676156222820282, "logits/rejected": 0.26664647459983826, "logps/chosen": -421.76019287109375, "logps/rejected": -394.2919921875, "loss": 0.2616, "rewards/accuracies": 0.4124999940395355, "rewards/chosen": -1.175879716873169, "rewards/margins": 0.085006944835186, "rewards/rejected": -1.260886788368225, "step": 490 }, { "epoch": 0.27, "learning_rate": 4.589761193149254e-06, "logits/chosen": 0.19543419778347015, "logits/rejected": 0.25549161434173584, "logps/chosen": -401.7788391113281, "logps/rejected": -381.381103515625, "loss": 0.2201, "rewards/accuracies": 0.4749999940395355, "rewards/chosen": -0.9598444104194641, "rewards/margins": 0.10931962728500366, "rewards/rejected": -1.0691639184951782, "step": 500 }, { "epoch": 0.27, "learning_rate": 4.563846952611112e-06, "logits/chosen": 0.19228951632976532, "logits/rejected": 0.24320659041404724, "logps/chosen": -398.7080993652344, "logps/rejected": -402.6463317871094, "loss": 0.292, "rewards/accuracies": 0.4437499940395355, "rewards/chosen": -0.6720465421676636, "rewards/margins": 0.05984077975153923, "rewards/rejected": -0.7318873405456543, "step": 510 }, { "epoch": 0.28, "learning_rate": 4.537217006304141e-06, "logits/chosen": 0.1390676200389862, "logits/rejected": 0.30597618222236633, "logps/chosen": -402.3499450683594, "logps/rejected": -367.5543212890625, "loss": 0.2573, "rewards/accuracies": 0.4312500059604645, "rewards/chosen": -0.2744489014148712, "rewards/margins": 0.06540031731128693, "rewards/rejected": -0.33984917402267456, "step": 520 }, { "epoch": 0.28, "learning_rate": 4.50988058902464e-06, "logits/chosen": 0.23019719123840332, "logits/rejected": 0.2985975742340088, "logps/chosen": -491.07464599609375, "logps/rejected": -475.8605041503906, "loss": 0.2694, "rewards/accuracies": 0.53125, "rewards/chosen": -0.052188098430633545, "rewards/margins": 0.07880185544490814, "rewards/rejected": -0.1309899389743805, "step": 530 }, { "epoch": 0.29, "learning_rate": 4.481847180560593e-06, "logits/chosen": 0.2160387486219406, "logits/rejected": 0.1532837599515915, "logps/chosen": -405.447265625, "logps/rejected": -394.2893371582031, "loss": 0.2777, "rewards/accuracies": 0.4625000059604645, "rewards/chosen": -0.043611008673906326, "rewards/margins": 0.0625891238451004, "rewards/rejected": -0.10620013624429703, "step": 540 }, { "epoch": 0.29, "learning_rate": 4.453126502404253e-06, "logits/chosen": 0.13828139007091522, "logits/rejected": 0.2699658274650574, "logps/chosen": -442.07635498046875, "logps/rejected": -409.4737243652344, "loss": 0.2209, "rewards/accuracies": 0.46875, "rewards/chosen": -0.09488610178232193, "rewards/margins": 0.10609780251979828, "rewards/rejected": -0.2009839117527008, "step": 550 }, { "epoch": 0.3, "learning_rate": 4.423728514380892e-06, "logits/chosen": 0.17167535424232483, "logits/rejected": 0.3206785023212433, "logps/chosen": -432.7518005371094, "logps/rejected": -453.96417236328125, "loss": 0.2193, "rewards/accuracies": 0.5, "rewards/chosen": -0.23770996928215027, "rewards/margins": 0.08862265199422836, "rewards/rejected": -0.32633259892463684, "step": 560 }, { "epoch": 0.3, "learning_rate": 4.393663411194918e-06, "logits/chosen": 0.2704005837440491, "logits/rejected": 0.23753610253334045, "logps/chosen": -449.93560791015625, "logps/rejected": -445.89898681640625, "loss": 0.2345, "rewards/accuracies": 0.53125, "rewards/chosen": -0.6195427775382996, "rewards/margins": 0.09155760705471039, "rewards/rejected": -0.7111002802848816, "step": 570 }, { "epoch": 0.31, "learning_rate": 4.362941618894523e-06, "logits/chosen": 0.20778074860572815, "logits/rejected": 0.2294239103794098, "logps/chosen": -382.36859130859375, "logps/rejected": -390.04559326171875, "loss": 0.2449, "rewards/accuracies": 0.5, "rewards/chosen": -0.5993614196777344, "rewards/margins": 0.12903143465518951, "rewards/rejected": -0.7283927798271179, "step": 580 }, { "epoch": 0.31, "learning_rate": 4.331573791256116e-06, "logits/chosen": 0.2194238007068634, "logits/rejected": 0.27026933431625366, "logps/chosen": -362.18963623046875, "logps/rejected": -365.9380187988281, "loss": 0.1998, "rewards/accuracies": 0.4437499940395355, "rewards/chosen": -0.7672492861747742, "rewards/margins": 0.09667088091373444, "rewards/rejected": -0.8639200925827026, "step": 590 }, { "epoch": 0.32, "learning_rate": 4.299570806089786e-06, "logits/chosen": 0.15812727808952332, "logits/rejected": 0.21042820811271667, "logps/chosen": -411.093994140625, "logps/rejected": -413.24359130859375, "loss": 0.2049, "rewards/accuracies": 0.45625001192092896, "rewards/chosen": -1.074034571647644, "rewards/margins": 0.11771629750728607, "rewards/rejected": -1.1917510032653809, "step": 600 }, { "epoch": 0.33, "learning_rate": 4.266943761467057e-06, "logits/chosen": 0.2622697055339813, "logits/rejected": 0.19035783410072327, "logps/chosen": -437.5669860839844, "logps/rejected": -422.769287109375, "loss": 0.1986, "rewards/accuracies": 0.5249999761581421, "rewards/chosen": -1.0040676593780518, "rewards/margins": 0.1156647801399231, "rewards/rejected": -1.1197324991226196, "step": 610 }, { "epoch": 0.33, "learning_rate": 4.233703971872287e-06, "logits/chosen": 0.15742294490337372, "logits/rejected": 0.34303316473960876, "logps/chosen": -441.45318603515625, "logps/rejected": -427.8976135253906, "loss": 0.1777, "rewards/accuracies": 0.5625, "rewards/chosen": -0.9649977684020996, "rewards/margins": 0.13918432593345642, "rewards/rejected": -1.1041821241378784, "step": 620 }, { "epoch": 0.34, "learning_rate": 4.1998629642789925e-06, "logits/chosen": 0.18381045758724213, "logits/rejected": 0.2369348555803299, "logps/chosen": -398.6405334472656, "logps/rejected": -413.9356384277344, "loss": 0.1679, "rewards/accuracies": 0.53125, "rewards/chosen": -1.1926090717315674, "rewards/margins": 0.12783730030059814, "rewards/rejected": -1.3204463720321655, "step": 630 }, { "epoch": 0.34, "learning_rate": 4.165432474152505e-06, "logits/chosen": 0.19464774429798126, "logits/rejected": 0.27594244480133057, "logps/chosen": -402.32025146484375, "logps/rejected": -386.37506103515625, "loss": 0.1886, "rewards/accuracies": 0.5, "rewards/chosen": -1.4002799987792969, "rewards/margins": 0.15942224860191345, "rewards/rejected": -1.5597022771835327, "step": 640 }, { "epoch": 0.35, "learning_rate": 4.130424441380308e-06, "logits/chosen": 0.19669343531131744, "logits/rejected": 0.280894935131073, "logps/chosen": -427.1275329589844, "logps/rejected": -399.8666687011719, "loss": 0.2517, "rewards/accuracies": 0.5249999761581421, "rewards/chosen": -1.7949119806289673, "rewards/margins": 0.12687289714813232, "rewards/rejected": -1.9217851161956787, "step": 650 }, { "epoch": 0.35, "learning_rate": 4.09485100613151e-06, "logits/chosen": 0.16887667775154114, "logits/rejected": 0.25499215722084045, "logps/chosen": -472.88970947265625, "logps/rejected": -476.806640625, "loss": 0.2186, "rewards/accuracies": 0.5, "rewards/chosen": -2.312155246734619, "rewards/margins": 0.11931940168142319, "rewards/rejected": -2.431474447250366, "step": 660 }, { "epoch": 0.36, "learning_rate": 4.058724504646834e-06, "logits/chosen": 0.22942841053009033, "logits/rejected": 0.21801376342773438, "logps/chosen": -404.1775817871094, "logps/rejected": -412.6273498535156, "loss": 0.211, "rewards/accuracies": 0.4625000059604645, "rewards/chosen": -2.2967944145202637, "rewards/margins": 0.11083470284938812, "rewards/rejected": -2.4076290130615234, "step": 670 }, { "epoch": 0.36, "learning_rate": 4.022057464960632e-06, "logits/chosen": 0.21228846907615662, "logits/rejected": 0.22944676876068115, "logps/chosen": -413.43914794921875, "logps/rejected": -405.81744384765625, "loss": 0.2846, "rewards/accuracies": 0.48124998807907104, "rewards/chosen": -2.1489250659942627, "rewards/margins": 0.11012575775384903, "rewards/rejected": -2.2590508460998535, "step": 680 }, { "epoch": 0.37, "learning_rate": 3.984862602556383e-06, "logits/chosen": 0.25439196825027466, "logits/rejected": 0.2606196105480194, "logps/chosen": -440.023681640625, "logps/rejected": -442.59649658203125, "loss": 0.2369, "rewards/accuracies": 0.48124998807907104, "rewards/chosen": -2.2106480598449707, "rewards/margins": 0.12710650265216827, "rewards/rejected": -2.337754487991333, "step": 690 }, { "epoch": 0.37, "learning_rate": 3.947152815957187e-06, "logits/chosen": 0.14478982985019684, "logits/rejected": 0.1685037910938263, "logps/chosen": -383.38726806640625, "logps/rejected": -375.55682373046875, "loss": 0.193, "rewards/accuracies": 0.4937500059604645, "rewards/chosen": -2.3591561317443848, "rewards/margins": 0.12616077065467834, "rewards/rejected": -2.4853172302246094, "step": 700 }, { "epoch": 0.38, "learning_rate": 3.908941182252785e-06, "logits/chosen": 0.17379993200302124, "logits/rejected": 0.27680641412734985, "logps/chosen": -435.744384765625, "logps/rejected": -435.990966796875, "loss": 0.2464, "rewards/accuracies": 0.48124998807907104, "rewards/chosen": -2.4536287784576416, "rewards/margins": 0.10799276828765869, "rewards/rejected": -2.56162166595459, "step": 710 }, { "epoch": 0.38, "learning_rate": 3.8702409525646535e-06, "logits/chosen": 0.17838087677955627, "logits/rejected": 0.20998401939868927, "logps/chosen": -420.6383361816406, "logps/rejected": -408.6434020996094, "loss": 0.2224, "rewards/accuracies": 0.44999998807907104, "rewards/chosen": -2.475475788116455, "rewards/margins": 0.10919404029846191, "rewards/rejected": -2.584669828414917, "step": 720 }, { "epoch": 0.39, "learning_rate": 3.8310655474507495e-06, "logits/chosen": 0.16479340195655823, "logits/rejected": 0.2636985778808594, "logps/chosen": -437.88427734375, "logps/rejected": -414.7217712402344, "loss": 0.2194, "rewards/accuracies": 0.5375000238418579, "rewards/chosen": -2.8833677768707275, "rewards/margins": 0.13075178861618042, "rewards/rejected": -3.0141196250915527, "step": 730 }, { "epoch": 0.39, "learning_rate": 3.7914285522515002e-06, "logits/chosen": 0.2210838496685028, "logits/rejected": 0.23706713318824768, "logps/chosen": -409.523193359375, "logps/rejected": -432.919189453125, "loss": 0.2153, "rewards/accuracies": 0.48750001192092896, "rewards/chosen": -2.8223652839660645, "rewards/margins": 0.0898015946149826, "rewards/rejected": -2.9121673107147217, "step": 740 }, { "epoch": 0.4, "learning_rate": 3.751343712378639e-06, "logits/chosen": 0.23539260029792786, "logits/rejected": 0.2847990095615387, "logps/chosen": -429.42828369140625, "logps/rejected": -439.2691955566406, "loss": 0.4669, "rewards/accuracies": 0.48124998807907104, "rewards/chosen": -3.1466052532196045, "rewards/margins": 0.06740974634885788, "rewards/rejected": -3.214015245437622, "step": 750 }, { "epoch": 0.41, "learning_rate": 3.710824928548546e-06, "logits/chosen": 0.18615157902240753, "logits/rejected": 0.26414626836776733, "logps/chosen": -444.6000061035156, "logps/rejected": -440.26611328125, "loss": 0.2051, "rewards/accuracies": 0.4937500059604645, "rewards/chosen": -2.4763169288635254, "rewards/margins": 0.12143449485301971, "rewards/rejected": -2.5977511405944824, "step": 760 }, { "epoch": 0.41, "learning_rate": 3.6698862519617225e-06, "logits/chosen": 0.22848829627037048, "logits/rejected": 0.23971283435821533, "logps/chosen": -435.1680603027344, "logps/rejected": -451.59930419921875, "loss": 0.2253, "rewards/accuracies": 0.550000011920929, "rewards/chosen": -1.9101835489273071, "rewards/margins": 0.16930529475212097, "rewards/rejected": -2.079489231109619, "step": 770 }, { "epoch": 0.42, "learning_rate": 3.6285418794300793e-06, "logits/chosen": 0.16407668590545654, "logits/rejected": 0.19629423320293427, "logps/chosen": -429.7174377441406, "logps/rejected": -416.3013610839844, "loss": 0.2255, "rewards/accuracies": 0.48124998807907104, "rewards/chosen": -1.8518050909042358, "rewards/margins": 0.09626470506191254, "rewards/rejected": -1.9480698108673096, "step": 780 }, { "epoch": 0.42, "learning_rate": 3.5868061484537365e-06, "logits/chosen": 0.2521757185459137, "logits/rejected": 0.3432837724685669, "logps/chosen": -433.750732421875, "logps/rejected": -421.22540283203125, "loss": 0.2085, "rewards/accuracies": 0.48124998807907104, "rewards/chosen": -1.9239532947540283, "rewards/margins": 0.10734070837497711, "rewards/rejected": -2.0312938690185547, "step": 790 }, { "epoch": 0.43, "learning_rate": 3.5446935322490285e-06, "logits/chosen": 0.2557790279388428, "logits/rejected": 0.30611905455589294, "logps/chosen": -436.35687255859375, "logps/rejected": -427.75958251953125, "loss": 0.2532, "rewards/accuracies": 0.48750001192092896, "rewards/chosen": -1.8817793130874634, "rewards/margins": 0.10908164829015732, "rewards/rejected": -1.990860939025879, "step": 800 }, { "epoch": 0.43, "learning_rate": 3.502218634729447e-06, "logits/chosen": 0.2341274917125702, "logits/rejected": 0.18765440583229065, "logps/chosen": -403.7873229980469, "logps/rejected": -405.88909912109375, "loss": 0.1786, "rewards/accuracies": 0.53125, "rewards/chosen": -1.8474352359771729, "rewards/margins": 0.12625011801719666, "rewards/rejected": -1.9736855030059814, "step": 810 }, { "epoch": 0.44, "learning_rate": 3.459396185441265e-06, "logits/chosen": 0.18527312576770782, "logits/rejected": 0.2313808649778366, "logps/chosen": -379.3541564941406, "logps/rejected": -352.1042175292969, "loss": 0.1985, "rewards/accuracies": 0.512499988079071, "rewards/chosen": -1.6922273635864258, "rewards/margins": 0.11197100579738617, "rewards/rejected": -1.8041980266571045, "step": 820 }, { "epoch": 0.44, "learning_rate": 3.4162410344555834e-06, "logits/chosen": 0.29786452651023865, "logits/rejected": 0.3404124677181244, "logps/chosen": -379.1360168457031, "logps/rejected": -397.03662109375, "loss": 0.1982, "rewards/accuracies": 0.512499988079071, "rewards/chosen": -1.7881301641464233, "rewards/margins": 0.11922701448202133, "rewards/rejected": -1.9073572158813477, "step": 830 }, { "epoch": 0.45, "learning_rate": 3.3727681472185937e-06, "logits/chosen": 0.2442360371351242, "logits/rejected": 0.22556254267692566, "logps/chosen": -422.0938415527344, "logps/rejected": -419.09063720703125, "loss": 0.2123, "rewards/accuracies": 0.5062500238418579, "rewards/chosen": -1.8410011529922485, "rewards/margins": 0.13928382098674774, "rewards/rejected": -1.9802849292755127, "step": 840 }, { "epoch": 0.45, "learning_rate": 3.3289925993618217e-06, "logits/chosen": 0.13701245188713074, "logits/rejected": 0.2810230851173401, "logps/chosen": -392.12860107421875, "logps/rejected": -401.2889099121094, "loss": 0.2089, "rewards/accuracies": 0.46875, "rewards/chosen": -1.8769737482070923, "rewards/margins": 0.12001794576644897, "rewards/rejected": -1.996991515159607, "step": 850 }, { "epoch": 0.46, "learning_rate": 3.2849295714741643e-06, "logits/chosen": 0.12385988235473633, "logits/rejected": 0.3134171962738037, "logps/chosen": -428.42242431640625, "logps/rejected": -409.5142517089844, "loss": 0.2026, "rewards/accuracies": 0.518750011920929, "rewards/chosen": -1.8102359771728516, "rewards/margins": 0.08851593732833862, "rewards/rejected": -1.898751974105835, "step": 860 }, { "epoch": 0.46, "learning_rate": 3.2405943438375287e-06, "logits/chosen": 0.12988132238388062, "logits/rejected": 0.2288062870502472, "logps/chosen": -376.00006103515625, "logps/rejected": -390.5958557128906, "loss": 0.2222, "rewards/accuracies": 0.48124998807907104, "rewards/chosen": -1.7014586925506592, "rewards/margins": 0.11990761756896973, "rewards/rejected": -1.8213660717010498, "step": 870 }, { "epoch": 0.47, "learning_rate": 3.1960022911279036e-06, "logits/chosen": 0.18507717549800873, "logits/rejected": 0.1665429174900055, "logps/chosen": -375.88677978515625, "logps/rejected": -396.2275695800781, "loss": 0.2264, "rewards/accuracies": 0.4375, "rewards/chosen": -1.7533376216888428, "rewards/margins": 0.07123322784900665, "rewards/rejected": -1.8245712518692017, "step": 880 }, { "epoch": 0.47, "learning_rate": 3.1511688770836844e-06, "logits/chosen": 0.31608790159225464, "logits/rejected": 0.16106575727462769, "logps/chosen": -444.12542724609375, "logps/rejected": -439.402587890625, "loss": 0.3027, "rewards/accuracies": 0.48750001192092896, "rewards/chosen": -1.820054292678833, "rewards/margins": 0.10226750373840332, "rewards/rejected": -1.9223216772079468, "step": 890 }, { "epoch": 0.48, "learning_rate": 3.1061096491431307e-06, "logits/chosen": 0.12085793167352676, "logits/rejected": 0.2093731164932251, "logps/chosen": -409.0577697753906, "logps/rejected": -399.948974609375, "loss": 0.248, "rewards/accuracies": 0.39375001192092896, "rewards/chosen": -1.4467195272445679, "rewards/margins": 0.04833744466304779, "rewards/rejected": -1.4950571060180664, "step": 900 }, { "epoch": 0.49, "learning_rate": 3.0608402330527796e-06, "logits/chosen": 0.18963545560836792, "logits/rejected": 0.2660353183746338, "logps/chosen": -424.49188232421875, "logps/rejected": -422.8685607910156, "loss": 0.1943, "rewards/accuracies": 0.4625000059604645, "rewards/chosen": -1.53090500831604, "rewards/margins": 0.08851666748523712, "rewards/rejected": -1.6194219589233398, "step": 910 }, { "epoch": 0.49, "learning_rate": 3.0153763274487176e-06, "logits/chosen": 0.20010657608509064, "logits/rejected": 0.2609160840511322, "logps/chosen": -418.9073181152344, "logps/rejected": -398.8419494628906, "loss": 0.2011, "rewards/accuracies": 0.5062500238418579, "rewards/chosen": -1.4970227479934692, "rewards/margins": 0.10633371025323868, "rewards/rejected": -1.6033565998077393, "step": 920 }, { "epoch": 0.5, "learning_rate": 2.9697336984125683e-06, "logits/chosen": 0.13921412825584412, "logits/rejected": 0.23919770121574402, "logps/chosen": -473.777099609375, "logps/rejected": -455.8279724121094, "loss": 0.2057, "rewards/accuracies": 0.4749999940395355, "rewards/chosen": -1.5798097848892212, "rewards/margins": 0.08555875718593597, "rewards/rejected": -1.6653684377670288, "step": 930 }, { "epoch": 0.5, "learning_rate": 2.923928174004094e-06, "logits/chosen": 0.29412585496902466, "logits/rejected": 0.19027197360992432, "logps/chosen": -455.6954650878906, "logps/rejected": -465.9042053222656, "loss": 0.2045, "rewards/accuracies": 0.44999998807907104, "rewards/chosen": -1.7673845291137695, "rewards/margins": 0.09541743248701096, "rewards/rejected": -1.8628019094467163, "step": 940 }, { "epoch": 0.51, "learning_rate": 2.8779756387723036e-06, "logits/chosen": 0.18643781542778015, "logits/rejected": 0.26781877875328064, "logps/chosen": -418.92578125, "logps/rejected": -428.0494079589844, "loss": 0.2276, "rewards/accuracies": 0.53125, "rewards/chosen": -1.4455392360687256, "rewards/margins": 0.11924241483211517, "rewards/rejected": -1.564781904220581, "step": 950 }, { "epoch": 0.51, "learning_rate": 2.831892028246968e-06, "logits/chosen": 0.23528459668159485, "logits/rejected": 0.16012924909591675, "logps/chosen": -420.2757263183594, "logps/rejected": -425.47686767578125, "loss": 0.2191, "rewards/accuracies": 0.5062500238418579, "rewards/chosen": -0.983513355255127, "rewards/margins": 0.13725998997688293, "rewards/rejected": -1.1207733154296875, "step": 960 }, { "epoch": 0.52, "learning_rate": 2.7856933234124617e-06, "logits/chosen": 0.2554696202278137, "logits/rejected": 0.2754512429237366, "logps/chosen": -383.97296142578125, "logps/rejected": -409.36566162109375, "loss": 0.2216, "rewards/accuracies": 0.45625001192092896, "rewards/chosen": -0.7281330227851868, "rewards/margins": 0.08484755456447601, "rewards/rejected": -0.8129806518554688, "step": 970 }, { "epoch": 0.52, "learning_rate": 2.7393955451658387e-06, "logits/chosen": 0.1969253122806549, "logits/rejected": 0.30447250604629517, "logps/chosen": -418.5328063964844, "logps/rejected": -383.39825439453125, "loss": 0.2054, "rewards/accuracies": 0.45625001192092896, "rewards/chosen": -0.7214577198028564, "rewards/margins": 0.08516426384449005, "rewards/rejected": -0.8066219091415405, "step": 980 }, { "epoch": 0.53, "learning_rate": 2.6930147487610667e-06, "logits/chosen": 0.14184299111366272, "logits/rejected": 0.2329244166612625, "logps/chosen": -436.410400390625, "logps/rejected": -420.66357421875, "loss": 0.217, "rewards/accuracies": 0.5375000238418579, "rewards/chosen": -0.7258288264274597, "rewards/margins": 0.1092677116394043, "rewards/rejected": -0.8350966572761536, "step": 990 }, { "epoch": 0.53, "learning_rate": 2.6465670182413487e-06, "logits/chosen": 0.26164674758911133, "logits/rejected": 0.23967313766479492, "logps/chosen": -431.079345703125, "logps/rejected": -406.0982360839844, "loss": 0.1821, "rewards/accuracies": 0.48750001192092896, "rewards/chosen": -0.6475759744644165, "rewards/margins": 0.10677828639745712, "rewards/rejected": -0.7543543577194214, "step": 1000 }, { "epoch": 0.54, "learning_rate": 2.6000684608614594e-06, "logits/chosen": 0.23683926463127136, "logits/rejected": 0.1576264351606369, "logps/chosen": -430.5428161621094, "logps/rejected": -448.5796813964844, "loss": 0.2441, "rewards/accuracies": 0.5625, "rewards/chosen": -0.7693523168563843, "rewards/margins": 0.13786748051643372, "rewards/rejected": -0.9072197675704956, "step": 1010 }, { "epoch": 0.54, "learning_rate": 2.5535352015020338e-06, "logits/chosen": 0.23495522141456604, "logits/rejected": 0.21038460731506348, "logps/chosen": -410.57904052734375, "logps/rejected": -398.42193603515625, "loss": 0.2056, "rewards/accuracies": 0.5062500238418579, "rewards/chosen": -0.8607665300369263, "rewards/margins": 0.11764762550592422, "rewards/rejected": -0.9784140586853027, "step": 1020 }, { "epoch": 0.55, "learning_rate": 2.506983377077741e-06, "logits/chosen": 0.24832454323768616, "logits/rejected": 0.2945554852485657, "logps/chosen": -438.53680419921875, "logps/rejected": -407.5857849121094, "loss": 0.2016, "rewards/accuracies": 0.518750011920929, "rewards/chosen": -1.1375848054885864, "rewards/margins": 0.11563225090503693, "rewards/rejected": -1.2532169818878174, "step": 1030 }, { "epoch": 0.55, "learning_rate": 2.460429130941289e-06, "logits/chosen": 0.17995227873325348, "logits/rejected": 0.2567993402481079, "logps/chosen": -371.36102294921875, "logps/rejected": -355.2597351074219, "loss": 0.2277, "rewards/accuracies": 0.48124998807907104, "rewards/chosen": -0.8424074053764343, "rewards/margins": 0.11410113424062729, "rewards/rejected": -0.9565085172653198, "step": 1040 }, { "epoch": 0.56, "learning_rate": 2.413888607285192e-06, "logits/chosen": 0.14955028891563416, "logits/rejected": 0.20088477432727814, "logps/chosen": -402.30767822265625, "logps/rejected": -393.7333068847656, "loss": 0.2652, "rewards/accuracies": 0.45625001192092896, "rewards/chosen": -1.1037840843200684, "rewards/margins": 0.11586949974298477, "rewards/rejected": -1.219653844833374, "step": 1050 }, { "epoch": 0.57, "learning_rate": 2.367377945543249e-06, "logits/chosen": 0.1821899712085724, "logits/rejected": 0.211843803524971, "logps/chosen": -395.53118896484375, "logps/rejected": -402.3675842285156, "loss": 0.1918, "rewards/accuracies": 0.46875, "rewards/chosen": -1.1062228679656982, "rewards/margins": 0.10730864852666855, "rewards/rejected": -1.2135313749313354, "step": 1060 }, { "epoch": 0.57, "learning_rate": 2.320913274793676e-06, "logits/chosen": 0.2403596192598343, "logits/rejected": 0.2753956615924835, "logps/chosen": -405.42047119140625, "logps/rejected": -420.23199462890625, "loss": 0.2069, "rewards/accuracies": 0.5, "rewards/chosen": -1.0861284732818604, "rewards/margins": 0.09204941987991333, "rewards/rejected": -1.178177833557129, "step": 1070 }, { "epoch": 0.58, "learning_rate": 2.27451070816582e-06, "logits/chosen": 0.21563324332237244, "logits/rejected": 0.3084403872489929, "logps/chosen": -478.88763427734375, "logps/rejected": -456.4349670410156, "loss": 0.21, "rewards/accuracies": 0.48750001192092896, "rewards/chosen": -1.1920944452285767, "rewards/margins": 0.10027352720499039, "rewards/rejected": -1.2923680543899536, "step": 1080 }, { "epoch": 0.58, "learning_rate": 2.228186337252414e-06, "logits/chosen": 0.13818100094795227, "logits/rejected": 0.3011969029903412, "logps/chosen": -417.41961669921875, "logps/rejected": -380.45306396484375, "loss": 0.1998, "rewards/accuracies": 0.42500001192092896, "rewards/chosen": -1.3052135705947876, "rewards/margins": 0.08900976181030273, "rewards/rejected": -1.3942232131958008, "step": 1090 }, { "epoch": 0.59, "learning_rate": 2.1819562265292946e-06, "logits/chosen": 0.16572540998458862, "logits/rejected": 0.22458620369434357, "logps/chosen": -473.16827392578125, "logps/rejected": -460.0870056152344, "loss": 0.1622, "rewards/accuracies": 0.5375000238418579, "rewards/chosen": -1.3903932571411133, "rewards/margins": 0.15418145060539246, "rewards/rejected": -1.5445747375488281, "step": 1100 }, { "epoch": 0.59, "learning_rate": 2.1358364077845236e-06, "logits/chosen": 0.1645340472459793, "logits/rejected": 0.34099048376083374, "logps/chosen": -454.2103576660156, "logps/rejected": -452.72991943359375, "loss": 0.2015, "rewards/accuracies": 0.5, "rewards/chosen": -1.5796992778778076, "rewards/margins": 0.07802970707416534, "rewards/rejected": -1.657728910446167, "step": 1110 }, { "epoch": 0.6, "learning_rate": 2.089842874558849e-06, "logits/chosen": 0.24456259608268738, "logits/rejected": 0.1955045759677887, "logps/chosen": -407.2257385253906, "logps/rejected": -420.85015869140625, "loss": 0.2256, "rewards/accuracies": 0.5, "rewards/chosen": -1.4515103101730347, "rewards/margins": 0.06924048811197281, "rewards/rejected": -1.520750880241394, "step": 1120 }, { "epoch": 0.6, "learning_rate": 2.0439915765994242e-06, "logits/chosen": 0.21577736735343933, "logits/rejected": 0.18421390652656555, "logps/chosen": -413.2845153808594, "logps/rejected": -417.5594177246094, "loss": 0.2097, "rewards/accuracies": 0.5375000238418579, "rewards/chosen": -1.4378559589385986, "rewards/margins": 0.12303999811410904, "rewards/rejected": -1.5608961582183838, "step": 1130 }, { "epoch": 0.61, "learning_rate": 1.9982984143287186e-06, "logits/chosen": 0.1870601922273636, "logits/rejected": 0.1403540074825287, "logps/chosen": -437.71875, "logps/rejected": -439.2840881347656, "loss": 0.2507, "rewards/accuracies": 0.4937500059604645, "rewards/chosen": -1.5519638061523438, "rewards/margins": 0.1572073996067047, "rewards/rejected": -1.7091710567474365, "step": 1140 }, { "epoch": 0.61, "learning_rate": 1.95277923333053e-06, "logits/chosen": 0.18075987696647644, "logits/rejected": 0.29444438219070435, "logps/chosen": -461.15533447265625, "logps/rejected": -489.20025634765625, "loss": 0.2153, "rewards/accuracies": 0.5062500238418579, "rewards/chosen": -1.6609361171722412, "rewards/margins": 0.12212083488702774, "rewards/rejected": -1.7830568552017212, "step": 1150 }, { "epoch": 0.62, "learning_rate": 1.9074498188550156e-06, "logits/chosen": 0.13356804847717285, "logits/rejected": 0.27090999484062195, "logps/chosen": -425.75665283203125, "logps/rejected": -410.96368408203125, "loss": 0.1966, "rewards/accuracies": 0.512499988079071, "rewards/chosen": -1.284343957901001, "rewards/margins": 0.11508820950984955, "rewards/rejected": -1.3994321823120117, "step": 1160 }, { "epoch": 0.62, "learning_rate": 1.862325890344643e-06, "logits/chosen": 0.18534113466739655, "logits/rejected": 0.2500341534614563, "logps/chosen": -387.5941467285156, "logps/rejected": -358.5582275390625, "loss": 0.2102, "rewards/accuracies": 0.4312500059604645, "rewards/chosen": -1.2002973556518555, "rewards/margins": 0.07647459208965302, "rewards/rejected": -1.276771903038025, "step": 1170 }, { "epoch": 0.63, "learning_rate": 1.817423095982972e-06, "logits/chosen": 0.2761315107345581, "logits/rejected": 0.1992635577917099, "logps/chosen": -410.59130859375, "logps/rejected": -429.77667236328125, "loss": 0.1798, "rewards/accuracies": 0.5062500238418579, "rewards/chosen": -1.3009172677993774, "rewards/margins": 0.12259545177221298, "rewards/rejected": -1.4235126972198486, "step": 1180 }, { "epoch": 0.63, "learning_rate": 1.7727570072681293e-06, "logits/chosen": 0.21099631488323212, "logits/rejected": 0.3009958863258362, "logps/chosen": -356.77490234375, "logps/rejected": -372.29718017578125, "loss": 0.1696, "rewards/accuracies": 0.53125, "rewards/chosen": -1.1670329570770264, "rewards/margins": 0.13822320103645325, "rewards/rejected": -1.3052562475204468, "step": 1190 }, { "epoch": 0.64, "learning_rate": 1.7283431136128961e-06, "logits/chosen": 0.16253753006458282, "logits/rejected": 0.29959315061569214, "logps/chosen": -471.822998046875, "logps/rejected": -446.8572692871094, "loss": 0.1791, "rewards/accuracies": 0.5, "rewards/chosen": -1.4546597003936768, "rewards/margins": 0.1095527857542038, "rewards/rejected": -1.5642122030258179, "step": 1200 }, { "epoch": 0.65, "learning_rate": 1.6841968169732478e-06, "logits/chosen": 0.21443840861320496, "logits/rejected": 0.2336159497499466, "logps/chosen": -424.06805419921875, "logps/rejected": -417.39141845703125, "loss": 0.2039, "rewards/accuracies": 0.5249999761581421, "rewards/chosen": -1.357987403869629, "rewards/margins": 0.11536715924739838, "rewards/rejected": -1.4733545780181885, "step": 1210 }, { "epoch": 0.65, "learning_rate": 1.6403334265072284e-06, "logits/chosen": 0.10831620544195175, "logits/rejected": 0.31975752115249634, "logps/chosen": -425.4805603027344, "logps/rejected": -405.4668273925781, "loss": 0.2061, "rewards/accuracies": 0.45625001192092896, "rewards/chosen": -1.2643564939498901, "rewards/margins": 0.12265012413263321, "rewards/rejected": -1.3870065212249756, "step": 1220 }, { "epoch": 0.66, "learning_rate": 1.5967681532660066e-06, "logits/chosen": 0.19904041290283203, "logits/rejected": 0.31774818897247314, "logps/chosen": -451.98895263671875, "logps/rejected": -464.37408447265625, "loss": 0.268, "rewards/accuracies": 0.4937500059604645, "rewards/chosen": -1.5613361597061157, "rewards/margins": 0.11387111991643906, "rewards/rejected": -1.6752071380615234, "step": 1230 }, { "epoch": 0.66, "learning_rate": 1.5535161049189463e-06, "logits/chosen": 0.1638219654560089, "logits/rejected": 0.24011917412281036, "logps/chosen": -435.06170654296875, "logps/rejected": -426.3074645996094, "loss": 0.2635, "rewards/accuracies": 0.48124998807907104, "rewards/chosen": -1.6264498233795166, "rewards/margins": 0.14013561606407166, "rewards/rejected": -1.766585350036621, "step": 1240 }, { "epoch": 0.67, "learning_rate": 1.5105922805145356e-06, "logits/chosen": 0.22532732784748077, "logits/rejected": 0.21466323733329773, "logps/chosen": -470.2998962402344, "logps/rejected": -450.73193359375, "loss": 0.1847, "rewards/accuracies": 0.5562499761581421, "rewards/chosen": -1.6772887706756592, "rewards/margins": 0.1449218988418579, "rewards/rejected": -1.822210669517517, "step": 1250 }, { "epoch": 0.67, "learning_rate": 1.4680115652789823e-06, "logits/chosen": 0.16722983121871948, "logits/rejected": 0.2960713505744934, "logps/chosen": -452.7391662597656, "logps/rejected": -429.21435546875, "loss": 0.187, "rewards/accuracies": 0.48750001192092896, "rewards/chosen": -1.6514288187026978, "rewards/margins": 0.11130253225564957, "rewards/rejected": -1.7627313137054443, "step": 1260 }, { "epoch": 0.68, "learning_rate": 1.4257887254542767e-06, "logits/chosen": 0.2482529580593109, "logits/rejected": 0.2227180004119873, "logps/chosen": -396.0767517089844, "logps/rejected": -405.37091064453125, "loss": 0.2139, "rewards/accuracies": 0.5062500238418579, "rewards/chosen": -1.2826321125030518, "rewards/margins": 0.13906559348106384, "rewards/rejected": -1.4216978549957275, "step": 1270 }, { "epoch": 0.68, "learning_rate": 1.3839384031775227e-06, "logits/chosen": 0.1646454632282257, "logits/rejected": 0.27516424655914307, "logps/chosen": -406.2199401855469, "logps/rejected": -418.5736389160156, "loss": 0.2454, "rewards/accuracies": 0.48124998807907104, "rewards/chosen": -1.3300670385360718, "rewards/margins": 0.11273592710494995, "rewards/rejected": -1.442803144454956, "step": 1280 }, { "epoch": 0.69, "learning_rate": 1.342475111403298e-06, "logits/chosen": 0.16665968298912048, "logits/rejected": 0.31475013494491577, "logps/chosen": -460.81640625, "logps/rejected": -437.21270751953125, "loss": 0.1881, "rewards/accuracies": 0.5375000238418579, "rewards/chosen": -1.377325415611267, "rewards/margins": 0.12428633123636246, "rewards/rejected": -1.5016119480133057, "step": 1290 }, { "epoch": 0.69, "learning_rate": 1.3014132288708209e-06, "logits/chosen": 0.22961941361427307, "logits/rejected": 0.27373284101486206, "logps/chosen": -383.945556640625, "logps/rejected": -404.753173828125, "loss": 0.2236, "rewards/accuracies": 0.4437499940395355, "rewards/chosen": -1.2235456705093384, "rewards/margins": 0.08865731954574585, "rewards/rejected": -1.312203049659729, "step": 1300 }, { "epoch": 0.7, "learning_rate": 1.2607669951176549e-06, "logits/chosen": 0.22940245270729065, "logits/rejected": 0.27775856852531433, "logps/chosen": -397.48736572265625, "logps/rejected": -366.0208740234375, "loss": 0.1997, "rewards/accuracies": 0.44999998807907104, "rewards/chosen": -1.242621660232544, "rewards/margins": 0.06689196825027466, "rewards/rejected": -1.3095134496688843, "step": 1310 }, { "epoch": 0.7, "learning_rate": 1.2205505055416891e-06, "logits/chosen": 0.18240857124328613, "logits/rejected": 0.3393421173095703, "logps/chosen": -417.5065002441406, "logps/rejected": -389.3405456542969, "loss": 0.1692, "rewards/accuracies": 0.518750011920929, "rewards/chosen": -1.4786723852157593, "rewards/margins": 0.12162399291992188, "rewards/rejected": -1.6002962589263916, "step": 1320 }, { "epoch": 0.71, "learning_rate": 1.1807777065131002e-06, "logits/chosen": 0.15853646397590637, "logits/rejected": 0.2295243740081787, "logps/chosen": -459.41912841796875, "logps/rejected": -431.26708984375, "loss": 0.2207, "rewards/accuracies": 0.5375000238418579, "rewards/chosen": -1.5732864141464233, "rewards/margins": 0.13572458922863007, "rewards/rejected": -1.7090113162994385, "step": 1330 }, { "epoch": 0.71, "learning_rate": 1.1414623905380012e-06, "logits/chosen": 0.1904124766588211, "logits/rejected": 0.2695844769477844, "logps/chosen": -402.2320251464844, "logps/rejected": -390.7575378417969, "loss": 0.1981, "rewards/accuracies": 0.5062500238418579, "rewards/chosen": -1.56125009059906, "rewards/margins": 0.12390075623989105, "rewards/rejected": -1.6851508617401123, "step": 1340 }, { "epoch": 0.72, "learning_rate": 1.1026181914754388e-06, "logits/chosen": 0.2069578617811203, "logits/rejected": 0.2751797139644623, "logps/chosen": -385.76312255859375, "logps/rejected": -406.4028625488281, "loss": 0.2131, "rewards/accuracies": 0.5062500238418579, "rewards/chosen": -1.6344740390777588, "rewards/margins": 0.0895460695028305, "rewards/rejected": -1.7240203619003296, "step": 1350 }, { "epoch": 0.73, "learning_rate": 1.0642585798094136e-06, "logits/chosen": 0.20105871558189392, "logits/rejected": 0.22045502066612244, "logps/chosen": -420.058349609375, "logps/rejected": -411.83612060546875, "loss": 0.167, "rewards/accuracies": 0.512499988079071, "rewards/chosen": -1.566814661026001, "rewards/margins": 0.12989051640033722, "rewards/rejected": -1.6967051029205322, "step": 1360 }, { "epoch": 0.73, "learning_rate": 1.0263968579775522e-06, "logits/chosen": 0.21636684238910675, "logits/rejected": 0.18984094262123108, "logps/chosen": -388.39642333984375, "logps/rejected": -390.4629211425781, "loss": 0.2022, "rewards/accuracies": 0.42500001192092896, "rewards/chosen": -1.5708179473876953, "rewards/margins": 0.07592545449733734, "rewards/rejected": -1.6467434167861938, "step": 1370 }, { "epoch": 0.74, "learning_rate": 9.89046155758058e-07, "logits/chosen": 0.24614374339580536, "logits/rejected": 0.3122064471244812, "logps/chosen": -384.8661193847656, "logps/rejected": -419.45513916015625, "loss": 0.2277, "rewards/accuracies": 0.45625001192092896, "rewards/chosen": -1.3731580972671509, "rewards/margins": 0.09444873034954071, "rewards/rejected": -1.467606782913208, "step": 1380 }, { "epoch": 0.74, "learning_rate": 9.52219425716534e-07, "logits/chosen": 0.17535480856895447, "logits/rejected": 0.2093926966190338, "logps/chosen": -457.5205078125, "logps/rejected": -446.33587646484375, "loss": 0.1623, "rewards/accuracies": 0.5687500238418579, "rewards/chosen": -1.650968313217163, "rewards/margins": 0.13586655259132385, "rewards/rejected": -1.7868350744247437, "step": 1390 }, { "epoch": 0.75, "learning_rate": 9.15929438714262e-07, "logits/chosen": 0.28407081961631775, "logits/rejected": 0.2300814837217331, "logps/chosen": -454.60650634765625, "logps/rejected": -465.85906982421875, "loss": 0.1815, "rewards/accuracies": 0.581250011920929, "rewards/chosen": -1.5347188711166382, "rewards/margins": 0.15691988170146942, "rewards/rejected": -1.691638708114624, "step": 1400 }, { "epoch": 0.75, "learning_rate": 8.801887794794911e-07, "logits/chosen": 0.2535427212715149, "logits/rejected": 0.211197167634964, "logps/chosen": -396.9275817871094, "logps/rejected": -406.4787902832031, "loss": 0.5444, "rewards/accuracies": 0.518750011920929, "rewards/chosen": -1.4354115724563599, "rewards/margins": 0.15605568885803223, "rewards/rejected": -1.591467261314392, "step": 1410 }, { "epoch": 0.76, "learning_rate": 8.450098422432787e-07, "logits/chosen": 0.24384653568267822, "logits/rejected": 0.2819363474845886, "logps/chosen": -444.5672912597656, "logps/rejected": -427.81134033203125, "loss": 0.1977, "rewards/accuracies": 0.4937500059604645, "rewards/chosen": -1.5881189107894897, "rewards/margins": 0.11528144031763077, "rewards/rejected": -1.7034003734588623, "step": 1420 }, { "epoch": 0.76, "learning_rate": 8.104048264413858e-07, "logits/chosen": 0.22661654651165009, "logits/rejected": 0.1742086112499237, "logps/chosen": -367.4610595703125, "logps/rejected": -356.89422607421875, "loss": 0.1993, "rewards/accuracies": 0.4625000059604645, "rewards/chosen": -1.2963422536849976, "rewards/margins": 0.09244900941848755, "rewards/rejected": -1.3887912034988403, "step": 1430 }, { "epoch": 0.77, "learning_rate": 7.763857324837321e-07, "logits/chosen": 0.24572880566120148, "logits/rejected": 0.21287715435028076, "logps/chosen": -424.8355407714844, "logps/rejected": -423.1205139160156, "loss": 0.2398, "rewards/accuracies": 0.42500001192092896, "rewards/chosen": -1.5864840745925903, "rewards/margins": 0.08626910299062729, "rewards/rejected": -1.672753095626831, "step": 1440 }, { "epoch": 0.77, "learning_rate": 7.429643575928605e-07, "logits/chosen": 0.16526217758655548, "logits/rejected": 0.2713525891304016, "logps/chosen": -448.96234130859375, "logps/rejected": -429.2787170410156, "loss": 0.2337, "rewards/accuracies": 0.518750011920929, "rewards/chosen": -1.5646464824676514, "rewards/margins": 0.11367311328649521, "rewards/rejected": -1.6783195734024048, "step": 1450 }, { "epoch": 0.78, "learning_rate": 7.101522917128709e-07, "logits/chosen": 0.26174360513687134, "logits/rejected": 0.17661890387535095, "logps/chosen": -409.60528564453125, "logps/rejected": -430.09033203125, "loss": 0.2186, "rewards/accuracies": 0.518750011920929, "rewards/chosen": -1.5871174335479736, "rewards/margins": 0.08051513135433197, "rewards/rejected": -1.6676326990127563, "step": 1460 }, { "epoch": 0.78, "learning_rate": 6.779609134902312e-07, "logits/chosen": 0.14115332067012787, "logits/rejected": 0.21945491433143616, "logps/chosen": -390.1023864746094, "logps/rejected": -373.8792419433594, "loss": 0.2011, "rewards/accuracies": 0.44999998807907104, "rewards/chosen": -1.3179562091827393, "rewards/margins": 0.11220277845859528, "rewards/rejected": -1.4301589727401733, "step": 1470 }, { "epoch": 0.79, "learning_rate": 6.464013863278629e-07, "logits/chosen": 0.105352483689785, "logits/rejected": 0.2724997103214264, "logps/chosen": -462.69989013671875, "logps/rejected": -417.04278564453125, "loss": 0.2099, "rewards/accuracies": 0.4937500059604645, "rewards/chosen": -1.6254713535308838, "rewards/margins": 0.10472464561462402, "rewards/rejected": -1.7301959991455078, "step": 1480 }, { "epoch": 0.79, "learning_rate": 6.154846545138696e-07, "logits/chosen": 0.2331116646528244, "logits/rejected": 0.163321390748024, "logps/chosen": -437.3831481933594, "logps/rejected": -436.51446533203125, "loss": 0.2165, "rewards/accuracies": 0.5375000238418579, "rewards/chosen": -1.76204514503479, "rewards/margins": 0.11055411398410797, "rewards/rejected": -1.8725992441177368, "step": 1490 }, { "epoch": 0.8, "learning_rate": 5.852214394262515e-07, "logits/chosen": 0.22001425921916962, "logits/rejected": 0.24056494235992432, "logps/chosen": -430.39471435546875, "logps/rejected": -417.5421447753906, "loss": 0.2663, "rewards/accuracies": 0.4749999940395355, "rewards/chosen": -1.5459119081497192, "rewards/margins": 0.06696294248104095, "rewards/rejected": -1.612874984741211, "step": 1500 }, { "epoch": 0.81, "learning_rate": 5.556222358149191e-07, "logits/chosen": 0.25534382462501526, "logits/rejected": 0.20129148662090302, "logps/chosen": -448.9977111816406, "logps/rejected": -448.968017578125, "loss": 0.2053, "rewards/accuracies": 0.512499988079071, "rewards/chosen": -1.7130622863769531, "rewards/margins": 0.10370539128780365, "rewards/rejected": -1.816767692565918, "step": 1510 }, { "epoch": 0.81, "learning_rate": 5.266973081622992e-07, "logits/chosen": 0.21835920214653015, "logits/rejected": 0.22384993731975555, "logps/chosen": -436.5664978027344, "logps/rejected": -419.2340393066406, "loss": 0.2178, "rewards/accuracies": 0.46875, "rewards/chosen": -1.622981071472168, "rewards/margins": 0.1084824651479721, "rewards/rejected": -1.7314636707305908, "step": 1520 }, { "epoch": 0.82, "learning_rate": 4.984566871237942e-07, "logits/chosen": 0.20313188433647156, "logits/rejected": 0.18856777250766754, "logps/chosen": -392.1644592285156, "logps/rejected": -397.46185302734375, "loss": 0.1923, "rewards/accuracies": 0.518750011920929, "rewards/chosen": -1.6037871837615967, "rewards/margins": 0.11552910506725311, "rewards/rejected": -1.7193162441253662, "step": 1530 }, { "epoch": 0.82, "learning_rate": 4.709101660493251e-07, "logits/chosen": 0.14777527749538422, "logits/rejected": 0.31510457396507263, "logps/chosen": -404.65362548828125, "logps/rejected": -411.5926818847656, "loss": 0.24, "rewards/accuracies": 0.5062500238418579, "rewards/chosen": -1.5332674980163574, "rewards/margins": 0.08915401995182037, "rewards/rejected": -1.6224216222763062, "step": 1540 }, { "epoch": 0.83, "learning_rate": 4.440672975871743e-07, "logits/chosen": 0.18385054171085358, "logits/rejected": 0.24470014870166779, "logps/chosen": -413.1792907714844, "logps/rejected": -391.4145202636719, "loss": 0.1848, "rewards/accuracies": 0.5062500238418579, "rewards/chosen": -1.3847424983978271, "rewards/margins": 0.12068633735179901, "rewards/rejected": -1.5054289102554321, "step": 1550 }, { "epoch": 0.83, "learning_rate": 4.1793739037129134e-07, "logits/chosen": 0.25826650857925415, "logits/rejected": 0.24141784012317657, "logps/chosen": -424.6695861816406, "logps/rejected": -403.15301513671875, "loss": 0.207, "rewards/accuracies": 0.5062500238418579, "rewards/chosen": -1.5093286037445068, "rewards/margins": 0.11515948921442032, "rewards/rejected": -1.624488115310669, "step": 1560 }, { "epoch": 0.84, "learning_rate": 3.9252950579322405e-07, "logits/chosen": 0.17827372252941132, "logits/rejected": 0.2383604496717453, "logps/chosen": -381.010498046875, "logps/rejected": -394.955322265625, "loss": 0.2787, "rewards/accuracies": 0.48750001192092896, "rewards/chosen": -1.4884707927703857, "rewards/margins": 0.07128787040710449, "rewards/rejected": -1.5597585439682007, "step": 1570 }, { "epoch": 0.84, "learning_rate": 3.6785245485978864e-07, "logits/chosen": 0.20974624156951904, "logits/rejected": 0.21026365458965302, "logps/chosen": -419.64788818359375, "logps/rejected": -432.1351013183594, "loss": 0.1838, "rewards/accuracies": 0.48750001192092896, "rewards/chosen": -1.6164900064468384, "rewards/margins": 0.08999641239643097, "rewards/rejected": -1.7064863443374634, "step": 1580 }, { "epoch": 0.85, "learning_rate": 3.43914795137566e-07, "logits/chosen": 0.16758783161640167, "logits/rejected": 0.19063523411750793, "logps/chosen": -434.1309509277344, "logps/rejected": -426.19061279296875, "loss": 0.1937, "rewards/accuracies": 0.512499988079071, "rewards/chosen": -1.5491647720336914, "rewards/margins": 0.10655365139245987, "rewards/rejected": -1.6557185649871826, "step": 1590 }, { "epoch": 0.85, "learning_rate": 3.207248277852901e-07, "logits/chosen": 0.29995113611221313, "logits/rejected": 0.23389451205730438, "logps/chosen": -431.3026428222656, "logps/rejected": -456.0216369628906, "loss": 0.2003, "rewards/accuracies": 0.5687500238418579, "rewards/chosen": -1.639478325843811, "rewards/margins": 0.15117011964321136, "rewards/rejected": -1.7906484603881836, "step": 1600 }, { "epoch": 0.86, "learning_rate": 2.9829059467515074e-07, "logits/chosen": 0.24911722540855408, "logits/rejected": 0.1267918050289154, "logps/chosen": -411.49237060546875, "logps/rejected": -393.44775390625, "loss": 0.1768, "rewards/accuracies": 0.512499988079071, "rewards/chosen": -1.5094853639602661, "rewards/margins": 0.12646006047725677, "rewards/rejected": -1.6359455585479736, "step": 1610 }, { "epoch": 0.86, "learning_rate": 2.766198756040153e-07, "logits/chosen": 0.18798711895942688, "logits/rejected": 0.22833366692066193, "logps/chosen": -468.82373046875, "logps/rejected": -442.8505859375, "loss": 0.1957, "rewards/accuracies": 0.5687500238418579, "rewards/chosen": -1.6605987548828125, "rewards/margins": 0.10856117308139801, "rewards/rejected": -1.7691596746444702, "step": 1620 }, { "epoch": 0.87, "learning_rate": 2.5572018559553155e-07, "logits/chosen": 0.1967056542634964, "logits/rejected": 0.3702329099178314, "logps/chosen": -437.220947265625, "logps/rejected": -398.580322265625, "loss": 0.2339, "rewards/accuracies": 0.45625001192092896, "rewards/chosen": -1.4941322803497314, "rewards/margins": 0.08855145424604416, "rewards/rejected": -1.582683801651001, "step": 1630 }, { "epoch": 0.87, "learning_rate": 2.3559877229404864e-07, "logits/chosen": 0.12015213072299957, "logits/rejected": 0.25555509328842163, "logps/chosen": -421.092529296875, "logps/rejected": -394.6724548339844, "loss": 0.1893, "rewards/accuracies": 0.5562499761581421, "rewards/chosen": -1.5294889211654663, "rewards/margins": 0.102900430560112, "rewards/rejected": -1.6323894262313843, "step": 1640 }, { "epoch": 0.88, "learning_rate": 2.1626261345126576e-07, "logits/chosen": 0.19819477200508118, "logits/rejected": 0.137999027967453, "logps/chosen": -409.06414794921875, "logps/rejected": -397.3619384765625, "loss": 0.2459, "rewards/accuracies": 0.4375, "rewards/chosen": -1.6599018573760986, "rewards/margins": 0.07884421199560165, "rewards/rejected": -1.738745927810669, "step": 1650 }, { "epoch": 0.89, "learning_rate": 1.9771841450646505e-07, "logits/chosen": 0.22391238808631897, "logits/rejected": 0.34880322217941284, "logps/chosen": -402.15704345703125, "logps/rejected": -385.95684814453125, "loss": 0.2246, "rewards/accuracies": 0.4749999940395355, "rewards/chosen": -1.4918997287750244, "rewards/margins": 0.1032886654138565, "rewards/rejected": -1.5951886177062988, "step": 1660 }, { "epoch": 0.89, "learning_rate": 1.7997260626118758e-07, "logits/chosen": 0.16321547329425812, "logits/rejected": 0.20088446140289307, "logps/chosen": -432.9225158691406, "logps/rejected": -423.73651123046875, "loss": 0.2085, "rewards/accuracies": 0.512499988079071, "rewards/chosen": -1.5294908285140991, "rewards/margins": 0.13079039752483368, "rewards/rejected": -1.6602811813354492, "step": 1670 }, { "epoch": 0.9, "learning_rate": 1.6303134264914365e-07, "logits/chosen": 0.2602766156196594, "logits/rejected": 0.26268571615219116, "logps/chosen": -391.55474853515625, "logps/rejected": -416.79736328125, "loss": 0.203, "rewards/accuracies": 0.512499988079071, "rewards/chosen": -1.5875943899154663, "rewards/margins": 0.08693927526473999, "rewards/rejected": -1.6745338439941406, "step": 1680 }, { "epoch": 0.9, "learning_rate": 1.469004986021355e-07, "logits/chosen": 0.18200412392616272, "logits/rejected": 0.23976688086986542, "logps/chosen": -433.8134765625, "logps/rejected": -419.9029235839844, "loss": 0.1764, "rewards/accuracies": 0.5, "rewards/chosen": -1.5231817960739136, "rewards/margins": 0.10605546087026596, "rewards/rejected": -1.629237413406372, "step": 1690 }, { "epoch": 0.91, "learning_rate": 1.315856680127367e-07, "logits/chosen": 0.22589726746082306, "logits/rejected": 0.2215690314769745, "logps/chosen": -458.8824768066406, "logps/rejected": -451.643310546875, "loss": 0.2159, "rewards/accuracies": 0.5249999761581421, "rewards/chosen": -1.6877247095108032, "rewards/margins": 0.08869151771068573, "rewards/rejected": -1.7764160633087158, "step": 1700 }, { "epoch": 0.91, "learning_rate": 1.1709216179442817e-07, "logits/chosen": 0.21676687896251678, "logits/rejected": 0.26381057500839233, "logps/chosen": -391.4685974121094, "logps/rejected": -410.5079040527344, "loss": 0.2025, "rewards/accuracies": 0.5625, "rewards/chosen": -1.452115535736084, "rewards/margins": 0.15328224003314972, "rewards/rejected": -1.6053978204727173, "step": 1710 }, { "epoch": 0.92, "learning_rate": 1.0342500603986421e-07, "logits/chosen": 0.24852243065834045, "logits/rejected": 0.22877268493175507, "logps/chosen": -399.13787841796875, "logps/rejected": -416.26031494140625, "loss": 0.1899, "rewards/accuracies": 0.46875, "rewards/chosen": -1.4324636459350586, "rewards/margins": 0.1060786023736, "rewards/rejected": -1.5385421514511108, "step": 1720 }, { "epoch": 0.92, "learning_rate": 9.058894027791643e-08, "logits/chosen": 0.08590670675039291, "logits/rejected": 0.2821047306060791, "logps/chosen": -400.29595947265625, "logps/rejected": -401.18670654296875, "loss": 0.2732, "rewards/accuracies": 0.46875, "rewards/chosen": -1.5163061618804932, "rewards/margins": 0.13003432750701904, "rewards/rejected": -1.6463406085968018, "step": 1730 }, { "epoch": 0.93, "learning_rate": 7.858841583008592e-08, "logits/chosen": 0.22975251078605652, "logits/rejected": 0.22381798923015594, "logps/chosen": -436.4466857910156, "logps/rejected": -396.7818908691406, "loss": 0.1904, "rewards/accuracies": 0.48750001192092896, "rewards/chosen": -1.4835584163665771, "rewards/margins": 0.10315178334712982, "rewards/rejected": -1.5867103338241577, "step": 1740 }, { "epoch": 0.93, "learning_rate": 6.742759426686313e-08, "logits/chosen": 0.2513951361179352, "logits/rejected": 0.2925378680229187, "logps/chosen": -405.58123779296875, "logps/rejected": -404.1107177734375, "loss": 0.2141, "rewards/accuracies": 0.5, "rewards/chosen": -1.4388840198516846, "rewards/margins": 0.10872511565685272, "rewards/rejected": -1.5476090908050537, "step": 1750 }, { "epoch": 0.94, "learning_rate": 5.7110345964571104e-08, "logits/chosen": 0.1784476488828659, "logits/rejected": 0.25003939867019653, "logps/chosen": -415.51593017578125, "logps/rejected": -410.5809020996094, "loss": 0.1926, "rewards/accuracies": 0.48750001192092896, "rewards/chosen": -1.519412875175476, "rewards/margins": 0.13301292061805725, "rewards/rejected": -1.65242600440979, "step": 1760 }, { "epoch": 0.94, "learning_rate": 4.764024876318357e-08, "logits/chosen": 0.07363792508840561, "logits/rejected": 0.30069494247436523, "logps/chosen": -460.7679138183594, "logps/rejected": -434.69757080078125, "loss": 0.2058, "rewards/accuracies": 0.543749988079071, "rewards/chosen": -1.5749928951263428, "rewards/margins": 0.14540357887744904, "rewards/rejected": -1.7203963994979858, "step": 1770 }, { "epoch": 0.95, "learning_rate": 3.902058672559633e-08, "logits/chosen": 0.14530272781848907, "logits/rejected": 0.1931276023387909, "logps/chosen": -424.07928466796875, "logps/rejected": -428.74932861328125, "loss": 0.1889, "rewards/accuracies": 0.5, "rewards/chosen": -1.6333696842193604, "rewards/margins": 0.10268044471740723, "rewards/rejected": -1.7360502481460571, "step": 1780 }, { "epoch": 0.95, "learning_rate": 3.125434899876933e-08, "logits/chosen": 0.17779022455215454, "logits/rejected": 0.2716149389743805, "logps/chosen": -426.74224853515625, "logps/rejected": -393.4443664550781, "loss": 0.2089, "rewards/accuracies": 0.512499988079071, "rewards/chosen": -1.4672796726226807, "rewards/margins": 0.13659346103668213, "rewards/rejected": -1.6038730144500732, "step": 1790 }, { "epoch": 0.96, "learning_rate": 2.4344228777145873e-08, "logits/chosen": 0.11182694137096405, "logits/rejected": 0.20840835571289062, "logps/chosen": -430.88555908203125, "logps/rejected": -420.53338623046875, "loss": 0.2144, "rewards/accuracies": 0.5, "rewards/chosen": -1.655738115310669, "rewards/margins": 0.0909801721572876, "rewards/rejected": -1.7467180490493774, "step": 1800 }, { "epoch": 0.97, "learning_rate": 1.829262236869772e-08, "logits/chosen": 0.21950674057006836, "logits/rejected": 0.22139541804790497, "logps/chosen": -420.4927673339844, "logps/rejected": -435.2838439941406, "loss": 0.1756, "rewards/accuracies": 0.581250011920929, "rewards/chosen": -1.5889238119125366, "rewards/margins": 0.13235324621200562, "rewards/rejected": -1.7212769985198975, "step": 1810 }, { "epoch": 0.97, "learning_rate": 1.3101628363929586e-08, "logits/chosen": 0.26217299699783325, "logits/rejected": 0.21528875827789307, "logps/chosen": -405.06927490234375, "logps/rejected": -418.52716064453125, "loss": 0.1739, "rewards/accuracies": 0.48124998807907104, "rewards/chosen": -1.4184885025024414, "rewards/margins": 0.11147931963205338, "rewards/rejected": -1.5299677848815918, "step": 1820 }, { "epoch": 0.98, "learning_rate": 8.773046908123195e-09, "logits/chosen": 0.2996518015861511, "logits/rejected": 0.27609384059906006, "logps/chosen": -472.917724609375, "logps/rejected": -475.19744873046875, "loss": 0.1954, "rewards/accuracies": 0.5375000238418579, "rewards/chosen": -1.7787902355194092, "rewards/margins": 0.10695469379425049, "rewards/rejected": -1.8857448101043701, "step": 1830 }, { "epoch": 0.98, "learning_rate": 5.308379077080817e-09, "logits/chosen": 0.19328948855400085, "logits/rejected": 0.27855589985847473, "logps/chosen": -399.41546630859375, "logps/rejected": -393.7513122558594, "loss": 0.1921, "rewards/accuracies": 0.4625000059604645, "rewards/chosen": -1.5310485363006592, "rewards/margins": 0.09220267832279205, "rewards/rejected": -1.6232513189315796, "step": 1840 }, { "epoch": 0.99, "learning_rate": 2.7088263565760996e-09, "logits/chosen": 0.17895421385765076, "logits/rejected": 0.3152690827846527, "logps/chosen": -428.0615234375, "logps/rejected": -412.861328125, "loss": 0.1983, "rewards/accuracies": 0.5562499761581421, "rewards/chosen": -1.5715734958648682, "rewards/margins": 0.13439974188804626, "rewards/rejected": -1.7059733867645264, "step": 1850 }, { "epoch": 0.99, "learning_rate": 9.752902257023633e-10, "logits/chosen": 0.16039499640464783, "logits/rejected": 0.25197163224220276, "logps/chosen": -391.1293640136719, "logps/rejected": -386.4063415527344, "loss": 0.2198, "rewards/accuracies": 0.45625001192092896, "rewards/chosen": -1.3501429557800293, "rewards/margins": 0.10042072832584381, "rewards/rejected": -1.450563669204712, "step": 1860 }, { "epoch": 1.0, "learning_rate": 1.083718442532189e-10, "logits/chosen": 0.18452344834804535, "logits/rejected": 0.27631479501724243, "logps/chosen": -442.81622314453125, "logps/rejected": -411.86212158203125, "loss": 0.157, "rewards/accuracies": 0.574999988079071, "rewards/chosen": -1.6199710369110107, "rewards/margins": 0.15131933987140656, "rewards/rejected": -1.7712904214859009, "step": 1870 }, { "epoch": 1.0, "step": 1875, "total_flos": 0.0, "train_loss": 0.22404387894471486, "train_runtime": 11451.6187, "train_samples_per_second": 2.62, "train_steps_per_second": 0.164 } ], "logging_steps": 10, "max_steps": 1875, "num_input_tokens_seen": 0, "num_train_epochs": 1, "save_steps": 100, "total_flos": 0.0, "train_batch_size": 4, "trial_name": null, "trial_params": null }