qlora-llama3b-iterative / trainer_log.jsonl
pakphum's picture
Training in progress, step 500
e841950 verified
raw
history blame
18.7 kB
{"current_steps": 10, "total_steps": 500, "loss": 2.1156, "lr": 4e-05, "epoch": 0.08888888888888889, "percentage": 2.0, "elapsed_time": "0:00:24", "remaining_time": "0:20:20"}
{"current_steps": 10, "total_steps": 500, "eval_loss": 1.5894473791122437, "epoch": 0.08888888888888889, "percentage": 2.0, "elapsed_time": "0:00:32", "remaining_time": "0:26:54"}
{"current_steps": 20, "total_steps": 500, "loss": 1.1893, "lr": 8e-05, "epoch": 0.17777777777777778, "percentage": 4.0, "elapsed_time": "0:00:57", "remaining_time": "0:22:50"}
{"current_steps": 20, "total_steps": 500, "eval_loss": 0.6867849826812744, "epoch": 0.17777777777777778, "percentage": 4.0, "elapsed_time": "0:01:05", "remaining_time": "0:26:03"}
{"current_steps": 30, "total_steps": 500, "loss": 0.5218, "lr": 0.00012, "epoch": 0.26666666666666666, "percentage": 6.0, "elapsed_time": "0:01:29", "remaining_time": "0:23:19"}
{"current_steps": 30, "total_steps": 500, "eval_loss": 0.45551854372024536, "epoch": 0.26666666666666666, "percentage": 6.0, "elapsed_time": "0:01:37", "remaining_time": "0:25:25"}
{"current_steps": 40, "total_steps": 500, "loss": 0.5292, "lr": 0.00016, "epoch": 0.35555555555555557, "percentage": 8.0, "elapsed_time": "0:02:01", "remaining_time": "0:23:17"}
{"current_steps": 40, "total_steps": 500, "eval_loss": 0.3795148730278015, "epoch": 0.35555555555555557, "percentage": 8.0, "elapsed_time": "0:02:09", "remaining_time": "0:24:49"}
{"current_steps": 50, "total_steps": 500, "loss": 0.3866, "lr": 0.0002, "epoch": 0.4444444444444444, "percentage": 10.0, "elapsed_time": "0:02:33", "remaining_time": "0:23:03"}
{"current_steps": 50, "total_steps": 500, "eval_loss": 0.30648669600486755, "epoch": 0.4444444444444444, "percentage": 10.0, "elapsed_time": "0:02:41", "remaining_time": "0:24:15"}
{"current_steps": 60, "total_steps": 500, "loss": 0.3232, "lr": 0.00019975640502598244, "epoch": 0.5333333333333333, "percentage": 12.0, "elapsed_time": "0:03:05", "remaining_time": "0:22:43"}
{"current_steps": 60, "total_steps": 500, "eval_loss": 0.20737296342849731, "epoch": 0.5333333333333333, "percentage": 12.0, "elapsed_time": "0:03:13", "remaining_time": "0:23:42"}
{"current_steps": 70, "total_steps": 500, "loss": 0.1802, "lr": 0.00019902680687415705, "epoch": 0.6222222222222222, "percentage": 14.0, "elapsed_time": "0:03:38", "remaining_time": "0:22:19"}
{"current_steps": 70, "total_steps": 500, "eval_loss": 0.15315091609954834, "epoch": 0.6222222222222222, "percentage": 14.0, "elapsed_time": "0:03:46", "remaining_time": "0:23:09"}
{"current_steps": 80, "total_steps": 500, "loss": 0.21, "lr": 0.00019781476007338058, "epoch": 0.7111111111111111, "percentage": 16.0, "elapsed_time": "0:04:10", "remaining_time": "0:21:53"}
{"current_steps": 80, "total_steps": 500, "eval_loss": 0.13480396568775177, "epoch": 0.7111111111111111, "percentage": 16.0, "elapsed_time": "0:04:18", "remaining_time": "0:22:36"}
{"current_steps": 90, "total_steps": 500, "loss": 0.158, "lr": 0.0001961261695938319, "epoch": 0.8, "percentage": 18.0, "elapsed_time": "0:04:42", "remaining_time": "0:21:26"}
{"current_steps": 90, "total_steps": 500, "eval_loss": 0.13721750676631927, "epoch": 0.8, "percentage": 18.0, "elapsed_time": "0:04:50", "remaining_time": "0:22:03"}
{"current_steps": 100, "total_steps": 500, "loss": 0.1629, "lr": 0.00019396926207859084, "epoch": 0.8888888888888888, "percentage": 20.0, "elapsed_time": "0:05:14", "remaining_time": "0:20:58"}
{"current_steps": 100, "total_steps": 500, "eval_loss": 0.12762245535850525, "epoch": 0.8888888888888888, "percentage": 20.0, "elapsed_time": "0:05:22", "remaining_time": "0:21:30"}
{"current_steps": 110, "total_steps": 500, "loss": 0.0966, "lr": 0.0001913545457642601, "epoch": 0.9777777777777777, "percentage": 22.0, "elapsed_time": "0:05:46", "remaining_time": "0:20:29"}
{"current_steps": 110, "total_steps": 500, "eval_loss": 0.10031093657016754, "epoch": 0.9777777777777777, "percentage": 22.0, "elapsed_time": "0:05:54", "remaining_time": "0:20:58"}
{"current_steps": 120, "total_steps": 500, "loss": 0.0643, "lr": 0.00018829475928589271, "epoch": 1.0666666666666667, "percentage": 24.0, "elapsed_time": "0:06:18", "remaining_time": "0:20:00"}
{"current_steps": 120, "total_steps": 500, "eval_loss": 0.08794313669204712, "epoch": 1.0666666666666667, "percentage": 24.0, "elapsed_time": "0:06:27", "remaining_time": "0:20:25"}
{"current_steps": 130, "total_steps": 500, "loss": 0.0726, "lr": 0.0001848048096156426, "epoch": 1.1555555555555554, "percentage": 26.0, "elapsed_time": "0:06:51", "remaining_time": "0:19:30"}
{"current_steps": 130, "total_steps": 500, "eval_loss": 0.08720792084932327, "epoch": 1.1555555555555554, "percentage": 26.0, "elapsed_time": "0:06:59", "remaining_time": "0:19:53"}
{"current_steps": 140, "total_steps": 500, "loss": 0.0493, "lr": 0.00018090169943749476, "epoch": 1.2444444444444445, "percentage": 28.0, "elapsed_time": "0:07:23", "remaining_time": "0:19:00"}
{"current_steps": 140, "total_steps": 500, "eval_loss": 0.09057007730007172, "epoch": 1.2444444444444445, "percentage": 28.0, "elapsed_time": "0:07:31", "remaining_time": "0:19:21"}
{"current_steps": 150, "total_steps": 500, "loss": 0.0746, "lr": 0.0001766044443118978, "epoch": 1.3333333333333333, "percentage": 30.0, "elapsed_time": "0:07:55", "remaining_time": "0:18:29"}
{"current_steps": 150, "total_steps": 500, "eval_loss": 0.058685798197984695, "epoch": 1.3333333333333333, "percentage": 30.0, "elapsed_time": "0:08:03", "remaining_time": "0:18:49"}
{"current_steps": 160, "total_steps": 500, "loss": 0.0473, "lr": 0.0001719339800338651, "epoch": 1.4222222222222223, "percentage": 32.0, "elapsed_time": "0:08:27", "remaining_time": "0:17:59"}
{"current_steps": 160, "total_steps": 500, "eval_loss": 0.0560651533305645, "epoch": 1.4222222222222223, "percentage": 32.0, "elapsed_time": "0:08:35", "remaining_time": "0:18:16"}
{"current_steps": 170, "total_steps": 500, "loss": 0.0644, "lr": 0.00016691306063588583, "epoch": 1.511111111111111, "percentage": 34.0, "elapsed_time": "0:08:59", "remaining_time": "0:17:27"}
{"current_steps": 170, "total_steps": 500, "eval_loss": 0.05025744438171387, "epoch": 1.511111111111111, "percentage": 34.0, "elapsed_time": "0:09:08", "remaining_time": "0:17:43"}
{"current_steps": 180, "total_steps": 500, "loss": 0.0366, "lr": 0.0001615661475325658, "epoch": 1.6, "percentage": 36.0, "elapsed_time": "0:09:31", "remaining_time": "0:16:56"}
{"current_steps": 180, "total_steps": 500, "eval_loss": 0.030684156343340874, "epoch": 1.6, "percentage": 36.0, "elapsed_time": "0:09:40", "remaining_time": "0:17:11"}
{"current_steps": 190, "total_steps": 500, "loss": 0.0247, "lr": 0.0001559192903470747, "epoch": 1.6888888888888889, "percentage": 38.0, "elapsed_time": "0:10:04", "remaining_time": "0:16:25"}
{"current_steps": 190, "total_steps": 500, "eval_loss": 0.023328043520450592, "epoch": 1.6888888888888889, "percentage": 38.0, "elapsed_time": "0:10:12", "remaining_time": "0:16:39"}
{"current_steps": 200, "total_steps": 500, "loss": 0.01, "lr": 0.00015000000000000001, "epoch": 1.7777777777777777, "percentage": 40.0, "elapsed_time": "0:10:36", "remaining_time": "0:15:54"}
{"current_steps": 200, "total_steps": 500, "eval_loss": 0.021545417606830597, "epoch": 1.7777777777777777, "percentage": 40.0, "elapsed_time": "0:10:44", "remaining_time": "0:16:06"}
{"current_steps": 210, "total_steps": 500, "loss": 0.0393, "lr": 0.00014383711467890774, "epoch": 1.8666666666666667, "percentage": 42.0, "elapsed_time": "0:11:08", "remaining_time": "0:15:23"}
{"current_steps": 210, "total_steps": 500, "eval_loss": 0.012232878245413303, "epoch": 1.8666666666666667, "percentage": 42.0, "elapsed_time": "0:11:16", "remaining_time": "0:15:34"}
{"current_steps": 220, "total_steps": 500, "loss": 0.0299, "lr": 0.00013746065934159123, "epoch": 1.9555555555555557, "percentage": 44.0, "elapsed_time": "0:11:40", "remaining_time": "0:14:51"}
{"current_steps": 220, "total_steps": 500, "eval_loss": 0.01798514649271965, "epoch": 1.9555555555555557, "percentage": 44.0, "elapsed_time": "0:11:49", "remaining_time": "0:15:02"}
{"current_steps": 230, "total_steps": 500, "loss": 0.0166, "lr": 0.00013090169943749476, "epoch": 2.0444444444444443, "percentage": 46.0, "elapsed_time": "0:12:12", "remaining_time": "0:14:20"}
{"current_steps": 230, "total_steps": 500, "eval_loss": 0.008207106962800026, "epoch": 2.0444444444444443, "percentage": 46.0, "elapsed_time": "0:12:21", "remaining_time": "0:14:30"}
{"current_steps": 240, "total_steps": 500, "loss": 0.0319, "lr": 0.00012419218955996676, "epoch": 2.1333333333333333, "percentage": 48.0, "elapsed_time": "0:12:45", "remaining_time": "0:13:49"}
{"current_steps": 240, "total_steps": 500, "eval_loss": 0.008276881650090218, "epoch": 2.1333333333333333, "percentage": 48.0, "elapsed_time": "0:12:53", "remaining_time": "0:13:57"}
{"current_steps": 250, "total_steps": 500, "loss": 0.0077, "lr": 0.00011736481776669306, "epoch": 2.2222222222222223, "percentage": 50.0, "elapsed_time": "0:13:17", "remaining_time": "0:13:17"}
{"current_steps": 250, "total_steps": 500, "eval_loss": 0.007150276098400354, "epoch": 2.2222222222222223, "percentage": 50.0, "elapsed_time": "0:13:25", "remaining_time": "0:13:25"}
{"current_steps": 260, "total_steps": 500, "loss": 0.0141, "lr": 0.00011045284632676536, "epoch": 2.311111111111111, "percentage": 52.0, "elapsed_time": "0:13:49", "remaining_time": "0:12:46"}
{"current_steps": 260, "total_steps": 500, "eval_loss": 0.003109171986579895, "epoch": 2.311111111111111, "percentage": 52.0, "elapsed_time": "0:13:57", "remaining_time": "0:12:53"}
{"current_steps": 270, "total_steps": 500, "loss": 0.0017, "lr": 0.00010348994967025012, "epoch": 2.4, "percentage": 54.0, "elapsed_time": "0:14:22", "remaining_time": "0:12:14"}
{"current_steps": 270, "total_steps": 500, "eval_loss": 0.012033730745315552, "epoch": 2.4, "percentage": 54.0, "elapsed_time": "0:14:30", "remaining_time": "0:12:21"}
{"current_steps": 280, "total_steps": 500, "loss": 0.0015, "lr": 9.651005032974994e-05, "epoch": 2.488888888888889, "percentage": 56.0, "elapsed_time": "0:14:54", "remaining_time": "0:11:42"}
{"current_steps": 280, "total_steps": 500, "eval_loss": 0.015280201099812984, "epoch": 2.488888888888889, "percentage": 56.0, "elapsed_time": "0:15:02", "remaining_time": "0:11:48"}
{"current_steps": 290, "total_steps": 500, "loss": 0.0126, "lr": 8.954715367323468e-05, "epoch": 2.5777777777777775, "percentage": 58.0, "elapsed_time": "0:15:26", "remaining_time": "0:11:10"}
{"current_steps": 290, "total_steps": 500, "eval_loss": 0.01406156551092863, "epoch": 2.5777777777777775, "percentage": 58.0, "elapsed_time": "0:15:34", "remaining_time": "0:11:16"}
{"current_steps": 300, "total_steps": 500, "loss": 0.0043, "lr": 8.263518223330697e-05, "epoch": 2.6666666666666665, "percentage": 60.0, "elapsed_time": "0:15:58", "remaining_time": "0:10:39"}
{"current_steps": 300, "total_steps": 500, "eval_loss": 0.0021963752806186676, "epoch": 2.6666666666666665, "percentage": 60.0, "elapsed_time": "0:16:06", "remaining_time": "0:10:44"}
{"current_steps": 310, "total_steps": 500, "loss": 0.0068, "lr": 7.580781044003324e-05, "epoch": 2.7555555555555555, "percentage": 62.0, "elapsed_time": "0:16:30", "remaining_time": "0:10:07"}
{"current_steps": 310, "total_steps": 500, "eval_loss": 0.001871286309324205, "epoch": 2.7555555555555555, "percentage": 62.0, "elapsed_time": "0:16:38", "remaining_time": "0:10:12"}
{"current_steps": 320, "total_steps": 500, "loss": 0.0018, "lr": 6.909830056250527e-05, "epoch": 2.8444444444444446, "percentage": 64.0, "elapsed_time": "0:17:03", "remaining_time": "0:09:35"}
{"current_steps": 320, "total_steps": 500, "eval_loss": 0.002184124430641532, "epoch": 2.8444444444444446, "percentage": 64.0, "elapsed_time": "0:17:11", "remaining_time": "0:09:39"}
{"current_steps": 330, "total_steps": 500, "loss": 0.0026, "lr": 6.25393406584088e-05, "epoch": 2.9333333333333336, "percentage": 66.0, "elapsed_time": "0:17:35", "remaining_time": "0:09:03"}
{"current_steps": 330, "total_steps": 500, "eval_loss": 0.0034216546919196844, "epoch": 2.9333333333333336, "percentage": 66.0, "elapsed_time": "0:17:43", "remaining_time": "0:09:07"}
{"current_steps": 340, "total_steps": 500, "loss": 0.0017, "lr": 5.616288532109225e-05, "epoch": 3.022222222222222, "percentage": 68.0, "elapsed_time": "0:18:07", "remaining_time": "0:08:31"}
{"current_steps": 340, "total_steps": 500, "eval_loss": 0.007565508596599102, "epoch": 3.022222222222222, "percentage": 68.0, "elapsed_time": "0:18:15", "remaining_time": "0:08:35"}
{"current_steps": 350, "total_steps": 500, "loss": 0.0002, "lr": 5.000000000000002e-05, "epoch": 3.111111111111111, "percentage": 70.0, "elapsed_time": "0:18:39", "remaining_time": "0:07:59"}
{"current_steps": 350, "total_steps": 500, "eval_loss": 0.010203778743743896, "epoch": 3.111111111111111, "percentage": 70.0, "elapsed_time": "0:18:47", "remaining_time": "0:08:03"}
{"current_steps": 360, "total_steps": 500, "loss": 0.0004, "lr": 4.4080709652925336e-05, "epoch": 3.2, "percentage": 72.0, "elapsed_time": "0:19:11", "remaining_time": "0:07:27"}
{"current_steps": 360, "total_steps": 500, "eval_loss": 0.011154056526720524, "epoch": 3.2, "percentage": 72.0, "elapsed_time": "0:19:19", "remaining_time": "0:07:31"}
{"current_steps": 370, "total_steps": 500, "loss": 0.006, "lr": 3.843385246743417e-05, "epoch": 3.2888888888888888, "percentage": 74.0, "elapsed_time": "0:19:43", "remaining_time": "0:06:55"}
{"current_steps": 370, "total_steps": 500, "eval_loss": 0.00937813799828291, "epoch": 3.2888888888888888, "percentage": 74.0, "elapsed_time": "0:19:51", "remaining_time": "0:06:58"}
{"current_steps": 380, "total_steps": 500, "loss": 0.0003, "lr": 3.308693936411421e-05, "epoch": 3.3777777777777778, "percentage": 76.0, "elapsed_time": "0:20:16", "remaining_time": "0:06:24"}
{"current_steps": 380, "total_steps": 500, "eval_loss": 0.007455301936715841, "epoch": 3.3777777777777778, "percentage": 76.0, "elapsed_time": "0:20:24", "remaining_time": "0:06:26"}
{"current_steps": 390, "total_steps": 500, "loss": 0.0003, "lr": 2.8066019966134904e-05, "epoch": 3.466666666666667, "percentage": 78.0, "elapsed_time": "0:20:48", "remaining_time": "0:05:52"}
{"current_steps": 390, "total_steps": 500, "eval_loss": 0.006887929514050484, "epoch": 3.466666666666667, "percentage": 78.0, "elapsed_time": "0:20:56", "remaining_time": "0:05:54"}
{"current_steps": 400, "total_steps": 500, "loss": 0.0002, "lr": 2.339555568810221e-05, "epoch": 3.5555555555555554, "percentage": 80.0, "elapsed_time": "0:21:20", "remaining_time": "0:05:20"}
{"current_steps": 400, "total_steps": 500, "eval_loss": 0.006745634134858847, "epoch": 3.5555555555555554, "percentage": 80.0, "elapsed_time": "0:21:28", "remaining_time": "0:05:22"}
{"current_steps": 410, "total_steps": 500, "loss": 0.0005, "lr": 1.9098300562505266e-05, "epoch": 3.6444444444444444, "percentage": 82.0, "elapsed_time": "0:21:52", "remaining_time": "0:04:48"}
{"current_steps": 410, "total_steps": 500, "eval_loss": 0.006609635427594185, "epoch": 3.6444444444444444, "percentage": 82.0, "elapsed_time": "0:22:00", "remaining_time": "0:04:49"}
{"current_steps": 420, "total_steps": 500, "loss": 0.0003, "lr": 1.5195190384357404e-05, "epoch": 3.7333333333333334, "percentage": 84.0, "elapsed_time": "0:22:24", "remaining_time": "0:04:16"}
{"current_steps": 420, "total_steps": 500, "eval_loss": 0.007157918065786362, "epoch": 3.7333333333333334, "percentage": 84.0, "elapsed_time": "0:22:32", "remaining_time": "0:04:17"}
{"current_steps": 430, "total_steps": 500, "loss": 0.0037, "lr": 1.1705240714107302e-05, "epoch": 3.822222222222222, "percentage": 86.0, "elapsed_time": "0:22:57", "remaining_time": "0:03:44"}
{"current_steps": 430, "total_steps": 500, "eval_loss": 0.006302958354353905, "epoch": 3.822222222222222, "percentage": 86.0, "elapsed_time": "0:23:05", "remaining_time": "0:03:45"}
{"current_steps": 440, "total_steps": 500, "loss": 0.004, "lr": 8.645454235739903e-06, "epoch": 3.911111111111111, "percentage": 88.0, "elapsed_time": "0:23:29", "remaining_time": "0:03:12"}
{"current_steps": 440, "total_steps": 500, "eval_loss": 0.005341523326933384, "epoch": 3.911111111111111, "percentage": 88.0, "elapsed_time": "0:23:37", "remaining_time": "0:03:13"}
{"current_steps": 450, "total_steps": 500, "loss": 0.0003, "lr": 6.030737921409169e-06, "epoch": 4.0, "percentage": 90.0, "elapsed_time": "0:24:01", "remaining_time": "0:02:40"}
{"current_steps": 450, "total_steps": 500, "eval_loss": 0.005209792871028185, "epoch": 4.0, "percentage": 90.0, "elapsed_time": "0:24:09", "remaining_time": "0:02:41"}
{"current_steps": 460, "total_steps": 500, "loss": 0.0002, "lr": 3.873830406168111e-06, "epoch": 4.088888888888889, "percentage": 92.0, "elapsed_time": "0:24:33", "remaining_time": "0:02:08"}
{"current_steps": 460, "total_steps": 500, "eval_loss": 0.005074501037597656, "epoch": 4.088888888888889, "percentage": 92.0, "elapsed_time": "0:24:41", "remaining_time": "0:02:08"}
{"current_steps": 470, "total_steps": 500, "loss": 0.0002, "lr": 2.1852399266194314e-06, "epoch": 4.177777777777778, "percentage": 94.0, "elapsed_time": "0:25:05", "remaining_time": "0:01:36"}
{"current_steps": 470, "total_steps": 500, "eval_loss": 0.004982742480933666, "epoch": 4.177777777777778, "percentage": 94.0, "elapsed_time": "0:25:13", "remaining_time": "0:01:36"}
{"current_steps": 480, "total_steps": 500, "loss": 0.0006, "lr": 9.731931258429638e-07, "epoch": 4.266666666666667, "percentage": 96.0, "elapsed_time": "0:25:38", "remaining_time": "0:01:04"}
{"current_steps": 480, "total_steps": 500, "eval_loss": 0.004874282516539097, "epoch": 4.266666666666667, "percentage": 96.0, "elapsed_time": "0:25:46", "remaining_time": "0:01:04"}
{"current_steps": 490, "total_steps": 500, "loss": 0.0005, "lr": 2.4359497401758024e-07, "epoch": 4.355555555555555, "percentage": 98.0, "elapsed_time": "0:26:10", "remaining_time": "0:00:32"}
{"current_steps": 490, "total_steps": 500, "eval_loss": 0.004786411300301552, "epoch": 4.355555555555555, "percentage": 98.0, "elapsed_time": "0:26:18", "remaining_time": "0:00:32"}
{"current_steps": 500, "total_steps": 500, "loss": 0.0002, "lr": 0.0, "epoch": 4.444444444444445, "percentage": 100.0, "elapsed_time": "0:26:42", "remaining_time": "0:00:00"}
{"current_steps": 500, "total_steps": 500, "eval_loss": 0.005105508491396904, "epoch": 4.444444444444445, "percentage": 100.0, "elapsed_time": "0:26:50", "remaining_time": "0:00:00"}
{"current_steps": 500, "total_steps": 500, "epoch": 4.444444444444445, "percentage": 100.0, "elapsed_time": "0:26:51", "remaining_time": "0:00:00"}