Training in progress, step 10000

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4120117d250ccbc68a45f04c769a8427711032a5093688f5e6017217743a6bc2
+oid sha256:c2ac9ebeb8dac5b552a8ccdae781ab5530a4491505556af0c9010fb8f32e60c3
 size 3141646744

runs/Apr25_13-44-28_ip-26-0-167-177/events.out.tfevents.1714052716.ip-26-0-167-177.156194.0

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cf71866770393a20788ee13f1fefaf1e41ffc168f00346f69ee3231932150974
-size 47364
+oid sha256:78f7f1750e79c17ed233ddec97d5a18467e60f4657fc9fc54778c0d6a0117e50
+size 89835

wandb/run-20240425_134518-etajcxpg/files/output.log

@@ -4768,5 +4768,4715 @@
  25%|██▌       | 5024/20000 [2:32:17<7:35:07,  1.82s/it]
 
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 25%|██▌       | 5049/20000 [2:33:03<7:32:43,  1.82s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 25%|██▌       | 5075/20000 [2:33:50<7:34:44,  1.83s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 25%|██▌       | 5099/20000 [2:34:34<7:30:34,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 26%|██▌       | 5124/20000 [2:35:19<7:30:22,  1.82s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 26%|██▌       | 5149/20000 [2:36:04<7:27:53,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 26%|██▌       | 5174/20000 [2:36:50<7:31:46,  1.83s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 26%|██▌       | 5199/20000 [2:37:35<7:29:45,  1.82s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 26%|██▌       | 5224/20000 [2:38:21<7:28:07,  1.82s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 26%|██▋       | 5250/20000 [2:39:08<7:26:20,  1.82s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 26%|██▋       | 5274/20000 [2:39:52<7:24:32,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 26%|██▋       | 5299/20000 [2:40:37<7:23:38,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 27%|██▋       | 5324/20000 [2:41:23<7:23:40,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 27%|██▋       | 5349/20000 [2:42:08<7:22:37,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 27%|██▋       | 5374/20000 [2:42:53<7:25:50,  1.83s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 27%|██▋       | 5399/20000 [2:43:39<7:22:30,  1.82s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 27%|██▋       | 5425/20000 [2:44:26<7:19:05,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 27%|██▋       | 5449/20000 [2:45:10<7:18:31,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 27%|██▋       | 5474/20000 [2:45:55<7:17:40,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 27%|██▋       | 5499/20000 [2:46:40<7:17:56,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|██▊       | 5524/20000 [2:47:25<7:18:23,  1.82s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|██▊       | 5549/20000 [2:48:11<7:17:43,  1.82s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|██▊       | 5574/20000 [2:48:56<7:15:27,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|██▊       | 5599/20000 [2:49:41<7:17:35,  1.82s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|██▊       | 5624/20000 [2:50:27<7:13:04,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|██▊       | 5649/20000 [2:51:12<7:12:32,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|██▊       | 5674/20000 [2:51:57<7:11:09,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 28%|██▊       | 5700/20000 [2:52:44<7:10:12,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|██▊       | 5724/20000 [2:53:28<7:11:26,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|██▊       | 5749/20000 [2:54:13<7:10:57,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|██▉       | 5774/20000 [2:54:58<7:09:36,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|██▉       | 5799/20000 [2:55:43<7:08:03,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|██▉       | 5825/20000 [2:56:30<7:07:08,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|██▉       | 5849/20000 [2:57:13<7:05:22,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|██▉       | 5875/20000 [2:58:00<7:05:08,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 29%|██▉       | 5899/20000 [2:58:44<7:05:27,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|██▉       | 5924/20000 [2:59:29<7:04:18,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|██▉       | 5949/20000 [3:00:14<7:02:06,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|██▉       | 5974/20000 [3:00:59<7:01:58,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|██▉       | 5999/20000 [3:01:44<7:00:25,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|███       | 6024/20000 [3:02:29<6:59:00,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|███       | 6049/20000 [3:03:14<6:59:22,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|███       | 6074/20000 [3:03:59<6:58:00,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 30%|███       | 6100/20000 [3:04:46<6:58:55,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|███       | 6124/20000 [3:05:30<6:57:08,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|███       | 6149/20000 [3:06:15<6:56:35,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|███       | 6174/20000 [3:07:00<6:56:34,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|███       | 6199/20000 [3:07:45<6:56:10,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|███       | 6225/20000 [3:08:32<6:54:11,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|███       | 6249/20000 [3:09:16<6:54:19,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|███▏      | 6274/20000 [3:10:01<6:54:28,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 31%|███▏      | 6299/20000 [3:10:46<6:51:37,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|███▏      | 6324/20000 [3:11:31<6:51:45,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|███▏      | 6349/20000 [3:12:16<6:51:04,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|███▏      | 6374/20000 [3:13:02<6:51:57,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|███▏      | 6400/20000 [3:13:49<6:49:26,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|███▏      | 6424/20000 [3:14:32<6:48:31,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|███▏      | 6449/20000 [3:15:17<6:47:34,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|███▏      | 6474/20000 [3:16:02<6:45:28,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 32%|███▏      | 6499/20000 [3:16:47<6:44:54,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|███▎      | 6524/20000 [3:17:32<6:44:34,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|███▎      | 6549/20000 [3:18:17<6:43:51,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|███▎      | 6574/20000 [3:19:02<6:43:38,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|███▎      | 6599/20000 [3:19:47<6:42:20,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|███▎      | 6624/20000 [3:20:32<6:44:06,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|███▎      | 6649/20000 [3:21:18<6:43:31,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|███▎      | 6674/20000 [3:22:03<6:43:55,  1.82s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 33%|███▎      | 6699/20000 [3:22:48<6:42:19,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|███▎      | 6724/20000 [3:23:34<6:40:28,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|███▍      | 6750/20000 [3:24:21<6:40:40,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|███▍      | 6774/20000 [3:25:04<6:39:02,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|███▍      | 6799/20000 [3:25:49<6:38:31,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|███▍      | 6824/20000 [3:26:34<6:36:28,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|███▍      | 6849/20000 [3:27:19<6:37:02,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|███▍      | 6875/20000 [3:28:06<6:34:41,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 34%|███▍      | 6899/20000 [3:28:50<6:36:05,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|███▍      | 6925/20000 [3:29:37<6:32:28,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|███▍      | 6949/20000 [3:30:20<6:31:52,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|███▍      | 6975/20000 [3:31:07<6:31:24,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|███▍      | 6999/20000 [3:31:50<6:29:58,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|███▌      | 7025/20000 [3:32:37<6:29:30,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|███▌      | 7049/20000 [3:33:20<6:27:50,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|███▌      | 7075/20000 [3:34:07<6:27:59,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 35%|███▌      | 7099/20000 [3:34:50<6:25:41,  1.79s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|███▌      | 7125/20000 [3:35:37<6:26:11,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|███▌      | 7149/20000 [3:36:20<6:26:26,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|███▌      | 7175/20000 [3:37:07<6:25:29,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|███▌      | 7199/20000 [3:37:50<6:24:31,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|███▌      | 7225/20000 [3:38:37<6:23:36,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|███▌      | 7249/20000 [3:39:20<6:23:37,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|███▋      | 7275/20000 [3:40:07<6:22:01,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 36%|███▋      | 7299/20000 [3:40:50<6:21:10,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|███▋      | 7324/20000 [3:41:35<6:21:53,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|███▋      | 7349/20000 [3:42:21<6:20:32,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|███▋      | 7374/20000 [3:43:06<6:18:08,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|███▋      | 7399/20000 [3:43:51<6:17:46,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|███▋      | 7424/20000 [3:44:35<6:16:26,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|███▋      | 7449/20000 [3:45:21<6:17:37,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|███▋      | 7474/20000 [3:46:05<6:15:29,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 37%|███▋      | 7499/20000 [3:46:50<6:14:08,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|███▊      | 7524/20000 [3:47:35<6:13:39,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|███▊      | 7549/20000 [3:48:20<6:11:35,  1.79s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|███▊      | 7575/20000 [3:49:07<6:12:19,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|███▊      | 7599/20000 [3:49:50<6:11:49,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|███▊      | 7625/20000 [3:50:37<6:13:21,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|███▊      | 7649/20000 [3:51:20<6:08:24,  1.79s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|███▊      | 7675/20000 [3:52:07<6:07:49,  1.79s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 38%|███▊      | 7699/20000 [3:52:50<6:09:27,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 39%|███▊      | 7724/20000 [3:53:35<6:06:23,  1.79s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 39%|███▊      | 7749/20000 [3:54:20<6:06:56,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 39%|███▉      | 7774/20000 [3:55:05<6:05:23,  1.79s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 39%|███▉      | 7800/20000 [3:55:51<6:04:06,  1.79s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 39%|███▉      | 7824/20000 [3:56:34<6:04:18,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 39%|███▉      | 7850/20000 [3:57:21<6:03:50,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 39%|███▉      | 7874/20000 [3:58:04<6:01:36,  1.79s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|███▉      | 7900/20000 [3:58:51<6:02:17,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|███▉      | 7924/20000 [3:59:34<6:02:41,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|███▉      | 7950/20000 [4:00:21<6:02:50,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|███▉      | 7974/20000 [4:01:04<6:00:31,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|████      | 8000/20000 [4:01:51<6:00:56,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|████      | 8024/20000 [4:02:35<5:59:09,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|████      | 8050/20000 [4:03:21<5:59:34,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|████      | 8074/20000 [4:04:05<5:57:34,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 40%|████      | 8100/20000 [4:04:51<5:52:52,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|████      | 8124/20000 [4:05:34<5:51:47,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|████      | 8149/20000 [4:06:19<5:51:18,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|████      | 8175/20000 [4:07:05<5:52:12,  1.79s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|████      | 8200/20000 [4:07:49<5:49:41,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|████      | 8224/20000 [4:08:32<5:49:38,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|████      | 8249/20000 [4:09:17<5:49:00,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|████▏     | 8275/20000 [4:10:03<5:50:13,  1.79s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 41%|████▏     | 8299/20000 [4:10:46<5:47:26,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|████▏     | 8324/20000 [4:11:30<5:46:44,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|████▏     | 8350/20000 [4:12:17<5:46:45,  1.79s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|████▏     | 8375/20000 [4:13:02<5:44:39,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|████▏     | 8399/20000 [4:13:44<5:44:50,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|████▏     | 8424/20000 [4:14:29<5:43:29,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|████▏     | 8450/20000 [4:15:15<5:43:06,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|████▏     | 8474/20000 [4:15:58<5:42:37,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 42%|████▏     | 8499/20000 [4:16:43<5:40:15,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 43%|████▎     | 8525/20000 [4:17:29<5:39:23,  1.77s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 43%|████▎     | 8550/20000 [4:18:13<5:40:31,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 43%|████▎     | 8574/20000 [4:18:56<5:37:43,  1.77s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 43%|████▎     | 8599/20000 [4:19:41<5:39:07,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 43%|████▎     | 8625/20000 [4:20:27<5:38:37,  1.79s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 43%|████▎     | 8650/20000 [4:21:12<5:36:21,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 43%|████▎     | 8674/20000 [4:21:54<5:37:26,  1.79s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 44%|████▎     | 8700/20000 [4:22:41<5:36:35,  1.79s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 44%|████▎     | 8725/20000 [4:23:25<5:33:57,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 44%|████▎     | 8749/20000 [4:24:08<5:33:45,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 44%|████▍     | 8774/20000 [4:24:53<5:33:43,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 44%|████▍     | 8800/20000 [4:25:39<5:32:52,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 44%|████▍     | 8825/20000 [4:26:24<5:31:41,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 44%|████▍     | 8849/20000 [4:27:06<5:31:36,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 44%|████▍     | 8874/20000 [4:27:51<5:31:30,  1.79s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 44%|████▍     | 8900/20000 [4:28:37<5:30:27,  1.79s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 45%|████▍     | 8925/20000 [4:29:22<5:28:51,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 45%|████▍     | 8949/20000 [4:30:05<5:29:21,  1.79s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 45%|████▍     | 8975/20000 [4:30:51<5:29:48,  1.79s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 45%|████▌     | 9000/20000 [4:31:36<5:26:14,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 45%|████▌     | 9024/20000 [4:32:18<5:25:26,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 45%|████▌     | 9049/20000 [4:33:03<5:24:41,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 45%|████▌     | 9075/20000 [4:33:50<5:24:45,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 45%|████▌     | 9099/20000 [4:34:32<5:24:40,  1.79s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 46%|████▌     | 9124/20000 [4:35:17<5:23:54,  1.79s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 46%|████▌     | 9150/20000 [4:36:03<5:29:12,  1.82s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 46%|████▌     | 9174/20000 [4:36:46<5:22:26,  1.79s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 46%|████▌     | 9199/20000 [4:37:31<5:20:49,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 46%|████▌     | 9225/20000 [4:38:17<5:20:25,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 46%|████▋     | 9250/20000 [4:39:02<5:20:33,  1.79s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 46%|████▋     | 9274/20000 [4:39:45<5:19:20,  1.79s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 46%|████▋     | 9300/20000 [4:40:31<5:18:41,  1.79s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 47%|████▋     | 9325/20000 [4:41:16<5:17:54,  1.79s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 47%|████▋     | 9349/20000 [4:41:59<5:21:56,  1.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 47%|████▋     | 9374/20000 [4:42:43<5:14:42,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 47%|████▋     | 9400/20000 [4:43:30<5:15:34,  1.79s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 47%|████▋     | 9424/20000 [4:44:13<5:14:26,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 47%|████▋     | 9449/20000 [4:44:57<5:13:20,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 47%|████▋     | 9475/20000 [4:45:44<5:13:40,  1.79s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 47%|████▋     | 9499/20000 [4:46:26<5:13:32,  1.79s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 48%|████▊     | 9524/20000 [4:47:11<5:11:23,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 48%|████▊     | 9550/20000 [4:47:57<5:10:43,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 48%|████▊     | 9575/20000 [4:48:42<5:08:55,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 48%|████▊     | 9599/20000 [4:49:25<5:08:29,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 48%|████▊     | 9625/20000 [4:50:11<5:07:44,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 48%|████▊     | 9650/20000 [4:50:56<5:07:26,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 48%|████▊     | 9674/20000 [4:51:39<5:08:06,  1.79s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 48%|████▊     | 9699/20000 [4:52:23<5:05:06,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 49%|████▊     | 9725/20000 [4:53:10<5:05:13,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 49%|████▉     | 9750/20000 [4:53:54<5:04:18,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 49%|████▉     | 9774/20000 [4:54:37<5:04:35,  1.79s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 49%|████▉     | 9800/20000 [4:55:23<5:02:53,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 49%|████▉     | 9825/20000 [4:56:08<5:01:38,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 49%|████▉     | 9850/20000 [4:56:52<5:01:15,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 49%|████▉     | 9874/20000 [4:57:35<5:00:40,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 50%|████▉     | 9900/20000 [4:58:22<4:59:14,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 50%|████▉     | 9925/20000 [4:59:06<4:58:48,  1.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 50%|████▉     | 9950/20000 [4:59:51<4:59:27,  1.79s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 50%|████▉     | 9975/20000 [5:00:35<5:00:09,  1.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 50%|█████     | 10000/20000 [5:01:20<4:57:06,  1.78s/it][INFO|trainer.py:3614] 2024-04-25 18:46:43,212 >> ***** Running Evaluation *****
+[INFO|trainer.py:3616] 2024-04-25 18:46:43,213 >>   Num examples = 639
+[INFO|trainer.py:3619] 2024-04-25 18:46:43,213 >>   Batch size = 32
+ 67%|██████▋   | 2/3 [00:00<00:00,  4.00it/s]
+[INFO|configuration_utils.py:471] 2024-04-25 18:46:44,750 >> Configuration saved in ./checkpoint-10000/config.json
+[INFO|configuration_utils.py:697] 2024-04-25 18:46:44,752 >> Configuration saved in ./checkpoint-10000/generation_config.json
+{'eval_loss': 1.0057789087295532, 'eval_runtime': 1.5251, 'eval_samples_per_second': 418.982, 'eval_steps_per_second': 1.967, 'epoch': 2.47}
+[INFO|modeling_utils.py:2590] 2024-04-25 18:46:48,667 >> Model weights saved in ./checkpoint-10000/model.safetensors
+[INFO|tokenization_utils_base.py:2488] 2024-04-25 18:46:48,687 >> tokenizer config file saved in ./checkpoint-10000/tokenizer_config.json
+[INFO|tokenization_utils_base.py:2497] 2024-04-25 18:46:48,689 >> Special tokens file saved in ./checkpoint-10000/special_tokens_map.json
+[INFO|tokenization_utils_base.py:2488] 2024-04-25 18:46:58,479 >> tokenizer config file saved in ./tokenizer_config.json
+[INFO|tokenization_utils_base.py:2497] 2024-04-25 18:46:58,481 >> Special tokens file saved in ./special_tokens_map.json
+[INFO|trainer.py:3397] 2024-04-25 18:46:58,508 >> Deleting older checkpoint [checkpoint-5000] due to args.save_total_limit
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 50%|█████     | 10025/20000 [5:02:20<4:58:08,  1.79s/it]
+
+
+
+
+
+
+
+
+
+
+
 
 

wandb/run-20240425_134518-etajcxpg/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 1.1348, "train/grad_norm": 0.8203125, "train/learning_rate": 7.67948717948718e-05, "train/epoch": 1.242274412855377, "train/global_step": 5025, "_timestamp": 1714061862.1020992, "_runtime": 9143.862290143967, "_step": 202, "eval/loss": 1.048352837562561, "eval/runtime": 1.5386, "eval/samples_per_second": 415.323, "eval/steps_per_second": 1.95}
+{"train/loss": 0.9694, "train/grad_norm": 0.765625, "train/learning_rate": 5.115384615384615e-05, "train/epoch": 2.478368355995056, "train/global_step": 10025, "_timestamp": 1714070863.3109484, "_runtime": 18145.071139335632, "_step": 403, "eval/loss": 1.0057789087295532, "eval/runtime": 1.5251, "eval/samples_per_second": 418.982, "eval/steps_per_second": 1.967}

wandb/run-20240425_134518-etajcxpg/run-etajcxpg.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ea7106ec7a3c9fc6ae7e330c0412d38729748b95cc3dd091bc3a351fe0c1f7e1
-size 2359339
+oid sha256:8d5c84a6349f083978b30c5d2558db9c712d66ea8f5e0264543ed8cac8646be3
+size 4787581