diff --git "a/run.log" "b/run.log"
--- "a/run.log"
+++ "b/run.log"
@@ -37385,3 +37385,1346 @@ Time to load utils op: 0.00040650367736816406 seconds
 [2022-12-20 20:49:16,059] [INFO] [engine.py:3269:_save_zero_checkpoint] zero checkpoint saved ./checkpoint-3000/global_step3036/zero_pp_rank_0_mp_rank_00_optim_states.pt
 [2022-12-20 20:49:16,059] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step3036 is ready now!
 [INFO|feature_extraction_utils.py:368] 2022-12-20 20:49:20,274 >> Feature extractor saved in ./preprocessor_config.json
+[2022-12-20 20:51:21,162] [INFO] [timer.py:197:stop] 0/6074, RunningAvgSamplesPerSec=5.875122686635279, CurrSamplesPerSec=5.118360486106803, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|██████    | 3001/5000 [10:29:01<241:59:39, 435.81s/it][2022-12-20 20:51:32,145] [INFO] [timer.py:197:stop] 0/6076, RunningAvgSamplesPerSec=5.875130666194641, CurrSamplesPerSec=5.367977786706339, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|██████    | 3002/5000 [10:29:12<171:08:17, 308.36s/it][2022-12-20 20:51:43,113] [INFO] [timer.py:197:stop] 0/6078, RunningAvgSamplesPerSec=5.875139350458805, CurrSamplesPerSec=5.370491693602941, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|██████    | 3003/5000 [10:29:23<121:33:41, 219.14s/it][2022-12-20 20:51:54,116] [INFO] [logging.py:68:log_dist] [Rank 0] step=3040, skipped=3, lr=[4.364444444444445e-06], mom=[[0.9, 0.999]]
+[2022-12-20 20:51:54,117] [INFO] [timer.py:197:stop] 0/6080, RunningAvgSamplesPerSec=5.875141544801326, CurrSamplesPerSec=5.335915225334919, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|██████    | 3004/5000 [10:29:34<86:52:34, 156.69s/it] [2022-12-20 20:52:05,113] [INFO] [timer.py:197:stop] 0/6082, RunningAvgSamplesPerSec=5.87514513650894, CurrSamplesPerSec=5.318208686681358, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|██████    | 3005/5000 [10:29:45<62:36:52, 112.99s/it][2022-12-20 20:52:16,146] [INFO] [timer.py:197:stop] 0/6084, RunningAvgSamplesPerSec=5.875142082756348, CurrSamplesPerSec=5.305457246489068, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|██████    | 3006/5000 [10:29:56<45:38:29, 82.40s/it] [2022-12-20 20:52:27,173] [INFO] [timer.py:197:stop] 0/6086, RunningAvgSamplesPerSec=5.875140137022045, CurrSamplesPerSec=5.3097983935669255, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|██████    | 3007/5000 [10:30:07<33:45:29, 60.98s/it][2022-12-20 20:52:38,135] [INFO] [timer.py:197:stop] 0/6088, RunningAvgSamplesPerSec=5.87514987678892, CurrSamplesPerSec=5.334614743635069, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|██████    | 3008/5000 [10:30:18<25:26:32, 45.98s/it][2022-12-20 20:52:49,105] [INFO] [timer.py:197:stop] 0/6090, RunningAvgSamplesPerSec=5.875158318778088, CurrSamplesPerSec=5.34843043902184, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|██████    | 3009/5000 [10:30:29<19:37:27, 35.48s/it][2022-12-20 20:53:00,099] [INFO] [timer.py:197:stop] 0/6092, RunningAvgSamplesPerSec=5.875162612941647, CurrSamplesPerSec=5.345334613176732, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|██████    | 3010/5000 [10:30:40<15:33:13, 28.14s/it][2022-12-20 20:53:11,101] [INFO] [timer.py:197:stop] 0/6094, RunningAvgSamplesPerSec=5.875165199593417, CurrSamplesPerSec=5.3400121085292485, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|██████    | 3011/5000 [10:30:51<12:41:56, 22.98s/it][2022-12-20 20:53:22,063] [INFO] [timer.py:197:stop] 0/6096, RunningAvgSamplesPerSec=5.875174976796689, CurrSamplesPerSec=5.340345689587291, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|██████    | 3012/5000 [10:31:02<10:42:00, 19.38s/it][2022-12-20 20:53:32,978] [INFO] [timer.py:197:stop] 0/6098, RunningAvgSamplesPerSec=5.875192878929816, CurrSamplesPerSec=5.378011358686893, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|██████    | 3013/5000 [10:31:13<9:18:14, 16.86s/it] [2022-12-20 20:53:44,006] [INFO] [logging.py:68:log_dist] [Rank 0] step=3050, skipped=3, lr=[4.3422222222222225e-06], mom=[[0.9, 0.999]]
+[2022-12-20 20:53:44,008] [INFO] [timer.py:197:stop] 0/6100, RunningAvgSamplesPerSec=5.875190564344091, CurrSamplesPerSec=5.33053439134569, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|██████    | 3014/5000 [10:31:24<8:20:05, 15.11s/it][2022-12-20 20:53:55,007] [INFO] [timer.py:197:stop] 0/6102, RunningAvgSamplesPerSec=5.875193671286669, CurrSamplesPerSec=5.358543320296199, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|██████    | 3015/5000 [10:31:35<7:39:05, 13.88s/it][2022-12-20 20:54:06,004] [INFO] [timer.py:197:stop] 0/6104, RunningAvgSamplesPerSec=5.875197024987366, CurrSamplesPerSec=5.363621403281703, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|██████    | 3016/5000 [10:31:46<7:09:40, 12.99s/it][2022-12-20 20:54:16,953] [INFO] [timer.py:197:stop] 0/6106, RunningAvgSamplesPerSec=5.875208802025289, CurrSamplesPerSec=5.34943446568799, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|██████    | 3017/5000 [10:31:57<6:49:33, 12.39s/it][2022-12-20 20:54:27,950] [INFO] [timer.py:197:stop] 0/6108, RunningAvgSamplesPerSec=5.875212103648999, CurrSamplesPerSec=5.341567546980284, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|██████    | 3018/5000 [10:32:08<6:35:41, 11.98s/it][2022-12-20 20:54:39,012] [INFO] [timer.py:197:stop] 0/6110, RunningAvgSamplesPerSec=5.875203971792002, CurrSamplesPerSec=5.29928927834003, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|██████    | 3019/5000 [10:32:19<6:25:54, 11.69s/it][2022-12-20 20:54:49,945] [INFO] [timer.py:197:stop] 0/6112, RunningAvgSamplesPerSec=5.87521868821656, CurrSamplesPerSec=5.367790798350218, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|██████    | 3020/5000 [10:32:30<6:18:38, 11.47s/it][2022-12-20 20:55:00,967] [INFO] [timer.py:197:stop] 0/6114, RunningAvgSamplesPerSec=5.875217580039601, CurrSamplesPerSec=5.325037000548145, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|██████    | 3021/5000 [10:32:41<6:14:09, 11.34s/it][2022-12-20 20:55:12,016] [INFO] [timer.py:197:stop] 0/6116, RunningAvgSamplesPerSec=5.875211745587569, CurrSamplesPerSec=5.317629249658798, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|██████    | 3022/5000 [10:32:52<6:10:37, 11.24s/it][2022-12-20 20:55:23,019] [INFO] [timer.py:197:stop] 0/6118, RunningAvgSamplesPerSec=5.875213952589403, CurrSamplesPerSec=5.31906606366517, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|██████    | 3023/5000 [10:33:03<6:08:05, 11.17s/it][2022-12-20 20:55:34,006] [INFO] [logging.py:68:log_dist] [Rank 0] step=3060, skipped=3, lr=[4.32e-06], mom=[[0.9, 0.999]]
+[2022-12-20 20:55:34,007] [INFO] [timer.py:197:stop] 0/6120, RunningAvgSamplesPerSec=5.875218848288103, CurrSamplesPerSec=5.33480557698934, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|██████    | 3024/5000 [10:33:14<6:06:23, 11.13s/it][2022-12-20 20:55:45,039] [INFO] [timer.py:197:stop] 0/6122, RunningAvgSamplesPerSec=5.875216046553836, CurrSamplesPerSec=5.323063425283123, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 60%|██████    | 3025/5000 [10:33:25<6:04:46, 11.08s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 4.3177777777777776e-06, 'epoch': 73.77}
+
 60%|██████    | 3025/5000 [10:33:25<6:04:46, 11.08s/it][2022-12-20 20:55:56,031] [INFO] [timer.py:197:stop] 0/6124, RunningAvgSamplesPerSec=5.875220431909395, CurrSamplesPerSec=5.313250684534988, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3026/5000 [10:33:36<6:04:05, 11.07s/it][2022-12-20 20:56:07,067] [INFO] [timer.py:197:stop] 0/6126, RunningAvgSamplesPerSec=5.8752169523771345, CurrSamplesPerSec=5.308794910052335, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3027/5000 [10:33:47<6:03:35, 11.06s/it][2022-12-20 20:56:18,102] [INFO] [timer.py:197:stop] 0/6128, RunningAvgSamplesPerSec=5.875214034065897, CurrSamplesPerSec=5.309694415228059, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3028/5000 [10:33:58<6:03:00, 11.04s/it][2022-12-20 20:56:29,100] [INFO] [timer.py:197:stop] 0/6130, RunningAvgSamplesPerSec=5.875217472758273, CurrSamplesPerSec=5.324754126645204, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3029/5000 [10:34:09<6:02:23, 11.03s/it][2022-12-20 20:56:40,148] [INFO] [timer.py:197:stop] 0/6132, RunningAvgSamplesPerSec=5.8752118857452365, CurrSamplesPerSec=5.282699170418777, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3030/5000 [10:34:20<6:02:23, 11.04s/it][2022-12-20 20:56:51,141] [INFO] [timer.py:197:stop] 0/6134, RunningAvgSamplesPerSec=5.87521646778979, CurrSamplesPerSec=5.332765427774333, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3031/5000 [10:34:31<6:01:54, 11.03s/it][2022-12-20 20:57:02,134] [INFO] [timer.py:197:stop] 0/6136, RunningAvgSamplesPerSec=5.875220844890712, CurrSamplesPerSec=5.347056961422694, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3032/5000 [10:34:42<6:01:13, 11.01s/it][2022-12-20 20:57:13,122] [INFO] [timer.py:197:stop] 0/6138, RunningAvgSamplesPerSec=5.8752259924068015, CurrSamplesPerSec=5.336429060612356, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3033/5000 [10:34:53<6:01:02, 11.01s/it][2022-12-20 20:57:24,164] [INFO] [logging.py:68:log_dist] [Rank 0] step=3070, skipped=3, lr=[4.297777777777778e-06], mom=[[0.9, 0.999]]
+[2022-12-20 20:57:24,166] [INFO] [timer.py:197:stop] 0/6140, RunningAvgSamplesPerSec=5.8752213918464085, CurrSamplesPerSec=5.309526798116403, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3034/5000 [10:35:04<6:00:42, 11.01s/it][2022-12-20 20:57:34,237] [INFO] [timer.py:197:stop] 0/6142, RunningAvgSamplesPerSec=5.8753859191633735, CurrSamplesPerSec=6.262189267073503, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-20 20:57:45,224] [INFO] [timer.py:197:stop] 0/6144, RunningAvgSamplesPerSec=5.875390684367076, CurrSamplesPerSec=5.333592957312771, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3035/5000 [10:35:20<6:51:03, 12.55s/it][2022-12-20 20:57:56,230] [INFO] [timer.py:197:stop] 0/6146, RunningAvgSamplesPerSec=5.875392736458048, CurrSamplesPerSec=5.3180021819155225, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3036/5000 [10:35:31<6:35:40, 12.09s/it][2022-12-20 20:58:07,244] [INFO] [timer.py:197:stop] 0/6148, RunningAvgSamplesPerSec=5.875392875779637, CurrSamplesPerSec=5.334751294112457, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3037/5000 [10:35:42<6:24:56, 11.77s/it][2022-12-20 20:58:18,262] [INFO] [timer.py:197:stop] 0/6150, RunningAvgSamplesPerSec=5.87539258499823, CurrSamplesPerSec=5.3246867399091276, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3038/5000 [10:35:53<6:17:24, 11.54s/it][2022-12-20 20:58:29,301] [INFO] [timer.py:197:stop] 0/6152, RunningAvgSamplesPerSec=5.875388592712671, CurrSamplesPerSec=5.323922579605136, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3039/5000 [10:36:04<6:12:17, 11.39s/it][2022-12-20 20:58:40,328] [INFO] [timer.py:197:stop] 0/6154, RunningAvgSamplesPerSec=5.875386891070727, CurrSamplesPerSec=5.332875608948258, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3040/5000 [10:36:15<6:08:31, 11.28s/it][2022-12-20 20:58:51,347] [INFO] [timer.py:197:stop] 0/6156, RunningAvgSamplesPerSec=5.875386699436804, CurrSamplesPerSec=5.325776544060893, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3041/5000 [10:36:26<6:05:46, 11.20s/it][2022-12-20 20:59:02,327] [INFO] [timer.py:197:stop] 0/6158, RunningAvgSamplesPerSec=5.875393301675989, CurrSamplesPerSec=5.3415337465722885, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3042/5000 [10:36:37<6:03:24, 11.14s/it][2022-12-20 20:59:13,277] [INFO] [logging.py:68:log_dist] [Rank 0] step=3080, skipped=3, lr=[4.275555555555556e-06], mom=[[0.9, 0.999]]
+[2022-12-20 20:59:13,278] [INFO] [timer.py:197:stop] 0/6160, RunningAvgSamplesPerSec=5.875405086218458, CurrSamplesPerSec=5.364144447640836, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3043/5000 [10:36:48<6:01:24, 11.08s/it][2022-12-20 20:59:24,296] [INFO] [timer.py:197:stop] 0/6162, RunningAvgSamplesPerSec=5.875405067480711, CurrSamplesPerSec=5.325974987969529, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3044/5000 [10:36:59<6:00:36, 11.06s/it][2022-12-20 20:59:35,294] [INFO] [timer.py:197:stop] 0/6164, RunningAvgSamplesPerSec=5.875408392859427, CurrSamplesPerSec=5.350447187676909, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3045/5000 [10:37:10<5:59:48, 11.04s/it][2022-12-20 20:59:46,293] [INFO] [timer.py:197:stop] 0/6166, RunningAvgSamplesPerSec=5.87541185261061, CurrSamplesPerSec=5.343934213471608, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3046/5000 [10:37:21<5:59:11, 11.03s/it][2022-12-20 20:59:57,320] [INFO] [timer.py:197:stop] 0/6168, RunningAvgSamplesPerSec=5.875410233910452, CurrSamplesPerSec=5.300251286394974, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3047/5000 [10:37:32<5:58:59, 11.03s/it][2022-12-20 21:00:08,340] [INFO] [timer.py:197:stop] 0/6170, RunningAvgSamplesPerSec=5.875409622782899, CurrSamplesPerSec=5.304067599513908, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3048/5000 [10:37:43<5:58:42, 11.03s/it][2022-12-20 21:00:19,371] [INFO] [timer.py:197:stop] 0/6172, RunningAvgSamplesPerSec=5.875406870020345, CurrSamplesPerSec=5.292491526132839, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3049/5000 [10:37:54<5:58:34, 11.03s/it][2022-12-20 21:00:30,329] [INFO] [timer.py:197:stop] 0/6174, RunningAvgSamplesPerSec=5.87541702171069, CurrSamplesPerSec=5.343037534419127, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3050/5000 [10:38:05<5:57:42, 11.01s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 4.26e-06, 'epoch': 74.39}
+
 61%|██████    | 3050/5000 [10:38:05<5:57:42, 11.01s/it][2022-12-20 21:00:41,295] [INFO] [timer.py:197:stop] 0/6176, RunningAvgSamplesPerSec=5.875425932188193, CurrSamplesPerSec=5.33471821602604, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3051/5000 [10:38:16<5:57:08, 10.99s/it][2022-12-20 21:00:52,346] [INFO] [timer.py:197:stop] 0/6178, RunningAvgSamplesPerSec=5.875419752058925, CurrSamplesPerSec=5.307340343606874, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3052/5000 [10:38:27<5:57:30, 11.01s/it][2022-12-20 21:01:03,380] [INFO] [logging.py:68:log_dist] [Rank 0] step=3090, skipped=3, lr=[4.253333333333334e-06], mom=[[0.9, 0.999]]
+[2022-12-20 21:01:03,382] [INFO] [timer.py:197:stop] 0/6180, RunningAvgSamplesPerSec=5.875416121444207, CurrSamplesPerSec=5.312549942060481, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██���███    | 3053/5000 [10:38:38<5:57:33, 11.02s/it][2022-12-20 21:01:14,397] [INFO] [timer.py:197:stop] 0/6182, RunningAvgSamplesPerSec=5.875416191920165, CurrSamplesPerSec=5.30522363089849, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3054/5000 [10:38:49<5:57:20, 11.02s/it][2022-12-20 21:01:25,423] [INFO] [timer.py:197:stop] 0/6184, RunningAvgSamplesPerSec=5.8754146101603455, CurrSamplesPerSec=5.327931688265228, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3055/5000 [10:39:01<5:57:14, 11.02s/it][2022-12-20 21:01:36,356] [INFO] [timer.py:197:stop] 0/6186, RunningAvgSamplesPerSec=5.875429429103313, CurrSamplesPerSec=5.360922059053901, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3056/5000 [10:39:11<5:56:12, 10.99s/it][2022-12-20 21:01:47,363] [INFO] [timer.py:197:stop] 0/6188, RunningAvgSamplesPerSec=5.875430814632638, CurrSamplesPerSec=5.329813632458138, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3057/5000 [10:39:22<5:56:09, 11.00s/it][2022-12-20 21:01:58,405] [INFO] [timer.py:197:stop] 0/6190, RunningAvgSamplesPerSec=5.875426277662624, CurrSamplesPerSec=5.306155279897708, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3058/5000 [10:39:34<5:56:23, 11.01s/it][2022-12-20 21:02:09,350] [INFO] [timer.py:197:stop] 0/6192, RunningAvgSamplesPerSec=5.875438891624383, CurrSamplesPerSec=5.353111865021125, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3059/5000 [10:39:44<5:55:33, 10.99s/it][2022-12-20 21:02:20,353] [INFO] [timer.py:197:stop] 0/6194, RunningAvgSamplesPerSec=5.875441272566205, CurrSamplesPerSec=5.3169366193947, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3060/5000 [10:39:55<5:55:29, 10.99s/it][2022-12-20 21:02:31,293] [INFO] [timer.py:197:stop] 0/6196, RunningAvgSamplesPerSec=5.875454789629599, CurrSamplesPerSec=5.372090313643407, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3061/5000 [10:40:06<5:54:47, 10.98s/it][2022-12-20 21:02:42,274] [INFO] [timer.py:197:stop] 0/6198, RunningAvgSamplesPerSec=5.875460861624813, CurrSamplesPerSec=5.359835376550583, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████    | 3062/5000 [10:40:17<5:54:37, 10.98s/it][2022-12-20 21:02:53,284] [INFO] [logging.py:68:log_dist] [Rank 0] step=3100, skipped=3, lr=[4.2311111111111114e-06], mom=[[0.9, 0.999]]
+[2022-12-20 21:02:53,286] [INFO] [timer.py:197:stop] 0/6200, RunningAvgSamplesPerSec=5.875461442902075, CurrSamplesPerSec=5.325920250632796, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████▏   | 3063/5000 [10:40:28<5:54:45, 10.99s/it][2022-12-20 21:03:04,347] [INFO] [timer.py:197:stop] 0/6202, RunningAvgSamplesPerSec=5.875453717439492, CurrSamplesPerSec=5.292645129550349, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████▏   | 3064/5000 [10:40:39<5:55:16, 11.01s/it][2022-12-20 21:03:15,302] [INFO] [timer.py:197:stop] 0/6204, RunningAvgSamplesPerSec=5.875464536084555, CurrSamplesPerSec=5.359400484486505, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████▏   | 3065/5000 [10:40:50<5:54:33, 10.99s/it][2022-12-20 21:03:26,431] [INFO] [timer.py:197:stop] 0/6206, RunningAvgSamplesPerSec=5.875445052505861, CurrSamplesPerSec=5.263628772277219, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████▏   | 3066/5000 [10:41:02<5:55:40, 11.03s/it][2022-12-20 21:03:37,439] [INFO] [timer.py:197:stop] 0/6208, RunningAvgSamplesPerSec=5.875446520257692, CurrSamplesPerSec=5.347423167340086, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████▏   | 3067/5000 [10:41:13<5:55:14, 11.03s/it][2022-12-20 21:03:48,485] [INFO] [timer.py:197:stop] 0/6210, RunningAvgSamplesPerSec=5.875441588749201, CurrSamplesPerSec=5.319366674725249, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████▏   | 3068/5000 [10:41:24<5:55:14, 11.03s/it][2022-12-20 21:03:59,513] [INFO] [timer.py:197:stop] 0/6212, RunningAvgSamplesPerSec=5.875439669538957, CurrSamplesPerSec=5.314527091143623, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████▏   | 3069/5000 [10:41:35<5:55:00, 11.03s/it][2022-12-20 21:04:10,561] [INFO] [timer.py:197:stop] 0/6214, RunningAvgSamplesPerSec=5.875434042459262, CurrSamplesPerSec=5.305987676395846, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████▏   | 3070/5000 [10:41:46<5:54:59, 11.04s/it][2022-12-20 21:04:21,592] [INFO] [timer.py:197:stop] 0/6216, RunningAvgSamplesPerSec=5.875431569680698, CurrSamplesPerSec=5.296343693144335, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████▏   | 3071/5000 [10:41:57<5:54:45, 11.03s/it][2022-12-20 21:04:32,559] [INFO] [timer.py:197:stop] 0/6218, RunningAvgSamplesPerSec=5.875440040521494, CurrSamplesPerSec=5.350358247269186, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████▏   | 3072/5000 [10:42:08<5:53:55, 11.01s/it][2022-12-20 21:04:43,603] [INFO] [logging.py:68:log_dist] [Rank 0] step=3110, skipped=3, lr=[4.208888888888889e-06], mom=[[0.9, 0.999]]
+[2022-12-20 21:04:43,605] [INFO] [timer.py:197:stop] 0/6220, RunningAvgSamplesPerSec=5.8754348591508885, CurrSamplesPerSec=5.310994332152723, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████▏   | 3073/5000 [10:42:19<5:54:02, 11.02s/it][2022-12-20 21:04:54,613] [INFO] [timer.py:197:stop] 0/6222, RunningAvgSamplesPerSec=5.875436265342186, CurrSamplesPerSec=5.331052484428889, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 61%|██████▏   | 3074/5000 [10:42:30<5:53:42, 11.02s/it][2022-12-20 21:05:05,621] [INFO] [timer.py:197:stop] 0/6224, RunningAvgSamplesPerSec=5.875437567245927, CurrSamplesPerSec=5.335063647806382, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3075/5000 [10:42:41<5:53:25, 11.02s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 4.204444444444445e-06, 'epoch': 74.99}
+
 62%|██████▏   | 3075/5000 [10:42:41<5:53:25, 11.02s/it][2022-12-20 21:05:15,656] [INFO] [timer.py:197:stop] 0/6226, RunningAvgSamplesPerSec=5.875606319456273, CurrSamplesPerSec=5.350819617755565, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3076/5000 [10:42:56<6:31:21, 12.20s/it][2022-12-20 21:05:26,634] [INFO] [timer.py:197:stop] 0/6228, RunningAvgSamplesPerSec=5.875612595127981, CurrSamplesPerSec=5.3595562841601625, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3077/5000 [10:43:07<6:19:35, 11.84s/it][2022-12-20 21:05:37,668] [INFO] [timer.py:197:stop] 0/6230, RunningAvgSamplesPerSec=5.875609332267287, CurrSamplesPerSec=5.330502847450952, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3078/5000 [10:43:18<6:11:25, 11.59s/it][2022-12-20 21:05:48,719] [INFO] [timer.py:197:stop] 0/6232, RunningAvgSamplesPerSec=5.875603210604129, CurrSamplesPerSec=5.298652665859183, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3079/5000 [10:43:29<6:05:40, 11.42s/it][2022-12-20 21:05:59,733] [INFO] [timer.py:197:stop] 0/6234, RunningAvgSamplesPerSec=5.875603403086806, CurrSamplesPerSec=5.300252123621727, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3080/5000 [10:43:40<6:02:03, 11.31s/it][2022-12-20 21:06:10,772] [INFO] [timer.py:197:stop] 0/6236, RunningAvgSamplesPerSec=5.875599611061532, CurrSamplesPerSec=5.324005152416805, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3081/5000 [10:43:51<5:59:16, 11.23s/it][2022-12-20 21:06:21,765] [INFO] [timer.py:197:stop] 0/6238, RunningAvgSamplesPerSec=5.875603325306536, CurrSamplesPerSec=5.369089466297792, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3082/5000 [10:44:02<5:56:22, 11.15s/it][2022-12-20 21:06:32,769] [INFO] [logging.py:68:log_dist] [Rank 0] step=3120, skipped=3, lr=[4.1866666666666675e-06], mom=[[0.9, 0.999]]
+[2022-12-20 21:06:32,771] [INFO] [timer.py:197:stop] 0/6240, RunningAvgSamplesPerSec=5.87560490750612, CurrSamplesPerSec=5.3187342925868535, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3083/5000 [10:44:13<5:54:50, 11.11s/it][2022-12-20 21:06:43,815] [INFO] [timer.py:197:stop] 0/6242, RunningAvgSamplesPerSec=5.875599870660265, CurrSamplesPerSec=5.2872399431228, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3084/5000 [10:44:24<5:54:26, 11.10s/it][2022-12-20 21:06:54,811] [INFO] [timer.py:197:stop] 0/6244, RunningAvgSamplesPerSec=5.875603062950178, CurrSamplesPerSec=5.364987106544963, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3085/5000 [10:44:35<5:53:14, 11.07s/it][2022-12-20 21:07:05,790] [INFO] [timer.py:197:stop] 0/6246, RunningAvgSamplesPerSec=5.875609273493416, CurrSamplesPerSec=5.377592688016897, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3086/5000 [10:44:46<5:52:16, 11.04s/it][2022-12-20 21:07:16,859] [INFO] [timer.py:197:stop] 0/6248, RunningAvgSamplesPerSec=5.875599941935591, CurrSamplesPerSec=5.303341197073117, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3087/5000 [10:44:57<5:52:17, 11.05s/it][2022-12-20 21:07:27,918] [INFO] [timer.py:197:stop] 0/6250, RunningAvgSamplesPerSec=5.875592298765122, CurrSamplesPerSec=5.307945249013958, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3088/5000 [10:45:08<5:52:05, 11.05s/it][2022-12-20 21:07:38,948] [INFO] [timer.py:197:stop] 0/6252, RunningAvgSamplesPerSec=5.875589689367221, CurrSamplesPerSec=5.322781394283794, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3089/5000 [10:45:19<5:51:51, 11.05s/it][2022-12-20 21:07:49,961] [INFO] [timer.py:197:stop] 0/6254, RunningAvgSamplesPerSec=5.875590040001721, CurrSamplesPerSec=5.350465317379412, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3090/5000 [10:45:30<5:50:47, 11.02s/it][2022-12-20 21:08:00,925] [INFO] [timer.py:197:stop] 0/6256, RunningAvgSamplesPerSec=5.875599070689484, CurrSamplesPerSec=5.3423603848873205, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3091/5000 [10:45:41<5:50:26, 11.01s/it][2022-12-20 21:08:11,925] [INFO] [timer.py:197:stop] 0/6258, RunningAvgSamplesPerSec=5.875601826769716, CurrSamplesPerSec=5.344804162694101, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3092/5000 [10:45:52<5:49:28, 10.99s/it][2022-12-20 21:08:22,850] [INFO] [logging.py:68:log_dist] [Rank 0] step=3130, skipped=3, lr=[4.164444444444445e-06], mom=[[0.9, 0.999]]
+[2022-12-20 21:08:22,851] [INFO] [timer.py:197:stop] 0/6260, RunningAvgSamplesPerSec=5.875617276772755, CurrSamplesPerSec=5.349955599639249, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3093/5000 [10:46:03<5:49:26, 10.99s/it][2022-12-20 21:08:33,882] [INFO] [timer.py:197:stop] 0/6262, RunningAvgSamplesPerSec=5.875614709231481, CurrSamplesPerSec=5.328375872267355, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3094/5000 [10:46:14<5:49:07, 10.99s/it][2022-12-20 21:08:44,859] [INFO] [timer.py:197:stop] 0/6264, RunningAvgSamplesPerSec=5.875621276986196, CurrSamplesPerSec=5.331503542419439, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3095/5000 [10:46:25<5:49:00, 10.99s/it][2022-12-20 21:08:55,869] [INFO] [timer.py:197:stop] 0/6266, RunningAvgSamplesPerSec=5.875622204226143, CurrSamplesPerSec=5.320642225310853, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3096/5000 [10:46:36<5:49:25, 11.01s/it][2022-12-20 21:09:06,914] [INFO] [timer.py:197:stop] 0/6268, RunningAvgSamplesPerSec=5.875616969970707, CurrSamplesPerSec=5.328927398031122, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3097/5000 [10:46:47<5:49:18, 11.01s/it][2022-12-20 21:09:17,961] [INFO] [timer.py:197:stop] 0/6270, RunningAvgSamplesPerSec=5.87561161308371, CurrSamplesPerSec=5.304475318754092, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3098/5000 [10:46:58<5:49:26, 11.02s/it][2022-12-20 21:09:28,987] [INFO] [timer.py:197:stop] 0/6272, RunningAvgSamplesPerSec=5.8756098384770015, CurrSamplesPerSec=5.323357943142148, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3099/5000 [10:47:09<5:49:15, 11.02s/it][2022-12-20 21:09:40,007] [INFO] [timer.py:197:stop] 0/6274, RunningAvgSamplesPerSec=5.875608993920647, CurrSamplesPerSec=5.325873122475926, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3100/5000 [10:47:20<5:49:11, 11.03s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 4.148888888888889e-06, 'epoch': 75.6}
+
 62%|██████▏   | 3100/5000 [10:47:20<5:49:11, 11.03s/it][2022-12-20 21:09:51,055] [INFO] [timer.py:197:stop] 0/6276, RunningAvgSamplesPerSec=5.875603524674878, CurrSamplesPerSec=5.31501155822723, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3101/5000 [10:47:31<5:48:47, 11.02s/it][2022-12-20 21:10:02,074] [INFO] [timer.py:197:stop] 0/6278, RunningAvgSamplesPerSec=5.8756028379582075, CurrSamplesPerSec=5.301434761111063, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3102/5000 [10:47:42<5:48:54, 11.03s/it][2022-12-20 21:10:13,087] [INFO] [logging.py:68:log_dist] [Rank 0] step=3140, skipped=3, lr=[4.142222222222223e-06], mom=[[0.9, 0.999]]
+[2022-12-20 21:10:13,088] [INFO] [timer.py:197:stop] 0/6280, RunningAvgSamplesPerSec=5.8756031157203745, CurrSamplesPerSec=5.334563220970156, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3103/5000 [10:47:53<5:48:47, 11.03s/it][2022-12-20 21:10:24,103] [INFO] [timer.py:197:stop] 0/6282, RunningAvgSamplesPerSec=5.875603600510592, CurrSamplesPerSec=5.356465739130754, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3104/5000 [10:48:04<5:48:13, 11.02s/it][2022-12-20 21:10:35,094] [INFO] [timer.py:197:stop] 0/6284, RunningAvgSamplesPerSec=5.875607888589488, CurrSamplesPerSec=5.355285561212265, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3105/5000 [10:48:15<5:47:48, 11.01s/it][2022-12-20 21:10:46,136] [INFO] [timer.py:197:stop] 0/6286, RunningAvgSamplesPerSec=5.87560332639956, CurrSamplesPerSec=5.313932256075216, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3106/5000 [10:48:26<5:48:10, 11.03s/it][2022-12-20 21:10:57,170] [INFO] [timer.py:197:stop] 0/6288, RunningAvgSamplesPerSec=5.875600017426598, CurrSamplesPerSec=5.346167752136601, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3107/5000 [10:48:37<5:47:30, 11.01s/it][2022-12-20 21:11:08,187] [INFO] [timer.py:197:stop] 0/6290, RunningAvgSamplesPerSec=5.875599787024507, CurrSamplesPerSec=5.313245846843472, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3108/5000 [10:48:48<5:47:10, 11.01s/it][2022-12-20 21:11:19,168] [INFO] [timer.py:197:stop] 0/6292, RunningAvgSamplesPerSec=5.875605681574856, CurrSamplesPerSec=5.32825826205645, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3109/5000 [10:48:59<5:46:54, 11.01s/it][2022-12-20 21:11:30,165] [INFO] [timer.py:197:stop] 0/6294, RunningAvgSamplesPerSec=5.875608850498775, CurrSamplesPerSec=5.3323327988197, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3110/5000 [10:49:10<5:46:38, 11.00s/it][2022-12-20 21:11:41,165] [INFO] [timer.py:197:stop] 0/6296, RunningAvgSamplesPerSec=5.87561154462533, CurrSamplesPerSec=5.330709689052558, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3111/5000 [10:49:21<5:46:34, 11.01s/it][2022-12-20 21:11:52,200] [INFO] [timer.py:197:stop] 0/6298, RunningAvgSamplesPerSec=5.875608168060725, CurrSamplesPerSec=5.3140008436346795, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3112/5000 [10:49:32<5:46:15, 11.00s/it][2022-12-20 21:12:03,195] [INFO] [logging.py:68:log_dist] [Rank 0] step=3150, skipped=3, lr=[4.12e-06], mom=[[0.9, 0.999]]
+[2022-12-20 21:12:03,197] [INFO] [timer.py:197:stop] 0/6300, RunningAvgSamplesPerSec=5.875611124523405, CurrSamplesPerSec=5.31139344838698, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3113/5000 [10:49:43<5:45:59, 11.00s/it][2022-12-20 21:12:14,166] [INFO] [timer.py:197:stop] 0/6302, RunningAvgSamplesPerSec=5.8756187493680985, CurrSamplesPerSec=5.334053348669427, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3114/5000 [10:49:54<5:45:44, 11.00s/it][2022-12-20 21:12:25,140] [INFO] [timer.py:197:stop] 0/6304, RunningAvgSamplesPerSec=5.875625798594866, CurrSamplesPerSec=5.352168564182508, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3115/5000 [10:50:05<5:45:30, 11.00s/it][2022-12-20 21:12:36,168] [INFO] [timer.py:197:stop] 0/6306, RunningAvgSamplesPerSec=5.875623428158685, CurrSamplesPerSec=5.321727209244086, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3116/5000 [10:50:16<5:45:31, 11.00s/it][2022-12-20 21:12:46,250] [INFO] [timer.py:197:stop] 0/6308, RunningAvgSamplesPerSec=5.87578141331627, CurrSamplesPerSec=6.2929277584795145, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-20 21:12:57,254] [INFO] [timer.py:197:stop] 0/6310, RunningAvgSamplesPerSec=5.875782690699931, CurrSamplesPerSec=5.3575266760207425, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3117/5000 [10:50:32<6:33:43, 12.55s/it][2022-12-20 21:13:08,247] [INFO] [timer.py:197:stop] 0/6312, RunningAvgSamplesPerSec=5.875786610263724, CurrSamplesPerSec=5.312993878394825, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3118/5000 [10:50:43<6:18:54, 12.08s/it][2022-12-20 21:13:19,263] [INFO] [timer.py:197:stop] 0/6314, RunningAvgSamplesPerSec=5.875786367770704, CurrSamplesPerSec=5.310380327315196, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3119/5000 [10:50:54<6:08:42, 11.76s/it][2022-12-20 21:13:30,277] [INFO] [timer.py:197:stop] 0/6316, RunningAvgSamplesPerSec=5.875786689064704, CurrSamplesPerSec=5.345114501761743, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3120/5000 [10:51:05<6:01:29, 11.54s/it][2022-12-20 21:13:41,256] [INFO] [timer.py:197:stop] 0/6318, RunningAvgSamplesPerSec=5.875792878768084, CurrSamplesPerSec=5.338806889715256, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3121/5000 [10:51:16<5:56:02, 11.37s/it][2022-12-20 21:13:52,300] [INFO] [logging.py:68:log_dist] [Rank 0] step=3160, skipped=3, lr=[4.097777777777778e-06], mom=[[0.9, 0.999]]
+[2022-12-20 21:13:52,301] [INFO] [timer.py:197:stop] 0/6320, RunningAvgSamplesPerSec=5.875787802305916, CurrSamplesPerSec=5.31391479388119, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3122/5000 [10:51:27<5:52:48, 11.27s/it][2022-12-20 21:14:03,306] [INFO] [timer.py:197:stop] 0/6322, RunningAvgSamplesPerSec=5.875789882997917, CurrSamplesPerSec=5.3473255927406775, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3123/5000 [10:51:38<5:50:07, 11.19s/it][2022-12-20 21:14:14,293] [INFO] [timer.py:197:stop] 0/6324, RunningAvgSamplesPerSec=5.875794807417099, CurrSamplesPerSec=5.358853545267827, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▏   | 3124/5000 [10:51:49<5:48:00, 11.13s/it][2022-12-20 21:14:25,308] [INFO] [timer.py:197:stop] 0/6326, RunningAvgSamplesPerSec=5.875794975849509, CurrSamplesPerSec=5.306514016341192, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 62%|██████▎   | 3125/5000 [10:52:00<5:46:44, 11.10s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 4.091111111111111e-06, 'epoch': 76.22}
+
 62%|██████▎   | 3125/5000 [10:52:00<5:46:44, 11.10s/it][2022-12-20 21:14:36,348] [INFO] [timer.py:197:stop] 0/6328, RunningAvgSamplesPerSec=5.875790888602485, CurrSamplesPerSec=5.310986136081386, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██���███▎   | 3126/5000 [10:52:11<5:46:02, 11.08s/it][2022-12-20 21:14:47,333] [INFO] [timer.py:197:stop] 0/6330, RunningAvgSamplesPerSec=5.875796103722073, CurrSamplesPerSec=5.34866510449421, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3127/5000 [10:52:22<5:44:58, 11.05s/it][2022-12-20 21:14:58,241] [INFO] [timer.py:197:stop] 0/6332, RunningAvgSamplesPerSec=5.875814304073243, CurrSamplesPerSec=5.385724130531104, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3128/5000 [10:52:33<5:43:27, 11.01s/it][2022-12-20 21:15:09,263] [INFO] [timer.py:197:stop] 0/6334, RunningAvgSamplesPerSec=5.875813232499461, CurrSamplesPerSec=5.319812172809861, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3129/5000 [10:52:44<5:43:23, 11.01s/it][2022-12-20 21:15:20,262] [INFO] [timer.py:197:stop] 0/6336, RunningAvgSamplesPerSec=5.875815891111635, CurrSamplesPerSec=5.347012866711517, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3130/5000 [10:52:55<5:43:05, 11.01s/it][2022-12-20 21:15:31,298] [INFO] [timer.py:197:stop] 0/6338, RunningAvgSamplesPerSec=5.875812305076677, CurrSamplesPerSec=5.304510328932238, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3131/5000 [10:53:06<5:43:09, 11.02s/it][2022-12-20 21:15:42,338] [INFO] [logging.py:68:log_dist] [Rank 0] step=3170, skipped=3, lr=[4.075555555555556e-06], mom=[[0.9, 0.999]]
+[2022-12-20 21:15:42,339] [INFO] [timer.py:197:stop] 0/6340, RunningAvgSamplesPerSec=5.875807642541231, CurrSamplesPerSec=5.325972663194898, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3132/5000 [10:53:17<5:43:12, 11.02s/it][2022-12-20 21:15:53,287] [INFO] [timer.py:197:stop] 0/6342, RunningAvgSamplesPerSec=5.875819078930652, CurrSamplesPerSec=5.3852042159215365, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3133/5000 [10:53:28<5:42:19, 11.00s/it][2022-12-20 21:16:04,273] [INFO] [timer.py:197:stop] 0/6344, RunningAvgSamplesPerSec=5.875824032444822, CurrSamplesPerSec=5.334759563698155, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3134/5000 [10:53:39<5:41:59, 11.00s/it][2022-12-20 21:16:15,294] [INFO] [timer.py:197:stop] 0/6346, RunningAvgSamplesPerSec=5.875823489378198, CurrSamplesPerSec=5.328289356337546, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3135/5000 [10:53:50<5:42:02, 11.00s/it][2022-12-20 21:16:26,282] [INFO] [timer.py:197:stop] 0/6348, RunningAvgSamplesPerSec=5.8758283019878395, CurrSamplesPerSec=5.312495059774893, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3136/5000 [10:54:01<5:41:42, 11.00s/it][2022-12-20 21:16:37,324] [INFO] [timer.py:197:stop] 0/6350, RunningAvgSamplesPerSec=5.8758240164387505, CurrSamplesPerSec=5.317266902317515, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3137/5000 [10:54:12<5:41:55, 11.01s/it][2022-12-20 21:16:48,321] [INFO] [timer.py:197:stop] 0/6352, RunningAvgSamplesPerSec=5.875827262274412, CurrSamplesPerSec=5.310300487699732, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3138/5000 [10:54:23<5:41:35, 11.01s/it][2022-12-20 21:16:59,316] [INFO] [timer.py:197:stop] 0/6354, RunningAvgSamplesPerSec=5.87583074297457, CurrSamplesPerSec=5.360196056622872, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3139/5000 [10:54:34<5:41:17, 11.00s/it][2022-12-20 21:17:10,309] [INFO] [timer.py:197:stop] 0/6356, RunningAvgSamplesPerSec=5.875834519934735, CurrSamplesPerSec=5.329756276468057, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3140/5000 [10:54:45<5:41:01, 11.00s/it][2022-12-20 21:17:21,269] [INFO] [timer.py:197:stop] 0/6358, RunningAvgSamplesPerSec=5.875844222828617, CurrSamplesPerSec=5.3575777877818815, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3141/5000 [10:54:56<5:40:27, 10.99s/it][2022-12-20 21:17:32,219] [INFO] [logging.py:68:log_dist] [Rank 0] step=3180, skipped=3, lr=[4.053333333333333e-06], mom=[[0.9, 0.999]]
+[2022-12-20 21:17:32,221] [INFO] [timer.py:197:stop] 0/6360, RunningAvgSamplesPerSec=5.875855027408668, CurrSamplesPerSec=5.326508894511631, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3142/5000 [10:55:07<5:39:55, 10.98s/it][2022-12-20 21:17:43,214] [INFO] [timer.py:197:stop] 0/6362, RunningAvgSamplesPerSec=5.875858775970194, CurrSamplesPerSec=5.367408274930487, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3143/5000 [10:55:18<5:39:53, 10.98s/it][2022-12-20 21:17:54,223] [INFO] [timer.py:197:stop] 0/6364, RunningAvgSamplesPerSec=5.875859907437065, CurrSamplesPerSec=5.348037244677403, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3144/5000 [10:55:29<5:39:57, 10.99s/it][2022-12-20 21:18:05,177] [INFO] [timer.py:197:stop] 0/6366, RunningAvgSamplesPerSec=5.875870309526034, CurrSamplesPerSec=5.346936181989491, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3145/5000 [10:55:40<5:39:26, 10.98s/it][2022-12-20 21:18:16,166] [INFO] [timer.py:197:stop] 0/6368, RunningAvgSamplesPerSec=5.8758746784421065, CurrSamplesPerSec=5.33175981123773, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3146/5000 [10:55:51<5:39:21, 10.98s/it][2022-12-20 21:18:26,508] [INFO] [stage_1_and_2.py:1765:step] [deepspeed] OVERFLOW! Rank 0 Skipping step. Attempted loss scale: 131072.0, reducing to 131072.0
+[2022-12-20 21:18:26,510] [INFO] [timer.py:197:stop] 0/6370, RunningAvgSamplesPerSec=5.875988284750518, CurrSamplesPerSec=5.9390995257622645, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3147/5000 [10:56:02<5:33:14, 10.79s/it][2022-12-20 21:18:36,903] [INFO] [stage_1_and_2.py:1765:step] [deepspeed] OVERFLOW! Rank 0 Skipping step. Attempted loss scale: 131072.0, reducing to 65536.0
+[2022-12-20 21:18:36,905] [INFO] [timer.py:197:stop] 0/6372, RunningAvgSamplesPerSec=5.876093041006348, CurrSamplesPerSec=5.932779524200002, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3148/5000 [10:56:12<5:29:24, 10.67s/it][2022-12-20 21:18:47,870] [INFO] [timer.py:197:stop] 0/6374, RunningAvgSamplesPerSec=5.876101254158571, CurrSamplesPerSec=5.355648834751575, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3149/5000 [10:56:23<5:31:56, 10.76s/it][2022-12-20 21:18:58,946] [INFO] [timer.py:197:stop] 0/6376, RunningAvgSamplesPerSec=5.876090683937146, CurrSamplesPerSec=5.291397777015493, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3150/5000 [10:56:34<5:34:41, 10.85s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 4.04e-06, 'epoch': 76.82}
+
 63%|██████▎   | 3150/5000 [10:56:34<5:34:41, 10.85s/it][2022-12-20 21:19:09,901] [INFO] [timer.py:197:stop] 0/6378, RunningAvgSamplesPerSec=5.8761012120645475, CurrSamplesPerSec=5.351963681953357, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3151/5000 [10:56:45<5:35:25, 10.88s/it][2022-12-20 21:19:20,947] [INFO] [logging.py:68:log_dist] [Rank 0] step=3190, skipped=5, lr=[4.035555555555556e-06], mom=[[0.9, 0.999]]
+[2022-12-20 21:19:20,949] [INFO] [timer.py:197:stop] 0/6380, RunningAvgSamplesPerSec=5.8760955506500965, CurrSamplesPerSec=5.3080676321433415, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3152/5000 [10:56:56<5:36:45, 10.93s/it][2022-12-20 21:19:31,931] [INFO] [timer.py:197:stop] 0/6382, RunningAvgSamplesPerSec=5.87610101982182, CurrSamplesPerSec=5.318680757803182, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3153/5000 [10:57:07<5:37:01, 10.95s/it][2022-12-20 21:19:42,935] [INFO] [timer.py:197:stop] 0/6384, RunningAvgSamplesPerSec=5.87610297034574, CurrSamplesPerSec=5.306512547730559, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3154/5000 [10:57:18<5:37:21, 10.96s/it][2022-12-20 21:19:53,985] [INFO] [timer.py:197:stop] 0/6386, RunningAvgSamplesPerSec=5.876097098631627, CurrSamplesPerSec=5.309603883869695, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3155/5000 [10:57:29<5:37:57, 10.99s/it][2022-12-20 21:20:04,988] [INFO] [timer.py:197:stop] 0/6388, RunningAvgSamplesPerSec=5.876099063138022, CurrSamplesPerSec=5.341944482398727, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3156/5000 [10:57:40<5:37:53, 10.99s/it][2022-12-20 21:20:15,981] [INFO] [timer.py:197:stop] 0/6390, RunningAvgSamplesPerSec=5.87610258844795, CurrSamplesPerSec=5.354727711755498, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3157/5000 [10:57:51<5:37:42, 10.99s/it][2022-12-20 21:20:26,081] [INFO] [timer.py:197:stop] 0/6392, RunningAvgSamplesPerSec=5.876255369318709, CurrSamplesPerSec=5.332155699675131, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3158/5000 [10:58:06<6:15:00, 12.22s/it][2022-12-20 21:20:37,126] [INFO] [timer.py:197:stop] 0/6394, RunningAvgSamplesPerSec=5.876250159910572, CurrSamplesPerSec=5.319182003597226, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3159/5000 [10:58:17<6:03:42, 11.85s/it][2022-12-20 21:20:48,105] [INFO] [timer.py:197:stop] 0/6396, RunningAvgSamplesPerSec=5.876255961658893, CurrSamplesPerSec=5.346189685957223, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3160/5000 [10:58:28<5:55:47, 11.60s/it][2022-12-20 21:20:59,198] [INFO] [timer.py:197:stop] 0/6398, RunningAvgSamplesPerSec=5.876242603432418, CurrSamplesPerSec=5.276474861999291, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3161/5000 [10:58:39<5:50:34, 11.44s/it][2022-12-20 21:21:10,212] [INFO] [logging.py:68:log_dist] [Rank 0] step=3200, skipped=5, lr=[4.013333333333334e-06], mom=[[0.9, 0.999]]
+[2022-12-20 21:21:10,213] [INFO] [timer.py:197:stop] 0/6400, RunningAvgSamplesPerSec=5.8762422826888905, CurrSamplesPerSec=5.312503470779569, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3162/5000 [10:58:50<5:46:30, 11.31s/it][2022-12-20 21:21:21,184] [INFO] [timer.py:197:stop] 0/6402, RunningAvgSamplesPerSec=5.876249427693909, CurrSamplesPerSec=5.352740823575111, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3163/5000 [10:59:01<5:43:10, 11.21s/it][2022-12-20 21:21:32,197] [INFO] [timer.py:197:stop] 0/6404, RunningAvgSamplesPerSec=5.876249531632792, CurrSamplesPerSec=5.313683378052247, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3164/5000 [10:59:12<5:41:30, 11.16s/it][2022-12-20 21:21:43,263] [INFO] [timer.py:197:stop] 0/6406, RunningAvgSamplesPerSec=5.876240704356938, CurrSamplesPerSec=5.296535978420769, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3165/5000 [10:59:23<5:40:18, 11.13s/it][2022-12-20 21:21:54,208] [INFO] [timer.py:197:stop] 0/6408, RunningAvgSamplesPerSec=5.876252614638468, CurrSamplesPerSec=5.392621080394686, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3166/5000 [10:59:34<5:38:27, 11.07s/it][2022-12-20 21:22:05,252] [INFO] [timer.py:197:stop] 0/6410, RunningAvgSamplesPerSec=5.8762475570118164, CurrSamplesPerSec=5.304394189159511, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3167/5000 [10:59:45<5:38:17, 11.07s/it][2022-12-20 21:22:16,327] [INFO] [timer.py:197:stop] 0/6412, RunningAvgSamplesPerSec=5.876237125687304, CurrSamplesPerSec=5.30281234368005, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3168/5000 [10:59:56<5:37:30, 11.05s/it][2022-12-20 21:22:27,323] [INFO] [timer.py:197:stop] 0/6414, RunningAvgSamplesPerSec=5.876240386141141, CurrSamplesPerSec=5.315047970486134, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3169/5000 [11:00:07<5:36:57, 11.04s/it][2022-12-20 21:22:38,332] [INFO] [timer.py:197:stop] 0/6416, RunningAvgSamplesPerSec=5.876241208720822, CurrSamplesPerSec=5.319394924635874, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3170/5000 [11:00:18<5:36:39, 11.04s/it][2022-12-20 21:22:49,363] [INFO] [timer.py:197:stop] 0/6418, RunningAvgSamplesPerSec=5.876238226311194, CurrSamplesPerSec=5.317464923233397, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3171/5000 [11:00:29<5:36:34, 11.04s/it][2022-12-20 21:23:00,437] [INFO] [logging.py:68:log_dist] [Rank 0] step=3210, skipped=5, lr=[3.991111111111112e-06], mom=[[0.9, 0.999]]
+[2022-12-20 21:23:00,438] [INFO] [timer.py:197:stop] 0/6420, RunningAvgSamplesPerSec=5.876228189137, CurrSamplesPerSec=5.293966570610825, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3172/5000 [11:00:40<5:36:41, 11.05s/it][2022-12-20 21:23:11,481] [INFO] [timer.py:197:stop] 0/6422, RunningAvgSamplesPerSec=5.876223259937705, CurrSamplesPerSec=5.320739672240535, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3173/5000 [11:00:52<5:36:28, 11.05s/it][2022-12-20 21:23:22,499] [INFO] [timer.py:197:stop] 0/6424, RunningAvgSamplesPerSec=5.876222503710531, CurrSamplesPerSec=5.347268285301791, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 63%|██████▎   | 3174/5000 [11:01:03<5:35:31, 11.03s/it][2022-12-20 21:23:33,470] [INFO] [timer.py:197:stop] 0/6426, RunningAvgSamplesPerSec=5.876229675108204, CurrSamplesPerSec=5.343587844823914, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▎   | 3175/5000 [11:01:14<5:35:13, 11.02s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 3.984444444444444e-06, 'epoch': 77.43}
+
 64%|██████▎   | 3175/5000 [11:01:14<5:35:13, 11.02s/it][2022-12-20 21:23:44,447] [INFO] [timer.py:197:stop] 0/6428, RunningAvgSamplesPerSec=5.876236012038305, CurrSamplesPerSec=5.376032351268899, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▎   | 3176/5000 [11:01:25<5:34:42, 11.01s/it][2022-12-20 21:23:55,485] [INFO] [timer.py:197:stop] 0/6430, RunningAvgSamplesPerSec=5.876231974081519, CurrSamplesPerSec=5.327617208471063, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▎   | 3177/5000 [11:01:36<5:34:20, 11.00s/it][2022-12-20 21:24:06,489] [INFO] [timer.py:197:stop] 0/6432, RunningAvgSamplesPerSec=5.87623357983966, CurrSamplesPerSec=5.315034710472819, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▎   | 3178/5000 [11:01:47<5:34:48, 11.03s/it][2022-12-20 21:24:17,573] [INFO] [timer.py:197:stop] 0/6434, RunningAvgSamplesPerSec=5.876221957645018, CurrSamplesPerSec=5.307427229935187, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▎   | 3179/5000 [11:01:58<5:34:50, 11.03s/it][2022-12-20 21:24:28,622] [INFO] [timer.py:197:stop] 0/6436, RunningAvgSamplesPerSec=5.8762160861004995, CurrSamplesPerSec=5.307512230183054, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▎   | 3180/5000 [11:02:09<5:34:16, 11.02s/it][2022-12-20 21:24:39,589] [INFO] [timer.py:197:stop] 0/6438, RunningAvgSamplesPerSec=5.8762242507962545, CurrSamplesPerSec=5.32945279705313, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▎   | 3181/5000 [11:02:20<5:33:21, 11.00s/it][2022-12-20 21:24:50,520] [INFO] [logging.py:68:log_dist] [Rank 0] step=3220, skipped=5, lr=[3.96888888888889e-06], mom=[[0.9, 0.999]]
+[2022-12-20 21:24:50,522] [INFO] [timer.py:197:stop] 0/6440, RunningAvgSamplesPerSec=5.876237790325284, CurrSamplesPerSec=5.334250501867993, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▎   | 3182/5000 [11:02:31<5:32:53, 10.99s/it][2022-12-20 21:25:01,511] [INFO] [timer.py:197:stop] 0/6442, RunningAvgSamplesPerSec=5.876244336048273, CurrSamplesPerSec=5.324572249741662, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▎   | 3183/5000 [11:02:42<5:33:03, 11.00s/it][2022-12-20 21:25:12,588] [INFO] [timer.py:197:stop] 0/6444, RunningAvgSamplesPerSec=5.8762339286776815, CurrSamplesPerSec=5.267048345117353, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▎   | 3184/5000 [11:02:53<5:33:04, 11.00s/it][2022-12-20 21:25:23,555] [INFO] [timer.py:197:stop] 0/6446, RunningAvgSamplesPerSec=5.876241751250619, CurrSamplesPerSec=5.312695248912615, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▎   | 3185/5000 [11:03:04<5:33:03, 11.01s/it][2022-12-20 21:25:34,568] [INFO] [timer.py:197:stop] 0/6448, RunningAvgSamplesPerSec=5.876241997190707, CurrSamplesPerSec=5.321844320294171, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▎   | 3186/5000 [11:03:15<5:33:00, 11.01s/it][2022-12-20 21:25:45,599] [INFO] [timer.py:197:stop] 0/6450, RunningAvgSamplesPerSec=5.876238983452693, CurrSamplesPerSec=5.315616318956953, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▎   | 3187/5000 [11:03:26<5:32:33, 11.01s/it][2022-12-20 21:25:56,593] [INFO] [timer.py:197:stop] 0/6452, RunningAvgSamplesPerSec=5.876242463045907, CurrSamplesPerSec=5.30939321611673, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3188/5000 [11:03:37<5:32:34, 11.01s/it][2022-12-20 21:26:07,596] [INFO] [timer.py:197:stop] 0/6454, RunningAvgSamplesPerSec=5.876244648670828, CurrSamplesPerSec=5.332781742776376, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3189/5000 [11:03:48<5:32:19, 11.01s/it][2022-12-20 21:26:18,583] [INFO] [timer.py:197:stop] 0/6456, RunningAvgSamplesPerSec=5.876249417688414, CurrSamplesPerSec=5.349450882843727, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3190/5000 [11:03:59<5:32:13, 11.01s/it][2022-12-20 21:26:29,658] [INFO] [timer.py:197:stop] 0/6458, RunningAvgSamplesPerSec=5.87623911382305, CurrSamplesPerSec=5.297866682466066, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3191/5000 [11:04:10<5:32:12, 11.02s/it][2022-12-20 21:26:40,663] [INFO] [logging.py:68:log_dist] [Rank 0] step=3230, skipped=5, lr=[3.946666666666667e-06], mom=[[0.9, 0.999]]
+[2022-12-20 21:26:40,665] [INFO] [timer.py:197:stop] 0/6460, RunningAvgSamplesPerSec=5.876240351516524, CurrSamplesPerSec=5.319230910684234, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3192/5000 [11:04:21<5:32:02, 11.02s/it][2022-12-20 21:26:51,718] [INFO] [timer.py:197:stop] 0/6462, RunningAvgSamplesPerSec=5.876233874451713, CurrSamplesPerSec=5.29088590253055, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3193/5000 [11:04:32<5:32:08, 11.03s/it][2022-12-20 21:27:02,702] [INFO] [timer.py:197:stop] 0/6464, RunningAvgSamplesPerSec=5.876239042473475, CurrSamplesPerSec=5.35012087404007, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3194/5000 [11:04:43<5:31:36, 11.02s/it][2022-12-20 21:27:13,710] [INFO] [timer.py:197:stop] 0/6466, RunningAvgSamplesPerSec=5.876240278084935, CurrSamplesPerSec=5.335167985906205, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3195/5000 [11:04:54<5:30:59, 11.00s/it][2022-12-20 21:27:24,684] [INFO] [timer.py:197:stop] 0/6468, RunningAvgSamplesPerSec=5.876247041142377, CurrSamplesPerSec=5.329727916397701, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3196/5000 [11:05:05<5:31:02, 11.01s/it][2022-12-20 21:27:35,722] [INFO] [timer.py:197:stop] 0/6470, RunningAvgSamplesPerSec=5.876243094220484, CurrSamplesPerSec=5.321609470357277, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3197/5000 [11:05:16<5:30:37, 11.00s/it][2022-12-20 21:27:46,705] [INFO] [timer.py:197:stop] 0/6472, RunningAvgSamplesPerSec=5.876248575700225, CurrSamplesPerSec=5.323194106963349, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3198/5000 [11:05:27<5:30:28, 11.00s/it][2022-12-20 21:27:56,802] [INFO] [timer.py:197:stop] 0/6474, RunningAvgSamplesPerSec=5.876399826978776, CurrSamplesPerSec=6.25827132075074, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-20 21:28:07,762] [INFO] [timer.py:197:stop] 0/6476, RunningAvgSamplesPerSec=5.876408493150059, CurrSamplesPerSec=5.31331126159194, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3199/5000 [11:05:43<6:16:25, 12.54s/it][2022-12-20 21:28:18,829] [INFO] [timer.py:197:stop] 0/6478, RunningAvgSamplesPerSec=5.876400711603962, CurrSamplesPerSec=5.292020336495706, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3200/5000 [11:05:54<6:02:57, 12.10s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 3.926666666666667e-06, 'epoch': 78.05}
+
 64%|██████▍   | 3200/5000 [11:05:54<6:02:57, 12.10s/it][2022-12-20 21:28:29,867] [INFO] [logging.py:68:log_dist] [Rank 0] step=3240, skipped=5, lr=[3.924444444444445e-06], mom=[[0.9, 0.999]]
+[2022-12-20 21:28:29,868] [INFO] [timer.py:197:stop] 0/6480, RunningAvgSamplesPerSec=5.876396856253623, CurrSamplesPerSec=5.320810123109742, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3201/5000 [11:06:05<5:53:13, 11.78s/it][2022-12-20 21:28:40,935] [INFO] [timer.py:197:stop] 0/6482, RunningAvgSamplesPerSec=5.876388225205631, CurrSamplesPerSec=5.2810870142081034, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3202/5000 [11:06:16<5:46:36, 11.57s/it][2022-12-20 21:28:51,981] [INFO] [timer.py:197:stop] 0/6484, RunningAvgSamplesPerSec=5.876382994866593, CurrSamplesPerSec=5.320551952985218, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3203/5000 [11:06:27<5:41:44, 11.41s/it][2022-12-20 21:29:02,983] [INFO] [timer.py:197:stop] 0/6486, RunningAvgSamplesPerSec=5.876385114794843, CurrSamplesPerSec=5.348985484882464, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3204/5000 [11:06:38<5:37:52, 11.29s/it][2022-12-20 21:29:13,925] [INFO] [timer.py:197:stop] 0/6488, RunningAvgSamplesPerSec=5.87639712521979, CurrSamplesPerSec=5.357480911505666, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3205/5000 [11:06:49<5:34:35, 11.18s/it][2022-12-20 21:29:24,933] [INFO] [timer.py:197:stop] 0/6490, RunningAvgSamplesPerSec=5.876397998877807, CurrSamplesPerSec=5.342515833372143, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3206/5000 [11:07:00<5:32:49, 11.13s/it][2022-12-20 21:29:35,903] [INFO] [timer.py:197:stop] 0/6492, RunningAvgSamplesPerSec=5.8764053479878, CurrSamplesPerSec=5.3902156492231486, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3207/5000 [11:07:11<5:31:11, 11.08s/it][2022-12-20 21:29:46,902] [INFO] [timer.py:197:stop] 0/6494, RunningAvgSamplesPerSec=5.876407775323178, CurrSamplesPerSec=5.3565191821667595, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3208/5000 [11:07:22<5:30:15, 11.06s/it][2022-12-20 21:29:57,913] [INFO] [timer.py:197:stop] 0/6496, RunningAvgSamplesPerSec=5.87640834462596, CurrSamplesPerSec=5.342688091917127, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3209/5000 [11:07:33<5:29:39, 11.04s/it][2022-12-20 21:30:08,898] [INFO] [timer.py:197:stop] 0/6498, RunningAvgSamplesPerSec=5.876413422244448, CurrSamplesPerSec=5.359413538781795, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3210/5000 [11:07:44<5:28:56, 11.03s/it][2022-12-20 21:30:19,927] [INFO] [logging.py:68:log_dist] [Rank 0] step=3250, skipped=5, lr=[3.9022222222222225e-06], mom=[[0.9, 0.999]]
+[2022-12-20 21:30:19,929] [INFO] [timer.py:197:stop] 0/6500, RunningAvgSamplesPerSec=5.876410622343033, CurrSamplesPerSec=5.284151909773724, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3211/5000 [11:07:55<5:28:47, 11.03s/it][2022-12-20 21:30:30,912] [INFO] [timer.py:197:stop] 0/6502, RunningAvgSamplesPerSec=5.8764157984207674, CurrSamplesPerSec=5.345617548961401, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3212/5000 [11:08:06<5:28:13, 11.01s/it][2022-12-20 21:30:41,935] [INFO] [timer.py:197:stop] 0/6504, RunningAvgSamplesPerSec=5.8764145410436806, CurrSamplesPerSec=5.313141102465751, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3213/5000 [11:08:17<5:28:06, 11.02s/it][2022-12-20 21:30:52,942] [INFO] [timer.py:197:stop] 0/6506, RunningAvgSamplesPerSec=5.8764156496928885, CurrSamplesPerSec=5.338994837452263, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3214/5000 [11:08:28<5:27:50, 11.01s/it][2022-12-20 21:31:04,002] [INFO] [timer.py:197:stop] 0/6508, RunningAvgSamplesPerSec=5.87640809004743, CurrSamplesPerSec=5.303552223092157, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3215/5000 [11:08:39<5:28:04, 11.03s/it][2022-12-20 21:31:15,065] [INFO] [timer.py:197:stop] 0/6510, RunningAvgSamplesPerSec=5.876400242994232, CurrSamplesPerSec=5.305167641708319, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3216/5000 [11:08:50<5:28:12, 11.04s/it][2022-12-20 21:31:26,075] [INFO] [timer.py:197:stop] 0/6512, RunningAvgSamplesPerSec=5.876401022357505, CurrSamplesPerSec=5.3060151550928785, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3217/5000 [11:09:01<5:27:46, 11.03s/it][2022-12-20 21:31:37,101] [INFO] [timer.py:197:stop] 0/6514, RunningAvgSamplesPerSec=5.876399312951286, CurrSamplesPerSec=5.31036351877559, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3218/5000 [11:09:12<5:27:32, 11.03s/it][2022-12-20 21:31:48,155] [INFO] [timer.py:197:stop] 0/6516, RunningAvgSamplesPerSec=5.876392715259438, CurrSamplesPerSec=5.321550391875264, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3219/5000 [11:09:23<5:27:35, 11.04s/it][2022-12-20 21:31:59,210] [INFO] [timer.py:197:stop] 0/6518, RunningAvgSamplesPerSec=5.8763860515865485, CurrSamplesPerSec=5.301317290237658, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3220/5000 [11:09:34<5:27:34, 11.04s/it][2022-12-20 21:32:10,200] [INFO] [logging.py:68:log_dist] [Rank 0] step=3260, skipped=5, lr=[3.88e-06], mom=[[0.9, 0.999]]
+[2022-12-20 21:32:10,202] [INFO] [timer.py:197:stop] 0/6520, RunningAvgSamplesPerSec=5.876389748218032, CurrSamplesPerSec=5.362526984391314, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3221/5000 [11:09:45<5:26:56, 11.03s/it][2022-12-20 21:32:21,180] [INFO] [timer.py:197:stop] 0/6522, RunningAvgSamplesPerSec=5.876395695277371, CurrSamplesPerSec=5.3575341609457325, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3222/5000 [11:09:56<5:26:19, 11.01s/it][2022-12-20 21:32:32,241] [INFO] [timer.py:197:stop] 0/6524, RunningAvgSamplesPerSec=5.876387794423687, CurrSamplesPerSec=5.304716625833253, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3223/5000 [11:10:07<5:26:34, 11.03s/it][2022-12-20 21:32:43,228] [INFO] [timer.py:197:stop] 0/6526, RunningAvgSamplesPerSec=5.876392398223364, CurrSamplesPerSec=5.3137812013751295, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3224/5000 [11:10:18<5:26:02, 11.01s/it][2022-12-20 21:32:54,302] [INFO] [timer.py:197:stop] 0/6528, RunningAvgSamplesPerSec=5.876382349249041, CurrSamplesPerSec=5.279760986657776, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 64%|██████▍   | 3225/5000 [11:10:29<5:26:22, 11.03s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 3.8711111111111115e-06, 'epoch': 78.65}
+
 64%|██████▍   | 3225/5000 [11:10:29<5:26:22, 11.03s/it][2022-12-20 21:33:05,363] [INFO] [timer.py:197:stop] 0/6530, RunningAvgSamplesPerSec=5.876374938086388, CurrSamplesPerSec=5.3116614510824585, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▍   | 3226/5000 [11:10:40<5:26:26, 11.04s/it][2022-12-20 21:33:16,300] [INFO] [timer.py:197:stop] 0/6532, RunningAvgSamplesPerSec=5.876387565440981, CurrSamplesPerSec=5.339519037326742, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▍   | 3227/5000 [11:10:51<5:25:20, 11.01s/it][2022-12-20 21:33:27,317] [INFO] [timer.py:197:stop] 0/6534, RunningAvgSamplesPerSec=5.876388430269386, CurrSamplesPerSec=5.311941463962921, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▍   | 3228/5000 [11:11:02<5:25:13, 11.01s/it][2022-12-20 21:33:38,352] [INFO] [timer.py:197:stop] 0/6536, RunningAvgSamplesPerSec=5.876385016128062, CurrSamplesPerSec=5.301398325737773, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▍   | 3229/5000 [11:11:13<5:25:14, 11.02s/it][2022-12-20 21:33:49,344] [INFO] [timer.py:197:stop] 0/6538, RunningAvgSamplesPerSec=5.876388471749313, CurrSamplesPerSec=5.308051677933515, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▍   | 3230/5000 [11:11:24<5:24:49, 11.01s/it][2022-12-20 21:34:00,303] [INFO] [logging.py:68:log_dist] [Rank 0] step=3270, skipped=5, lr=[3.857777777777778e-06], mom=[[0.9, 0.999]]
+[2022-12-20 21:34:00,304] [INFO] [timer.py:197:stop] 0/6540, RunningAvgSamplesPerSec=5.876397246598861, CurrSamplesPerSec=5.354624316198671, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▍   | 3231/5000 [11:11:35<5:24:11, 11.00s/it][2022-12-20 21:34:11,372] [INFO] [timer.py:197:stop] 0/6542, RunningAvgSamplesPerSec=5.876388462193976, CurrSamplesPerSec=5.29607451721135, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▍   | 3232/5000 [11:11:46<5:24:38, 11.02s/it][2022-12-20 21:34:22,392] [INFO] [timer.py:197:stop] 0/6544, RunningAvgSamplesPerSec=5.876387569397221, CurrSamplesPerSec=5.306126750894322, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▍   | 3233/5000 [11:11:57<5:24:28, 11.02s/it][2022-12-20 21:34:33,368] [INFO] [timer.py:197:stop] 0/6546, RunningAvgSamplesPerSec=5.8763938087732495, CurrSamplesPerSec=5.337822344784141, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▍   | 3234/5000 [11:12:08<5:23:55, 11.01s/it][2022-12-20 21:34:44,346] [INFO] [timer.py:197:stop] 0/6548, RunningAvgSamplesPerSec=5.876399626194584, CurrSamplesPerSec=5.325906936315606, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▍   | 3235/5000 [11:12:19<5:23:30, 11.00s/it][2022-12-20 21:34:55,362] [INFO] [timer.py:197:stop] 0/6550, RunningAvgSamplesPerSec=5.876399184821528, CurrSamplesPerSec=5.333549084383961, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▍   | 3236/5000 [11:12:30<5:23:29, 11.00s/it][2022-12-20 21:35:06,370] [INFO] [timer.py:197:stop] 0/6552, RunningAvgSamplesPerSec=5.87640043527191, CurrSamplesPerSec=5.3092819029180855, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▍   | 3237/5000 [11:12:41<5:23:20, 11.00s/it][2022-12-20 21:35:17,383] [INFO] [timer.py:197:stop] 0/6554, RunningAvgSamplesPerSec=5.876400762828286, CurrSamplesPerSec=5.330019785581214, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▍   | 3238/5000 [11:12:52<5:23:14, 11.01s/it][2022-12-20 21:35:28,385] [INFO] [timer.py:197:stop] 0/6556, RunningAvgSamplesPerSec=5.876402775800579, CurrSamplesPerSec=5.3355609869674865, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▍   | 3239/5000 [11:13:03<5:23:00, 11.01s/it][2022-12-20 21:35:38,488] [INFO] [timer.py:197:stop] 0/6558, RunningAvgSamplesPerSec=5.876550910219189, CurrSamplesPerSec=5.316438744890436, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▍   | 3240/5000 [11:13:18<5:58:03, 12.21s/it][2022-12-20 21:35:49,478] [INFO] [logging.py:68:log_dist] [Rank 0] step=3280, skipped=5, lr=[3.835555555555555e-06], mom=[[0.9, 0.999]]
+[2022-12-20 21:35:49,480] [INFO] [timer.py:197:stop] 0/6560, RunningAvgSamplesPerSec=5.876554339001798, CurrSamplesPerSec=5.3136370973051426, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▍   | 3241/5000 [11:13:29<5:47:05, 11.84s/it][2022-12-20 21:36:00,470] [INFO] [timer.py:197:stop] 0/6562, RunningAvgSamplesPerSec=5.876557983145742, CurrSamplesPerSec=5.3071789608508455, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▍   | 3242/5000 [11:13:41<5:40:01, 11.60s/it][2022-12-20 21:36:11,541] [INFO] [timer.py:197:stop] 0/6564, RunningAvgSamplesPerSec=5.876548197565742, CurrSamplesPerSec=5.294785234676136, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▍   | 3243/5000 [11:13:52<5:35:06, 11.44s/it][2022-12-20 21:36:22,562] [INFO] [timer.py:197:stop] 0/6566, RunningAvgSamplesPerSec=5.876546748445909, CurrSamplesPerSec=5.336025960988943, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▍   | 3244/5000 [11:14:03<5:31:11, 11.32s/it][2022-12-20 21:36:33,584] [INFO] [timer.py:197:stop] 0/6568, RunningAvgSamplesPerSec=5.876545372900037, CurrSamplesPerSec=5.333681129246278, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▍   | 3245/5000 [11:14:14<5:27:56, 11.21s/it][2022-12-20 21:36:44,590] [INFO] [timer.py:197:stop] 0/6570, RunningAvgSamplesPerSec=5.876546435656051, CurrSamplesPerSec=5.3005847328727835, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▍   | 3246/5000 [11:14:25<5:26:25, 11.17s/it][2022-12-20 21:36:55,601] [INFO] [timer.py:197:stop] 0/6572, RunningAvgSamplesPerSec=5.876547010076143, CurrSamplesPerSec=5.344196147343519, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▍   | 3247/5000 [11:14:36<5:24:34, 11.11s/it][2022-12-20 21:37:06,571] [INFO] [timer.py:197:stop] 0/6574, RunningAvgSamplesPerSec=5.876554167933507, CurrSamplesPerSec=5.34926475658042, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▍   | 3248/5000 [11:14:47<5:22:56, 11.06s/it][2022-12-20 21:37:17,558] [INFO] [timer.py:197:stop] 0/6576, RunningAvgSamplesPerSec=5.876558557123496, CurrSamplesPerSec=5.3119759420645245, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▍   | 3249/5000 [11:14:58<5:22:48, 11.06s/it][2022-12-20 21:37:28,590] [INFO] [timer.py:197:stop] 0/6578, RunningAvgSamplesPerSec=5.876555537843151, CurrSamplesPerSec=5.342273839514454, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▌   | 3250/5000 [11:15:09<5:21:53, 11.04s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 3.8155555555555555e-06, 'epoch': 79.27}
+
 65%|██████▌   | 3250/5000 [11:15:09<5:21:53, 11.04s/it][2022-12-20 21:37:39,582] [INFO] [logging.py:68:log_dist] [Rank 0] step=3290, skipped=5, lr=[3.813333333333334e-06], mom=[[0.9, 0.999]]
+[2022-12-20 21:37:39,584] [INFO] [timer.py:197:stop] 0/6580, RunningAvgSamplesPerSec=5.876558780307483, CurrSamplesPerSec=5.326824300524972, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▌   | 3251/5000 [11:15:20<5:21:31, 11.03s/it][2022-12-20 21:37:50,600] [INFO] [timer.py:197:stop] 0/6582, RunningAvgSamplesPerSec=5.87655816160739, CurrSamplesPerSec=5.325796831577375, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▌   | 3252/5000 [11:15:31<5:21:04, 11.02s/it][2022-12-20 21:38:01,608] [INFO] [timer.py:197:stop] 0/6584, RunningAvgSamplesPerSec=5.8765590232739795, CurrSamplesPerSec=5.318052963743396, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▌   | 3253/5000 [11:15:42<5:21:03, 11.03s/it][2022-12-20 21:38:12,623] [INFO] [timer.py:197:stop] 0/6586, RunningAvgSamplesPerSec=5.876558580498142, CurrSamplesPerSec=5.340697800917922, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▌   | 3254/5000 [11:15:53<5:20:29, 11.01s/it][2022-12-20 21:38:23,592] [INFO] [timer.py:197:stop] 0/6588, RunningAvgSamplesPerSec=5.876565901240078, CurrSamplesPerSec=5.354105903231055, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▌   | 3255/5000 [11:16:04<5:20:07, 11.01s/it][2022-12-20 21:38:34,673] [INFO] [timer.py:197:stop] 0/6590, RunningAvgSamplesPerSec=5.876554843997878, CurrSamplesPerSec=5.275271610143361, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▌   | 3256/5000 [11:16:15<5:20:34, 11.03s/it][2022-12-20 21:38:45,727] [INFO] [timer.py:197:stop] 0/6592, RunningAvgSamplesPerSec=5.876548124294219, CurrSamplesPerSec=5.297321357916446, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▌   | 3257/5000 [11:16:26<5:20:34, 11.04s/it][2022-12-20 21:38:56,760] [INFO] [timer.py:197:stop] 0/6594, RunningAvgSamplesPerSec=5.8765449056449555, CurrSamplesPerSec=5.311292560261655, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▌   | 3258/5000 [11:16:37<5:20:24, 11.04s/it][2022-12-20 21:39:07,797] [INFO] [timer.py:197:stop] 0/6596, RunningAvgSamplesPerSec=5.8765410933148035, CurrSamplesPerSec=5.312962121129353, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▌   | 3259/5000 [11:16:48<5:19:59, 11.03s/it][2022-12-20 21:39:18,786] [INFO] [timer.py:197:stop] 0/6598, RunningAvgSamplesPerSec=5.876545062811377, CurrSamplesPerSec=5.330784850417162, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▌   | 3260/5000 [11:16:59<5:19:29, 11.02s/it][2022-12-20 21:39:29,806] [INFO] [logging.py:68:log_dist] [Rank 0] step=3300, skipped=5, lr=[3.7911111111111114e-06], mom=[[0.9, 0.999]]
+[2022-12-20 21:39:29,808] [INFO] [timer.py:197:stop] 0/6600, RunningAvgSamplesPerSec=5.876543300860448, CurrSamplesPerSec=5.30208565663978, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▌   | 3261/5000 [11:17:10<5:19:37, 11.03s/it][2022-12-20 21:39:40,833] [INFO] [timer.py:197:stop] 0/6602, RunningAvgSamplesPerSec=5.876541238006011, CurrSamplesPerSec=5.327376139484198, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▌   | 3262/5000 [11:17:21<5:19:24, 11.03s/it][2022-12-20 21:39:51,890] [INFO] [timer.py:197:stop] 0/6604, RunningAvgSamplesPerSec=5.8765339166448, CurrSamplesPerSec=5.299084449419083, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▌   | 3263/5000 [11:17:32<5:19:33, 11.04s/it][2022-12-20 21:40:02,921] [INFO] [timer.py:197:stop] 0/6606, RunningAvgSamplesPerSec=5.876530956457931, CurrSamplesPerSec=5.329343815321253, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▌   | 3264/5000 [11:17:43<5:19:10, 11.03s/it][2022-12-20 21:40:13,888] [INFO] [timer.py:197:stop] 0/6608, RunningAvgSamplesPerSec=5.8765383976262555, CurrSamplesPerSec=5.373233597618539, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▌   | 3265/5000 [11:17:54<5:18:06, 11.00s/it][2022-12-20 21:40:24,847] [INFO] [timer.py:197:stop] 0/6610, RunningAvgSamplesPerSec=5.876547153461613, CurrSamplesPerSec=5.346157317663753, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▌   | 3266/5000 [11:18:05<5:17:58, 11.00s/it][2022-12-20 21:40:35,879] [INFO] [timer.py:197:stop] 0/6612, RunningAvgSamplesPerSec=5.876543951073196, CurrSamplesPerSec=5.322873219958154, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▌   | 3267/5000 [11:18:16<5:18:03, 11.01s/it][2022-12-20 21:40:46,883] [INFO] [timer.py:197:stop] 0/6614, RunningAvgSamplesPerSec=5.876545375620576, CurrSamplesPerSec=5.349590112903691, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▌   | 3268/5000 [11:18:27<5:17:42, 11.01s/it][2022-12-20 21:40:57,905] [INFO] [timer.py:197:stop] 0/6616, RunningAvgSamplesPerSec=5.876543873461056, CurrSamplesPerSec=5.324003040548273, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▌   | 3269/5000 [11:18:38<5:17:49, 11.02s/it][2022-12-20 21:41:08,944] [INFO] [timer.py:197:stop] 0/6618, RunningAvgSamplesPerSec=5.876539577107041, CurrSamplesPerSec=5.325619954993742, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▌   | 3270/5000 [11:18:49<5:17:39, 11.02s/it][2022-12-20 21:41:19,987] [INFO] [logging.py:68:log_dist] [Rank 0] step=3310, skipped=5, lr=[3.768888888888889e-06], mom=[[0.9, 0.999]]
+[2022-12-20 21:41:19,989] [INFO] [timer.py:197:stop] 0/6620, RunningAvgSamplesPerSec=5.876534285353074, CurrSamplesPerSec=5.301392253224249, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▌   | 3271/5000 [11:19:00<5:17:53, 11.03s/it][2022-12-20 21:41:31,013] [INFO] [timer.py:197:stop] 0/6622, RunningAvgSamplesPerSec=5.876532513786425, CurrSamplesPerSec=5.338259049788918, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▌   | 3272/5000 [11:19:11<5:17:18, 11.02s/it][2022-12-20 21:41:42,015] [INFO] [timer.py:197:stop] 0/6624, RunningAvgSamplesPerSec=5.876534237102872, CurrSamplesPerSec=5.323622931503264, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▌   | 3273/5000 [11:19:22<5:17:20, 11.03s/it][2022-12-20 21:41:52,976] [INFO] [timer.py:197:stop] 0/6626, RunningAvgSamplesPerSec=5.876542765766015, CurrSamplesPerSec=5.397361336496472, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 65%|██████▌   | 3274/5000 [11:19:33<5:16:06, 10.99s/it][2022-12-20 21:42:03,900] [INFO] [timer.py:197:stop] 0/6628, RunningAvgSamplesPerSec=5.876557352650253, CurrSamplesPerSec=5.378238498100474, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3275/5000 [11:19:44<5:15:19, 10.97s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 3.7600000000000004e-06, 'epoch': 79.87}
+
 66%|██████▌   | 3275/5000 [11:19:44<5:15:19, 10.97s/it][2022-12-20 21:42:14,870] [INFO] [timer.py:197:stop] 0/6630, RunningAvgSamplesPerSec=5.876564822090592, CurrSamplesPerSec=5.335096306041123, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3276/5000 [11:19:55<5:15:27, 10.98s/it][2022-12-20 21:42:25,822] [INFO] [timer.py:197:stop] 0/6632, RunningAvgSamplesPerSec=5.876574575337659, CurrSamplesPerSec=5.379633002775436, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3277/5000 [11:20:06<5:15:13, 10.98s/it][2022-12-20 21:42:36,886] [INFO] [timer.py:197:stop] 0/6634, RunningAvgSamplesPerSec=5.8765665002906635, CurrSamplesPerSec=5.299294718343364, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3278/5000 [11:20:17<5:15:20, 10.99s/it][2022-12-20 21:42:47,839] [INFO] [timer.py:197:stop] 0/6636, RunningAvgSamplesPerSec=5.8765763268209, CurrSamplesPerSec=5.351635903011341, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3279/5000 [11:20:28<5:14:56, 10.98s/it][2022-12-20 21:42:58,839] [INFO] [timer.py:197:stop] 0/6638, RunningAvgSamplesPerSec=5.876578632059887, CurrSamplesPerSec=5.317842046403018, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3280/5000 [11:20:39<5:15:08, 10.99s/it][2022-12-20 21:43:08,931] [INFO] [logging.py:68:log_dist] [Rank 0] step=3320, skipped=5, lr=[3.7466666666666667e-06], mom=[[0.9, 0.999]]
+[2022-12-20 21:43:08,932] [INFO] [timer.py:197:stop] 0/6640, RunningAvgSamplesPerSec=5.876726397002825, CurrSamplesPerSec=6.277170043707559, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-20 21:43:19,929] [INFO] [timer.py:197:stop] 0/6642, RunningAvgSamplesPerSec=5.876728559817958, CurrSamplesPerSec=5.360211255484006, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3281/5000 [11:20:55<5:59:15, 12.54s/it][2022-12-20 21:43:30,934] [INFO] [timer.py:197:stop] 0/6644, RunningAvgSamplesPerSec=5.876730206125147, CurrSamplesPerSec=5.316164153506739, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3282/5000 [11:21:06<5:45:51, 12.08s/it][2022-12-20 21:43:41,942] [INFO] [timer.py:197:stop] 0/6646, RunningAvgSamplesPerSec=5.876731119625821, CurrSamplesPerSec=5.328501103235164, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3283/5000 [11:21:17<5:36:28, 11.76s/it][2022-12-20 21:43:52,920] [INFO] [timer.py:197:stop] 0/6648, RunningAvgSamplesPerSec=5.876736925053151, CurrSamplesPerSec=5.332237045189865, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3284/5000 [11:21:28<5:29:34, 11.52s/it][2022-12-20 21:44:03,896] [INFO] [timer.py:197:stop] 0/6650, RunningAvgSamplesPerSec=5.876742971772372, CurrSamplesPerSec=5.34092966299962, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3285/5000 [11:21:39<5:24:41, 11.36s/it][2022-12-20 21:44:14,840] [INFO] [timer.py:197:stop] 0/6652, RunningAvgSamplesPerSec=5.87675413179437, CurrSamplesPerSec=5.4003378185356, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3286/5000 [11:21:50<5:20:56, 11.23s/it][2022-12-20 21:44:25,842] [INFO] [timer.py:197:stop] 0/6654, RunningAvgSamplesPerSec=5.876756032385161, CurrSamplesPerSec=5.303465254029104, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3287/5000 [11:22:01<5:18:45, 11.16s/it][2022-12-20 21:44:36,826] [INFO] [timer.py:197:stop] 0/6656, RunningAvgSamplesPerSec=5.876760816521527, CurrSamplesPerSec=5.368887152297971, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3288/5000 [11:22:12<5:17:01, 11.11s/it][2022-12-20 21:44:47,737] [INFO] [timer.py:197:stop] 0/6658, RunningAvgSamplesPerSec=5.876777452096817, CurrSamplesPerSec=5.363080459795835, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3289/5000 [11:22:23<5:15:08, 11.05s/it][2022-12-20 21:44:58,818] [INFO] [logging.py:68:log_dist] [Rank 0] step=3330, skipped=5, lr=[3.724444444444445e-06], mom=[[0.9, 0.999]]
+[2022-12-20 21:44:58,819] [INFO] [timer.py:197:stop] 0/6660, RunningAvgSamplesPerSec=5.876766480894791, CurrSamplesPerSec=5.266381226222015, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3290/5000 [11:22:34<5:15:13, 11.06s/it][2022-12-20 21:45:09,869] [INFO] [timer.py:197:stop] 0/6662, RunningAvgSamplesPerSec=5.876760939341473, CurrSamplesPerSec=5.31578642613019, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3291/5000 [11:22:45<5:14:56, 11.06s/it][2022-12-20 21:45:20,839] [INFO] [timer.py:197:stop] 0/6664, RunningAvgSamplesPerSec=5.876768163474889, CurrSamplesPerSec=5.329391851379955, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3292/5000 [11:22:56<5:14:00, 11.03s/it][2022-12-20 21:45:31,887] [INFO] [timer.py:197:stop] 0/6666, RunningAvgSamplesPerSec=5.876762712738285, CurrSamplesPerSec=5.302318995944538, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3293/5000 [11:23:07<5:13:58, 11.04s/it][2022-12-20 21:45:42,920] [INFO] [timer.py:197:stop] 0/6668, RunningAvgSamplesPerSec=5.876759638250988, CurrSamplesPerSec=5.296096877834512, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3294/5000 [11:23:18<5:13:46, 11.04s/it][2022-12-20 21:45:53,927] [INFO] [timer.py:197:stop] 0/6670, RunningAvgSamplesPerSec=5.876760616592619, CurrSamplesPerSec=5.3396084674368085, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3295/5000 [11:23:29<5:13:20, 11.03s/it][2022-12-20 21:46:04,959] [INFO] [timer.py:197:stop] 0/6672, RunningAvgSamplesPerSec=5.876757649743471, CurrSamplesPerSec=5.3000457552300135, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3296/5000 [11:23:40<5:13:12, 11.03s/it][2022-12-20 21:46:16,016] [INFO] [timer.py:197:stop] 0/6674, RunningAvgSamplesPerSec=5.876750535936039, CurrSamplesPerSec=5.272061271014854, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3297/5000 [11:23:51<5:13:16, 11.04s/it][2022-12-20 21:46:27,054] [INFO] [timer.py:197:stop] 0/6676, RunningAvgSamplesPerSec=5.876746739841108, CurrSamplesPerSec=5.320910318427502, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3298/5000 [11:24:02<5:13:05, 11.04s/it][2022-12-20 21:46:38,078] [INFO] [timer.py:197:stop] 0/6678, RunningAvgSamplesPerSec=5.876745132964561, CurrSamplesPerSec=5.329512474375459, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3299/5000 [11:24:13<5:12:47, 11.03s/it][2022-12-20 21:46:49,105] [INFO] [logging.py:68:log_dist] [Rank 0] step=3340, skipped=5, lr=[3.7022222222222227e-06], mom=[[0.9, 0.999]]
+[2022-12-20 21:46:49,107] [INFO] [timer.py:197:stop] 0/6680, RunningAvgSamplesPerSec=5.876742562181444, CurrSamplesPerSec=5.3035467743442135, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3300/5000 [11:24:24<5:12:34, 11.03s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 3.7022222222222227e-06, 'epoch': 80.48}
+
 66%|██████▌   | 3300/5000 [11:24:24<5:12:34, 11.03s/it][2022-12-20 21:47:00,032] [INFO] [timer.py:197:stop] 0/6682, RunningAvgSamplesPerSec=5.876757031091633, CurrSamplesPerSec=5.352762384414847, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3301/5000 [11:24:35<5:11:29, 11.00s/it][2022-12-20 21:47:11,041] [INFO] [timer.py:197:stop] 0/6684, RunningAvgSamplesPerSec=5.87675779998473, CurrSamplesPerSec=5.332021823919215, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3302/5000 [11:24:46<5:11:22, 11.00s/it][2022-12-20 21:47:22,067] [INFO] [timer.py:197:stop] 0/6686, RunningAvgSamplesPerSec=5.876755966507399, CurrSamplesPerSec=5.325087916817084, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3303/5000 [11:24:57<5:11:23, 11.01s/it][2022-12-20 21:47:33,079] [INFO] [timer.py:197:stop] 0/6688, RunningAvgSamplesPerSec=5.876756270043796, CurrSamplesPerSec=5.331858724883191, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3304/5000 [11:25:08<5:11:13, 11.01s/it][2022-12-20 21:47:44,075] [INFO] [timer.py:197:stop] 0/6690, RunningAvgSamplesPerSec=5.876759072874179, CurrSamplesPerSec=5.326167951339064, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3305/5000 [11:25:19<5:10:55, 11.01s/it][2022-12-20 21:47:55,125] [INFO] [timer.py:197:stop] 0/6692, RunningAvgSamplesPerSec=5.8767533765865725, CurrSamplesPerSec=5.31607719144456, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3306/5000 [11:25:30<5:11:06, 11.02s/it][2022-12-20 21:48:06,160] [INFO] [timer.py:197:stop] 0/6694, RunningAvgSamplesPerSec=5.876750000661042, CurrSamplesPerSec=5.332687032403595, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3307/5000 [11:25:41<5:11:03, 11.02s/it][2022-12-20 21:48:17,161] [INFO] [timer.py:197:stop] 0/6696, RunningAvgSamplesPerSec=5.876752343569615, CurrSamplesPerSec=5.357814326036864, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3308/5000 [11:25:52<5:10:40, 11.02s/it][2022-12-20 21:48:28,203] [INFO] [timer.py:197:stop] 0/6698, RunningAvgSamplesPerSec=5.876747705340313, CurrSamplesPerSec=5.308215842939329, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3309/5000 [11:26:03<5:10:42, 11.02s/it][2022-12-20 21:48:39,180] [INFO] [logging.py:68:log_dist] [Rank 0] step=3350, skipped=5, lr=[3.6800000000000003e-06], mom=[[0.9, 0.999]]
+[2022-12-20 21:48:39,182] [INFO] [timer.py:197:stop] 0/6700, RunningAvgSamplesPerSec=5.876753085715051, CurrSamplesPerSec=5.303998429718006, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3310/5000 [11:26:14<5:10:08, 11.01s/it][2022-12-20 21:48:50,205] [INFO] [timer.py:197:stop] 0/6702, RunningAvgSamplesPerSec=5.876751489110395, CurrSamplesPerSec=5.308860425236555, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3311/5000 [11:26:25<5:10:03, 11.01s/it][2022-12-20 21:49:01,244] [INFO] [timer.py:197:stop] 0/6704, RunningAvgSamplesPerSec=5.876747369995446, CurrSamplesPerSec=5.308913132536044, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▌   | 3312/5000 [11:26:36<5:10:05, 11.02s/it][2022-12-20 21:49:12,254] [INFO] [timer.py:197:stop] 0/6706, RunningAvgSamplesPerSec=5.876748017035954, CurrSamplesPerSec=5.3470558963244805, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▋   | 3313/5000 [11:26:47<5:09:47, 11.02s/it][2022-12-20 21:49:23,296] [INFO] [timer.py:197:stop] 0/6708, RunningAvgSamplesPerSec=5.876743598750022, CurrSamplesPerSec=5.323761876182243, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▋   | 3314/5000 [11:26:58<5:09:48, 11.03s/it][2022-12-20 21:49:34,284] [INFO] [timer.py:197:stop] 0/6710, RunningAvgSamplesPerSec=5.876747493184284, CurrSamplesPerSec=5.337908533695213, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▋   | 3315/5000 [11:27:09<5:09:18, 11.01s/it][2022-12-20 21:49:45,270] [INFO] [timer.py:197:stop] 0/6712, RunningAvgSamplesPerSec=5.87675190671935, CurrSamplesPerSec=5.333694270505217, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▋   | 3316/5000 [11:27:20<5:08:53, 11.01s/it][2022-12-20 21:49:56,309] [INFO] [timer.py:197:stop] 0/6714, RunningAvgSamplesPerSec=5.876747995074545, CurrSamplesPerSec=5.313293593140977, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▋   | 3317/5000 [11:27:31<5:08:59, 11.02s/it][2022-12-20 21:50:07,292] [INFO] [timer.py:197:stop] 0/6716, RunningAvgSamplesPerSec=5.87675279207203, CurrSamplesPerSec=5.330138320264769, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▋   | 3318/5000 [11:27:42<5:08:31, 11.01s/it][2022-12-20 21:50:18,248] [INFO] [timer.py:197:stop] 0/6718, RunningAvgSamplesPerSec=5.876761904722673, CurrSamplesPerSec=5.361325288359859, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▋   | 3319/5000 [11:27:53<5:07:55, 10.99s/it][2022-12-20 21:50:29,213] [INFO] [logging.py:68:log_dist] [Rank 0] step=3360, skipped=5, lr=[3.657777777777778e-06], mom=[[0.9, 0.999]]
+[2022-12-20 21:50:29,215] [INFO] [timer.py:197:stop] 0/6720, RunningAvgSamplesPerSec=5.876769155030542, CurrSamplesPerSec=5.3359264684004994, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▋   | 3320/5000 [11:28:04<5:07:32, 10.98s/it][2022-12-20 21:50:40,239] [INFO] [timer.py:197:stop] 0/6722, RunningAvgSamplesPerSec=5.8767675753111, CurrSamplesPerSec=5.338921780488556, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▋   | 3321/5000 [11:28:15<5:07:42, 11.00s/it][2022-12-20 21:50:50,322] [INFO] [timer.py:197:stop] 0/6724, RunningAvgSamplesPerSec=5.876915327060252, CurrSamplesPerSec=5.354080914289216, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▋   | 3322/5000 [11:28:30<5:41:08, 12.20s/it][2022-12-20 21:51:01,272] [INFO] [timer.py:197:stop] 0/6726, RunningAvgSamplesPerSec=5.87692531356035, CurrSamplesPerSec=5.3646590170285045, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▋   | 3323/5000 [11:28:41<5:30:50, 11.84s/it][2022-12-20 21:51:12,322] [INFO] [timer.py:197:stop] 0/6728, RunningAvgSamplesPerSec=5.876919204131609, CurrSamplesPerSec=5.313316309742369, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▋   | 3324/5000 [11:28:52<5:23:49, 11.59s/it][2022-12-20 21:51:23,332] [INFO] [timer.py:197:stop] 0/6730, RunningAvgSamplesPerSec=5.876919415076658, CurrSamplesPerSec=5.32425985604738, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 66%|██████▋   | 3325/5000 [11:29:03<5:18:52, 11.42s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 3.6466666666666668e-06, 'epoch': 81.1}
+
 66%|██████▋   | 3325/5000 [11:29:03<5:18:52, 11.42s/it][2022-12-20 21:51:34,370] [INFO] [timer.py:197:stop] 0/6732, RunningAvgSamplesPerSec=5.876915537877461, CurrSamplesPerSec=5.314782151422378, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3326/5000 [11:29:14<5:15:32, 11.31s/it][2022-12-20 21:51:45,411] [INFO] [timer.py:197:stop] 0/6734, RunningAvgSamplesPerSec=5.876910843716765, CurrSamplesPerSec=5.31816211634895, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3327/5000 [11:29:25<5:13:00, 11.23s/it][2022-12-20 21:51:56,433] [INFO] [timer.py:197:stop] 0/6736, RunningAvgSamplesPerSec=5.876909317866746, CurrSamplesPerSec=5.326181689680577, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3328/5000 [11:29:36<5:10:39, 11.15s/it][2022-12-20 21:52:07,442] [INFO] [timer.py:197:stop] 0/6738, RunningAvgSamplesPerSec=5.876909771595456, CurrSamplesPerSec=5.288438034450293, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3329/5000 [11:29:47<5:09:39, 11.12s/it][2022-12-20 21:52:18,460] [INFO] [logging.py:68:log_dist] [Rank 0] step=3370, skipped=5, lr=[3.635555555555556e-06], mom=[[0.9, 0.999]]
+[2022-12-20 21:52:18,461] [INFO] [timer.py:197:stop] 0/6740, RunningAvgSamplesPerSec=5.876908507086498, CurrSamplesPerSec=5.314929895553247, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3330/5000 [11:29:58<5:08:28, 11.08s/it][2022-12-20 21:52:29,452] [INFO] [timer.py:197:stop] 0/6742, RunningAvgSamplesPerSec=5.876911926709899, CurrSamplesPerSec=5.323124015183574, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3331/5000 [11:30:09<5:07:41, 11.06s/it][2022-12-20 21:52:40,433] [INFO] [timer.py:197:stop] 0/6744, RunningAvgSamplesPerSec=5.876916954619886, CurrSamplesPerSec=5.35051010893587, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3332/5000 [11:30:20<5:07:03, 11.05s/it][2022-12-20 21:52:51,422] [INFO] [timer.py:197:stop] 0/6746, RunningAvgSamplesPerSec=5.876920663450869, CurrSamplesPerSec=5.366228849025981, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3333/5000 [11:30:31<5:06:01, 11.01s/it][2022-12-20 21:53:02,422] [INFO] [timer.py:197:stop] 0/6748, RunningAvgSamplesPerSec=5.876922843960271, CurrSamplesPerSec=5.316866060441772, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3334/5000 [11:30:42<5:05:32, 11.00s/it][2022-12-20 21:53:13,345] [INFO] [timer.py:197:stop] 0/6750, RunningAvgSamplesPerSec=5.876937271852891, CurrSamplesPerSec=5.36610376929793, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3335/5000 [11:30:53<5:04:46, 10.98s/it][2022-12-20 21:53:24,296] [INFO] [timer.py:197:stop] 0/6752, RunningAvgSamplesPerSec=5.876947297139029, CurrSamplesPerSec=5.351655321087609, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3336/5000 [11:31:04<5:04:49, 10.99s/it][2022-12-20 21:53:35,329] [INFO] [timer.py:197:stop] 0/6754, RunningAvgSamplesPerSec=5.876943987266448, CurrSamplesPerSec=5.33209850509613, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3337/5000 [11:31:15<5:04:45, 11.00s/it][2022-12-20 21:53:46,284] [INFO] [timer.py:197:stop] 0/6756, RunningAvgSamplesPerSec=5.876953159231845, CurrSamplesPerSec=5.377035782531397, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3338/5000 [11:31:26<5:03:53, 10.97s/it][2022-12-20 21:53:57,248] [INFO] [timer.py:197:stop] 0/6758, RunningAvgSamplesPerSec=5.876960977729, CurrSamplesPerSec=5.3324774950106, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3339/5000 [11:31:37<5:04:10, 10.99s/it][2022-12-20 21:54:08,247] [INFO] [logging.py:68:log_dist] [Rank 0] step=3380, skipped=5, lr=[3.6133333333333336e-06], mom=[[0.9, 0.999]]
+[2022-12-20 21:54:08,248] [INFO] [timer.py:197:stop] 0/6760, RunningAvgSamplesPerSec=5.876963161466556, CurrSamplesPerSec=5.355239407632347, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3340/5000 [11:31:48<5:03:33, 10.97s/it][2022-12-20 21:54:19,201] [INFO] [timer.py:197:stop] 0/6762, RunningAvgSamplesPerSec=5.8769726984845985, CurrSamplesPerSec=5.340655298595116, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3341/5000 [11:31:59<5:03:25, 10.97s/it][2022-12-20 21:54:30,166] [INFO] [timer.py:197:stop] 0/6764, RunningAvgSamplesPerSec=5.876980176540531, CurrSamplesPerSec=5.350759462307742, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3342/5000 [11:32:10<5:03:29, 10.98s/it][2022-12-20 21:54:41,182] [INFO] [timer.py:197:stop] 0/6766, RunningAvgSamplesPerSec=5.876979711191236, CurrSamplesPerSec=5.340551595766481, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3343/5000 [11:32:21<5:03:05, 10.98s/it][2022-12-20 21:54:52,148] [INFO] [timer.py:197:stop] 0/6768, RunningAvgSamplesPerSec=5.876987266749327, CurrSamplesPerSec=5.333559681643645, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3344/5000 [11:32:32<5:02:36, 10.96s/it][2022-12-20 21:55:03,075] [INFO] [timer.py:197:stop] 0/6770, RunningAvgSamplesPerSec=5.8770011927924815, CurrSamplesPerSec=5.344420226508184, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3345/5000 [11:32:43<5:02:44, 10.98s/it][2022-12-20 21:55:14,059] [INFO] [timer.py:197:stop] 0/6772, RunningAvgSamplesPerSec=5.8770058066882, CurrSamplesPerSec=5.359845008325232, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3346/5000 [11:32:54<5:02:46, 10.98s/it][2022-12-20 21:55:25,058] [INFO] [timer.py:197:stop] 0/6774, RunningAvgSamplesPerSec=5.877008053417263, CurrSamplesPerSec=5.36154866452755, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3347/5000 [11:33:05<5:02:40, 10.99s/it][2022-12-20 21:55:36,105] [INFO] [timer.py:197:stop] 0/6776, RunningAvgSamplesPerSec=5.877002772526877, CurrSamplesPerSec=5.314598640236406, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3348/5000 [11:33:16<5:02:34, 10.99s/it][2022-12-20 21:55:47,100] [INFO] [timer.py:197:stop] 0/6778, RunningAvgSamplesPerSec=5.877005866219796, CurrSamplesPerSec=5.315986652853264, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3349/5000 [11:33:27<5:02:41, 11.00s/it][2022-12-20 21:55:58,083] [INFO] [logging.py:68:log_dist] [Rank 0] step=3390, skipped=5, lr=[3.5911111111111112e-06], mom=[[0.9, 0.999]]
+[2022-12-20 21:55:58,085] [INFO] [timer.py:197:stop] 0/6780, RunningAvgSamplesPerSec=5.877010222566821, CurrSamplesPerSec=5.351602401826418, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3350/5000 [11:33:38<5:02:15, 10.99s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 3.5911111111111112e-06, 'epoch': 81.7}
+
 67%|██████▋   | 3350/5000 [11:33:38<5:02:15, 10.99s/it][2022-12-20 21:56:09,093] [INFO] [timer.py:197:stop] 0/6782, RunningAvgSamplesPerSec=5.877011138842423, CurrSamplesPerSec=5.318804058036418, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3351/5000 [11:33:49<5:02:39, 11.01s/it][2022-12-20 21:56:20,139] [INFO] [timer.py:197:stop] 0/6784, RunningAvgSamplesPerSec=5.877006093928275, CurrSamplesPerSec=5.3323476282292, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3352/5000 [11:34:00<5:02:09, 11.00s/it][2022-12-20 21:56:31,091] [INFO] [timer.py:197:stop] 0/6786, RunningAvgSamplesPerSec=5.8770159787752165, CurrSamplesPerSec=5.352823865782948, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3353/5000 [11:34:11<5:01:46, 10.99s/it][2022-12-20 21:56:42,082] [INFO] [timer.py:197:stop] 0/6788, RunningAvgSamplesPerSec=5.8770198090831665, CurrSamplesPerSec=5.340828712322784, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3354/5000 [11:34:22<5:01:31, 10.99s/it][2022-12-20 21:56:53,088] [INFO] [timer.py:197:stop] 0/6790, RunningAvgSamplesPerSec=5.877021013294528, CurrSamplesPerSec=5.320727649374366, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3355/5000 [11:34:33<5:01:22, 10.99s/it][2022-12-20 21:57:04,105] [INFO] [timer.py:197:stop] 0/6792, RunningAvgSamplesPerSec=5.877020632971732, CurrSamplesPerSec=5.302481340160647, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3356/5000 [11:34:44<5:01:47, 11.01s/it][2022-12-20 21:57:15,096] [INFO] [timer.py:197:stop] 0/6794, RunningAvgSamplesPerSec=5.877024227980055, CurrSamplesPerSec=5.367933346272797, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3357/5000 [11:34:55<5:01:19, 11.00s/it][2022-12-20 21:57:26,059] [INFO] [timer.py:197:stop] 0/6796, RunningAvgSamplesPerSec=5.877032077616335, CurrSamplesPerSec=5.382901030909974, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3358/5000 [11:35:06<5:00:57, 11.00s/it][2022-12-20 21:57:37,103] [INFO] [timer.py:197:stop] 0/6798, RunningAvgSamplesPerSec=5.877026958059406, CurrSamplesPerSec=5.326881170596992, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3359/5000 [11:35:17<5:01:00, 11.01s/it][2022-12-20 21:57:48,142] [INFO] [logging.py:68:log_dist] [Rank 0] step=3400, skipped=5, lr=[3.568888888888889e-06], mom=[[0.9, 0.999]]
+[2022-12-20 21:57:48,143] [INFO] [timer.py:197:stop] 0/6800, RunningAvgSamplesPerSec=5.877022429298849, CurrSamplesPerSec=5.314393888699931, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3360/5000 [11:35:28<5:01:05, 11.02s/it][2022-12-20 21:57:59,169] [INFO] [timer.py:197:stop] 0/6802, RunningAvgSamplesPerSec=5.877020192535571, CurrSamplesPerSec=5.325660950431907, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3361/5000 [11:35:39<5:00:58, 11.02s/it][2022-12-20 21:58:10,218] [INFO] [timer.py:197:stop] 0/6804, RunningAvgSamplesPerSec=5.877014102959561, CurrSamplesPerSec=5.302982470426443, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3362/5000 [11:35:50<5:01:03, 11.03s/it][2022-12-20 21:58:20,341] [INFO] [timer.py:197:stop] 0/6806, RunningAvgSamplesPerSec=5.87715343379233, CurrSamplesPerSec=6.253252729519738, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-20 21:58:31,343] [INFO] [timer.py:197:stop] 0/6808, RunningAvgSamplesPerSec=5.877154685523359, CurrSamplesPerSec=5.323364066081431, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3363/5000 [11:36:06<5:42:58, 12.57s/it][2022-12-20 21:58:42,329] [INFO] [timer.py:197:stop] 0/6810, RunningAvgSamplesPerSec=5.877158908485091, CurrSamplesPerSec=5.327855972857088, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3364/5000 [11:36:17<5:29:48, 12.10s/it][2022-12-20 21:58:53,367] [INFO] [timer.py:197:stop] 0/6812, RunningAvgSamplesPerSec=5.877155035318235, CurrSamplesPerSec=5.324107791246855, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3365/5000 [11:36:28<5:20:57, 11.78s/it][2022-12-20 21:59:04,351] [INFO] [timer.py:197:stop] 0/6814, RunningAvgSamplesPerSec=5.877159614862972, CurrSamplesPerSec=5.323887312714259, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3366/5000 [11:36:39<5:14:16, 11.54s/it][2022-12-20 21:59:15,357] [INFO] [timer.py:197:stop] 0/6816, RunningAvgSamplesPerSec=5.8771608161028945, CurrSamplesPerSec=5.321547859969634, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3367/5000 [11:36:50<5:09:43, 11.38s/it][2022-12-20 21:59:26,255] [INFO] [timer.py:197:stop] 0/6818, RunningAvgSamplesPerSec=5.877178988848708, CurrSamplesPerSec=5.39475109406134, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3368/5000 [11:37:01<5:05:35, 11.24s/it][2022-12-20 21:59:37,272] [INFO] [logging.py:68:log_dist] [Rank 0] step=3410, skipped=5, lr=[3.5466666666666673e-06], mom=[[0.9, 0.999]]
+[2022-12-20 21:59:37,274] [INFO] [timer.py:197:stop] 0/6820, RunningAvgSamplesPerSec=5.877178042302333, CurrSamplesPerSec=5.303204363952804, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3369/5000 [11:37:12<5:03:38, 11.17s/it][2022-12-20 21:59:48,311] [INFO] [timer.py:197:stop] 0/6822, RunningAvgSamplesPerSec=5.877174079218388, CurrSamplesPerSec=5.313250894869602, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3370/5000 [11:37:23<5:02:22, 11.13s/it][2022-12-20 21:59:59,362] [INFO] [timer.py:197:stop] 0/6824, RunningAvgSamplesPerSec=5.8771680926985015, CurrSamplesPerSec=5.316782445241601, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3371/5000 [11:37:34<5:01:32, 11.11s/it][2022-12-20 22:00:10,305] [INFO] [timer.py:197:stop] 0/6826, RunningAvgSamplesPerSec=5.877179085885858, CurrSamplesPerSec=5.357220667423125, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3372/5000 [11:37:45<5:00:01, 11.06s/it][2022-12-20 22:00:21,348] [INFO] [timer.py:197:stop] 0/6828, RunningAvgSamplesPerSec=5.877174308869121, CurrSamplesPerSec=5.312875684402559, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3373/5000 [11:37:56<4:59:43, 11.05s/it][2022-12-20 22:00:32,359] [INFO] [timer.py:197:stop] 0/6830, RunningAvgSamplesPerSec=5.87717456571719, CurrSamplesPerSec=5.329425286680841, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 67%|██████▋   | 3374/5000 [11:38:07<4:59:12, 11.04s/it][2022-12-20 22:00:43,334] [INFO] [timer.py:197:stop] 0/6832, RunningAvgSamplesPerSec=5.877180575723231, CurrSamplesPerSec=5.344966139014561, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3375/5000 [11:38:18<4:58:28, 11.02s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 3.5333333333333335e-06, 'epoch': 82.31}
+
 68%|██████▊   | 3375/5000 [11:38:18<4:58:28, 11.02s/it][2022-12-20 22:00:54,342] [INFO] [timer.py:197:stop] 0/6834, RunningAvgSamplesPerSec=5.877181478525889, CurrSamplesPerSec=5.330518725203653, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3376/5000 [11:38:29<4:58:11, 11.02s/it][2022-12-20 22:01:05,332] [INFO] [timer.py:197:stop] 0/6836, RunningAvgSamplesPerSec=5.877185312715307, CurrSamplesPerSec=5.347984823132789, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3377/5000 [11:38:40<4:57:47, 11.01s/it][2022-12-20 22:01:16,320] [INFO] [timer.py:197:stop] 0/6838, RunningAvgSamplesPerSec=5.877189140223879, CurrSamplesPerSec=5.312320957959642, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3378/5000 [11:38:51<4:57:26, 11.00s/it][2022-12-20 22:01:27,298] [INFO] [logging.py:68:log_dist] [Rank 0] step=3420, skipped=5, lr=[3.524444444444445e-06], mom=[[0.9, 0.999]]
+[2022-12-20 22:01:27,299] [INFO] [timer.py:197:stop] 0/6840, RunningAvgSamplesPerSec=5.877194666974939, CurrSamplesPerSec=5.344442784426168, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3379/5000 [11:39:02<4:57:03, 11.00s/it][2022-12-20 22:01:38,343] [INFO] [timer.py:197:stop] 0/6842, RunningAvgSamplesPerSec=5.877189747891343, CurrSamplesPerSec=5.306849090616358, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3380/5000 [11:39:13<4:57:16, 11.01s/it][2022-12-20 22:01:49,297] [INFO] [timer.py:197:stop] 0/6844, RunningAvgSamplesPerSec=5.877199091220301, CurrSamplesPerSec=5.389259011321249, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3381/5000 [11:39:24<4:56:38, 10.99s/it][2022-12-20 22:02:00,357] [INFO] [timer.py:197:stop] 0/6846, RunningAvgSamplesPerSec=5.87719174167605, CurrSamplesPerSec=5.309576998121164, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3382/5000 [11:39:35<4:56:59, 11.01s/it][2022-12-20 22:02:11,398] [INFO] [timer.py:197:stop] 0/6848, RunningAvgSamplesPerSec=5.877187428854836, CurrSamplesPerSec=5.302465838477172, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3383/5000 [11:39:46<4:57:01, 11.02s/it][2022-12-20 22:02:22,441] [INFO] [timer.py:197:stop] 0/6850, RunningAvgSamplesPerSec=5.877182739670573, CurrSamplesPerSec=5.336479346258684, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3384/5000 [11:39:58<4:57:01, 11.03s/it][2022-12-20 22:02:33,462] [INFO] [timer.py:197:stop] 0/6852, RunningAvgSamplesPerSec=5.877181529855066, CurrSamplesPerSec=5.320433422480445, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3385/5000 [11:40:09<4:56:46, 11.03s/it][2022-12-20 22:02:44,444] [INFO] [timer.py:197:stop] 0/6854, RunningAvgSamplesPerSec=5.877186436927475, CurrSamplesPerSec=5.338868475651671, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3386/5000 [11:40:20<4:56:14, 11.01s/it][2022-12-20 22:02:55,471] [INFO] [timer.py:197:stop] 0/6856, RunningAvgSamplesPerSec=5.877184222371062, CurrSamplesPerSec=5.334600325669707, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3387/5000 [11:40:31<4:56:10, 11.02s/it][2022-12-20 22:03:06,405] [INFO] [timer.py:197:stop] 0/6858, RunningAvgSamplesPerSec=5.877196842011702, CurrSamplesPerSec=5.365163390818276, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3388/5000 [11:40:42<4:55:19, 10.99s/it][2022-12-20 22:03:17,400] [INFO] [logging.py:68:log_dist] [Rank 0] step=3430, skipped=5, lr=[3.5022222222222225e-06], mom=[[0.9, 0.999]]
+[2022-12-20 22:03:17,402] [INFO] [timer.py:197:stop] 0/6860, RunningAvgSamplesPerSec=5.87719921296691, CurrSamplesPerSec=5.34382655354221, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3389/5000 [11:40:52<4:55:10, 10.99s/it][2022-12-20 22:03:28,457] [INFO] [timer.py:197:stop] 0/6862, RunningAvgSamplesPerSec=5.877192567504644, CurrSamplesPerSec=5.304330461192507, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3390/5000 [11:41:04<4:55:29, 11.01s/it][2022-12-20 22:03:39,513] [INFO] [timer.py:197:stop] 0/6864, RunningAvgSamplesPerSec=5.877185822870018, CurrSamplesPerSec=5.296505462651388, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3391/5000 [11:41:15<4:55:39, 11.03s/it][2022-12-20 22:03:50,546] [INFO] [timer.py:197:stop] 0/6866, RunningAvgSamplesPerSec=5.877182587108042, CurrSamplesPerSec=5.315120586018002, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3392/5000 [11:41:26<4:55:32, 11.03s/it][2022-12-20 22:04:01,520] [INFO] [timer.py:197:stop] 0/6868, RunningAvgSamplesPerSec=5.877188798932847, CurrSamplesPerSec=5.365697890343939, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3393/5000 [11:41:37<4:54:55, 11.01s/it][2022-12-20 22:04:12,526] [INFO] [timer.py:197:stop] 0/6870, RunningAvgSamplesPerSec=5.877189667819138, CurrSamplesPerSec=5.317951822471385, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3394/5000 [11:41:48<4:54:41, 11.01s/it][2022-12-20 22:04:23,575] [INFO] [timer.py:197:stop] 0/6872, RunningAvgSamplesPerSec=5.877183964858576, CurrSamplesPerSec=5.30589748114639, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3395/5000 [11:41:59<4:54:49, 11.02s/it][2022-12-20 22:04:34,604] [INFO] [timer.py:197:stop] 0/6874, RunningAvgSamplesPerSec=5.8771813348065285, CurrSamplesPerSec=5.322737487954535, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3396/5000 [11:42:10<4:54:42, 11.02s/it][2022-12-20 22:04:45,637] [INFO] [timer.py:197:stop] 0/6876, RunningAvgSamplesPerSec=5.877178199969104, CurrSamplesPerSec=5.3268259918107095, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3397/5000 [11:42:21<4:54:35, 11.03s/it][2022-12-20 22:04:56,675] [INFO] [timer.py:197:stop] 0/6878, RunningAvgSamplesPerSec=5.877174091521779, CurrSamplesPerSec=5.315441591488052, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3398/5000 [11:42:32<4:54:29, 11.03s/it][2022-12-20 22:05:07,623] [INFO] [logging.py:68:log_dist] [Rank 0] step=3440, skipped=5, lr=[3.48e-06], mom=[[0.9, 0.999]]
+[2022-12-20 22:05:07,625] [INFO] [timer.py:197:stop] 0/6880, RunningAvgSamplesPerSec=5.877183610872277, CurrSamplesPerSec=5.354042897372087, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3399/5000 [11:42:43<4:53:40, 11.01s/it][2022-12-20 22:05:18,695] [INFO] [timer.py:197:stop] 0/6882, RunningAvgSamplesPerSec=5.877174669428006, CurrSamplesPerSec=5.305327434208992, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3400/5000 [11:42:54<4:54:00, 11.03s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 3.4777777777777784e-06, 'epoch': 82.92}
+
 68%|██████▊   | 3400/5000 [11:42:54<4:54:00, 11.03s/it][2022-12-20 22:05:29,703] [INFO] [timer.py:197:stop] 0/6884, RunningAvgSamplesPerSec=5.877175480106373, CurrSamplesPerSec=5.301469521761593, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3401/5000 [11:43:05<4:53:41, 11.02s/it][2022-12-20 22:05:40,777] [INFO] [timer.py:197:stop] 0/6886, RunningAvgSamplesPerSec=5.877165702040161, CurrSamplesPerSec=5.2787243944397915, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3402/5000 [11:43:16<4:53:55, 11.04s/it][2022-12-20 22:05:51,759] [INFO] [timer.py:197:stop] 0/6888, RunningAvgSamplesPerSec=5.877170358936834, CurrSamplesPerSec=5.3579554890153265, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3403/5000 [11:43:27<4:53:18, 11.02s/it][2022-12-20 22:06:01,833] [INFO] [timer.py:197:stop] 0/6890, RunningAvgSamplesPerSec=5.877315810523471, CurrSamplesPerSec=5.340438759763765, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3404/5000 [11:43:42<5:24:45, 12.21s/it][2022-12-20 22:06:12,807] [INFO] [timer.py:197:stop] 0/6892, RunningAvgSamplesPerSec=5.8773216716347445, CurrSamplesPerSec=5.331350216492948, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3405/5000 [11:43:53<5:14:32, 11.83s/it][2022-12-20 22:06:23,739] [INFO] [timer.py:197:stop] 0/6894, RunningAvgSamplesPerSec=5.877334025669836, CurrSamplesPerSec=5.351716990418064, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3406/5000 [11:44:04<5:07:17, 11.57s/it][2022-12-20 22:06:34,752] [INFO] [timer.py:197:stop] 0/6896, RunningAvgSamplesPerSec=5.877334027925976, CurrSamplesPerSec=5.2937738454589445, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3407/5000 [11:44:15<5:03:02, 11.41s/it][2022-12-20 22:06:45,781] [INFO] [timer.py:197:stop] 0/6898, RunningAvgSamplesPerSec=5.877331708515138, CurrSamplesPerSec=5.318920829681682, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3408/5000 [11:44:26<4:59:37, 11.29s/it][2022-12-20 22:06:56,800] [INFO] [logging.py:68:log_dist] [Rank 0] step=3450, skipped=5, lr=[3.457777777777778e-06], mom=[[0.9, 0.999]]
+[2022-12-20 22:06:56,802] [INFO] [timer.py:197:stop] 0/6900, RunningAvgSamplesPerSec=5.877330466214841, CurrSamplesPerSec=5.30837770866021, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3409/5000 [11:44:37<4:57:40, 11.23s/it][2022-12-20 22:07:07,874] [INFO] [timer.py:197:stop] 0/6902, RunningAvgSamplesPerSec=5.877321256762561, CurrSamplesPerSec=5.30653059071755, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3410/5000 [11:44:48<4:56:01, 11.17s/it][2022-12-20 22:07:18,918] [INFO] [timer.py:197:stop] 0/6904, RunningAvgSamplesPerSec=5.87731650635782, CurrSamplesPerSec=5.30544738976029, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3411/5000 [11:44:59<4:54:59, 11.14s/it][2022-12-20 22:07:29,988] [INFO] [timer.py:197:stop] 0/6906, RunningAvgSamplesPerSec=5.877307861837475, CurrSamplesPerSec=5.3009561156185665, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3412/5000 [11:45:10<4:54:15, 11.12s/it][2022-12-20 22:07:41,006] [INFO] [timer.py:197:stop] 0/6908, RunningAvgSamplesPerSec=5.8773070544836505, CurrSamplesPerSec=5.345843237802084, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3413/5000 [11:45:21<4:52:45, 11.07s/it][2022-12-20 22:07:51,986] [INFO] [timer.py:197:stop] 0/6910, RunningAvgSamplesPerSec=5.8773121307858425, CurrSamplesPerSec=5.320537610935107, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3414/5000 [11:45:32<4:51:57, 11.04s/it][2022-12-20 22:08:02,989] [INFO] [timer.py:197:stop] 0/6912, RunningAvgSamplesPerSec=5.877313812463177, CurrSamplesPerSec=5.309628039266594, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3415/5000 [11:45:43<4:51:55, 11.05s/it][2022-12-20 22:08:14,050] [INFO] [timer.py:197:stop] 0/6914, RunningAvgSamplesPerSec=5.877307545383985, CurrSamplesPerSec=5.31239307846239, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3416/5000 [11:45:54<4:51:24, 11.04s/it][2022-12-20 22:08:25,006] [INFO] [timer.py:197:stop] 0/6916, RunningAvgSamplesPerSec=5.877316099520914, CurrSamplesPerSec=5.359670785574039, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3417/5000 [11:46:05<4:50:43, 11.02s/it][2022-12-20 22:08:35,982] [INFO] [timer.py:197:stop] 0/6918, RunningAvgSamplesPerSec=5.877321686583255, CurrSamplesPerSec=5.357892606406187, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3418/5000 [11:46:16<4:50:22, 11.01s/it][2022-12-20 22:08:46,988] [INFO] [logging.py:68:log_dist] [Rank 0] step=3460, skipped=5, lr=[3.435555555555556e-06], mom=[[0.9, 0.999]]
+[2022-12-20 22:08:46,990] [INFO] [timer.py:197:stop] 0/6920, RunningAvgSamplesPerSec=5.87732227539434, CurrSamplesPerSec=5.349147714739017, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3419/5000 [11:46:27<4:50:07, 11.01s/it][2022-12-20 22:08:57,993] [INFO] [timer.py:197:stop] 0/6922, RunningAvgSamplesPerSec=5.877323720942297, CurrSamplesPerSec=5.350932039540956, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3420/5000 [11:46:38<4:49:33, 11.00s/it][2022-12-20 22:09:08,973] [INFO] [timer.py:197:stop] 0/6924, RunningAvgSamplesPerSec=5.877328750587015, CurrSamplesPerSec=5.3352971417675095, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3421/5000 [11:46:49<4:49:25, 11.00s/it][2022-12-20 22:09:19,940] [INFO] [timer.py:197:stop] 0/6926, RunningAvgSamplesPerSec=5.877335502436876, CurrSamplesPerSec=5.365227516559954, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3422/5000 [11:47:00<4:49:04, 10.99s/it][2022-12-20 22:09:30,940] [INFO] [timer.py:197:stop] 0/6928, RunningAvgSamplesPerSec=5.877337508857378, CurrSamplesPerSec=5.345756579550044, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3423/5000 [11:47:11<4:48:58, 10.99s/it][2022-12-20 22:09:41,940] [INFO] [timer.py:197:stop] 0/6930, RunningAvgSamplesPerSec=5.877339467387723, CurrSamplesPerSec=5.348187056626314, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3424/5000 [11:47:22<4:48:32, 10.99s/it][2022-12-20 22:09:52,938] [INFO] [timer.py:197:stop] 0/6932, RunningAvgSamplesPerSec=5.8773416822668, CurrSamplesPerSec=5.316915978074382, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 68%|██████▊   | 3425/5000 [11:47:33<4:48:39, 11.00s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 3.4222222222222224e-06, 'epoch': 83.53}
+
 68%|██████▊   | 3425/5000 [11:47:33<4:48:39, 11.00s/it][2022-12-20 22:10:03,960] [INFO] [timer.py:197:stop] 0/6934, RunningAvgSamplesPerSec=5.877340372004912, CurrSamplesPerSec=5.318010188944311, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▊   | 3426/5000 [11:47:44<4:48:37, 11.00s/it][2022-12-20 22:10:14,971] [INFO] [timer.py:197:stop] 0/6936, RunningAvgSamplesPerSec=5.87734058442746, CurrSamplesPerSec=5.3208337477710055, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▊   | 3427/5000 [11:47:55<4:48:37, 11.01s/it][2022-12-20 22:10:25,949] [INFO] [timer.py:197:stop] 0/6938, RunningAvgSamplesPerSec=5.8773461176789725, CurrSamplesPerSec=5.363545527449814, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▊   | 3428/5000 [11:48:06<4:47:50, 10.99s/it][2022-12-20 22:10:36,915] [INFO] [logging.py:68:log_dist] [Rank 0] step=3470, skipped=5, lr=[3.4133333333333334e-06], mom=[[0.9, 0.999]]
+[2022-12-20 22:10:36,916] [INFO] [timer.py:197:stop] 0/6940, RunningAvgSamplesPerSec=5.877353156269629, CurrSamplesPerSec=5.331998311624465, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▊   | 3429/5000 [11:48:17<4:47:56, 11.00s/it][2022-12-20 22:10:47,939] [INFO] [timer.py:197:stop] 0/6942, RunningAvgSamplesPerSec=5.877351457895751, CurrSamplesPerSec=5.3316217195243825, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▊   | 3430/5000 [11:48:28<4:48:08, 11.01s/it][2022-12-20 22:10:58,979] [INFO] [timer.py:197:stop] 0/6944, RunningAvgSamplesPerSec=5.877347136855028, CurrSamplesPerSec=5.338147372190629, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▊   | 3431/5000 [11:48:39<4:47:15, 10.99s/it][2022-12-20 22:11:09,910] [INFO] [timer.py:197:stop] 0/6946, RunningAvgSamplesPerSec=5.877359823604158, CurrSamplesPerSec=5.3311636536065645, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▊   | 3432/5000 [11:48:50<4:46:58, 10.98s/it][2022-12-20 22:11:20,881] [INFO] [timer.py:197:stop] 0/6948, RunningAvgSamplesPerSec=5.8773660226005235, CurrSamplesPerSec=5.331086999315391, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▊   | 3433/5000 [11:49:01<4:47:04, 10.99s/it][2022-12-20 22:11:31,922] [INFO] [timer.py:197:stop] 0/6950, RunningAvgSamplesPerSec=5.8773615738378995, CurrSamplesPerSec=5.309442783495795, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▊   | 3434/5000 [11:49:12<4:47:22, 11.01s/it][2022-12-20 22:11:42,963] [INFO] [timer.py:197:stop] 0/6952, RunningAvgSamplesPerSec=5.877357190817917, CurrSamplesPerSec=5.321452493278688, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▊   | 3435/5000 [11:49:23<4:47:24, 11.02s/it][2022-12-20 22:11:53,985] [INFO] [timer.py:197:stop] 0/6954, RunningAvgSamplesPerSec=5.877355543265012, CurrSamplesPerSec=5.33426915797919, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▊   | 3436/5000 [11:49:34<4:47:14, 11.02s/it][2022-12-20 22:12:04,978] [INFO] [timer.py:197:stop] 0/6956, RunningAvgSamplesPerSec=5.877358286494414, CurrSamplesPerSec=5.359462546457848, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▊   | 3437/5000 [11:49:45<4:46:37, 11.00s/it][2022-12-20 22:12:15,980] [INFO] [timer.py:197:stop] 0/6958, RunningAvgSamplesPerSec=5.877359875690221, CurrSamplesPerSec=5.326138784018202, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3438/5000 [11:49:56<4:46:33, 11.01s/it][2022-12-20 22:12:26,975] [INFO] [logging.py:68:log_dist] [Rank 0] step=3480, skipped=5, lr=[3.391111111111111e-06], mom=[[0.9, 0.999]]
+[2022-12-20 22:12:26,976] [INFO] [timer.py:197:stop] 0/6960, RunningAvgSamplesPerSec=5.877362221208545, CurrSamplesPerSec=5.345126209359258, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3439/5000 [11:50:07<4:46:18, 11.00s/it][2022-12-20 22:12:38,017] [INFO] [timer.py:197:stop] 0/6962, RunningAvgSamplesPerSec=5.8773576711673305, CurrSamplesPerSec=5.308471137596698, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3440/5000 [11:50:18<4:46:24, 11.02s/it][2022-12-20 22:12:49,061] [INFO] [timer.py:197:stop] 0/6964, RunningAvgSamplesPerSec=5.877352550160243, CurrSamplesPerSec=5.304979342166683, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3441/5000 [11:50:29<4:46:17, 11.02s/it][2022-12-20 22:13:00,036] [INFO] [timer.py:197:stop] 0/6966, RunningAvgSamplesPerSec=5.8773581789319955, CurrSamplesPerSec=5.3496449114079105, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3442/5000 [11:50:40<4:45:54, 11.01s/it][2022-12-20 22:13:11,036] [INFO] [timer.py:197:stop] 0/6968, RunningAvgSamplesPerSec=5.8773600034051725, CurrSamplesPerSec=5.343082201697642, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3443/5000 [11:50:51<4:45:42, 11.01s/it][2022-12-20 22:13:22,079] [INFO] [timer.py:197:stop] 0/6970, RunningAvgSamplesPerSec=5.8773551879723875, CurrSamplesPerSec=5.313720403230395, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3444/5000 [11:51:02<4:45:44, 11.02s/it][2022-12-20 22:13:32,205] [INFO] [timer.py:197:stop] 0/6972, RunningAvgSamplesPerSec=5.877490869911894, CurrSamplesPerSec=6.248447254125932, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-20 22:13:43,165] [INFO] [timer.py:197:stop] 0/6974, RunningAvgSamplesPerSec=5.877498574830126, CurrSamplesPerSec=5.322725244972642, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3445/5000 [11:51:18<5:25:20, 12.55s/it][2022-12-20 22:13:54,209] [INFO] [timer.py:197:stop] 0/6976, RunningAvgSamplesPerSec=5.877493587797274, CurrSamplesPerSec=5.316052766776635, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3446/5000 [11:51:29<5:13:24, 12.10s/it][2022-12-20 22:14:05,171] [INFO] [timer.py:197:stop] 0/6978, RunningAvgSamplesPerSec=5.877501197279952, CurrSamplesPerSec=5.343533808555723, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3447/5000 [11:51:40<5:04:21, 11.76s/it][2022-12-20 22:14:16,242] [INFO] [logging.py:68:log_dist] [Rank 0] step=3490, skipped=5, lr=[3.3688888888888895e-06], mom=[[0.9, 0.999]]
+[2022-12-20 22:14:16,243] [INFO] [timer.py:197:stop] 0/6980, RunningAvgSamplesPerSec=5.877491669252841, CurrSamplesPerSec=5.30939153588281, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3448/5000 [11:51:51<4:58:50, 11.55s/it][2022-12-20 22:14:27,293] [INFO] [timer.py:197:stop] 0/6982, RunningAvgSamplesPerSec=5.877485675721347, CurrSamplesPerSec=5.305356583747886, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3449/5000 [11:52:02<4:54:44, 11.40s/it][2022-12-20 22:14:38,330] [INFO] [timer.py:197:stop] 0/6984, RunningAvgSamplesPerSec=5.87748152849938, CurrSamplesPerSec=5.2929792895173025, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3450/5000 [11:52:13<4:51:43, 11.29s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 3.3644444444444447e-06, 'epoch': 84.14}
+
 69%|██████▉   | 3450/5000 [11:52:13<4:51:43, 11.29s/it][2022-12-20 22:14:49,349] [INFO] [timer.py:197:stop] 0/6986, RunningAvgSamplesPerSec=5.8774805628949025, CurrSamplesPerSec=5.348528906494639, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3451/5000 [11:52:24<4:49:24, 11.21s/it][2022-12-20 22:15:00,353] [INFO] [timer.py:197:stop] 0/6988, RunningAvgSamplesPerSec=5.877481684872852, CurrSamplesPerSec=5.328756448799737, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3452/5000 [11:52:35<4:47:38, 11.15s/it][2022-12-20 22:15:11,421] [INFO] [timer.py:197:stop] 0/6990, RunningAvgSamplesPerSec=5.8774728428873955, CurrSamplesPerSec=5.29729208753428, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3453/5000 [11:52:47<4:46:49, 11.12s/it][2022-12-20 22:15:22,439] [INFO] [timer.py:197:stop] 0/6992, RunningAvgSamplesPerSec=5.877471801103504, CurrSamplesPerSec=5.31107965685935, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3454/5000 [11:52:58<4:45:48, 11.09s/it][2022-12-20 22:15:33,513] [INFO] [timer.py:197:stop] 0/6994, RunningAvgSamplesPerSec=5.8774620629374, CurrSamplesPerSec=5.295767965837891, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3455/5000 [11:53:09<4:45:29, 11.09s/it][2022-12-20 22:15:44,588] [INFO] [timer.py:197:stop] 0/6996, RunningAvgSamplesPerSec=5.8774523689021265, CurrSamplesPerSec=5.280769313544631, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3456/5000 [11:53:20<4:45:12, 11.08s/it][2022-12-20 22:15:55,610] [INFO] [timer.py:197:stop] 0/6998, RunningAvgSamplesPerSec=5.877450942068787, CurrSamplesPerSec=5.349364747201171, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3457/5000 [11:53:31<4:44:33, 11.06s/it][2022-12-20 22:16:06,647] [INFO] [logging.py:68:log_dist] [Rank 0] step=3500, skipped=5, lr=[3.346666666666667e-06], mom=[[0.9, 0.999]]
+[2022-12-20 22:16:06,648] [INFO] [timer.py:197:stop] 0/7000, RunningAvgSamplesPerSec=5.877446973922996, CurrSamplesPerSec=5.331681445430516, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3458/5000 [11:53:42<4:44:09, 11.06s/it][2022-12-20 22:16:17,675] [INFO] [timer.py:197:stop] 0/7002, RunningAvgSamplesPerSec=5.87744472253816, CurrSamplesPerSec=5.341243378300105, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3459/5000 [11:53:53<4:43:44, 11.05s/it][2022-12-20 22:16:28,708] [INFO] [timer.py:197:stop] 0/7004, RunningAvgSamplesPerSec=5.8774416771214275, CurrSamplesPerSec=5.323021414280375, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3460/5000 [11:54:04<4:43:26, 11.04s/it][2022-12-20 22:16:39,693] [INFO] [timer.py:197:stop] 0/7006, RunningAvgSamplesPerSec=5.8774459871268565, CurrSamplesPerSec=5.32867055518247, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3461/5000 [11:54:15<4:42:48, 11.03s/it][2022-12-20 22:16:50,746] [INFO] [timer.py:197:stop] 0/7008, RunningAvgSamplesPerSec=5.877439716528593, CurrSamplesPerSec=5.32086791952723, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3462/5000 [11:54:26<4:42:50, 11.03s/it][2022-12-20 22:17:01,761] [INFO] [timer.py:197:stop] 0/7010, RunningAvgSamplesPerSec=5.877439540836382, CurrSamplesPerSec=5.312082742927017, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3463/5000 [11:54:37<4:42:30, 11.03s/it][2022-12-20 22:17:12,775] [INFO] [timer.py:197:stop] 0/7012, RunningAvgSamplesPerSec=5.877439543277474, CurrSamplesPerSec=5.338243338209372, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3464/5000 [11:54:48<4:42:12, 11.02s/it][2022-12-20 22:17:23,779] [INFO] [timer.py:197:stop] 0/7014, RunningAvgSamplesPerSec=5.877440924391854, CurrSamplesPerSec=5.342185808522981, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3465/5000 [11:54:59<4:41:52, 11.02s/it][2022-12-20 22:17:34,768] [INFO] [timer.py:197:stop] 0/7016, RunningAvgSamplesPerSec=5.877444642637558, CurrSamplesPerSec=5.362222760859554, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3466/5000 [11:55:10<4:41:28, 11.01s/it][2022-12-20 22:17:45,774] [INFO] [timer.py:197:stop] 0/7018, RunningAvgSamplesPerSec=5.877445710505525, CurrSamplesPerSec=5.320061625288502, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3467/5000 [11:55:21<4:41:15, 11.01s/it][2022-12-20 22:17:56,775] [INFO] [logging.py:68:log_dist] [Rank 0] step=3510, skipped=5, lr=[3.3244444444444447e-06], mom=[[0.9, 0.999]]
+[2022-12-20 22:17:56,777] [INFO] [timer.py:197:stop] 0/7020, RunningAvgSamplesPerSec=5.877447211724594, CurrSamplesPerSec=5.354907809005693, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3468/5000 [11:55:32<4:41:02, 11.01s/it][2022-12-20 22:18:07,799] [INFO] [timer.py:197:stop] 0/7022, RunningAvgSamplesPerSec=5.877445783028536, CurrSamplesPerSec=5.320336619455907, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3469/5000 [11:55:43<4:40:58, 11.01s/it][2022-12-20 22:18:18,738] [INFO] [timer.py:197:stop] 0/7024, RunningAvgSamplesPerSec=5.877457066854889, CurrSamplesPerSec=5.363328628719668, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3470/5000 [11:55:54<4:40:13, 10.99s/it][2022-12-20 22:18:29,758] [INFO] [timer.py:197:stop] 0/7026, RunningAvgSamplesPerSec=5.8774561109742365, CurrSamplesPerSec=5.331781203385353, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3471/5000 [11:56:05<4:40:16, 11.00s/it][2022-12-20 22:18:40,769] [INFO] [timer.py:197:stop] 0/7028, RunningAvgSamplesPerSec=5.877456183605429, CurrSamplesPerSec=5.313672228525821, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3472/5000 [11:56:16<4:40:11, 11.00s/it][2022-12-20 22:18:51,767] [INFO] [timer.py:197:stop] 0/7030, RunningAvgSamplesPerSec=5.87745845528586, CurrSamplesPerSec=5.356988244001654, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3473/5000 [11:56:27<4:39:58, 11.00s/it][2022-12-20 22:19:02,728] [INFO] [timer.py:197:stop] 0/7032, RunningAvgSamplesPerSec=5.877466353741475, CurrSamplesPerSec=5.391105280846113, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 69%|██████▉   | 3474/5000 [11:56:38<4:39:29, 10.99s/it][2022-12-20 22:19:13,681] [INFO] [timer.py:197:stop] 0/7034, RunningAvgSamplesPerSec=5.877475532310344, CurrSamplesPerSec=5.320628937369722, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|██████▉   | 3475/5000 [11:56:49<4:39:01, 10.98s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 3.308888888888889e-06, 'epoch': 84.75}
+
 70%|██████▉   | 3475/5000 [11:56:49<4:39:01, 10.98s/it][2022-12-20 22:19:24,749] [INFO] [timer.py:197:stop] 0/7036, RunningAvgSamplesPerSec=5.8774672265804115, CurrSamplesPerSec=5.312515666783656, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|██████▉   | 3476/5000 [11:57:00<4:39:31, 11.01s/it][2022-12-20 22:19:35,715] [INFO] [timer.py:197:stop] 0/7038, RunningAvgSamplesPerSec=5.877474541192782, CurrSamplesPerSec=5.351275520558105, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|██████▉   | 3477/5000 [11:57:11<4:39:03, 10.99s/it][2022-12-20 22:19:46,728] [INFO] [logging.py:68:log_dist] [Rank 0] step=3520, skipped=5, lr=[3.3022222222222223e-06], mom=[[0.9, 0.999]]
+[2022-12-20 22:19:46,729] [INFO] [timer.py:197:stop] 0/7040, RunningAvgSamplesPerSec=5.877474247885195, CurrSamplesPerSec=5.321180970116635, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|██████▉   | 3478/5000 [11:57:22<4:39:01, 11.00s/it][2022-12-20 22:19:57,672] [INFO] [timer.py:197:stop] 0/7042, RunningAvgSamplesPerSec=5.8774851341137335, CurrSamplesPerSec=5.354193900336188, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|██████▉   | 3479/5000 [11:57:33<4:38:24, 10.98s/it][2022-12-20 22:20:08,630] [INFO] [timer.py:197:stop] 0/7044, RunningAvgSamplesPerSec=5.877493624973739, CurrSamplesPerSec=5.375719057651177, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|██████▉   | 3480/5000 [11:57:44<4:38:02, 10.98s/it][2022-12-20 22:20:19,636] [INFO] [timer.py:197:stop] 0/7046, RunningAvgSamplesPerSec=5.877494576234701, CurrSamplesPerSec=5.328568163453796, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|██████▉   | 3481/5000 [11:57:55<4:38:05, 10.98s/it][2022-12-20 22:20:30,624] [INFO] [timer.py:197:stop] 0/7048, RunningAvgSamplesPerSec=5.8774982361315065, CurrSamplesPerSec=5.324680402686268, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|██████▉   | 3482/5000 [11:58:06<4:37:56, 10.99s/it][2022-12-20 22:20:41,571] [INFO] [timer.py:197:stop] 0/7050, RunningAvgSamplesPerSec=5.877508339212149, CurrSamplesPerSec=5.36354767080543, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|██████▉   | 3483/5000 [11:58:17<4:37:27, 10.97s/it][2022-12-20 22:20:52,577] [INFO] [timer.py:197:stop] 0/7052, RunningAvgSamplesPerSec=5.8775094210481065, CurrSamplesPerSec=5.316602376096449, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|██████▉   | 3484/5000 [11:58:28<4:37:31, 10.98s/it][2022-12-20 22:21:03,559] [INFO] [timer.py:197:stop] 0/7054, RunningAvgSamplesPerSec=5.877514208826281, CurrSamplesPerSec=5.3564197789727075, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|██████▉   | 3485/5000 [11:58:39<4:37:19, 10.98s/it][2022-12-20 22:21:13,628] [INFO] [timer.py:197:stop] 0/7056, RunningAvgSamplesPerSec=5.8776571352838305, CurrSamplesPerSec=5.339012889626352, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|██████▉   | 3486/5000 [11:58:54<5:07:35, 12.19s/it][2022-12-20 22:21:24,651] [INFO] [timer.py:197:stop] 0/7058, RunningAvgSamplesPerSec=5.877655314727172, CurrSamplesPerSec=5.312251362479783, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|██████▉   | 3487/5000 [11:59:05<4:58:35, 11.84s/it][2022-12-20 22:21:35,672] [INFO] [logging.py:68:log_dist] [Rank 0] step=3530, skipped=5, lr=[3.2800000000000004e-06], mom=[[0.9, 0.999]]
+[2022-12-20 22:21:35,674] [INFO] [timer.py:197:stop] 0/7060, RunningAvgSamplesPerSec=5.877653715399471, CurrSamplesPerSec=5.317598911708139, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|██████▉   | 3488/5000 [11:59:16<4:51:54, 11.58s/it][2022-12-20 22:21:46,646] [INFO] [timer.py:197:stop] 0/7062, RunningAvgSamplesPerSec=5.87765967397203, CurrSamplesPerSec=5.326693651947659, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|██████▉   | 3489/5000 [11:59:27<4:47:08, 11.40s/it][2022-12-20 22:21:57,612] [INFO] [timer.py:197:stop] 0/7064, RunningAvgSamplesPerSec=5.877666521024726, CurrSamplesPerSec=5.3363323111485075, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|██████▉   | 3490/5000 [11:59:38<4:43:37, 11.27s/it][2022-12-20 22:22:08,581] [INFO] [timer.py:197:stop] 0/7066, RunningAvgSamplesPerSec=5.877672971855654, CurrSamplesPerSec=5.3313442869381085, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|██████▉   | 3491/5000 [11:59:49<4:41:25, 11.19s/it][2022-12-20 22:22:19,605] [INFO] [timer.py:197:stop] 0/7068, RunningAvgSamplesPerSec=5.877671065308999, CurrSamplesPerSec=5.312624171630035, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|██████▉   | 3492/5000 [12:00:00<4:40:02, 11.14s/it][2022-12-20 22:22:30,605] [INFO] [timer.py:197:stop] 0/7070, RunningAvgSamplesPerSec=5.8776729225194995, CurrSamplesPerSec=5.341195128359186, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|██████▉   | 3493/5000 [12:00:11<4:38:37, 11.09s/it][2022-12-20 22:22:41,606] [INFO] [timer.py:197:stop] 0/7072, RunningAvgSamplesPerSec=5.877674629740443, CurrSamplesPerSec=5.321400380624145, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|██████▉   | 3494/5000 [12:00:22<4:37:59, 11.08s/it][2022-12-20 22:22:52,613] [INFO] [timer.py:197:stop] 0/7074, RunningAvgSamplesPerSec=5.877675350635434, CurrSamplesPerSec=5.344193168259925, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|██████▉   | 3495/5000 [12:00:33<4:37:11, 11.05s/it][2022-12-20 22:23:03,564] [INFO] [timer.py:197:stop] 0/7076, RunningAvgSamplesPerSec=5.877684793483298, CurrSamplesPerSec=5.383221423642713, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|██████▉   | 3496/5000 [12:00:44<4:36:00, 11.01s/it][2022-12-20 22:23:14,495] [INFO] [timer.py:197:stop] 0/7078, RunningAvgSamplesPerSec=5.877697172324611, CurrSamplesPerSec=5.3711794328447535, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|██████▉   | 3497/5000 [12:00:55<4:35:28, 11.00s/it][2022-12-20 22:23:25,472] [INFO] [logging.py:68:log_dist] [Rank 0] step=3540, skipped=5, lr=[3.257777777777778e-06], mom=[[0.9, 0.999]]
+[2022-12-20 22:23:25,473] [INFO] [timer.py:197:stop] 0/7080, RunningAvgSamplesPerSec=5.877702089924552, CurrSamplesPerSec=5.358249388533631, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|██████▉   | 3498/5000 [12:01:06<4:35:11, 10.99s/it][2022-12-20 22:23:36,526] [INFO] [timer.py:197:stop] 0/7082, RunningAvgSamplesPerSec=5.877696122526558, CurrSamplesPerSec=5.2985487052325855, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|██████▉   | 3499/5000 [12:01:17<4:35:29, 11.01s/it][2022-12-20 22:23:47,579] [INFO] [timer.py:197:stop] 0/7084, RunningAvgSamplesPerSec=5.877689998628411, CurrSamplesPerSec=5.302050259562705, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|███████   | 3500/5000 [12:01:28<4:35:41, 11.03s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 3.2533333333333332e-06, 'epoch': 85.36}
+
 70%|███████   | 3500/5000 [12:01:28<4:35:41, 11.03s/it][2022-12-20 22:23:58,603] [INFO] [timer.py:197:stop] 0/7086, RunningAvgSamplesPerSec=5.877688186615257, CurrSamplesPerSec=5.335869405029359, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|███████   | 3501/5000 [12:01:39<4:35:29, 11.03s/it][2022-12-20 22:24:09,642] [INFO] [timer.py:197:stop] 0/7088, RunningAvgSamplesPerSec=5.877683984337105, CurrSamplesPerSec=5.323403548821259, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|███████   | 3502/5000 [12:01:50<4:35:15, 11.02s/it][2022-12-20 22:24:20,681] [INFO] [timer.py:197:stop] 0/7090, RunningAvgSamplesPerSec=5.8776796667779605, CurrSamplesPerSec=5.306378068402197, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|███████   | 3503/5000 [12:02:01<4:35:24, 11.04s/it][2022-12-20 22:24:31,708] [INFO] [timer.py:197:stop] 0/7092, RunningAvgSamplesPerSec=5.8776771634151626, CurrSamplesPerSec=5.344237003682219, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|███████   | 3504/5000 [12:02:12<4:34:56, 11.03s/it][2022-12-20 22:24:42,738] [INFO] [timer.py:197:stop] 0/7094, RunningAvgSamplesPerSec=5.877674349049392, CurrSamplesPerSec=5.318041163730092, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|███████   | 3505/5000 [12:02:23<4:34:19, 11.01s/it][2022-12-20 22:24:53,727] [INFO] [timer.py:197:stop] 0/7096, RunningAvgSamplesPerSec=5.877677810319436, CurrSamplesPerSec=5.301913911925116, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|███████   | 3506/5000 [12:02:34<4:34:29, 11.02s/it][2022-12-20 22:25:04,790] [INFO] [timer.py:197:stop] 0/7098, RunningAvgSamplesPerSec=5.877669958957125, CurrSamplesPerSec=5.296598683160091, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|███████   | 3507/5000 [12:02:45<4:34:13, 11.02s/it][2022-12-20 22:25:15,780] [INFO] [logging.py:68:log_dist] [Rank 0] step=3550, skipped=5, lr=[3.2355555555555556e-06], mom=[[0.9, 0.999]]
+[2022-12-20 22:25:15,782] [INFO] [timer.py:197:stop] 0/7100, RunningAvgSamplesPerSec=5.8776727394301815, CurrSamplesPerSec=5.314050917879063, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|███████   | 3508/5000 [12:02:56<4:34:00, 11.02s/it][2022-12-20 22:25:26,803] [INFO] [timer.py:197:stop] 0/7102, RunningAvgSamplesPerSec=5.877671050061761, CurrSamplesPerSec=5.309400987212443, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|███████   | 3509/5000 [12:03:07<4:33:40, 11.01s/it][2022-12-20 22:25:37,745] [INFO] [timer.py:197:stop] 0/7104, RunningAvgSamplesPerSec=5.8776817427717685, CurrSamplesPerSec=5.361654897883952, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|███████   | 3510/5000 [12:03:18<4:33:16, 11.00s/it][2022-12-20 22:25:48,787] [INFO] [timer.py:197:stop] 0/7106, RunningAvgSamplesPerSec=5.877677144189214, CurrSamplesPerSec=5.309730124501472, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|███████   | 3511/5000 [12:03:29<4:33:20, 11.01s/it][2022-12-20 22:25:59,782] [INFO] [timer.py:197:stop] 0/7108, RunningAvgSamplesPerSec=5.877679893653706, CurrSamplesPerSec=5.348395486072259, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|███████   | 3512/5000 [12:03:40<4:33:03, 11.01s/it][2022-12-20 22:26:10,827] [INFO] [timer.py:197:stop] 0/7110, RunningAvgSamplesPerSec=5.877674698799316, CurrSamplesPerSec=5.307483056951617, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|███████   | 3513/5000 [12:03:51<4:33:15, 11.03s/it][2022-12-20 22:26:21,871] [INFO] [timer.py:197:stop] 0/7112, RunningAvgSamplesPerSec=5.877669801342578, CurrSamplesPerSec=5.324243170735287, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|███████   | 3514/5000 [12:04:02<4:32:54, 11.02s/it][2022-12-20 22:26:32,889] [INFO] [timer.py:197:stop] 0/7114, RunningAvgSamplesPerSec=5.877668691859985, CurrSamplesPerSec=5.310946206864896, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|███████   | 3515/5000 [12:04:13<4:32:58, 11.03s/it][2022-12-20 22:26:44,030] [INFO] [timer.py:197:stop] 0/7116, RunningAvgSamplesPerSec=5.877648971453292, CurrSamplesPerSec=5.234629936950541, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|███████   | 3516/5000 [12:04:24<4:33:34, 11.06s/it][2022-12-20 22:26:55,070] [INFO] [timer.py:197:stop] 0/7118, RunningAvgSamplesPerSec=5.877644675927699, CurrSamplesPerSec=5.317994385621187, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|███████   | 3517/5000 [12:04:35<4:33:10, 11.05s/it][2022-12-20 22:27:06,131] [INFO] [logging.py:68:log_dist] [Rank 0] step=3560, skipped=5, lr=[3.213333333333334e-06], mom=[[0.9, 0.999]]
+[2022-12-20 22:27:06,133] [INFO] [timer.py:197:stop] 0/7120, RunningAvgSamplesPerSec=5.877636627018262, CurrSamplesPerSec=5.289335031214186, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|███████   | 3518/5000 [12:04:46<4:32:54, 11.05s/it][2022-12-20 22:27:17,067] [INFO] [timer.py:197:stop] 0/7122, RunningAvgSamplesPerSec=5.877648209443444, CurrSamplesPerSec=5.384742945985685, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|███████   | 3519/5000 [12:04:57<4:31:43, 11.01s/it][2022-12-20 22:27:28,054] [INFO] [timer.py:197:stop] 0/7124, RunningAvgSamplesPerSec=5.8776519509022656, CurrSamplesPerSec=5.319924138864816, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|███████   | 3520/5000 [12:05:08<4:31:48, 11.02s/it][2022-12-20 22:27:39,060] [INFO] [timer.py:197:stop] 0/7126, RunningAvgSamplesPerSec=5.8776526506763265, CurrSamplesPerSec=5.353181894723441, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|███████   | 3521/5000 [12:05:19<4:31:23, 11.01s/it][2022-12-20 22:27:50,074] [INFO] [timer.py:197:stop] 0/7128, RunningAvgSamplesPerSec=5.877652281075482, CurrSamplesPerSec=5.3304933208447345, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|███████   | 3522/5000 [12:05:30<4:31:03, 11.00s/it][2022-12-20 22:28:01,069] [INFO] [timer.py:197:stop] 0/7130, RunningAvgSamplesPerSec=5.87765492605659, CurrSamplesPerSec=5.327145664099563, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|███████   | 3523/5000 [12:05:41<4:30:48, 11.00s/it][2022-12-20 22:28:12,055] [INFO] [timer.py:197:stop] 0/7132, RunningAvgSamplesPerSec=5.877658583682138, CurrSamplesPerSec=5.3326690229818405, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|███████   | 3524/5000 [12:05:52<4:30:50, 11.01s/it][2022-12-20 22:28:23,076] [INFO] [timer.py:197:stop] 0/7134, RunningAvgSamplesPerSec=5.87765711447172, CurrSamplesPerSec=5.340683350052266, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 70%|███████   | 3525/5000 [12:06:03<4:30:46, 11.01s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 3.197777777777778e-06, 'epoch': 85.96}
+
 70%|███████   | 3525/5000 [12:06:03<4:30:46, 11.01s/it][2022-12-20 22:28:34,127] [INFO] [timer.py:197:stop] 0/7136, RunningAvgSamplesPerSec=5.877651390789766, CurrSamplesPerSec=5.3199833921116495, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3526/5000 [12:06:14<4:30:39, 11.02s/it][2022-12-20 22:28:44,233] [INFO] [timer.py:197:stop] 0/7138, RunningAvgSamplesPerSec=5.877786769108438, CurrSamplesPerSec=6.264182824637949, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-20 22:28:55,270] [INFO] [logging.py:68:log_dist] [Rank 0] step=3570, skipped=5, lr=[3.1911111111111117e-06], mom=[[0.9, 0.999]]
+[2022-12-20 22:28:55,272] [INFO] [timer.py:197:stop] 0/7140, RunningAvgSamplesPerSec=5.877781944884518, CurrSamplesPerSec=5.3111095001378645, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3527/5000 [12:06:30<5:08:39, 12.57s/it][2022-12-20 22:29:06,249] [INFO] [timer.py:197:stop] 0/7142, RunningAvgSamplesPerSec=5.8777870559258085, CurrSamplesPerSec=5.363731577096455, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3528/5000 [12:06:41<4:56:41, 12.09s/it][2022-12-20 22:29:17,233] [INFO] [timer.py:197:stop] 0/7144, RunningAvgSamplesPerSec=5.877791170999546, CurrSamplesPerSec=5.3301186346754585, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3529/5000 [12:06:52<4:48:20, 11.76s/it][2022-12-20 22:29:28,278] [INFO] [timer.py:197:stop] 0/7146, RunningAvgSamplesPerSec=5.8777860286597114, CurrSamplesPerSec=5.3143107720748874, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3530/5000 [12:07:03<4:42:52, 11.55s/it][2022-12-20 22:29:39,330] [INFO] [timer.py:197:stop] 0/7148, RunningAvgSamplesPerSec=5.877779888836587, CurrSamplesPerSec=5.310253845666899, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3531/5000 [12:07:14<4:39:03, 11.40s/it][2022-12-20 22:29:50,337] [INFO] [timer.py:197:stop] 0/7150, RunningAvgSamplesPerSec=5.877780560936945, CurrSamplesPerSec=5.341362412719359, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3532/5000 [12:07:25<4:35:59, 11.28s/it][2022-12-20 22:30:01,353] [INFO] [timer.py:197:stop] 0/7152, RunningAvgSamplesPerSec=5.877779704756266, CurrSamplesPerSec=5.329207329924256, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3533/5000 [12:07:36<4:33:52, 11.20s/it][2022-12-20 22:30:12,285] [INFO] [timer.py:197:stop] 0/7154, RunningAvgSamplesPerSec=5.877791702046096, CurrSamplesPerSec=5.3291430043470855, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3534/5000 [12:07:47<4:31:42, 11.12s/it][2022-12-20 22:30:23,312] [INFO] [timer.py:197:stop] 0/7156, RunningAvgSamplesPerSec=5.8777892056737135, CurrSamplesPerSec=5.312989882426634, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3535/5000 [12:07:58<4:30:50, 11.09s/it][2022-12-20 22:30:34,323] [INFO] [timer.py:197:stop] 0/7158, RunningAvgSamplesPerSec=5.87778925220122, CurrSamplesPerSec=5.311349729728767, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3536/5000 [12:08:09<4:30:03, 11.07s/it][2022-12-20 22:30:45,329] [INFO] [logging.py:68:log_dist] [Rank 0] step=3580, skipped=5, lr=[3.1688888888888893e-06], mom=[[0.9, 0.999]]
+[2022-12-20 22:30:45,331] [INFO] [timer.py:197:stop] 0/7160, RunningAvgSamplesPerSec=5.877789568586745, CurrSamplesPerSec=5.299858865472995, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3537/5000 [12:08:20<4:29:26, 11.05s/it][2022-12-20 22:30:56,393] [INFO] [timer.py:197:stop] 0/7162, RunningAvgSamplesPerSec=5.877781900051507, CurrSamplesPerSec=5.304261703787576, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3538/5000 [12:08:31<4:29:20, 11.05s/it][2022-12-20 22:31:07,444] [INFO] [timer.py:197:stop] 0/7164, RunningAvgSamplesPerSec=5.877775869269952, CurrSamplesPerSec=5.299717398901105, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3539/5000 [12:08:43<4:29:08, 11.05s/it][2022-12-20 22:31:18,437] [INFO] [timer.py:197:stop] 0/7166, RunningAvgSamplesPerSec=5.8777786338945015, CurrSamplesPerSec=5.338908825828695, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3540/5000 [12:08:54<4:28:30, 11.03s/it][2022-12-20 22:31:29,407] [INFO] [timer.py:197:stop] 0/7168, RunningAvgSamplesPerSec=5.877784727835519, CurrSamplesPerSec=5.34269851285209, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3541/5000 [12:09:05<4:27:51, 11.02s/it][2022-12-20 22:31:40,440] [INFO] [timer.py:197:stop] 0/7170, RunningAvgSamplesPerSec=5.877781392208103, CurrSamplesPerSec=5.310631207414719, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3542/5000 [12:09:16<4:27:48, 11.02s/it][2022-12-20 22:31:51,406] [INFO] [timer.py:197:stop] 0/7172, RunningAvgSamplesPerSec=5.877788081665298, CurrSamplesPerSec=5.33450576271111, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3543/5000 [12:09:27<4:27:13, 11.00s/it][2022-12-20 22:32:02,427] [INFO] [timer.py:197:stop] 0/7174, RunningAvgSamplesPerSec=5.8777864796720065, CurrSamplesPerSec=5.338527010154311, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3544/5000 [12:09:38<4:27:09, 11.01s/it][2022-12-20 22:32:13,464] [INFO] [timer.py:197:stop] 0/7176, RunningAvgSamplesPerSec=5.877782595509925, CurrSamplesPerSec=5.285765938053256, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3545/5000 [12:09:49<4:27:10, 11.02s/it][2022-12-20 22:32:24,426] [INFO] [timer.py:197:stop] 0/7178, RunningAvgSamplesPerSec=5.87779018756586, CurrSamplesPerSec=5.354629656780461, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3546/5000 [12:10:00<4:26:35, 11.00s/it][2022-12-20 22:32:35,378] [INFO] [logging.py:68:log_dist] [Rank 0] step=3590, skipped=5, lr=[3.146666666666667e-06], mom=[[0.9, 0.999]]
+[2022-12-20 22:32:35,380] [INFO] [timer.py:197:stop] 0/7180, RunningAvgSamplesPerSec=5.877798796286612, CurrSamplesPerSec=5.346318737128904, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3547/5000 [12:10:10<4:26:03, 10.99s/it][2022-12-20 22:32:46,398] [INFO] [timer.py:197:stop] 0/7182, RunningAvgSamplesPerSec=5.877798035717452, CurrSamplesPerSec=5.32761847731292, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3548/5000 [12:10:21<4:26:06, 11.00s/it][2022-12-20 22:32:57,406] [INFO] [timer.py:197:stop] 0/7184, RunningAvgSamplesPerSec=5.877798543859734, CurrSamplesPerSec=5.319566117079837, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3549/5000 [12:10:33<4:26:00, 11.00s/it][2022-12-20 22:33:08,392] [INFO] [timer.py:197:stop] 0/7186, RunningAvgSamplesPerSec=5.877802448455616, CurrSamplesPerSec=5.319569279605488, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3550/5000 [12:10:43<4:25:43, 11.00s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 3.1400000000000004e-06, 'epoch': 86.58}
+
 71%|███████   | 3550/5000 [12:10:43<4:25:43, 11.00s/it][2022-12-20 22:33:19,404] [INFO] [timer.py:197:stop] 0/7188, RunningAvgSamplesPerSec=5.877802770504072, CurrSamplesPerSec=5.3441872101026995, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3551/5000 [12:10:54<4:25:39, 11.00s/it][2022-12-20 22:33:30,389] [INFO] [timer.py:197:stop] 0/7190, RunningAvgSamplesPerSec=5.877806727692985, CurrSamplesPerSec=5.361848741771878, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3552/5000 [12:11:05<4:25:21, 11.00s/it][2022-12-20 22:33:41,383] [INFO] [timer.py:197:stop] 0/7192, RunningAvgSamplesPerSec=5.8778093415532435, CurrSamplesPerSec=5.364391857433138, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3553/5000 [12:11:16<4:25:10, 11.00s/it][2022-12-20 22:33:52,385] [INFO] [timer.py:197:stop] 0/7194, RunningAvgSamplesPerSec=5.87781080382385, CurrSamplesPerSec=5.3500882448655105, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3554/5000 [12:11:27<4:25:02, 11.00s/it][2022-12-20 22:34:03,380] [INFO] [timer.py:197:stop] 0/7196, RunningAvgSamplesPerSec=5.877813247252966, CurrSamplesPerSec=5.354332095802918, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3555/5000 [12:11:38<4:24:50, 11.00s/it][2022-12-20 22:34:14,406] [INFO] [timer.py:197:stop] 0/7198, RunningAvgSamplesPerSec=5.877811163977937, CurrSamplesPerSec=5.3052112586536255, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3556/5000 [12:11:50<4:24:51, 11.01s/it][2022-12-20 22:34:25,413] [INFO] [logging.py:68:log_dist] [Rank 0] step=3600, skipped=5, lr=[3.124444444444445e-06], mom=[[0.9, 0.999]]
+[2022-12-20 22:34:25,414] [INFO] [timer.py:197:stop] 0/7200, RunningAvgSamplesPerSec=5.877811530666679, CurrSamplesPerSec=5.31947061057673, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3557/5000 [12:12:01<4:24:42, 11.01s/it][2022-12-20 22:34:36,461] [INFO] [timer.py:197:stop] 0/7202, RunningAvgSamplesPerSec=5.87780610863096, CurrSamplesPerSec=5.318934741446929, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3558/5000 [12:12:12<4:24:48, 11.02s/it][2022-12-20 22:34:47,410] [INFO] [timer.py:197:stop] 0/7204, RunningAvgSamplesPerSec=5.877815696622176, CurrSamplesPerSec=5.374079975527542, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3559/5000 [12:12:23<4:24:07, 11.00s/it][2022-12-20 22:34:58,442] [INFO] [timer.py:197:stop] 0/7206, RunningAvgSamplesPerSec=5.877812637694012, CurrSamplesPerSec=5.281099689782276, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3560/5000 [12:12:34<4:24:11, 11.01s/it][2022-12-20 22:35:09,454] [INFO] [timer.py:197:stop] 0/7208, RunningAvgSamplesPerSec=5.877812600273247, CurrSamplesPerSec=5.313562629067976, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3561/5000 [12:12:45<4:24:02, 11.01s/it][2022-12-20 22:35:20,437] [INFO] [timer.py:197:stop] 0/7210, RunningAvgSamplesPerSec=5.877816945700204, CurrSamplesPerSec=5.325273420837471, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████   | 3562/5000 [12:12:56<4:23:39, 11.00s/it][2022-12-20 22:35:31,435] [INFO] [timer.py:197:stop] 0/7212, RunningAvgSamplesPerSec=5.87781895269703, CurrSamplesPerSec=5.31551148109718, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████▏  | 3563/5000 [12:13:07<4:23:27, 11.00s/it][2022-12-20 22:35:42,476] [INFO] [timer.py:197:stop] 0/7214, RunningAvgSamplesPerSec=5.877814463094394, CurrSamplesPerSec=5.321133292918677, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████▏  | 3564/5000 [12:13:18<4:23:33, 11.01s/it][2022-12-20 22:35:53,527] [INFO] [timer.py:197:stop] 0/7216, RunningAvgSamplesPerSec=5.877808634928754, CurrSamplesPerSec=5.301649195523724, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████▏  | 3565/5000 [12:13:29<4:23:39, 11.02s/it][2022-12-20 22:36:04,536] [INFO] [timer.py:197:stop] 0/7218, RunningAvgSamplesPerSec=5.877808835922161, CurrSamplesPerSec=5.335293112179795, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████▏  | 3566/5000 [12:13:40<4:23:22, 11.02s/it][2022-12-20 22:36:15,588] [INFO] [logging.py:68:log_dist] [Rank 0] step=3610, skipped=5, lr=[3.1022222222222225e-06], mom=[[0.9, 0.999]]
+[2022-12-20 22:36:15,589] [INFO] [timer.py:197:stop] 0/7220, RunningAvgSamplesPerSec=5.8778026151772265, CurrSamplesPerSec=5.304230470067423, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████▏  | 3567/5000 [12:13:51<4:23:25, 11.03s/it][2022-12-20 22:36:25,733] [INFO] [timer.py:197:stop] 0/7222, RunningAvgSamplesPerSec=5.877930934202359, CurrSamplesPerSec=5.299606281786281, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████▏  | 3568/5000 [12:14:06<4:52:11, 12.24s/it][2022-12-20 22:36:36,750] [INFO] [timer.py:197:stop] 0/7224, RunningAvgSamplesPerSec=5.8779300665411744, CurrSamplesPerSec=5.3113684361819296, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████▏  | 3569/5000 [12:14:17<4:43:30, 11.89s/it][2022-12-20 22:36:47,815] [INFO] [timer.py:197:stop] 0/7226, RunningAvgSamplesPerSec=5.8779218227636845, CurrSamplesPerSec=5.305229083092511, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████▏  | 3570/5000 [12:14:28<4:37:21, 11.64s/it][2022-12-20 22:36:58,812] [INFO] [timer.py:197:stop] 0/7228, RunningAvgSamplesPerSec=5.87792406941612, CurrSamplesPerSec=5.357573724468116, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████▏  | 3571/5000 [12:14:39<4:32:24, 11.44s/it][2022-12-20 22:37:09,821] [INFO] [timer.py:197:stop] 0/7230, RunningAvgSamplesPerSec=5.877924334691541, CurrSamplesPerSec=5.323491806475145, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████▏  | 3572/5000 [12:14:50<4:29:03, 11.30s/it][2022-12-20 22:37:20,827] [INFO] [timer.py:197:stop] 0/7232, RunningAvgSamplesPerSec=5.877925096212432, CurrSamplesPerSec=5.3143185575724425, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████▏  | 3573/5000 [12:15:01<4:27:07, 11.23s/it][2022-12-20 22:37:31,910] [INFO] [timer.py:197:stop] 0/7234, RunningAvgSamplesPerSec=5.877914312907908, CurrSamplesPerSec=5.292963634610542, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 71%|███████▏  | 3574/5000 [12:15:12<4:25:54, 11.19s/it][2022-12-20 22:37:42,939] [INFO] [timer.py:197:stop] 0/7236, RunningAvgSamplesPerSec=5.877911675204993, CurrSamplesPerSec=5.345352495364455, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3575/5000 [12:15:23<4:24:29, 11.14s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 3.084444444444445e-06, 'epoch': 87.19}
+
 72%|███████▏  | 3575/5000 [12:15:23<4:24:29, 11.14s/it][2022-12-20 22:37:53,979] [INFO] [timer.py:197:stop] 0/7238, RunningAvgSamplesPerSec=5.87790781182473, CurrSamplesPerSec=5.326336198063375, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3576/5000 [12:15:34<4:23:32, 11.10s/it][2022-12-20 22:38:04,984] [INFO] [logging.py:68:log_dist] [Rank 0] step=3620, skipped=5, lr=[3.08e-06], mom=[[0.9, 0.999]]
+[2022-12-20 22:38:04,986] [INFO] [timer.py:197:stop] 0/7240, RunningAvgSamplesPerSec=5.877908749197606, CurrSamplesPerSec=5.345189431272094, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3577/5000 [12:15:45<4:22:39, 11.07s/it][2022-12-20 22:38:16,009] [INFO] [timer.py:197:stop] 0/7242, RunningAvgSamplesPerSec=5.877907157899447, CurrSamplesPerSec=5.328868791675421, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3578/5000 [12:15:56<4:21:55, 11.05s/it][2022-12-20 22:38:26,978] [INFO] [timer.py:197:stop] 0/7244, RunningAvgSamplesPerSec=5.877913442633043, CurrSamplesPerSec=5.354414119360696, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3579/5000 [12:16:07<4:21:23, 11.04s/it][2022-12-20 22:38:37,979] [INFO] [timer.py:197:stop] 0/7246, RunningAvgSamplesPerSec=5.877915166272913, CurrSamplesPerSec=5.355103729261289, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3580/5000 [12:16:18<4:21:02, 11.03s/it][2022-12-20 22:38:49,017] [INFO] [timer.py:197:stop] 0/7248, RunningAvgSamplesPerSec=5.877911263879821, CurrSamplesPerSec=5.333715466220601, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3581/5000 [12:16:29<4:20:36, 11.02s/it][2022-12-20 22:39:00,001] [INFO] [timer.py:197:stop] 0/7250, RunningAvgSamplesPerSec=5.877915516301028, CurrSamplesPerSec=5.344528123178367, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3582/5000 [12:16:40<4:20:18, 11.01s/it][2022-12-20 22:39:11,036] [INFO] [timer.py:197:stop] 0/7252, RunningAvgSamplesPerSec=5.8779119862530385, CurrSamplesPerSec=5.316016972410149, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3583/5000 [12:16:51<4:20:17, 11.02s/it][2022-12-20 22:39:22,047] [INFO] [timer.py:197:stop] 0/7254, RunningAvgSamplesPerSec=5.877911942114626, CurrSamplesPerSec=5.338225928188709, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3584/5000 [12:17:02<4:20:10, 11.02s/it][2022-12-20 22:39:33,068] [INFO] [timer.py:197:stop] 0/7256, RunningAvgSamplesPerSec=5.8779103874626, CurrSamplesPerSec=5.347282771824609, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3585/5000 [12:17:13<4:19:30, 11.00s/it][2022-12-20 22:39:44,050] [INFO] [timer.py:197:stop] 0/7258, RunningAvgSamplesPerSec=5.877914499419193, CurrSamplesPerSec=5.324840316555253, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3586/5000 [12:17:24<4:19:11, 11.00s/it][2022-12-20 22:39:55,042] [INFO] [logging.py:68:log_dist] [Rank 0] step=3630, skipped=5, lr=[3.0577777777777778e-06], mom=[[0.9, 0.999]]
+[2022-12-20 22:39:55,043] [INFO] [timer.py:197:stop] 0/7260, RunningAvgSamplesPerSec=5.8779172340732915, CurrSamplesPerSec=5.318222173264763, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3587/5000 [12:17:35<4:19:07, 11.00s/it][2022-12-20 22:40:06,094] [INFO] [timer.py:197:stop] 0/7262, RunningAvgSamplesPerSec=5.8779112343776, CurrSamplesPerSec=5.285345063621912, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3588/5000 [12:17:46<4:19:08, 11.01s/it][2022-12-20 22:40:17,107] [INFO] [timer.py:197:stop] 0/7264, RunningAvgSamplesPerSec=5.877910911468005, CurrSamplesPerSec=5.301912026983879, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3589/5000 [12:17:57<4:19:13, 11.02s/it][2022-12-20 22:40:28,126] [INFO] [timer.py:197:stop] 0/7266, RunningAvgSamplesPerSec=5.877909984424591, CurrSamplesPerSec=5.329839241988884, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3590/5000 [12:18:08<4:18:44, 11.01s/it][2022-12-20 22:40:39,104] [INFO] [timer.py:197:stop] 0/7268, RunningAvgSamplesPerSec=5.877915049431346, CurrSamplesPerSec=5.330822326030224, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3591/5000 [12:18:19<4:18:34, 11.01s/it][2022-12-20 22:40:50,155] [INFO] [timer.py:197:stop] 0/7270, RunningAvgSamplesPerSec=5.877909400851563, CurrSamplesPerSec=5.297541314279018, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3592/5000 [12:18:30<4:18:43, 11.02s/it][2022-12-20 22:41:01,211] [INFO] [timer.py:197:stop] 0/7272, RunningAvgSamplesPerSec=5.877902804687505, CurrSamplesPerSec=5.298294573190515, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3593/5000 [12:18:41<4:18:28, 11.02s/it][2022-12-20 22:41:12,221] [INFO] [timer.py:197:stop] 0/7274, RunningAvgSamplesPerSec=5.877903397779202, CurrSamplesPerSec=5.305886364269773, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3594/5000 [12:18:52<4:18:16, 11.02s/it][2022-12-20 22:41:23,241] [INFO] [timer.py:197:stop] 0/7276, RunningAvgSamplesPerSec=5.877902025459138, CurrSamplesPerSec=5.303008241809258, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3595/5000 [12:19:03<4:18:20, 11.03s/it][2022-12-20 22:41:34,259] [INFO] [timer.py:197:stop] 0/7278, RunningAvgSamplesPerSec=5.877901068744853, CurrSamplesPerSec=5.338285802150679, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3596/5000 [12:19:14<4:17:39, 11.01s/it][2022-12-20 22:41:45,257] [INFO] [logging.py:68:log_dist] [Rank 0] step=3640, skipped=5, lr=[3.0355555555555562e-06], mom=[[0.9, 0.999]]
+[2022-12-20 22:41:45,259] [INFO] [timer.py:197:stop] 0/7280, RunningAvgSamplesPerSec=5.877902571373462, CurrSamplesPerSec=5.304908890540668, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3597/5000 [12:19:25<4:17:40, 11.02s/it][2022-12-20 22:41:56,264] [INFO] [timer.py:197:stop] 0/7282, RunningAvgSamplesPerSec=5.877903173070877, CurrSamplesPerSec=5.334335727439525, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3598/5000 [12:19:36<4:17:26, 11.02s/it][2022-12-20 22:42:07,290] [INFO] [timer.py:197:stop] 0/7284, RunningAvgSamplesPerSec=5.877900954673123, CurrSamplesPerSec=5.3225249324580695, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3599/5000 [12:19:47<4:17:24, 11.02s/it][2022-12-20 22:42:18,343] [INFO] [timer.py:197:stop] 0/7286, RunningAvgSamplesPerSec=5.8778947413767355, CurrSamplesPerSec=5.312329788936401, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3600/5000 [12:19:58<4:17:07, 11.02s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 3.028888888888889e-06, 'epoch': 87.8}
+
 72%|███████▏  | 3600/5000 [12:19:58<4:17:07, 11.02s/it][2022-12-20 22:42:29,374] [INFO] [timer.py:197:stop] 0/7288, RunningAvgSamplesPerSec=5.877892056451469, CurrSamplesPerSec=5.29351703959998, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3601/5000 [12:20:09<4:17:12, 11.03s/it][2022-12-20 22:42:40,413] [INFO] [timer.py:197:stop] 0/7290, RunningAvgSamplesPerSec=5.877887988269642, CurrSamplesPerSec=5.308599424033712, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3602/5000 [12:20:20<4:16:54, 11.03s/it][2022-12-20 22:42:51,413] [INFO] [timer.py:197:stop] 0/7292, RunningAvgSamplesPerSec=5.877889597550178, CurrSamplesPerSec=5.320303298096859, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3603/5000 [12:20:31<4:16:53, 11.03s/it][2022-12-20 22:43:02,408] [INFO] [timer.py:197:stop] 0/7294, RunningAvgSamplesPerSec=5.877891995313751, CurrSamplesPerSec=5.3705101742920505, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3604/5000 [12:20:42<4:16:05, 11.01s/it][2022-12-20 22:43:13,372] [INFO] [timer.py:197:stop] 0/7296, RunningAvgSamplesPerSec=5.877898883611844, CurrSamplesPerSec=5.353014082709496, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3605/5000 [12:20:53<4:15:45, 11.00s/it][2022-12-20 22:43:24,393] [INFO] [timer.py:197:stop] 0/7298, RunningAvgSamplesPerSec=5.877897444271325, CurrSamplesPerSec=5.321478022522421, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3606/5000 [12:21:04<4:15:58, 11.02s/it][2022-12-20 22:43:35,406] [INFO] [logging.py:68:log_dist] [Rank 0] step=3650, skipped=5, lr=[3.013333333333334e-06], mom=[[0.9, 0.999]]
+[2022-12-20 22:43:35,407] [INFO] [timer.py:197:stop] 0/7300, RunningAvgSamplesPerSec=5.877896844100633, CurrSamplesPerSec=5.360514180341661, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3607/5000 [12:21:15<4:15:35, 11.01s/it][2022-12-20 22:43:46,425] [INFO] [timer.py:197:stop] 0/7302, RunningAvgSamplesPerSec=5.87789575555824, CurrSamplesPerSec=5.334672204241832, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3608/5000 [12:21:26<4:15:21, 11.01s/it][2022-12-20 22:43:56,543] [INFO] [timer.py:197:stop] 0/7304, RunningAvgSamplesPerSec=5.878026146968897, CurrSamplesPerSec=6.2443486339058145, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-20 22:44:07,539] [INFO] [timer.py:197:stop] 0/7306, RunningAvgSamplesPerSec=5.878027842535741, CurrSamplesPerSec=5.334809393795689, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3609/5000 [12:21:43<4:51:06, 12.56s/it][2022-12-20 22:44:18,522] [INFO] [timer.py:197:stop] 0/7308, RunningAvgSamplesPerSec=5.878031980968481, CurrSamplesPerSec=5.316654394818856, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3610/5000 [12:21:54<4:39:57, 12.08s/it][2022-12-20 22:44:29,506] [INFO] [timer.py:197:stop] 0/7310, RunningAvgSamplesPerSec=5.878036092730679, CurrSamplesPerSec=5.364719056554497, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3611/5000 [12:22:05<4:32:06, 11.75s/it][2022-12-20 22:44:40,529] [INFO] [timer.py:197:stop] 0/7312, RunningAvgSamplesPerSec=5.878034498206484, CurrSamplesPerSec=5.331615365783328, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3612/5000 [12:22:16<4:26:50, 11.53s/it][2022-12-20 22:44:51,566] [INFO] [timer.py:197:stop] 0/7314, RunningAvgSamplesPerSec=5.878030823276953, CurrSamplesPerSec=5.3287898761283525, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3613/5000 [12:22:27<4:23:11, 11.39s/it][2022-12-20 22:45:02,458] [INFO] [timer.py:197:stop] 0/7316, RunningAvgSamplesPerSec=5.87804834884731, CurrSamplesPerSec=5.40590934456368, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3614/5000 [12:22:38<4:19:35, 11.24s/it][2022-12-20 22:45:13,439] [INFO] [timer.py:197:stop] 0/7318, RunningAvgSamplesPerSec=5.878052890057602, CurrSamplesPerSec=5.371450063538423, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3615/5000 [12:22:49<4:17:37, 11.16s/it][2022-12-20 22:45:24,444] [INFO] [logging.py:68:log_dist] [Rank 0] step=3660, skipped=5, lr=[2.9911111111111115e-06], mom=[[0.9, 0.999]]
+[2022-12-20 22:45:24,446] [INFO] [timer.py:197:stop] 0/7320, RunningAvgSamplesPerSec=5.8780533728260975, CurrSamplesPerSec=5.337269400867251, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3616/5000 [12:23:00<4:16:22, 11.11s/it][2022-12-20 22:45:35,463] [INFO] [timer.py:197:stop] 0/7322, RunningAvgSamplesPerSec=5.878052364916596, CurrSamplesPerSec=5.331462245204065, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3617/5000 [12:23:11<4:15:31, 11.09s/it][2022-12-20 22:45:46,453] [INFO] [timer.py:197:stop] 0/7324, RunningAvgSamplesPerSec=5.878055378081517, CurrSamplesPerSec=5.331975646815904, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3618/5000 [12:23:22<4:14:40, 11.06s/it][2022-12-20 22:45:57,441] [INFO] [timer.py:197:stop] 0/7326, RunningAvgSamplesPerSec=5.8780586345905075, CurrSamplesPerSec=5.3712213476487225, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3619/5000 [12:23:33<4:14:00, 11.04s/it][2022-12-20 22:46:08,466] [INFO] [timer.py:197:stop] 0/7328, RunningAvgSamplesPerSec=5.878056452421843, CurrSamplesPerSec=5.330106992728701, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3620/5000 [12:23:44<4:13:45, 11.03s/it][2022-12-20 22:46:19,511] [INFO] [timer.py:197:stop] 0/7330, RunningAvgSamplesPerSec=5.87805140526741, CurrSamplesPerSec=5.318821973930595, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3621/5000 [12:23:55<4:13:39, 11.04s/it][2022-12-20 22:46:30,503] [INFO] [timer.py:197:stop] 0/7332, RunningAvgSamplesPerSec=5.8780541307577785, CurrSamplesPerSec=5.3348390802537216, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3622/5000 [12:24:06<4:13:09, 11.02s/it][2022-12-20 22:46:41,472] [INFO] [timer.py:197:stop] 0/7334, RunningAvgSamplesPerSec=5.878060196639519, CurrSamplesPerSec=5.350546156018779, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3623/5000 [12:24:17<4:12:36, 11.01s/it][2022-12-20 22:46:52,493] [INFO] [timer.py:197:stop] 0/7336, RunningAvgSamplesPerSec=5.878058521840436, CurrSamplesPerSec=5.340344202190716, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▏  | 3624/5000 [12:24:28<4:12:31, 11.01s/it][2022-12-20 22:47:03,563] [INFO] [timer.py:197:stop] 0/7338, RunningAvgSamplesPerSec=5.878049806912412, CurrSamplesPerSec=5.310230104766655, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 72%|███████▎  | 3625/5000 [12:24:39<4:12:44, 11.03s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 2.9711111111111112e-06, 'epoch': 88.41}
+
 72%|███████▎  | 3625/5000 [12:24:39<4:12:44, 11.03s/it][2022-12-20 22:47:14,558] [INFO] [logging.py:68:log_dist] [Rank 0] step=3670, skipped=5, lr=[2.968888888888889e-06], mom=[[0.9, 0.999]]
+[2022-12-20 22:47:14,560] [INFO] [timer.py:197:stop] 0/7340, RunningAvgSamplesPerSec=5.878052333548965, CurrSamplesPerSec=5.347223974072178, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3626/5000 [12:24:50<4:12:20, 11.02s/it][2022-12-20 22:47:25,601] [INFO] [timer.py:197:stop] 0/7342, RunningAvgSamplesPerSec=5.878047900381191, CurrSamplesPerSec=5.303931776894248, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3627/5000 [12:25:01<4:12:18, 11.03s/it][2022-12-20 22:47:36,614] [INFO] [timer.py:197:stop] 0/7344, RunningAvgSamplesPerSec=5.878047730698672, CurrSamplesPerSec=5.331107750811973, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3628/5000 [12:25:12<4:12:01, 11.02s/it][2022-12-20 22:47:47,738] [INFO] [timer.py:197:stop] 0/7346, RunningAvgSamplesPerSec=5.8780310561933975, CurrSamplesPerSec=5.235020312277251, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3629/5000 [12:25:23<4:12:33, 11.05s/it][2022-12-20 22:47:58,765] [INFO] [timer.py:197:stop] 0/7348, RunningAvgSamplesPerSec=5.878028691253228, CurrSamplesPerSec=5.327920901860552, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3630/5000 [12:25:34<4:12:11, 11.05s/it][2022-12-20 22:48:09,788] [INFO] [timer.py:197:stop] 0/7350, RunningAvgSamplesPerSec=5.87802699583217, CurrSamplesPerSec=5.333532764686408, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3631/5000 [12:25:45<4:11:51, 11.04s/it][2022-12-20 22:48:20,777] [INFO] [timer.py:197:stop] 0/7352, RunningAvgSamplesPerSec=5.878030309104839, CurrSamplesPerSec=5.3392248525194175, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3632/5000 [12:25:56<4:11:19, 11.02s/it][2022-12-20 22:48:31,735] [INFO] [timer.py:197:stop] 0/7354, RunningAvgSamplesPerSec=5.878038024013735, CurrSamplesPerSec=5.363030957343603, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3633/5000 [12:26:07<4:10:42, 11.00s/it][2022-12-20 22:48:42,797] [INFO] [timer.py:197:stop] 0/7356, RunningAvgSamplesPerSec=5.878030863530207, CurrSamplesPerSec=5.301377176699093, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3634/5000 [12:26:18<4:10:54, 11.02s/it][2022-12-20 22:48:53,867] [INFO] [timer.py:197:stop] 0/7358, RunningAvgSamplesPerSec=5.87802229856793, CurrSamplesPerSec=5.300160658162993, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3635/5000 [12:26:29<4:11:03, 11.04s/it][2022-12-20 22:49:04,926] [INFO] [logging.py:68:log_dist] [Rank 0] step=3680, skipped=5, lr=[2.946666666666667e-06], mom=[[0.9, 0.999]]
+[2022-12-20 22:49:04,928] [INFO] [timer.py:197:stop] 0/7360, RunningAvgSamplesPerSec=5.878015273498023, CurrSamplesPerSec=5.301714743879037, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3636/5000 [12:26:40<4:11:03, 11.04s/it][2022-12-20 22:49:15,949] [INFO] [timer.py:197:stop] 0/7362, RunningAvgSamplesPerSec=5.878014029393253, CurrSamplesPerSec=5.326591547569705, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3637/5000 [12:26:51<4:10:43, 11.04s/it][2022-12-20 22:49:26,968] [INFO] [timer.py:197:stop] 0/7364, RunningAvgSamplesPerSec=5.878012949924097, CurrSamplesPerSec=5.322669730089044, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3638/5000 [12:27:02<4:10:24, 11.03s/it][2022-12-20 22:49:37,947] [INFO] [timer.py:197:stop] 0/7366, RunningAvgSamplesPerSec=5.878017645845158, CurrSamplesPerSec=5.3439699592295415, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3639/5000 [12:27:13<4:09:52, 11.02s/it][2022-12-20 22:49:48,934] [INFO] [timer.py:197:stop] 0/7368, RunningAvgSamplesPerSec=5.8780212121197195, CurrSamplesPerSec=5.313443358020031, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3640/5000 [12:27:24<4:09:29, 11.01s/it][2022-12-20 22:50:00,024] [INFO] [timer.py:197:stop] 0/7370, RunningAvgSamplesPerSec=5.878009794764269, CurrSamplesPerSec=5.283415145251304, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3641/5000 [12:27:35<4:09:52, 11.03s/it][2022-12-20 22:50:11,028] [INFO] [timer.py:197:stop] 0/7372, RunningAvgSamplesPerSec=5.878010841181725, CurrSamplesPerSec=5.33514444585645, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3642/5000 [12:27:46<4:09:30, 11.02s/it][2022-12-20 22:50:21,968] [INFO] [timer.py:197:stop] 0/7374, RunningAvgSamplesPerSec=5.8780214076935575, CurrSamplesPerSec=5.358752985652896, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3643/5000 [12:27:57<4:08:44, 11.00s/it][2022-12-20 22:50:32,917] [INFO] [timer.py:197:stop] 0/7376, RunningAvgSamplesPerSec=5.8780305800332115, CurrSamplesPerSec=5.36434790515311, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3644/5000 [12:28:08<4:08:13, 10.98s/it][2022-12-20 22:50:43,847] [INFO] [timer.py:197:stop] 0/7378, RunningAvgSamplesPerSec=5.878042347007752, CurrSamplesPerSec=5.369321222619378, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3645/5000 [12:28:19<4:07:41, 10.97s/it][2022-12-20 22:50:54,832] [INFO] [logging.py:68:log_dist] [Rank 0] step=3690, skipped=5, lr=[2.9244444444444447e-06], mom=[[0.9, 0.999]]
+[2022-12-20 22:50:54,833] [INFO] [timer.py:197:stop] 0/7380, RunningAvgSamplesPerSec=5.878045830509898, CurrSamplesPerSec=5.331595033913721, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3646/5000 [12:28:30<4:07:37, 10.97s/it][2022-12-20 22:51:05,792] [INFO] [timer.py:197:stop] 0/7382, RunningAvgSamplesPerSec=5.87805330293477, CurrSamplesPerSec=5.371309263388221, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3647/5000 [12:28:41<4:07:20, 10.97s/it][2022-12-20 22:51:16,824] [INFO] [timer.py:197:stop] 0/7384, RunningAvgSamplesPerSec=5.878050257471396, CurrSamplesPerSec=5.304262332658203, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3648/5000 [12:28:52<4:07:35, 10.99s/it][2022-12-20 22:51:27,805] [INFO] [timer.py:197:stop] 0/7386, RunningAvgSamplesPerSec=5.878054638963181, CurrSamplesPerSec=5.353333062644151, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3649/5000 [12:29:03<4:07:21, 10.99s/it][2022-12-20 22:51:37,930] [INFO] [timer.py:197:stop] 0/7388, RunningAvgSamplesPerSec=5.878182748346063, CurrSamplesPerSec=5.3158426396809615, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3650/5000 [12:29:18<4:34:26, 12.20s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 2.915555555555556e-06, 'epoch': 89.02}
+
 73%|███████▎  | 3650/5000 [12:29:18<4:34:26, 12.20s/it][2022-12-20 22:51:48,897] [INFO] [timer.py:197:stop] 0/7390, RunningAvgSamplesPerSec=5.87818926627693, CurrSamplesPerSec=5.332236833349364, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3651/5000 [12:29:29<4:26:09, 11.84s/it][2022-12-20 22:51:59,920] [INFO] [timer.py:197:stop] 0/7392, RunningAvgSamplesPerSec=5.878187285979901, CurrSamplesPerSec=5.309254180356859, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3652/5000 [12:29:40<4:20:22, 11.59s/it][2022-12-20 22:52:10,883] [INFO] [timer.py:197:stop] 0/7394, RunningAvgSamplesPerSec=5.8781941906563215, CurrSamplesPerSec=5.350409862205678, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3653/5000 [12:29:51<4:15:50, 11.40s/it][2022-12-20 22:52:21,846] [INFO] [timer.py:197:stop] 0/7396, RunningAvgSamplesPerSec=5.878200977133208, CurrSamplesPerSec=5.335010844103868, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3654/5000 [12:30:02<4:12:59, 11.28s/it][2022-12-20 22:52:32,817] [INFO] [timer.py:197:stop] 0/7398, RunningAvgSamplesPerSec=5.878206764855284, CurrSamplesPerSec=5.361465137223794, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3655/5000 [12:30:13<4:10:29, 11.17s/it][2022-12-20 22:52:43,798] [INFO] [logging.py:68:log_dist] [Rank 0] step=3700, skipped=5, lr=[2.9022222222222223e-06], mom=[[0.9, 0.999]]
+[2022-12-20 22:52:43,800] [INFO] [timer.py:197:stop] 0/7400, RunningAvgSamplesPerSec=5.878210684548515, CurrSamplesPerSec=5.317834461275866, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3656/5000 [12:30:24<4:09:16, 11.13s/it][2022-12-20 22:52:54,847] [INFO] [timer.py:197:stop] 0/7402, RunningAvgSamplesPerSec=5.8782050604732845, CurrSamplesPerSec=5.292844451623324, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3657/5000 [12:30:35<4:08:37, 11.11s/it][2022-12-20 22:53:05,824] [INFO] [timer.py:197:stop] 0/7404, RunningAvgSamplesPerSec=5.8782097900660215, CurrSamplesPerSec=5.366779869972342, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3658/5000 [12:30:46<4:07:29, 11.07s/it][2022-12-20 22:53:16,829] [INFO] [timer.py:197:stop] 0/7406, RunningAvgSamplesPerSec=5.878210688049898, CurrSamplesPerSec=5.3328508177871665, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3659/5000 [12:30:57<4:06:55, 11.05s/it][2022-12-20 22:53:27,842] [INFO] [timer.py:197:stop] 0/7408, RunningAvgSamplesPerSec=5.878210425180106, CurrSamplesPerSec=5.328186979257869, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3660/5000 [12:31:08<4:06:40, 11.04s/it][2022-12-20 22:53:38,858] [INFO] [timer.py:197:stop] 0/7410, RunningAvgSamplesPerSec=5.878209737227267, CurrSamplesPerSec=5.347450437733798, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3661/5000 [12:31:19<4:06:14, 11.03s/it][2022-12-20 22:53:49,874] [INFO] [timer.py:197:stop] 0/7412, RunningAvgSamplesPerSec=5.878208894589859, CurrSamplesPerSec=5.34074455425436, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3662/5000 [12:31:30<4:05:54, 11.03s/it][2022-12-20 22:54:00,846] [INFO] [timer.py:197:stop] 0/7414, RunningAvgSamplesPerSec=5.878214784505318, CurrSamplesPerSec=5.376476838677448, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3663/5000 [12:31:41<4:05:07, 11.00s/it][2022-12-20 22:54:11,855] [INFO] [timer.py:197:stop] 0/7416, RunningAvgSamplesPerSec=5.87821516827551, CurrSamplesPerSec=5.313422323117286, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3664/5000 [12:31:52<4:05:08, 11.01s/it][2022-12-20 22:54:22,887] [INFO] [timer.py:197:stop] 0/7418, RunningAvgSamplesPerSec=5.878212142178648, CurrSamplesPerSec=5.312883886306952, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3665/5000 [12:32:03<4:05:06, 11.02s/it][2022-12-20 22:54:33,877] [INFO] [logging.py:68:log_dist] [Rank 0] step=3710, skipped=5, lr=[2.88e-06], mom=[[0.9, 0.999]]
+[2022-12-20 22:54:33,878] [INFO] [timer.py:197:stop] 0/7420, RunningAvgSamplesPerSec=5.8782150698215725, CurrSamplesPerSec=5.348794275202794, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3666/5000 [12:32:14<4:04:31, 11.00s/it][2022-12-20 22:54:44,885] [INFO] [timer.py:197:stop] 0/7422, RunningAvgSamplesPerSec=5.878215546432252, CurrSamplesPerSec=5.301923546090147, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3667/5000 [12:32:25<4:04:40, 11.01s/it][2022-12-20 22:54:55,918] [INFO] [timer.py:197:stop] 0/7424, RunningAvgSamplesPerSec=5.878212275573226, CurrSamplesPerSec=5.314704704789132, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3668/5000 [12:32:36<4:04:36, 11.02s/it][2022-12-20 22:55:06,958] [INFO] [timer.py:197:stop] 0/7426, RunningAvgSamplesPerSec=5.878208159805311, CurrSamplesPerSec=5.30888373391323, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3669/5000 [12:32:47<4:04:33, 11.02s/it][2022-12-20 22:55:17,944] [INFO] [timer.py:197:stop] 0/7428, RunningAvgSamplesPerSec=5.878211773794939, CurrSamplesPerSec=5.355146248200109, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3670/5000 [12:32:58<4:03:59, 11.01s/it][2022-12-20 22:55:28,976] [INFO] [timer.py:197:stop] 0/7430, RunningAvgSamplesPerSec=5.878208823479481, CurrSamplesPerSec=5.297335575075926, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3671/5000 [12:33:09<4:04:08, 11.02s/it][2022-12-20 22:55:40,014] [INFO] [timer.py:197:stop] 0/7432, RunningAvgSamplesPerSec=5.878204866157317, CurrSamplesPerSec=5.3134643930893235, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3672/5000 [12:33:20<4:04:01, 11.02s/it][2022-12-20 22:55:51,033] [INFO] [timer.py:197:stop] 0/7434, RunningAvgSamplesPerSec=5.878203642722273, CurrSamplesPerSec=5.324958198449896, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3673/5000 [12:33:31<4:03:57, 11.03s/it][2022-12-20 22:56:02,090] [INFO] [timer.py:197:stop] 0/7436, RunningAvgSamplesPerSec=5.878196980080248, CurrSamplesPerSec=5.313976227650446, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 73%|███████▎  | 3674/5000 [12:33:42<4:03:57, 11.04s/it][2022-12-20 22:56:13,161] [INFO] [timer.py:197:stop] 0/7438, RunningAvgSamplesPerSec=5.878188150274712, CurrSamplesPerSec=5.300909637589686, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▎  | 3675/5000 [12:33:53<4:03:35, 11.03s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 2.86e-06, 'epoch': 89.63}
+
 74%|███████▎  | 3675/5000 [12:33:53<4:03:35, 11.03s/it][2022-12-20 22:56:24,182] [INFO] [logging.py:68:log_dist] [Rank 0] step=3720, skipped=5, lr=[2.8577777777777784e-06], mom=[[0.9, 0.999]]
+[2022-12-20 22:56:24,183] [INFO] [timer.py:197:stop] 0/7440, RunningAvgSamplesPerSec=5.878186611237144, CurrSamplesPerSec=5.292870542011028, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▎  | 3676/5000 [12:34:04<4:03:34, 11.04s/it][2022-12-20 22:56:35,228] [INFO] [timer.py:197:stop] 0/7442, RunningAvgSamplesPerSec=5.878181684090456, CurrSamplesPerSec=5.302484901371005, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▎  | 3677/5000 [12:34:15<4:03:34, 11.05s/it][2022-12-20 22:56:46,248] [INFO] [timer.py:197:stop] 0/7444, RunningAvgSamplesPerSec=5.878180223552527, CurrSamplesPerSec=5.342893115329626, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▎  | 3678/5000 [12:34:26<4:03:06, 11.03s/it][2022-12-20 22:56:57,259] [INFO] [timer.py:197:stop] 0/7446, RunningAvgSamplesPerSec=5.87818018684361, CurrSamplesPerSec=5.335865798835011, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▎  | 3679/5000 [12:34:37<4:02:53, 11.03s/it][2022-12-20 22:57:08,279] [INFO] [timer.py:197:stop] 0/7448, RunningAvgSamplesPerSec=5.878178891789989, CurrSamplesPerSec=5.3446281496239285, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▎  | 3680/5000 [12:34:48<4:02:38, 11.03s/it][2022-12-20 22:57:19,324] [INFO] [timer.py:197:stop] 0/7450, RunningAvgSamplesPerSec=5.878173834211167, CurrSamplesPerSec=5.321941389215166, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▎  | 3681/5000 [12:34:59<4:02:17, 11.02s/it][2022-12-20 22:57:30,341] [INFO] [timer.py:197:stop] 0/7452, RunningAvgSamplesPerSec=5.878172652450014, CurrSamplesPerSec=5.311729769820751, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▎  | 3682/5000 [12:35:10<4:01:58, 11.02s/it][2022-12-20 22:57:41,355] [INFO] [timer.py:197:stop] 0/7454, RunningAvgSamplesPerSec=5.878172186743465, CurrSamplesPerSec=5.300643556096169, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▎  | 3683/5000 [12:35:21<4:02:01, 11.03s/it][2022-12-20 22:57:52,347] [INFO] [timer.py:197:stop] 0/7456, RunningAvgSamplesPerSec=5.878174806794226, CurrSamplesPerSec=5.353260253300206, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▎  | 3684/5000 [12:35:32<4:01:42, 11.02s/it][2022-12-20 22:58:03,344] [INFO] [timer.py:197:stop] 0/7458, RunningAvgSamplesPerSec=5.8781769894239915, CurrSamplesPerSec=5.363932002158878, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▎  | 3685/5000 [12:35:43<4:01:16, 11.01s/it][2022-12-20 22:58:14,362] [INFO] [logging.py:68:log_dist] [Rank 0] step=3730, skipped=5, lr=[2.835555555555556e-06], mom=[[0.9, 0.999]]
+[2022-12-20 22:58:14,363] [INFO] [timer.py:197:stop] 0/7460, RunningAvgSamplesPerSec=5.878175673460642, CurrSamplesPerSec=5.328023691727129, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▎  | 3686/5000 [12:35:54<4:00:55, 11.00s/it][2022-12-20 22:58:25,345] [INFO] [timer.py:197:stop] 0/7462, RunningAvgSamplesPerSec=5.8781799166570705, CurrSamplesPerSec=5.331557971008736, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▎  | 3687/5000 [12:36:05<4:01:00, 11.01s/it][2022-12-20 22:58:36,400] [INFO] [timer.py:197:stop] 0/7464, RunningAvgSamplesPerSec=5.8781736174222, CurrSamplesPerSec=5.320273562299765, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3688/5000 [12:36:16<4:00:48, 11.01s/it][2022-12-20 22:58:47,412] [INFO] [timer.py:197:stop] 0/7466, RunningAvgSamplesPerSec=5.878173538778539, CurrSamplesPerSec=5.3193152353601025, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3689/5000 [12:36:27<4:00:22, 11.00s/it][2022-12-20 22:58:58,391] [INFO] [timer.py:197:stop] 0/7468, RunningAvgSamplesPerSec=5.878178142466483, CurrSamplesPerSec=5.3138081297177076, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3690/5000 [12:36:38<4:00:19, 11.01s/it][2022-12-20 22:59:08,491] [INFO] [timer.py:197:stop] 0/7470, RunningAvgSamplesPerSec=5.878308404306301, CurrSamplesPerSec=6.261764181850644, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-20 22:59:19,523] [INFO] [timer.py:197:stop] 0/7472, RunningAvgSamplesPerSec=5.878304989022367, CurrSamplesPerSec=5.31860825571978, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3691/5000 [12:36:55<4:34:06, 12.56s/it][2022-12-20 22:59:30,546] [INFO] [timer.py:197:stop] 0/7474, RunningAvgSamplesPerSec=5.878303353055128, CurrSamplesPerSec=5.311923594391239, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3692/5000 [12:37:06<4:23:48, 12.10s/it][2022-12-20 22:59:41,509] [INFO] [timer.py:197:stop] 0/7476, RunningAvgSamplesPerSec=5.878310437952915, CurrSamplesPerSec=5.353786190523981, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3693/5000 [12:37:17<4:16:10, 11.76s/it][2022-12-20 22:59:52,580] [INFO] [timer.py:197:stop] 0/7478, RunningAvgSamplesPerSec=5.878301738271722, CurrSamplesPerSec=5.305817776268907, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3694/5000 [12:37:28<4:11:28, 11.55s/it][2022-12-20 23:00:03,589] [INFO] [logging.py:68:log_dist] [Rank 0] step=3740, skipped=5, lr=[2.8133333333333336e-06], mom=[[0.9, 0.999]]
+[2022-12-20 23:00:03,590] [INFO] [timer.py:197:stop] 0/7480, RunningAvgSamplesPerSec=5.878301723692942, CurrSamplesPerSec=5.3055344238442705, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3695/5000 [12:37:39<4:07:44, 11.39s/it][2022-12-20 23:00:14,618] [INFO] [timer.py:197:stop] 0/7482, RunningAvgSamplesPerSec=5.878299272050411, CurrSamplesPerSec=5.329698075157004, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3696/5000 [12:37:50<4:05:11, 11.28s/it][2022-12-20 23:00:25,593] [INFO] [timer.py:197:stop] 0/7484, RunningAvgSamplesPerSec=5.878304467366728, CurrSamplesPerSec=5.370003935348891, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3697/5000 [12:38:01<4:03:00, 11.19s/it][2022-12-20 23:00:36,624] [INFO] [timer.py:197:stop] 0/7486, RunningAvgSamplesPerSec=5.878301585058994, CurrSamplesPerSec=5.30574394651884, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3698/5000 [12:38:12<4:01:46, 11.14s/it][2022-12-20 23:00:47,708] [INFO] [timer.py:197:stop] 0/7488, RunningAvgSamplesPerSec=5.878291097964548, CurrSamplesPerSec=5.283247519455194, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3699/5000 [12:38:23<4:01:13, 11.12s/it][2022-12-20 23:00:58,793] [INFO] [timer.py:197:stop] 0/7490, RunningAvgSamplesPerSec=5.87828052563925, CurrSamplesPerSec=5.313727135136402, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3700/5000 [12:38:34<4:00:46, 11.11s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 2.8022222222222225e-06, 'epoch': 90.24}
+
 74%|███████▍  | 3700/5000 [12:38:34<4:00:46, 11.11s/it][2022-12-20 23:01:09,877] [INFO] [timer.py:197:stop] 0/7492, RunningAvgSamplesPerSec=5.878270118897539, CurrSamplesPerSec=5.299302459905826, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3701/5000 [12:38:45<4:00:24, 11.10s/it][2022-12-20 23:01:20,906] [INFO] [timer.py:197:stop] 0/7494, RunningAvgSamplesPerSec=5.878267525492624, CurrSamplesPerSec=5.3045229075522045, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3702/5000 [12:38:56<3:59:43, 11.08s/it][2022-12-20 23:01:31,958] [INFO] [timer.py:197:stop] 0/7496, RunningAvgSamplesPerSec=5.878261753173252, CurrSamplesPerSec=5.300910265665725, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3703/5000 [12:39:07<3:59:21, 11.07s/it][2022-12-20 23:01:42,897] [INFO] [timer.py:197:stop] 0/7498, RunningAvgSamplesPerSec=5.878272261372583, CurrSamplesPerSec=5.352530347669919, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3704/5000 [12:39:18<3:58:18, 11.03s/it][2022-12-20 23:01:53,908] [INFO] [logging.py:68:log_dist] [Rank 0] step=3750, skipped=5, lr=[2.7911111111111113e-06], mom=[[0.9, 0.999]]
+[2022-12-20 23:01:53,910] [INFO] [timer.py:197:stop] 0/7500, RunningAvgSamplesPerSec=5.878271941932843, CurrSamplesPerSec=5.351049586155769, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3705/5000 [12:39:29<3:57:59, 11.03s/it][2022-12-20 23:02:04,874] [INFO] [timer.py:197:stop] 0/7502, RunningAvgSamplesPerSec=5.878278677706603, CurrSamplesPerSec=5.341788641959979, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3706/5000 [12:39:40<3:57:24, 11.01s/it][2022-12-20 23:02:15,874] [INFO] [timer.py:197:stop] 0/7504, RunningAvgSamplesPerSec=5.8782803357034235, CurrSamplesPerSec=5.358951541034085, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3707/5000 [12:39:51<3:57:10, 11.01s/it][2022-12-20 23:02:26,928] [INFO] [timer.py:197:stop] 0/7506, RunningAvgSamplesPerSec=5.878274027798656, CurrSamplesPerSec=5.303792188672462, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3708/5000 [12:40:02<3:57:18, 11.02s/it][2022-12-20 23:02:37,968] [INFO] [timer.py:197:stop] 0/7508, RunningAvgSamplesPerSec=5.878269937813006, CurrSamplesPerSec=5.3118720886514295, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3709/5000 [12:40:13<3:57:14, 11.03s/it][2022-12-20 23:02:49,016] [INFO] [timer.py:197:stop] 0/7510, RunningAvgSamplesPerSec=5.878265098243694, CurrSamplesPerSec=5.333810637058999, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3710/5000 [12:40:24<3:57:12, 11.03s/it][2022-12-20 23:03:00,003] [INFO] [timer.py:197:stop] 0/7512, RunningAvgSamplesPerSec=5.878268643141073, CurrSamplesPerSec=5.337335832986836, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3711/5000 [12:40:35<3:56:43, 11.02s/it][2022-12-20 23:03:11,036] [INFO] [timer.py:197:stop] 0/7514, RunningAvgSamplesPerSec=5.878265530385083, CurrSamplesPerSec=5.315779689005386, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3712/5000 [12:40:46<3:56:38, 11.02s/it][2022-12-20 23:03:22,004] [INFO] [timer.py:197:stop] 0/7516, RunningAvgSamplesPerSec=5.878271810608499, CurrSamplesPerSec=5.3113722195255555, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3713/5000 [12:40:57<3:56:05, 11.01s/it][2022-12-20 23:03:33,052] [INFO] [timer.py:197:stop] 0/7518, RunningAvgSamplesPerSec=5.87826670738317, CurrSamplesPerSec=5.318396662104756, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3714/5000 [12:41:08<3:56:10, 11.02s/it][2022-12-20 23:03:44,036] [INFO] [logging.py:68:log_dist] [Rank 0] step=3760, skipped=5, lr=[2.7688888888888893e-06], mom=[[0.9, 0.999]]
+[2022-12-20 23:03:44,037] [INFO] [timer.py:197:stop] 0/7520, RunningAvgSamplesPerSec=5.878270256643285, CurrSamplesPerSec=5.318512783607798, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3715/5000 [12:41:19<3:55:46, 11.01s/it][2022-12-20 23:03:55,081] [INFO] [timer.py:197:stop] 0/7522, RunningAvgSamplesPerSec=5.8782656238565165, CurrSamplesPerSec=5.320530439939047, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3716/5000 [12:41:30<3:55:48, 11.02s/it][2022-12-20 23:04:06,100] [INFO] [timer.py:197:stop] 0/7524, RunningAvgSamplesPerSec=5.878264496594915, CurrSamplesPerSec=5.330162239509823, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3717/5000 [12:41:41<3:55:37, 11.02s/it][2022-12-20 23:04:17,091] [INFO] [timer.py:197:stop] 0/7526, RunningAvgSamplesPerSec=5.878267221105005, CurrSamplesPerSec=5.330617381093487, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3718/5000 [12:41:52<3:55:16, 11.01s/it][2022-12-20 23:04:28,102] [INFO] [timer.py:197:stop] 0/7528, RunningAvgSamplesPerSec=5.8782673045270135, CurrSamplesPerSec=5.344393625434488, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3719/5000 [12:42:03<3:55:04, 11.01s/it][2022-12-20 23:04:39,135] [INFO] [timer.py:197:stop] 0/7530, RunningAvgSamplesPerSec=5.878264161433213, CurrSamplesPerSec=5.313576092097527, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3720/5000 [12:42:14<3:55:02, 11.02s/it][2022-12-20 23:04:50,091] [INFO] [timer.py:197:stop] 0/7532, RunningAvgSamplesPerSec=5.8782720957249825, CurrSamplesPerSec=5.361969767438701, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3721/5000 [12:42:25<3:54:27, 11.00s/it][2022-12-20 23:05:01,073] [INFO] [timer.py:197:stop] 0/7534, RunningAvgSamplesPerSec=5.878276078235258, CurrSamplesPerSec=5.337995362284487, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3722/5000 [12:42:36<3:54:10, 10.99s/it][2022-12-20 23:05:12,008] [INFO] [timer.py:197:stop] 0/7536, RunningAvgSamplesPerSec=5.878286766695985, CurrSamplesPerSec=5.362018179235223, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3723/5000 [12:42:47<3:53:36, 10.98s/it][2022-12-20 23:05:22,961] [INFO] [timer.py:197:stop] 0/7538, RunningAvgSamplesPerSec=5.878295039241417, CurrSamplesPerSec=5.361974480053053, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3724/5000 [12:42:58<3:53:16, 10.97s/it][2022-12-20 23:05:34,006] [INFO] [logging.py:68:log_dist] [Rank 0] step=3770, skipped=5, lr=[2.746666666666667e-06], mom=[[0.9, 0.999]]
+[2022-12-20 23:05:34,007] [INFO] [timer.py:197:stop] 0/7540, RunningAvgSamplesPerSec=5.87828989330566, CurrSamplesPerSec=5.278788339068505, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 74%|███████▍  | 3725/5000 [12:43:09<3:53:35, 10.99s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 2.746666666666667e-06, 'epoch': 90.84}
+
 74%|███████▍  | 3725/5000 [12:43:09<3:53:35, 10.99s/it][2022-12-20 23:05:44,968] [INFO] [timer.py:197:stop] 0/7542, RunningAvgSamplesPerSec=5.878297361248722, CurrSamplesPerSec=5.361328286574438, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▍  | 3726/5000 [12:43:20<3:53:12, 10.98s/it][2022-12-20 23:05:55,908] [INFO] [timer.py:197:stop] 0/7544, RunningAvgSamplesPerSec=5.878307531489085, CurrSamplesPerSec=5.375669752221967, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▍  | 3727/5000 [12:43:31<3:52:44, 10.97s/it][2022-12-20 23:06:06,902] [INFO] [timer.py:197:stop] 0/7546, RunningAvgSamplesPerSec=5.8783096388257565, CurrSamplesPerSec=5.3144612255936305, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▍  | 3728/5000 [12:43:42<3:52:43, 10.98s/it][2022-12-20 23:06:17,926] [INFO] [timer.py:197:stop] 0/7548, RunningAvgSamplesPerSec=5.878307625348882, CurrSamplesPerSec=5.337223557301823, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▍  | 3729/5000 [12:43:53<3:52:49, 10.99s/it][2022-12-20 23:06:28,937] [INFO] [timer.py:197:stop] 0/7550, RunningAvgSamplesPerSec=5.878307688067831, CurrSamplesPerSec=5.339900994920523, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▍  | 3730/5000 [12:44:04<3:52:46, 11.00s/it][2022-12-20 23:06:39,903] [INFO] [timer.py:197:stop] 0/7552, RunningAvgSamplesPerSec=5.8783138604667, CurrSamplesPerSec=5.355189408818497, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▍  | 3731/5000 [12:44:15<3:52:23, 10.99s/it][2022-12-20 23:06:49,969] [INFO] [timer.py:197:stop] 0/7554, RunningAvgSamplesPerSec=5.878447486439268, CurrSamplesPerSec=5.359047186646252, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▍  | 3732/5000 [12:44:30<4:17:31, 12.19s/it][2022-12-20 23:07:00,979] [INFO] [timer.py:197:stop] 0/7556, RunningAvgSamplesPerSec=5.878447592823564, CurrSamplesPerSec=5.30527689512011, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▍  | 3733/5000 [12:44:41<4:09:49, 11.83s/it][2022-12-20 23:07:11,965] [INFO] [timer.py:197:stop] 0/7558, RunningAvgSamplesPerSec=5.8784509578912365, CurrSamplesPerSec=5.320025566045121, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▍  | 3734/5000 [12:44:52<4:04:37, 11.59s/it][2022-12-20 23:07:23,013] [INFO] [logging.py:68:log_dist] [Rank 0] step=3780, skipped=5, lr=[2.7244444444444445e-06], mom=[[0.9, 0.999]]
+[2022-12-20 23:07:23,014] [INFO] [timer.py:197:stop] 0/7560, RunningAvgSamplesPerSec=5.878445182145888, CurrSamplesPerSec=5.312091573111797, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▍  | 3735/5000 [12:45:03<4:00:41, 11.42s/it][2022-12-20 23:07:34,031] [INFO] [timer.py:197:stop] 0/7562, RunningAvgSamplesPerSec=5.878444256546486, CurrSamplesPerSec=5.2989787981987435, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▍  | 3736/5000 [12:45:14<3:58:10, 11.31s/it][2022-12-20 23:07:45,076] [INFO] [timer.py:197:stop] 0/7564, RunningAvgSamplesPerSec=5.878439404771691, CurrSamplesPerSec=5.303660781250123, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▍  | 3737/5000 [12:45:25<3:56:06, 11.22s/it][2022-12-20 23:07:56,041] [INFO] [timer.py:197:stop] 0/7566, RunningAvgSamplesPerSec=5.87844571088064, CurrSamplesPerSec=5.340764743448151, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▍  | 3738/5000 [12:45:36<3:54:36, 11.15s/it][2022-12-20 23:08:07,067] [INFO] [timer.py:197:stop] 0/7568, RunningAvgSamplesPerSec=5.8784435764863625, CurrSamplesPerSec=5.324779053832397, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▍  | 3739/5000 [12:45:47<3:53:02, 11.09s/it][2022-12-20 23:08:18,021] [INFO] [timer.py:197:stop] 0/7570, RunningAvgSamplesPerSec=5.878451607713328, CurrSamplesPerSec=5.308075609284219, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▍  | 3740/5000 [12:45:58<3:52:46, 11.08s/it][2022-12-20 23:08:29,069] [INFO] [timer.py:197:stop] 0/7572, RunningAvgSamplesPerSec=5.878446125690778, CurrSamplesPerSec=5.330949789403662, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▍  | 3741/5000 [12:46:09<3:51:55, 11.05s/it][2022-12-20 23:08:40,066] [INFO] [timer.py:197:stop] 0/7574, RunningAvgSamplesPerSec=5.8784479987905955, CurrSamplesPerSec=5.3168972325243855, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▍  | 3742/5000 [12:46:20<3:51:38, 11.05s/it][2022-12-20 23:08:51,185] [INFO] [timer.py:197:stop] 0/7576, RunningAvgSamplesPerSec=5.878432517080829, CurrSamplesPerSec=5.245709404510057, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▍  | 3743/5000 [12:46:31<3:51:44, 11.06s/it][2022-12-20 23:09:02,232] [INFO] [timer.py:197:stop] 0/7578, RunningAvgSamplesPerSec=5.878427278657847, CurrSamplesPerSec=5.285034550385853, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▍  | 3744/5000 [12:46:42<3:51:45, 11.07s/it][2022-12-20 23:09:13,275] [INFO] [logging.py:68:log_dist] [Rank 0] step=3790, skipped=5, lr=[2.702222222222222e-06], mom=[[0.9, 0.999]]
+[2022-12-20 23:09:13,276] [INFO] [timer.py:197:stop] 0/7580, RunningAvgSamplesPerSec=5.878422344671622, CurrSamplesPerSec=5.3279310537696265, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▍  | 3745/5000 [12:46:53<3:51:09, 11.05s/it][2022-12-20 23:09:24,247] [INFO] [timer.py:197:stop] 0/7582, RunningAvgSamplesPerSec=5.878428146982947, CurrSamplesPerSec=5.359324300031728, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▍  | 3746/5000 [12:47:04<3:50:33, 11.03s/it][2022-12-20 23:09:35,282] [INFO] [timer.py:197:stop] 0/7584, RunningAvgSamplesPerSec=5.878424526920814, CurrSamplesPerSec=5.313822225130846, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▍  | 3747/5000 [12:47:15<3:50:15, 11.03s/it][2022-12-20 23:09:46,235] [INFO] [timer.py:197:stop] 0/7586, RunningAvgSamplesPerSec=5.878432695988677, CurrSamplesPerSec=5.366853476741881, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▍  | 3748/5000 [12:47:26<3:49:37, 11.00s/it][2022-12-20 23:09:57,265] [INFO] [timer.py:197:stop] 0/7588, RunningAvgSamplesPerSec=5.878429971909111, CurrSamplesPerSec=5.299998456016633, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▍  | 3749/5000 [12:47:37<3:49:52, 11.02s/it][2022-12-20 23:10:08,345] [INFO] [timer.py:197:stop] 0/7590, RunningAvgSamplesPerSec=5.878420082765468, CurrSamplesPerSec=5.2934155768350974, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▌  | 3750/5000 [12:47:48<3:49:44, 11.03s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 2.6911111111111114e-06, 'epoch': 91.46}
+
 75%|███████▌  | 3750/5000 [12:47:48<3:49:44, 11.03s/it][2022-12-20 23:10:19,354] [INFO] [timer.py:197:stop] 0/7592, RunningAvgSamplesPerSec=5.878420417573296, CurrSamplesPerSec=5.316504870059522, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▌  | 3751/5000 [12:47:59<3:49:38, 11.03s/it][2022-12-20 23:10:30,401] [INFO] [timer.py:197:stop] 0/7594, RunningAvgSamplesPerSec=5.878415245318523, CurrSamplesPerSec=5.310415835704934, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▌  | 3752/5000 [12:48:10<3:49:36, 11.04s/it][2022-12-20 23:10:41,412] [INFO] [timer.py:197:stop] 0/7596, RunningAvgSamplesPerSec=5.8784152978270185, CurrSamplesPerSec=5.351687542602, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▌  | 3753/5000 [12:48:21<3:49:06, 11.02s/it][2022-12-20 23:10:52,414] [INFO] [timer.py:197:stop] 0/7598, RunningAvgSamplesPerSec=5.878416438312663, CurrSamplesPerSec=5.337076693404328, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▌  | 3754/5000 [12:48:32<3:48:41, 11.01s/it][2022-12-20 23:11:03,443] [INFO] [logging.py:68:log_dist] [Rank 0] step=3800, skipped=5, lr=[2.68e-06], mom=[[0.9, 0.999]]
+[2022-12-20 23:11:03,445] [INFO] [timer.py:197:stop] 0/7600, RunningAvgSamplesPerSec=5.878413575259489, CurrSamplesPerSec=5.29718985280888, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▌  | 3755/5000 [12:48:44<3:48:54, 11.03s/it][2022-12-20 23:11:14,490] [INFO] [timer.py:197:stop] 0/7602, RunningAvgSamplesPerSec=5.878408840050538, CurrSamplesPerSec=5.328150175236068, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▌  | 3756/5000 [12:48:54<3:48:22, 11.01s/it][2022-12-20 23:11:25,484] [INFO] [timer.py:197:stop] 0/7604, RunningAvgSamplesPerSec=5.878411182779143, CurrSamplesPerSec=5.309906577373378, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▌  | 3757/5000 [12:49:06<3:48:10, 11.01s/it][2022-12-20 23:11:36,437] [INFO] [timer.py:197:stop] 0/7606, RunningAvgSamplesPerSec=5.878419213635451, CurrSamplesPerSec=5.363168537846921, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▌  | 3758/5000 [12:49:16<3:47:50, 11.01s/it][2022-12-20 23:11:47,469] [INFO] [timer.py:197:stop] 0/7608, RunningAvgSamplesPerSec=5.878416188182511, CurrSamplesPerSec=5.325658837249563, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▌  | 3759/5000 [12:49:28<3:47:42, 11.01s/it][2022-12-20 23:11:58,487] [INFO] [timer.py:197:stop] 0/7610, RunningAvgSamplesPerSec=5.878415005700013, CurrSamplesPerSec=5.323341896885336, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▌  | 3760/5000 [12:49:38<3:47:17, 11.00s/it][2022-12-20 23:12:09,419] [INFO] [timer.py:197:stop] 0/7612, RunningAvgSamplesPerSec=5.878426013345752, CurrSamplesPerSec=5.358954322625611, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▌  | 3761/5000 [12:49:49<3:46:44, 10.98s/it][2022-12-20 23:12:20,367] [INFO] [timer.py:197:stop] 0/7614, RunningAvgSamplesPerSec=5.878434786467438, CurrSamplesPerSec=5.350085898992479, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▌  | 3762/5000 [12:50:00<3:46:36, 10.98s/it][2022-12-20 23:12:31,431] [INFO] [timer.py:197:stop] 0/7616, RunningAvgSamplesPerSec=5.878427200673542, CurrSamplesPerSec=5.283833424233183, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▌  | 3763/5000 [12:50:11<3:46:38, 10.99s/it][2022-12-20 23:12:42,413] [INFO] [timer.py:197:stop] 0/7618, RunningAvgSamplesPerSec=5.878431104278507, CurrSamplesPerSec=5.3149065337550585, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▌  | 3764/5000 [12:50:22<3:46:42, 11.01s/it][2022-12-20 23:12:53,406] [INFO] [logging.py:68:log_dist] [Rank 0] step=3810, skipped=5, lr=[2.6577777777777782e-06], mom=[[0.9, 0.999]]
+[2022-12-20 23:12:53,408] [INFO] [timer.py:197:stop] 0/7620, RunningAvgSamplesPerSec=5.878433118504083, CurrSamplesPerSec=5.348069848839359, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▌  | 3765/5000 [12:50:33<3:46:27, 11.00s/it][2022-12-20 23:13:04,438] [INFO] [timer.py:197:stop] 0/7622, RunningAvgSamplesPerSec=5.87843023776621, CurrSamplesPerSec=5.316054030116023, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▌  | 3766/5000 [12:50:44<3:46:26, 11.01s/it][2022-12-20 23:13:15,462] [INFO] [timer.py:197:stop] 0/7624, RunningAvgSamplesPerSec=5.878428356265862, CurrSamplesPerSec=5.321885046605942, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▌  | 3767/5000 [12:50:56<3:46:19, 11.01s/it][2022-12-20 23:13:26,454] [INFO] [timer.py:197:stop] 0/7626, RunningAvgSamplesPerSec=5.878430907917786, CurrSamplesPerSec=5.347921535076297, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▌  | 3768/5000 [12:51:07<3:46:06, 11.01s/it][2022-12-20 23:13:37,472] [INFO] [timer.py:197:stop] 0/7628, RunningAvgSamplesPerSec=5.878429871828313, CurrSamplesPerSec=5.339071294555368, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▌  | 3769/5000 [12:51:17<3:45:44, 11.00s/it][2022-12-20 23:13:48,456] [INFO] [timer.py:197:stop] 0/7630, RunningAvgSamplesPerSec=5.878433763802678, CurrSamplesPerSec=5.33728871482736, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▌  | 3770/5000 [12:51:28<3:45:25, 11.00s/it][2022-12-20 23:13:59,442] [INFO] [timer.py:197:stop] 0/7632, RunningAvgSamplesPerSec=5.878437055470631, CurrSamplesPerSec=5.333166339593109, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▌  | 3771/5000 [12:51:39<3:45:22, 11.00s/it][2022-12-20 23:14:10,435] [INFO] [timer.py:197:stop] 0/7634, RunningAvgSamplesPerSec=5.87843933398549, CurrSamplesPerSec=5.355107147844475, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▌  | 3772/5000 [12:51:50<3:45:06, 11.00s/it][2022-12-20 23:14:20,533] [INFO] [timer.py:197:stop] 0/7636, RunningAvgSamplesPerSec=5.878566986777598, CurrSamplesPerSec=6.280583818739152, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-20 23:14:31,586] [INFO] [timer.py:197:stop] 0/7638, RunningAvgSamplesPerSec=5.878560216601736, CurrSamplesPerSec=5.2951411818922045, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▌  | 3773/5000 [12:52:07<4:16:50, 12.56s/it][2022-12-20 23:14:42,540] [INFO] [logging.py:68:log_dist] [Rank 0] step=3820, skipped=5, lr=[2.635555555555556e-06], mom=[[0.9, 0.999]]
+[2022-12-20 23:14:42,542] [INFO] [timer.py:197:stop] 0/7640, RunningAvgSamplesPerSec=5.878567919694184, CurrSamplesPerSec=5.342437576369452, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 75%|███████▌  | 3774/5000 [12:52:18<4:06:47, 12.08s/it][2022-12-20 23:14:53,485] [INFO] [timer.py:197:stop] 0/7642, RunningAvgSamplesPerSec=5.878577431240252, CurrSamplesPerSec=5.327569627337699, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3775/5000 [12:52:29<3:59:38, 11.74s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 2.6333333333333332e-06, 'epoch': 92.07}
+
 76%|███████▌  | 3775/5000 [12:52:29<3:59:38, 11.74s/it][2022-12-20 23:15:04,433] [INFO] [timer.py:197:stop] 0/7644, RunningAvgSamplesPerSec=5.878586291757545, CurrSamplesPerSec=5.355597973489142, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3776/5000 [12:52:40<3:54:37, 11.50s/it][2022-12-20 23:15:15,445] [INFO] [timer.py:197:stop] 0/7646, RunningAvgSamplesPerSec=5.878585997334733, CurrSamplesPerSec=5.324745888049905, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3777/5000 [12:52:51<3:51:26, 11.35s/it][2022-12-20 23:15:26,396] [INFO] [timer.py:197:stop] 0/7648, RunningAvgSamplesPerSec=5.878594537053175, CurrSamplesPerSec=5.351348489285001, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3778/5000 [12:53:01<3:48:46, 11.23s/it][2022-12-20 23:15:37,428] [INFO] [timer.py:197:stop] 0/7650, RunningAvgSamplesPerSec=5.878591396775819, CurrSamplesPerSec=5.333393521458311, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3779/5000 [12:53:13<3:47:22, 11.17s/it][2022-12-20 23:15:48,394] [INFO] [timer.py:197:stop] 0/7652, RunningAvgSamplesPerSec=5.87859779128862, CurrSamplesPerSec=5.337131448074324, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3780/5000 [12:53:23<3:45:55, 11.11s/it][2022-12-20 23:15:59,371] [INFO] [timer.py:197:stop] 0/7654, RunningAvgSamplesPerSec=5.878602394097781, CurrSamplesPerSec=5.3624134321341375, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3781/5000 [12:53:34<3:44:55, 11.07s/it][2022-12-20 23:16:10,378] [INFO] [timer.py:197:stop] 0/7656, RunningAvgSamplesPerSec=5.878602919379653, CurrSamplesPerSec=5.300531981483919, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3782/5000 [12:53:45<3:44:20, 11.05s/it][2022-12-20 23:16:21,383] [INFO] [timer.py:197:stop] 0/7658, RunningAvgSamplesPerSec=5.878603912729005, CurrSamplesPerSec=5.337015148798557, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3783/5000 [12:53:56<3:43:52, 11.04s/it][2022-12-20 23:16:32,457] [INFO] [logging.py:68:log_dist] [Rank 0] step=3830, skipped=5, lr=[2.6133333333333334e-06], mom=[[0.9, 0.999]]
+[2022-12-20 23:16:32,458] [INFO] [timer.py:197:stop] 0/7660, RunningAvgSamplesPerSec=5.8785949973467995, CurrSamplesPerSec=5.294370232244625, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3784/5000 [12:54:08<3:43:55, 11.05s/it][2022-12-20 23:16:43,469] [INFO] [timer.py:197:stop] 0/7662, RunningAvgSamplesPerSec=5.878594951985115, CurrSamplesPerSec=5.350608866302717, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3785/5000 [12:54:19<3:43:30, 11.04s/it][2022-12-20 23:16:54,510] [INFO] [timer.py:197:stop] 0/7664, RunningAvgSamplesPerSec=5.87859055680708, CurrSamplesPerSec=5.326493674790825, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3786/5000 [12:54:30<3:43:20, 11.04s/it][2022-12-20 23:17:05,507] [INFO] [timer.py:197:stop] 0/7666, RunningAvgSamplesPerSec=5.878592709841305, CurrSamplesPerSec=5.3628333852298065, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3787/5000 [12:54:41<3:42:54, 11.03s/it][2022-12-20 23:17:16,476] [INFO] [timer.py:197:stop] 0/7668, RunningAvgSamplesPerSec=5.878598451275768, CurrSamplesPerSec=5.372889657985085, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3788/5000 [12:54:52<3:42:22, 11.01s/it][2022-12-20 23:17:27,469] [INFO] [timer.py:197:stop] 0/7670, RunningAvgSamplesPerSec=5.878600911663718, CurrSamplesPerSec=5.335066404654903, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3789/5000 [12:55:03<3:42:05, 11.00s/it][2022-12-20 23:17:38,463] [INFO] [timer.py:197:stop] 0/7672, RunningAvgSamplesPerSec=5.878602977237013, CurrSamplesPerSec=5.331597363599247, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3790/5000 [12:55:14<3:41:51, 11.00s/it][2022-12-20 23:17:49,524] [INFO] [timer.py:197:stop] 0/7674, RunningAvgSamplesPerSec=5.87859584127065, CurrSamplesPerSec=5.3280801643912445, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3791/5000 [12:55:25<3:42:02, 11.02s/it][2022-12-20 23:18:00,485] [INFO] [timer.py:197:stop] 0/7676, RunningAvgSamplesPerSec=5.878602697648732, CurrSamplesPerSec=5.349374554584891, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3792/5000 [12:55:36<3:41:30, 11.00s/it][2022-12-20 23:18:11,535] [INFO] [timer.py:197:stop] 0/7678, RunningAvgSamplesPerSec=5.8785971077164065, CurrSamplesPerSec=5.298085220168338, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3793/5000 [12:55:47<3:41:36, 11.02s/it][2022-12-20 23:18:22,570] [INFO] [logging.py:68:log_dist] [Rank 0] step=3840, skipped=5, lr=[2.5911111111111115e-06], mom=[[0.9, 0.999]]
+[2022-12-20 23:18:22,571] [INFO] [timer.py:197:stop] 0/7680, RunningAvgSamplesPerSec=5.878593255098724, CurrSamplesPerSec=5.295094387976816, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3794/5000 [12:55:58<3:41:32, 11.02s/it][2022-12-20 23:18:33,528] [INFO] [timer.py:197:stop] 0/7682, RunningAvgSamplesPerSec=5.878600691072159, CurrSamplesPerSec=5.318931790460341, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3795/5000 [12:56:09<3:40:58, 11.00s/it][2022-12-20 23:18:44,593] [INFO] [timer.py:197:stop] 0/7684, RunningAvgSamplesPerSec=5.878593226341898, CurrSamplesPerSec=5.301164858077294, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3796/5000 [12:56:20<3:41:09, 11.02s/it][2022-12-20 23:18:55,639] [INFO] [timer.py:197:stop] 0/7686, RunningAvgSamplesPerSec=5.8785882053688, CurrSamplesPerSec=5.315990021675839, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3797/5000 [12:56:31<3:41:07, 11.03s/it][2022-12-20 23:19:06,623] [INFO] [timer.py:197:stop] 0/7688, RunningAvgSamplesPerSec=5.878591872748984, CurrSamplesPerSec=5.364671024826193, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3798/5000 [12:56:42<3:40:40, 11.02s/it][2022-12-20 23:19:17,650] [INFO] [timer.py:197:stop] 0/7690, RunningAvgSamplesPerSec=5.878589557902313, CurrSamplesPerSec=5.323912865325096, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3799/5000 [12:56:53<3:40:33, 11.02s/it][2022-12-20 23:19:28,666] [INFO] [timer.py:197:stop] 0/7692, RunningAvgSamplesPerSec=5.878588644239332, CurrSamplesPerSec=5.352910326346408, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3800/5000 [12:57:04<3:40:21, 11.02s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 2.577777777777778e-06, 'epoch': 92.67}
+
 76%|███████▌  | 3800/5000 [12:57:04<3:40:21, 11.02s/it][2022-12-20 23:19:39,618] [INFO] [timer.py:197:stop] 0/7694, RunningAvgSamplesPerSec=5.878596966314001, CurrSamplesPerSec=5.351007985583689, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3801/5000 [12:57:15<3:39:46, 11.00s/it][2022-12-20 23:19:50,671] [INFO] [timer.py:197:stop] 0/7696, RunningAvgSamplesPerSec=5.87859079335074, CurrSamplesPerSec=5.315875694646052, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3802/5000 [12:57:26<3:39:55, 11.01s/it][2022-12-20 23:20:01,730] [INFO] [timer.py:197:stop] 0/7698, RunningAvgSamplesPerSec=5.87858399454067, CurrSamplesPerSec=5.313520557540341, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3803/5000 [12:57:37<3:40:00, 11.03s/it][2022-12-20 23:20:12,787] [INFO] [logging.py:68:log_dist] [Rank 0] step=3850, skipped=5, lr=[2.568888888888889e-06], mom=[[0.9, 0.999]]
+[2022-12-20 23:20:12,789] [INFO] [timer.py:197:stop] 0/7700, RunningAvgSamplesPerSec=5.878577034653145, CurrSamplesPerSec=5.301813174166168, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3804/5000 [12:57:48<3:40:00, 11.04s/it][2022-12-20 23:20:23,835] [INFO] [timer.py:197:stop] 0/7702, RunningAvgSamplesPerSec=5.878572037207572, CurrSamplesPerSec=5.306017042952742, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3805/5000 [12:57:59<3:39:52, 11.04s/it][2022-12-20 23:20:34,877] [INFO] [timer.py:197:stop] 0/7704, RunningAvgSamplesPerSec=5.878567540699114, CurrSamplesPerSec=5.306785933117945, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3806/5000 [12:58:10<3:39:42, 11.04s/it][2022-12-20 23:20:45,901] [INFO] [timer.py:197:stop] 0/7706, RunningAvgSamplesPerSec=5.8785655929271226, CurrSamplesPerSec=5.307793694881325, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3807/5000 [12:58:21<3:39:25, 11.04s/it][2022-12-20 23:20:56,898] [INFO] [timer.py:197:stop] 0/7708, RunningAvgSamplesPerSec=5.8785676051345215, CurrSamplesPerSec=5.332346780832149, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3808/5000 [12:58:32<3:39:00, 11.02s/it][2022-12-20 23:21:07,931] [INFO] [timer.py:197:stop] 0/7710, RunningAvgSamplesPerSec=5.878564400617195, CurrSamplesPerSec=5.286267244907392, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3809/5000 [12:58:43<3:38:52, 11.03s/it][2022-12-20 23:21:18,951] [INFO] [timer.py:197:stop] 0/7712, RunningAvgSamplesPerSec=5.878562904670783, CurrSamplesPerSec=5.31072702717546, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3810/5000 [12:58:54<3:38:39, 11.02s/it][2022-12-20 23:21:29,927] [INFO] [timer.py:197:stop] 0/7714, RunningAvgSamplesPerSec=5.878567572074026, CurrSamplesPerSec=5.3486646781985705, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3811/5000 [12:59:05<3:38:11, 11.01s/it][2022-12-20 23:21:40,935] [INFO] [timer.py:197:stop] 0/7716, RunningAvgSamplesPerSec=5.878567933335444, CurrSamplesPerSec=5.311430231469416, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▌  | 3812/5000 [12:59:16<3:37:59, 11.01s/it][2022-12-20 23:21:51,961] [INFO] [timer.py:197:stop] 0/7718, RunningAvgSamplesPerSec=5.87856563588244, CurrSamplesPerSec=5.319589309021926, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▋  | 3813/5000 [12:59:27<3:37:54, 11.01s/it][2022-12-20 23:22:02,069] [INFO] [logging.py:68:log_dist] [Rank 0] step=3860, skipped=5, lr=[2.5466666666666667e-06], mom=[[0.9, 0.999]]
+[2022-12-20 23:22:02,070] [INFO] [timer.py:197:stop] 0/7720, RunningAvgSamplesPerSec=5.878690290838976, CurrSamplesPerSec=5.313593762427347, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▋  | 3814/5000 [12:59:42<4:01:37, 12.22s/it][2022-12-20 23:22:13,103] [INFO] [timer.py:197:stop] 0/7722, RunningAvgSamplesPerSec=5.878687284216578, CurrSamplesPerSec=5.305649774620705, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▋  | 3815/5000 [12:59:53<3:54:34, 11.88s/it][2022-12-20 23:22:24,089] [INFO] [timer.py:197:stop] 0/7724, RunningAvgSamplesPerSec=5.878690616464378, CurrSamplesPerSec=5.378639163623882, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▋  | 3816/5000 [13:00:04<3:48:58, 11.60s/it][2022-12-20 23:22:35,103] [INFO] [timer.py:197:stop] 0/7726, RunningAvgSamplesPerSec=5.878690063750731, CurrSamplesPerSec=5.335016569756026, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▋  | 3817/5000 [13:00:15<3:45:24, 11.43s/it][2022-12-20 23:22:46,178] [INFO] [timer.py:197:stop] 0/7728, RunningAvgSamplesPerSec=5.878680968754474, CurrSamplesPerSec=5.296797466997161, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▋  | 3818/5000 [13:00:26<3:42:59, 11.32s/it][2022-12-20 23:22:57,166] [INFO] [timer.py:197:stop] 0/7730, RunningAvgSamplesPerSec=5.878684206717754, CurrSamplesPerSec=5.357013046275805, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▋  | 3819/5000 [13:00:37<3:40:50, 11.22s/it][2022-12-20 23:23:08,179] [INFO] [timer.py:197:stop] 0/7732, RunningAvgSamplesPerSec=5.8786838219750726, CurrSamplesPerSec=5.334283786157653, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▋  | 3820/5000 [13:00:48<3:39:11, 11.15s/it][2022-12-20 23:23:19,134] [INFO] [timer.py:197:stop] 0/7734, RunningAvgSamplesPerSec=5.87869146120634, CurrSamplesPerSec=5.349372848950356, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▋  | 3821/5000 [13:00:59<3:38:01, 11.10s/it][2022-12-20 23:23:30,149] [INFO] [timer.py:197:stop] 0/7736, RunningAvgSamplesPerSec=5.878691012062508, CurrSamplesPerSec=5.317719001456355, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▋  | 3822/5000 [13:01:10<3:37:32, 11.08s/it][2022-12-20 23:23:41,214] [INFO] [timer.py:197:stop] 0/7738, RunningAvgSamplesPerSec=5.8786832082061045, CurrSamplesPerSec=5.298355646320129, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▋  | 3823/5000 [13:01:21<3:37:00, 11.06s/it][2022-12-20 23:23:52,179] [INFO] [logging.py:68:log_dist] [Rank 0] step=3870, skipped=5, lr=[2.5244444444444447e-06], mom=[[0.9, 0.999]]
+[2022-12-20 23:23:52,181] [INFO] [timer.py:197:stop] 0/7740, RunningAvgSamplesPerSec=5.878689034021425, CurrSamplesPerSec=5.346106636517864, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▋  | 3824/5000 [13:01:32<3:36:25, 11.04s/it][2022-12-20 23:24:03,180] [INFO] [timer.py:197:stop] 0/7742, RunningAvgSamplesPerSec=5.878690594964675, CurrSamplesPerSec=5.342014858073612, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 76%|███████▋  | 3825/5000 [13:01:43<3:35:54, 11.02s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 2.5222222222222226e-06, 'epoch': 93.29}
+
 76%|███████▋  | 3825/5000 [13:01:43<3:35:54, 11.02s/it][2022-12-20 23:24:14,158] [INFO] [timer.py:197:stop] 0/7744, RunningAvgSamplesPerSec=5.878695382177821, CurrSamplesPerSec=5.349904633206649, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3826/5000 [13:01:54<3:35:24, 11.01s/it][2022-12-20 23:24:25,188] [INFO] [timer.py:197:stop] 0/7746, RunningAvgSamplesPerSec=5.878692574073494, CurrSamplesPerSec=5.297778854210213, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3827/5000 [13:02:05<3:35:29, 11.02s/it][2022-12-20 23:24:36,199] [INFO] [timer.py:197:stop] 0/7748, RunningAvgSamplesPerSec=5.878692494612233, CurrSamplesPerSec=5.335400852724543, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3828/5000 [13:02:16<3:35:05, 11.01s/it][2022-12-20 23:24:47,249] [INFO] [timer.py:197:stop] 0/7750, RunningAvgSamplesPerSec=5.878686802274691, CurrSamplesPerSec=5.277901349469991, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3829/5000 [13:02:27<3:34:59, 11.02s/it][2022-12-20 23:24:58,221] [INFO] [timer.py:197:stop] 0/7752, RunningAvgSamplesPerSec=5.878691952741859, CurrSamplesPerSec=5.32447783074356, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3830/5000 [13:02:38<3:34:59, 11.03s/it][2022-12-20 23:25:09,236] [INFO] [timer.py:197:stop] 0/7754, RunningAvgSamplesPerSec=5.878691361289831, CurrSamplesPerSec=5.3540691674313425, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3831/5000 [13:02:49<3:34:27, 11.01s/it][2022-12-20 23:25:20,199] [INFO] [timer.py:197:stop] 0/7756, RunningAvgSamplesPerSec=5.878697798645662, CurrSamplesPerSec=5.355799714599917, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3832/5000 [13:03:00<3:34:09, 11.00s/it][2022-12-20 23:25:31,229] [INFO] [timer.py:197:stop] 0/7758, RunningAvgSamplesPerSec=5.878694874581168, CurrSamplesPerSec=5.318756844964015, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3833/5000 [13:03:11<3:34:12, 11.01s/it][2022-12-20 23:25:42,266] [INFO] [logging.py:68:log_dist] [Rank 0] step=3880, skipped=5, lr=[2.5022222222222224e-06], mom=[[0.9, 0.999]]
+[2022-12-20 23:25:42,268] [INFO] [timer.py:197:stop] 0/7760, RunningAvgSamplesPerSec=5.878690638549906, CurrSamplesPerSec=5.320031470449167, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3834/5000 [13:03:22<3:34:03, 11.01s/it][2022-12-20 23:25:53,227] [INFO] [timer.py:197:stop] 0/7762, RunningAvgSamplesPerSec=5.878697615316082, CurrSamplesPerSec=5.372930308967938, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3835/5000 [13:03:33<3:33:35, 11.00s/it][2022-12-20 23:26:04,240] [INFO] [timer.py:197:stop] 0/7764, RunningAvgSamplesPerSec=5.878697105599595, CurrSamplesPerSec=5.331084034829067, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3836/5000 [13:03:44<3:33:34, 11.01s/it][2022-12-20 23:26:15,255] [INFO] [timer.py:197:stop] 0/7766, RunningAvgSamplesPerSec=5.878696336007531, CurrSamplesPerSec=5.343238330338344, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3837/5000 [13:03:55<3:33:03, 10.99s/it][2022-12-20 23:26:26,225] [INFO] [timer.py:197:stop] 0/7768, RunningAvgSamplesPerSec=5.878701818308288, CurrSamplesPerSec=5.326377204529283, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3838/5000 [13:04:06<3:32:53, 10.99s/it][2022-12-20 23:26:37,210] [INFO] [timer.py:197:stop] 0/7770, RunningAvgSamplesPerSec=5.878705273640628, CurrSamplesPerSec=5.335321955678633, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3839/5000 [13:04:17<3:32:51, 11.00s/it][2022-12-20 23:26:48,231] [INFO] [timer.py:197:stop] 0/7772, RunningAvgSamplesPerSec=5.87870380643178, CurrSamplesPerSec=5.333633439737585, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3840/5000 [13:04:28<3:32:51, 11.01s/it][2022-12-20 23:26:59,242] [INFO] [timer.py:197:stop] 0/7774, RunningAvgSamplesPerSec=5.878703564924367, CurrSamplesPerSec=5.352393952753218, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3841/5000 [13:04:39<3:32:33, 11.00s/it][2022-12-20 23:27:10,239] [INFO] [timer.py:197:stop] 0/7776, RunningAvgSamplesPerSec=5.878705403333957, CurrSamplesPerSec=5.346318524168043, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3842/5000 [13:04:50<3:32:21, 11.00s/it][2022-12-20 23:27:21,233] [INFO] [timer.py:197:stop] 0/7778, RunningAvgSamplesPerSec=5.878707602723294, CurrSamplesPerSec=5.351187619354196, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3843/5000 [13:05:01<3:32:07, 11.00s/it][2022-12-20 23:27:32,210] [INFO] [logging.py:68:log_dist] [Rank 0] step=3890, skipped=5, lr=[2.4800000000000004e-06], mom=[[0.9, 0.999]]
+[2022-12-20 23:27:32,211] [INFO] [timer.py:197:stop] 0/7780, RunningAvgSamplesPerSec=5.878711716009359, CurrSamplesPerSec=5.365132293602501, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3844/5000 [13:05:12<3:31:44, 10.99s/it][2022-12-20 23:27:43,200] [INFO] [timer.py:197:stop] 0/7782, RunningAvgSamplesPerSec=5.878714437552027, CurrSamplesPerSec=5.345178787747728, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3845/5000 [13:05:23<3:31:34, 10.99s/it][2022-12-20 23:27:54,211] [INFO] [timer.py:197:stop] 0/7784, RunningAvgSamplesPerSec=5.878714088006636, CurrSamplesPerSec=5.327882621051407, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3846/5000 [13:05:34<3:31:32, 11.00s/it][2022-12-20 23:28:05,240] [INFO] [timer.py:197:stop] 0/7786, RunningAvgSamplesPerSec=5.878711294867211, CurrSamplesPerSec=5.319041611524896, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3847/5000 [13:05:45<3:31:31, 11.01s/it][2022-12-20 23:28:16,268] [INFO] [timer.py:197:stop] 0/7788, RunningAvgSamplesPerSec=5.878708684225889, CurrSamplesPerSec=5.317142198841507, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3848/5000 [13:05:56<3:31:25, 11.01s/it][2022-12-20 23:28:27,289] [INFO] [timer.py:197:stop] 0/7790, RunningAvgSamplesPerSec=5.878707173442277, CurrSamplesPerSec=5.319617561297324, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3849/5000 [13:06:07<3:31:25, 11.02s/it][2022-12-20 23:28:38,333] [INFO] [timer.py:197:stop] 0/7792, RunningAvgSamplesPerSec=5.878702397662285, CurrSamplesPerSec=5.317858270219842, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3850/5000 [13:06:18<3:30:51, 11.00s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 2.466666666666667e-06, 'epoch': 93.89}
+
 77%|███████▋  | 3850/5000 [13:06:18<3:30:51, 11.00s/it][2022-12-20 23:28:49,284] [INFO] [timer.py:197:stop] 0/7794, RunningAvgSamplesPerSec=5.8787110497235275, CurrSamplesPerSec=5.323519044476014, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3851/5000 [13:06:29<3:30:51, 11.01s/it][2022-12-20 23:29:00,258] [INFO] [timer.py:197:stop] 0/7796, RunningAvgSamplesPerSec=5.878716097044405, CurrSamplesPerSec=5.375528300180477, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3852/5000 [13:06:40<3:30:24, 11.00s/it][2022-12-20 23:29:11,282] [INFO] [timer.py:197:stop] 0/7798, RunningAvgSamplesPerSec=5.878714039258267, CurrSamplesPerSec=5.321443420961807, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3853/5000 [13:06:51<3:30:23, 11.01s/it][2022-12-20 23:29:22,301] [INFO] [logging.py:68:log_dist] [Rank 0] step=3900, skipped=5, lr=[2.457777777777778e-06], mom=[[0.9, 0.999]]
+[2022-12-20 23:29:22,302] [INFO] [timer.py:197:stop] 0/7800, RunningAvgSamplesPerSec=5.878712487563296, CurrSamplesPerSec=5.327456281928265, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3854/5000 [13:07:02<3:30:16, 11.01s/it][2022-12-20 23:29:32,394] [INFO] [timer.py:197:stop] 0/7802, RunningAvgSamplesPerSec=5.878838202380105, CurrSamplesPerSec=6.283705344006855, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-20 23:29:43,397] [INFO] [timer.py:197:stop] 0/7804, RunningAvgSamplesPerSec=5.878838781812679, CurrSamplesPerSec=5.316228166140369, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3855/5000 [13:07:18<3:59:30, 12.55s/it][2022-12-20 23:29:54,493] [INFO] [timer.py:197:stop] 0/7806, RunningAvgSamplesPerSec=5.878826764871694, CurrSamplesPerSec=5.261058189103867, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3856/5000 [13:07:30<3:50:59, 12.11s/it][2022-12-20 23:30:05,472] [INFO] [timer.py:197:stop] 0/7808, RunningAvgSamplesPerSec=5.87883102743074, CurrSamplesPerSec=5.327462414299021, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3857/5000 [13:07:41<3:44:17, 11.77s/it][2022-12-20 23:30:16,475] [INFO] [timer.py:197:stop] 0/7810, RunningAvgSamplesPerSec=5.878832168099429, CurrSamplesPerSec=5.344308290977387, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3858/5000 [13:07:52<3:39:41, 11.54s/it][2022-12-20 23:30:27,524] [INFO] [timer.py:197:stop] 0/7812, RunningAvgSamplesPerSec=5.878826658369451, CurrSamplesPerSec=5.312717329520715, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3859/5000 [13:08:03<3:36:41, 11.39s/it][2022-12-20 23:30:38,500] [INFO] [timer.py:197:stop] 0/7814, RunningAvgSamplesPerSec=5.878831396450725, CurrSamplesPerSec=5.34191599247152, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3860/5000 [13:08:14<3:34:06, 11.27s/it][2022-12-20 23:30:49,529] [INFO] [timer.py:197:stop] 0/7816, RunningAvgSamplesPerSec=5.87882868387825, CurrSamplesPerSec=5.313649298514762, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3861/5000 [13:08:25<3:32:33, 11.20s/it][2022-12-20 23:31:00,534] [INFO] [timer.py:197:stop] 0/7818, RunningAvgSamplesPerSec=5.878829414144464, CurrSamplesPerSec=5.306470377972026, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3862/5000 [13:08:36<3:31:16, 11.14s/it][2022-12-20 23:31:11,499] [INFO] [logging.py:68:log_dist] [Rank 0] step=3910, skipped=5, lr=[2.4355555555555556e-06], mom=[[0.9, 0.999]]
+[2022-12-20 23:31:11,500] [INFO] [timer.py:197:stop] 0/7820, RunningAvgSamplesPerSec=5.878835327409504, CurrSamplesPerSec=5.340538208194829, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3863/5000 [13:08:47<3:30:06, 11.09s/it][2022-12-20 23:31:22,535] [INFO] [timer.py:197:stop] 0/7822, RunningAvgSamplesPerSec=5.878831790766284, CurrSamplesPerSec=5.309960145744297, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3864/5000 [13:08:58<3:29:37, 11.07s/it][2022-12-20 23:31:33,531] [INFO] [timer.py:197:stop] 0/7824, RunningAvgSamplesPerSec=5.878833681674794, CurrSamplesPerSec=5.330578638109024, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3865/5000 [13:09:09<3:29:00, 11.05s/it][2022-12-20 23:31:44,540] [INFO] [timer.py:197:stop] 0/7826, RunningAvgSamplesPerSec=5.87883370143866, CurrSamplesPerSec=5.3376796933919985, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3866/5000 [13:09:20<3:28:36, 11.04s/it][2022-12-20 23:31:55,470] [INFO] [timer.py:197:stop] 0/7828, RunningAvgSamplesPerSec=5.8788447908938055, CurrSamplesPerSec=5.338036336169464, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3867/5000 [13:09:31<3:27:48, 11.00s/it][2022-12-20 23:32:06,545] [INFO] [timer.py:197:stop] 0/7830, RunningAvgSamplesPerSec=5.878836551761894, CurrSamplesPerSec=5.297575396604553, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3868/5000 [13:09:42<3:28:01, 11.03s/it][2022-12-20 23:32:17,566] [INFO] [timer.py:197:stop] 0/7832, RunningAvgSamplesPerSec=5.878835123760748, CurrSamplesPerSec=5.326752210468932, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3869/5000 [13:09:53<3:27:48, 11.02s/it][2022-12-20 23:32:28,548] [INFO] [timer.py:197:stop] 0/7834, RunningAvgSamplesPerSec=5.878838960687376, CurrSamplesPerSec=5.306401355286229, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3870/5000 [13:10:04<3:27:23, 11.01s/it][2022-12-20 23:32:39,566] [INFO] [timer.py:197:stop] 0/7836, RunningAvgSamplesPerSec=5.878837959573247, CurrSamplesPerSec=5.328541085307449, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3871/5000 [13:10:15<3:27:14, 11.01s/it][2022-12-20 23:32:50,596] [INFO] [timer.py:197:stop] 0/7838, RunningAvgSamplesPerSec=5.878835123463715, CurrSamplesPerSec=5.33660177561313, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3872/5000 [13:10:26<3:27:08, 11.02s/it][2022-12-20 23:33:01,627] [INFO] [logging.py:68:log_dist] [Rank 0] step=3920, skipped=5, lr=[2.4133333333333337e-06], mom=[[0.9, 0.999]]
+[2022-12-20 23:33:01,628] [INFO] [timer.py:197:stop] 0/7840, RunningAvgSamplesPerSec=5.87883185203176, CurrSamplesPerSec=5.294645292011416, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3873/5000 [13:10:37<3:27:02, 11.02s/it][2022-12-20 23:33:12,661] [INFO] [timer.py:197:stop] 0/7842, RunningAvgSamplesPerSec=5.878828715323466, CurrSamplesPerSec=5.312198378624068, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 77%|███████▋  | 3874/5000 [13:10:48<3:26:54, 11.03s/it][2022-12-20 23:33:23,648] [INFO] [timer.py:197:stop] 0/7844, RunningAvgSamplesPerSec=5.878831850766411, CurrSamplesPerSec=5.359287706583335, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3875/5000 [13:10:59<3:26:30, 11.01s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 2.4088888888888894e-06, 'epoch': 94.51}
+
 78%|███████▊  | 3875/5000 [13:10:59<3:26:30, 11.01s/it][2022-12-20 23:33:34,675] [INFO] [timer.py:197:stop] 0/7846, RunningAvgSamplesPerSec=5.878829898324588, CurrSamplesPerSec=5.311153845347203, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3876/5000 [13:11:10<3:26:24, 11.02s/it][2022-12-20 23:33:45,681] [INFO] [timer.py:197:stop] 0/7848, RunningAvgSamplesPerSec=5.878830387191428, CurrSamplesPerSec=5.36021275396794, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3877/5000 [13:11:21<3:26:09, 11.01s/it][2022-12-20 23:33:56,726] [INFO] [timer.py:197:stop] 0/7850, RunningAvgSamplesPerSec=5.878825538888365, CurrSamplesPerSec=5.30296864198283, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3878/5000 [13:11:32<3:26:08, 11.02s/it][2022-12-20 23:34:07,731] [INFO] [timer.py:197:stop] 0/7852, RunningAvgSamplesPerSec=5.878826005293767, CurrSamplesPerSec=5.337938891536074, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3879/5000 [13:11:43<3:25:51, 11.02s/it][2022-12-20 23:34:18,783] [INFO] [timer.py:197:stop] 0/7854, RunningAvgSamplesPerSec=5.878820233112816, CurrSamplesPerSec=5.316866271062725, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3880/5000 [13:11:54<3:25:51, 11.03s/it][2022-12-20 23:34:29,770] [INFO] [timer.py:197:stop] 0/7856, RunningAvgSamplesPerSec=5.878823313426732, CurrSamplesPerSec=5.3257868991281185, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3881/5000 [13:12:05<3:25:26, 11.02s/it][2022-12-20 23:34:40,765] [INFO] [timer.py:197:stop] 0/7858, RunningAvgSamplesPerSec=5.878825198992642, CurrSamplesPerSec=5.3520661211071605, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3882/5000 [13:12:16<3:25:08, 11.01s/it][2022-12-20 23:34:51,839] [INFO] [logging.py:68:log_dist] [Rank 0] step=3930, skipped=5, lr=[2.3911111111111113e-06], mom=[[0.9, 0.999]]
+[2022-12-20 23:34:51,841] [INFO] [timer.py:197:stop] 0/7860, RunningAvgSamplesPerSec=5.878815953332355, CurrSamplesPerSec=5.305035117697726, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3883/5000 [13:12:27<3:25:19, 11.03s/it][2022-12-20 23:35:02,885] [INFO] [timer.py:197:stop] 0/7862, RunningAvgSamplesPerSec=5.878811202483665, CurrSamplesPerSec=5.321673403192416, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3884/5000 [13:12:38<3:25:13, 11.03s/it][2022-12-20 23:35:13,855] [INFO] [timer.py:197:stop] 0/7864, RunningAvgSamplesPerSec=5.87881657635384, CurrSamplesPerSec=5.357456960295431, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3885/5000 [13:12:49<3:24:41, 11.01s/it][2022-12-20 23:35:24,929] [INFO] [timer.py:197:stop] 0/7866, RunningAvgSamplesPerSec=5.878807855383179, CurrSamplesPerSec=5.300307171861124, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3886/5000 [13:13:00<3:24:50, 11.03s/it][2022-12-20 23:35:35,894] [INFO] [timer.py:197:stop] 0/7868, RunningAvgSamplesPerSec=5.878813976675247, CurrSamplesPerSec=5.344225087185568, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3887/5000 [13:13:11<3:24:16, 11.01s/it][2022-12-20 23:35:46,937] [INFO] [timer.py:197:stop] 0/7870, RunningAvgSamplesPerSec=5.878809561246004, CurrSamplesPerSec=5.308943791446665, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3888/5000 [13:13:22<3:24:15, 11.02s/it][2022-12-20 23:35:57,998] [INFO] [timer.py:197:stop] 0/7872, RunningAvgSamplesPerSec=5.878802458509132, CurrSamplesPerSec=5.31367117668612, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3889/5000 [13:13:33<3:24:17, 11.03s/it][2022-12-20 23:36:09,049] [INFO] [timer.py:197:stop] 0/7874, RunningAvgSamplesPerSec=5.878796671891111, CurrSamplesPerSec=5.30551722652179, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3890/5000 [13:13:44<3:24:12, 11.04s/it][2022-12-20 23:36:20,110] [INFO] [timer.py:197:stop] 0/7876, RunningAvgSamplesPerSec=5.8787896511832916, CurrSamplesPerSec=5.301572759392883, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3891/5000 [13:13:55<3:24:09, 11.05s/it][2022-12-20 23:36:31,156] [INFO] [timer.py:197:stop] 0/7878, RunningAvgSamplesPerSec=5.878784565907606, CurrSamplesPerSec=5.27618654587394, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3892/5000 [13:14:06<3:23:58, 11.05s/it][2022-12-20 23:36:42,194] [INFO] [logging.py:68:log_dist] [Rank 0] step=3940, skipped=5, lr=[2.3688888888888893e-06], mom=[[0.9, 0.999]]
+[2022-12-20 23:36:42,195] [INFO] [timer.py:197:stop] 0/7880, RunningAvgSamplesPerSec=5.878780401181073, CurrSamplesPerSec=5.332000641662437, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3893/5000 [13:14:17<3:23:45, 11.04s/it][2022-12-20 23:36:53,223] [INFO] [timer.py:197:stop] 0/7882, RunningAvgSamplesPerSec=5.878778221558751, CurrSamplesPerSec=5.323814668615089, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3894/5000 [13:14:28<3:23:28, 11.04s/it][2022-12-20 23:37:04,167] [INFO] [timer.py:197:stop] 0/7884, RunningAvgSamplesPerSec=5.878787270369022, CurrSamplesPerSec=5.360912423407893, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3895/5000 [13:14:39<3:22:46, 11.01s/it][2022-12-20 23:37:14,263] [INFO] [timer.py:197:stop] 0/7886, RunningAvgSamplesPerSec=5.8789111115507, CurrSamplesPerSec=5.319943749223865, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3896/5000 [13:14:54<3:44:56, 12.22s/it][2022-12-20 23:37:25,341] [INFO] [timer.py:197:stop] 0/7888, RunningAvgSamplesPerSec=5.878901803549298, CurrSamplesPerSec=5.28752530250932, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3897/5000 [13:15:05<3:38:22, 11.88s/it][2022-12-20 23:37:36,370] [INFO] [timer.py:197:stop] 0/7890, RunningAvgSamplesPerSec=5.878899109607377, CurrSamplesPerSec=5.326257780239931, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3898/5000 [13:15:16<3:33:31, 11.63s/it][2022-12-20 23:37:47,428] [INFO] [timer.py:197:stop] 0/7892, RunningAvgSamplesPerSec=5.8788926945396, CurrSamplesPerSec=5.306178145276303, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3899/5000 [13:15:27<3:30:10, 11.45s/it][2022-12-20 23:37:58,458] [INFO] [timer.py:197:stop] 0/7894, RunningAvgSamplesPerSec=5.87888989066829, CurrSamplesPerSec=5.326595564024068, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3900/5000 [13:15:39<3:27:40, 11.33s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 2.3533333333333334e-06, 'epoch': 95.12}
+
 78%|███████▊  | 3900/5000 [13:15:39<3:27:40, 11.33s/it][2022-12-20 23:38:09,470] [INFO] [timer.py:197:stop] 0/7896, RunningAvgSamplesPerSec=5.878889686922154, CurrSamplesPerSec=5.344928889977026, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3901/5000 [13:15:50<3:25:52, 11.24s/it][2022-12-20 23:38:20,565] [INFO] [timer.py:197:stop] 0/7898, RunningAvgSamplesPerSec=5.878877871250663, CurrSamplesPerSec=5.290292385637862, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3902/5000 [13:16:01<3:24:46, 11.19s/it][2022-12-20 23:38:31,592] [INFO] [logging.py:68:log_dist] [Rank 0] step=3950, skipped=5, lr=[2.346666666666667e-06], mom=[[0.9, 0.999]]
+[2022-12-20 23:38:31,593] [INFO] [timer.py:197:stop] 0/7900, RunningAvgSamplesPerSec=5.878875086481545, CurrSamplesPerSec=5.328518238335531, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3903/5000 [13:16:12<3:23:36, 11.14s/it][2022-12-20 23:38:42,570] [INFO] [timer.py:197:stop] 0/7902, RunningAvgSamplesPerSec=5.8788794440153, CurrSamplesPerSec=5.361598139619385, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3904/5000 [13:16:23<3:22:26, 11.08s/it][2022-12-20 23:38:53,556] [INFO] [timer.py:197:stop] 0/7904, RunningAvgSamplesPerSec=5.878882357836643, CurrSamplesPerSec=5.336097029395148, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3905/5000 [13:16:34<3:21:51, 11.06s/it][2022-12-20 23:39:04,563] [INFO] [timer.py:197:stop] 0/7906, RunningAvgSamplesPerSec=5.878882518073066, CurrSamplesPerSec=5.337262609178235, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3906/5000 [13:16:45<3:21:28, 11.05s/it][2022-12-20 23:39:15,609] [INFO] [timer.py:197:stop] 0/7908, RunningAvgSamplesPerSec=5.878877264090868, CurrSamplesPerSec=5.318523953457095, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3907/5000 [13:16:56<3:21:13, 11.05s/it][2022-12-20 23:39:26,638] [INFO] [timer.py:197:stop] 0/7910, RunningAvgSamplesPerSec=5.878874390716469, CurrSamplesPerSec=5.326485219427959, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3908/5000 [13:17:07<3:20:53, 11.04s/it][2022-12-20 23:39:37,633] [INFO] [timer.py:197:stop] 0/7912, RunningAvgSamplesPerSec=5.878876239394385, CurrSamplesPerSec=5.348085405213583, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3909/5000 [13:17:18<3:20:21, 11.02s/it][2022-12-20 23:39:48,617] [INFO] [timer.py:197:stop] 0/7914, RunningAvgSamplesPerSec=5.878879531774835, CurrSamplesPerSec=5.339378419316736, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3910/5000 [13:17:29<3:19:59, 11.01s/it][2022-12-20 23:39:59,606] [INFO] [timer.py:197:stop] 0/7916, RunningAvgSamplesPerSec=5.878882352361135, CurrSamplesPerSec=5.335342740157891, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3911/5000 [13:17:40<3:19:52, 11.01s/it][2022-12-20 23:40:10,605] [INFO] [timer.py:197:stop] 0/7918, RunningAvgSamplesPerSec=5.878883693135093, CurrSamplesPerSec=5.355027453259119, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3912/5000 [13:17:51<3:19:19, 10.99s/it][2022-12-20 23:40:21,629] [INFO] [logging.py:68:log_dist] [Rank 0] step=3960, skipped=5, lr=[2.3244444444444445e-06], mom=[[0.9, 0.999]]
+[2022-12-20 23:40:21,631] [INFO] [timer.py:197:stop] 0/7920, RunningAvgSamplesPerSec=5.878881316859216, CurrSamplesPerSec=5.283400794735814, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3913/5000 [13:18:02<3:19:24, 11.01s/it][2022-12-20 23:40:32,626] [INFO] [timer.py:197:stop] 0/7922, RunningAvgSamplesPerSec=5.878883307961018, CurrSamplesPerSec=5.324480365438023, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3914/5000 [13:18:13<3:19:22, 11.01s/it][2022-12-20 23:40:43,671] [INFO] [timer.py:197:stop] 0/7924, RunningAvgSamplesPerSec=5.87887845229876, CurrSamplesPerSec=5.314681976314301, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3915/5000 [13:18:24<3:19:07, 11.01s/it][2022-12-20 23:40:54,676] [INFO] [timer.py:197:stop] 0/7926, RunningAvgSamplesPerSec=5.87887896934134, CurrSamplesPerSec=5.312508096843493, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3916/5000 [13:18:35<3:19:02, 11.02s/it][2022-12-20 23:41:05,663] [INFO] [timer.py:197:stop] 0/7928, RunningAvgSamplesPerSec=5.8788820159449795, CurrSamplesPerSec=5.350831776999772, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3917/5000 [13:18:46<3:18:40, 11.01s/it][2022-12-20 23:41:16,646] [INFO] [timer.py:197:stop] 0/7930, RunningAvgSamplesPerSec=5.878885598039494, CurrSamplesPerSec=5.3520481939721645, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3918/5000 [13:18:57<3:18:14, 10.99s/it][2022-12-20 23:41:27,665] [INFO] [timer.py:197:stop] 0/7932, RunningAvgSamplesPerSec=5.87888417912698, CurrSamplesPerSec=5.29993295009323, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3919/5000 [13:19:08<3:18:22, 11.01s/it][2022-12-20 23:41:38,664] [INFO] [timer.py:197:stop] 0/7934, RunningAvgSamplesPerSec=5.878885590039562, CurrSamplesPerSec=5.346700390249608, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3920/5000 [13:19:19<3:17:57, 11.00s/it][2022-12-20 23:41:49,669] [INFO] [timer.py:197:stop] 0/7936, RunningAvgSamplesPerSec=5.878886035599633, CurrSamplesPerSec=5.313889968308931, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3921/5000 [13:19:30<3:17:54, 11.01s/it][2022-12-20 23:42:00,637] [INFO] [timer.py:197:stop] 0/7938, RunningAvgSamplesPerSec=5.878891675688125, CurrSamplesPerSec=5.362221903940613, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3922/5000 [13:19:41<3:17:32, 10.99s/it][2022-12-20 23:42:11,653] [INFO] [logging.py:68:log_dist] [Rank 0] step=3970, skipped=5, lr=[2.302222222222222e-06], mom=[[0.9, 0.999]]
+[2022-12-20 23:42:11,655] [INFO] [timer.py:197:stop] 0/7940, RunningAvgSamplesPerSec=5.878890561891558, CurrSamplesPerSec=5.320429837120422, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3923/5000 [13:19:52<3:17:16, 10.99s/it][2022-12-20 23:42:22,644] [INFO] [timer.py:197:stop] 0/7942, RunningAvgSamplesPerSec=5.878893277883253, CurrSamplesPerSec=5.311251995908265, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3924/5000 [13:20:03<3:17:12, 11.00s/it][2022-12-20 23:42:33,633] [INFO] [timer.py:197:stop] 0/7944, RunningAvgSamplesPerSec=5.878896089454, CurrSamplesPerSec=5.332834926138469, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 78%|███████▊  | 3925/5000 [13:20:14<3:17:05, 11.00s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 2.297777777777778e-06, 'epoch': 95.72}
+
 78%|███████▊  | 3925/5000 [13:20:14<3:17:05, 11.00s/it][2022-12-20 23:42:44,609] [INFO] [timer.py:197:stop] 0/7946, RunningAvgSamplesPerSec=5.878900982390727, CurrSamplesPerSec=5.3621908408139065, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▊  | 3926/5000 [13:20:25<3:16:39, 10.99s/it][2022-12-20 23:42:55,612] [INFO] [timer.py:197:stop] 0/7948, RunningAvgSamplesPerSec=5.878901890515244, CurrSamplesPerSec=5.318572216175446, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▊  | 3927/5000 [13:20:36<3:16:33, 10.99s/it][2022-12-20 23:43:06,618] [INFO] [timer.py:197:stop] 0/7950, RunningAvgSamplesPerSec=5.878902450514792, CurrSamplesPerSec=5.3153559160710815, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▊  | 3928/5000 [13:20:47<3:16:37, 11.00s/it][2022-12-20 23:43:17,615] [INFO] [timer.py:197:stop] 0/7952, RunningAvgSamplesPerSec=5.878903966784523, CurrSamplesPerSec=5.350079927688591, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▊  | 3929/5000 [13:20:58<3:16:19, 11.00s/it][2022-12-20 23:43:28,583] [INFO] [timer.py:197:stop] 0/7954, RunningAvgSamplesPerSec=5.878909573692298, CurrSamplesPerSec=5.36369235127001, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▊  | 3930/5000 [13:21:09<3:16:06, 11.00s/it][2022-12-20 23:43:39,629] [INFO] [timer.py:197:stop] 0/7956, RunningAvgSamplesPerSec=5.878904800100759, CurrSamplesPerSec=5.316927141217511, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▊  | 3931/5000 [13:21:20<3:15:57, 11.00s/it][2022-12-20 23:43:50,621] [INFO] [timer.py:197:stop] 0/7958, RunningAvgSamplesPerSec=5.878907263834069, CurrSamplesPerSec=5.327062148125951, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▊  | 3932/5000 [13:21:31<3:15:54, 11.01s/it][2022-12-20 23:44:01,650] [INFO] [logging.py:68:log_dist] [Rank 0] step=3980, skipped=5, lr=[2.28e-06], mom=[[0.9, 0.999]]
+[2022-12-20 23:44:01,651] [INFO] [timer.py:197:stop] 0/7960, RunningAvgSamplesPerSec=5.878904354090247, CurrSamplesPerSec=5.321340040930845, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▊  | 3933/5000 [13:21:42<3:15:43, 11.01s/it][2022-12-20 23:44:12,671] [INFO] [timer.py:197:stop] 0/7962, RunningAvgSamplesPerSec=5.878902854476758, CurrSamplesPerSec=5.306947291971816, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▊  | 3934/5000 [13:21:53<3:15:49, 11.02s/it][2022-12-20 23:44:23,697] [INFO] [timer.py:197:stop] 0/7964, RunningAvgSamplesPerSec=5.878900636471695, CurrSamplesPerSec=5.3380885628621675, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▊  | 3935/5000 [13:22:04<3:15:22, 11.01s/it][2022-12-20 23:44:34,690] [INFO] [timer.py:197:stop] 0/7966, RunningAvgSamplesPerSec=5.878902889348789, CurrSamplesPerSec=5.320294229697979, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▊  | 3936/5000 [13:22:15<3:15:23, 11.02s/it][2022-12-20 23:44:44,818] [INFO] [timer.py:197:stop] 0/7968, RunningAvgSamplesPerSec=5.879021055678216, CurrSamplesPerSec=6.261743732470191, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+[2022-12-20 23:44:55,867] [INFO] [timer.py:197:stop] 0/7970, RunningAvgSamplesPerSec=5.8790154241969095, CurrSamplesPerSec=5.3110330011382, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▊  | 3937/5000 [13:22:31<3:42:48, 12.58s/it][2022-12-20 23:45:06,832] [INFO] [timer.py:197:stop] 0/7972, RunningAvgSamplesPerSec=5.879021501316945, CurrSamplesPerSec=5.35462153910035, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3938/5000 [13:22:42<3:34:02, 12.09s/it][2022-12-20 23:45:17,840] [INFO] [timer.py:197:stop] 0/7974, RunningAvgSamplesPerSec=5.879021763991139, CurrSamplesPerSec=5.315892327554655, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3939/5000 [13:22:53<3:28:05, 11.77s/it][2022-12-20 23:45:28,880] [INFO] [timer.py:197:stop] 0/7976, RunningAvgSamplesPerSec=5.879017599329839, CurrSamplesPerSec=5.307782989861259, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3940/5000 [13:23:04<3:24:02, 11.55s/it][2022-12-20 23:45:39,882] [INFO] [timer.py:197:stop] 0/7978, RunningAvgSamplesPerSec=5.879018702932782, CurrSamplesPerSec=5.342815485319329, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3941/5000 [13:23:15<3:20:56, 11.39s/it][2022-12-20 23:45:50,870] [INFO] [logging.py:68:log_dist] [Rank 0] step=3990, skipped=5, lr=[2.257777777777778e-06], mom=[[0.9, 0.999]]
+[2022-12-20 23:45:50,871] [INFO] [timer.py:197:stop] 0/7980, RunningAvgSamplesPerSec=5.8790213438661, CurrSamplesPerSec=5.3508345501684715, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3942/5000 [13:23:26<3:18:39, 11.27s/it][2022-12-20 23:46:01,861] [INFO] [timer.py:197:stop] 0/7982, RunningAvgSamplesPerSec=5.879023982058138, CurrSamplesPerSec=5.322068006172942, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3943/5000 [13:23:37<3:17:00, 11.18s/it][2022-12-20 23:46:12,850] [INFO] [timer.py:197:stop] 0/7984, RunningAvgSamplesPerSec=5.879026710812216, CurrSamplesPerSec=5.338748702929006, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3944/5000 [13:23:48<3:15:48, 11.13s/it][2022-12-20 23:46:23,882] [INFO] [timer.py:197:stop] 0/7986, RunningAvgSamplesPerSec=5.879023570951474, CurrSamplesPerSec=5.321751475074357, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3945/5000 [13:23:59<3:15:07, 11.10s/it][2022-12-20 23:46:34,905] [INFO] [timer.py:197:stop] 0/7988, RunningAvgSamplesPerSec=5.879021763185896, CurrSamplesPerSec=5.359743341338609, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3946/5000 [13:24:10<3:14:33, 11.07s/it][2022-12-20 23:46:45,936] [INFO] [timer.py:197:stop] 0/7990, RunningAvgSamplesPerSec=5.879018890392515, CurrSamplesPerSec=5.302349788212537, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3947/5000 [13:24:21<3:14:08, 11.06s/it][2022-12-20 23:46:57,006] [INFO] [timer.py:197:stop] 0/7992, RunningAvgSamplesPerSec=5.879010862722608, CurrSamplesPerSec=5.297363382387859, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3948/5000 [13:24:32<3:13:59, 11.06s/it][2022-12-20 23:47:08,048] [INFO] [timer.py:197:stop] 0/7994, RunningAvgSamplesPerSec=5.879006538187522, CurrSamplesPerSec=5.305335822532682, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3949/5000 [13:24:43<3:13:41, 11.06s/it][2022-12-20 23:47:19,009] [INFO] [timer.py:197:stop] 0/7996, RunningAvgSamplesPerSec=5.8790130042525535, CurrSamplesPerSec=5.314405672533149, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3950/5000 [13:24:54<3:13:00, 11.03s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 2.24e-06, 'epoch': 96.34}
+
 79%|███████▉  | 3950/5000 [13:24:54<3:13:00, 11.03s/it][2022-12-20 23:47:30,063] [INFO] [timer.py:197:stop] 0/7998, RunningAvgSamplesPerSec=5.879007201965908, CurrSamplesPerSec=5.307726736739969, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3951/5000 [13:25:05<3:12:57, 11.04s/it][2022-12-20 23:47:41,111] [INFO] [logging.py:68:log_dist] [Rank 0] step=4000, skipped=5, lr=[2.235555555555556e-06], mom=[[0.9, 0.999]]
+[2022-12-20 23:47:41,112] [INFO] [timer.py:197:stop] 0/8000, RunningAvgSamplesPerSec=5.879001748581906, CurrSamplesPerSec=5.316080560381886, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3952/5000 [13:25:16<3:12:49, 11.04s/it][2022-12-20 23:47:52,189] [INFO] [timer.py:197:stop] 0/8002, RunningAvgSamplesPerSec=5.87899255494425, CurrSamplesPerSec=5.302552146299819, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3953/5000 [13:25:27<3:12:50, 11.05s/it][2022-12-20 23:48:03,145] [INFO] [timer.py:197:stop] 0/8004, RunningAvgSamplesPerSec=5.878999989229112, CurrSamplesPerSec=5.327957702715014, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3954/5000 [13:25:38<3:12:09, 11.02s/it][2022-12-20 23:48:14,153] [INFO] [timer.py:197:stop] 0/8006, RunningAvgSamplesPerSec=5.879000276817181, CurrSamplesPerSec=5.3453982657947305, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3955/5000 [13:25:49<3:11:53, 11.02s/it][2022-12-20 23:48:25,099] [INFO] [timer.py:197:stop] 0/8008, RunningAvgSamplesPerSec=5.879008876892938, CurrSamplesPerSec=5.357384894106234, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3956/5000 [13:26:00<3:11:20, 11.00s/it][2022-12-20 23:48:36,070] [INFO] [timer.py:197:stop] 0/8010, RunningAvgSamplesPerSec=5.87901425063256, CurrSamplesPerSec=5.33043256306842, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3957/5000 [13:26:11<3:11:01, 10.99s/it][2022-12-20 23:48:47,060] [INFO] [timer.py:197:stop] 0/8012, RunningAvgSamplesPerSec=5.879016874857364, CurrSamplesPerSec=5.370808461611475, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3958/5000 [13:26:22<3:10:50, 10.99s/it][2022-12-20 23:48:58,047] [INFO] [timer.py:197:stop] 0/8014, RunningAvgSamplesPerSec=5.879019720350586, CurrSamplesPerSec=5.34755099971453, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3959/5000 [13:26:33<3:10:39, 10.99s/it][2022-12-20 23:49:09,015] [INFO] [timer.py:197:stop] 0/8016, RunningAvgSamplesPerSec=5.87902509367868, CurrSamplesPerSec=5.33241605643001, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3960/5000 [13:26:44<3:10:21, 10.98s/it][2022-12-20 23:49:20,048] [INFO] [timer.py:197:stop] 0/8018, RunningAvgSamplesPerSec=5.87902175360841, CurrSamplesPerSec=5.321361560524614, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3961/5000 [13:26:55<3:10:26, 11.00s/it][2022-12-20 23:49:31,096] [INFO] [logging.py:68:log_dist] [Rank 0] step=4010, skipped=5, lr=[2.2133333333333335e-06], mom=[[0.9, 0.999]]
+[2022-12-20 23:49:31,098] [INFO] [timer.py:197:stop] 0/8020, RunningAvgSamplesPerSec=5.8790163010814975, CurrSamplesPerSec=5.310760228688641, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3962/5000 [13:27:06<3:10:31, 11.01s/it][2022-12-20 23:49:42,079] [INFO] [timer.py:197:stop] 0/8022, RunningAvgSamplesPerSec=5.879020066849477, CurrSamplesPerSec=5.355524247743994, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3963/5000 [13:27:17<3:10:10, 11.00s/it][2022-12-20 23:49:53,059] [INFO] [timer.py:197:stop] 0/8024, RunningAvgSamplesPerSec=5.879023871320158, CurrSamplesPerSec=5.364921485431285, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3964/5000 [13:27:28<3:09:52, 11.00s/it][2022-12-20 23:50:03,994] [INFO] [timer.py:197:stop] 0/8026, RunningAvgSamplesPerSec=5.879033777735862, CurrSamplesPerSec=5.361430442031898, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3965/5000 [13:27:39<3:09:22, 10.98s/it][2022-12-20 23:50:14,980] [INFO] [timer.py:197:stop] 0/8028, RunningAvgSamplesPerSec=5.879036850889962, CurrSamplesPerSec=5.330978585960009, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3966/5000 [13:27:50<3:09:13, 10.98s/it][2022-12-20 23:50:25,986] [INFO] [timer.py:197:stop] 0/8030, RunningAvgSamplesPerSec=5.879037041637896, CurrSamplesPerSec=5.309151693401624, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3967/5000 [13:28:01<3:09:10, 10.99s/it][2022-12-20 23:50:37,031] [INFO] [timer.py:197:stop] 0/8032, RunningAvgSamplesPerSec=5.879032263377862, CurrSamplesPerSec=5.296480172451452, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3968/5000 [13:28:12<3:09:17, 11.01s/it][2022-12-20 23:50:48,013] [INFO] [timer.py:197:stop] 0/8034, RunningAvgSamplesPerSec=5.879035718639782, CurrSamplesPerSec=5.358451971236659, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3969/5000 [13:28:23<3:08:59, 11.00s/it][2022-12-20 23:50:59,143] [INFO] [timer.py:197:stop] 0/8036, RunningAvgSamplesPerSec=5.879019486700361, CurrSamplesPerSec=5.2303435203746025, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3970/5000 [13:28:34<3:09:29, 11.04s/it][2022-12-20 23:51:10,153] [INFO] [timer.py:197:stop] 0/8038, RunningAvgSamplesPerSec=5.879019682155673, CurrSamplesPerSec=5.344526633451091, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3971/5000 [13:28:45<3:09:09, 11.03s/it][2022-12-20 23:51:21,146] [INFO] [logging.py:68:log_dist] [Rank 0] step=4020, skipped=5, lr=[2.1911111111111115e-06], mom=[[0.9, 0.999]]
+[2022-12-20 23:51:21,148] [INFO] [timer.py:197:stop] 0/8040, RunningAvgSamplesPerSec=5.879021766128096, CurrSamplesPerSec=5.323278768471806, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3972/5000 [13:28:56<3:08:47, 11.02s/it][2022-12-20 23:51:32,141] [INFO] [timer.py:197:stop] 0/8042, RunningAvgSamplesPerSec=5.87902404473087, CurrSamplesPerSec=5.36313853526619, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3973/5000 [13:29:07<3:08:28, 11.01s/it][2022-12-20 23:51:43,150] [INFO] [timer.py:197:stop] 0/8044, RunningAvgSamplesPerSec=5.87902416724383, CurrSamplesPerSec=5.308165878601749, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 79%|███████▉  | 3974/5000 [13:29:18<3:08:16, 11.01s/it][2022-12-20 23:51:54,107] [INFO] [timer.py:197:stop] 0/8046, RunningAvgSamplesPerSec=5.879031237520502, CurrSamplesPerSec=5.357375698856583, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|███████▉  | 3975/5000 [13:29:29<3:07:49, 10.99s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 2.1844444444444446e-06, 'epoch': 96.94}
+
 80%|███████▉  | 3975/5000 [13:29:29<3:07:49, 10.99s/it][2022-12-20 23:52:05,122] [INFO] [timer.py:197:stop] 0/8048, RunningAvgSamplesPerSec=5.87903079854065, CurrSamplesPerSec=5.325618898418627, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|███████▉  | 3976/5000 [13:29:40<3:07:44, 11.00s/it][2022-12-20 23:52:16,091] [INFO] [timer.py:197:stop] 0/8050, RunningAvgSamplesPerSec=5.87903638791964, CurrSamplesPerSec=5.326353741989549, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|███████▉  | 3977/5000 [13:29:51<3:07:23, 10.99s/it][2022-12-20 23:52:26,190] [INFO] [timer.py:197:stop] 0/8052, RunningAvgSamplesPerSec=5.879157287656026, CurrSamplesPerSec=5.311837191452507, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|███████▉  | 3978/5000 [13:30:06<3:27:44, 12.20s/it][2022-12-20 23:52:37,195] [INFO] [timer.py:197:stop] 0/8054, RunningAvgSamplesPerSec=5.879157810484929, CurrSamplesPerSec=5.304329413047954, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|███████▉  | 3979/5000 [13:30:17<3:21:32, 11.84s/it][2022-12-20 23:52:48,211] [INFO] [timer.py:197:stop] 0/8056, RunningAvgSamplesPerSec=5.879156832409986, CurrSamplesPerSec=5.3089267820073145, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|███████▉  | 3980/5000 [13:30:28<3:17:25, 11.61s/it][2022-12-20 23:52:59,244] [INFO] [timer.py:197:stop] 0/8058, RunningAvgSamplesPerSec=5.879153641081622, CurrSamplesPerSec=5.347164964584492, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|███████▉  | 3981/5000 [13:30:39<3:13:52, 11.42s/it][2022-12-20 23:53:10,244] [INFO] [logging.py:68:log_dist] [Rank 0] step=4030, skipped=5, lr=[2.168888888888889e-06], mom=[[0.9, 0.999]]
+[2022-12-20 23:53:10,245] [INFO] [timer.py:197:stop] 0/8060, RunningAvgSamplesPerSec=5.879154740709566, CurrSamplesPerSec=5.3066789255054525, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|███████▉  | 3982/5000 [13:30:50<3:11:41, 11.30s/it][2022-12-20 23:53:21,233] [INFO] [timer.py:197:stop] 0/8062, RunningAvgSamplesPerSec=5.879157468337238, CurrSamplesPerSec=5.337938466948506, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|███████▉  | 3983/5000 [13:31:01<3:10:05, 11.21s/it][2022-12-20 23:53:32,245] [INFO] [timer.py:197:stop] 0/8064, RunningAvgSamplesPerSec=5.879157050450306, CurrSamplesPerSec=5.345096408302825, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|███████▉  | 3984/5000 [13:31:12<3:08:42, 11.14s/it][2022-12-20 23:53:43,254] [INFO] [timer.py:197:stop] 0/8066, RunningAvgSamplesPerSec=5.87915736363983, CurrSamplesPerSec=5.319964835792752, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|███████▉  | 3985/5000 [13:31:23<3:07:44, 11.10s/it][2022-12-20 23:53:54,283] [INFO] [timer.py:197:stop] 0/8068, RunningAvgSamplesPerSec=5.879154752741992, CurrSamplesPerSec=5.286399249320175, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|███████▉  | 3986/5000 [13:31:34<3:07:18, 11.08s/it][2022-12-20 23:54:05,325] [INFO] [timer.py:197:stop] 0/8070, RunningAvgSamplesPerSec=5.879150298300576, CurrSamplesPerSec=5.291553820393454, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|███████▉  | 3987/5000 [13:31:45<3:07:04, 11.08s/it][2022-12-20 23:54:16,352] [INFO] [timer.py:197:stop] 0/8072, RunningAvgSamplesPerSec=5.879147922641214, CurrSamplesPerSec=5.333284590424864, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|███████▉  | 3988/5000 [13:31:56<3:06:18, 11.05s/it][2022-12-20 23:54:27,328] [INFO] [timer.py:197:stop] 0/8074, RunningAvgSamplesPerSec=5.879152456155177, CurrSamplesPerSec=5.32535772591076, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|███████▉  | 3989/5000 [13:32:07<3:06:02, 11.04s/it][2022-12-20 23:54:38,389] [INFO] [timer.py:197:stop] 0/8076, RunningAvgSamplesPerSec=5.879145528684287, CurrSamplesPerSec=5.296573601086199, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|███████▉  | 3990/5000 [13:32:18<3:05:58, 11.05s/it][2022-12-20 23:54:49,448] [INFO] [timer.py:197:stop] 0/8078, RunningAvgSamplesPerSec=5.879138868083944, CurrSamplesPerSec=5.302382047162607, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|███████▉  | 3991/5000 [13:32:29<3:05:48, 11.05s/it][2022-12-20 23:55:00,468] [INFO] [logging.py:68:log_dist] [Rank 0] step=4040, skipped=5, lr=[2.1466666666666667e-06], mom=[[0.9, 0.999]]
+[2022-12-20 23:55:00,470] [INFO] [timer.py:197:stop] 0/8080, RunningAvgSamplesPerSec=5.879137156472995, CurrSamplesPerSec=5.328352392116208, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|███████▉  | 3992/5000 [13:32:41<3:05:28, 11.04s/it][2022-12-20 23:55:11,463] [INFO] [timer.py:197:stop] 0/8082, RunningAvgSamplesPerSec=5.879139299929784, CurrSamplesPerSec=5.350232200101593, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|███████▉  | 3993/5000 [13:32:52<3:05:09, 11.03s/it][2022-12-20 23:55:22,501] [INFO] [timer.py:197:stop] 0/8084, RunningAvgSamplesPerSec=5.879135299194591, CurrSamplesPerSec=5.328872599991503, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|███████▉  | 3994/5000 [13:33:03<3:04:58, 11.03s/it][2022-12-20 23:55:33,537] [INFO] [timer.py:197:stop] 0/8086, RunningAvgSamplesPerSec=5.879131861713902, CurrSamplesPerSec=5.327379099848214, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|███████▉  | 3995/5000 [13:33:14<3:04:36, 11.02s/it][2022-12-20 23:55:44,542] [INFO] [timer.py:197:stop] 0/8088, RunningAvgSamplesPerSec=5.879132454490535, CurrSamplesPerSec=5.320064155780077, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|███████▉  | 3996/5000 [13:33:25<3:04:22, 11.02s/it][2022-12-20 23:55:55,552] [INFO] [timer.py:197:stop] 0/8090, RunningAvgSamplesPerSec=5.87913239966328, CurrSamplesPerSec=5.319070490369771, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|███████▉  | 3997/5000 [13:33:36<3:04:20, 11.03s/it][2022-12-20 23:56:06,581] [INFO] [timer.py:197:stop] 0/8092, RunningAvgSamplesPerSec=5.87912963128783, CurrSamplesPerSec=5.3362958188408625, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|███████▉  | 3998/5000 [13:33:47<3:04:01, 11.02s/it][2022-12-20 23:56:17,557] [INFO] [timer.py:197:stop] 0/8094, RunningAvgSamplesPerSec=5.879133926343883, CurrSamplesPerSec=5.359716587509476, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|███████▉  | 3999/5000 [13:33:58<3:03:28, 11.00s/it][2022-12-20 23:56:28,506] [INFO] [timer.py:197:stop] 0/8096, RunningAvgSamplesPerSec=5.879141902984215, CurrSamplesPerSec=5.35595829720102, MemAllocated=3.0GB, MaxMemAllocated=19.53GB
+
 80%|████████  | 4000/5000 [13:34:09<3:03:08, 10.99s/it]
                                                        
{'loss': 0.0001, 'learning_rate': 2.128888888888889e-06, 'epoch': 97.55}
+
 80%|████████  | 4000/5000 [13:34:09<3:03:08, 10.99s/it][INFO|trainer.py:2955] 2022-12-20 23:56:33,442 >> ***** Running Evaluation *****
+[INFO|trainer.py:2957] 2022-12-20 23:56:33,442 >>   Num examples = 1325
+[INFO|trainer.py:2960] 2022-12-20 23:56:33,442 >>   Batch size = 8
+
+
  0%|          | 0/166 [00:00<?, ?it/s]
+
  1%|          | 2/166 [00:05<07:53,  2.88s/it]
+
  2%|▏         | 3/166 [00:11<11:09,  4.11s/it]
+
  2%|▏         | 4/166 [00:17<12:55,  4.79s/it]
+
  3%|▎         | 5/166 [00:24<15:06,  5.63s/it]
+
  4%|▎         | 6/166 [00:32<16:30,  6.19s/it]
+
  4%|▍         | 7/166 [00:38<16:30,  6.23s/it]
+
  5%|▍         | 8/166 [00:44<16:06,  6.12s/it]
+
  5%|▌         | 9/166 [00:50<15:51,  6.06s/it]
+
  6%|▌         | 10/166 [00:55<15:31,  5.97s/it]
+
  7%|▋         | 11/166 [01:01<15:08,  5.86s/it]
+
  7%|▋         | 12/166 [01:07<15:08,  5.90s/it]
+
  8%|▊         | 13/166 [01:14<15:29,  6.07s/it]
+
  8%|▊         | 14/166 [01:20<16:02,  6.33s/it]
+
  9%|▉         | 15/166 [01:26<15:25,  6.13s/it]
+
 10%|▉         | 16/166 [01:33<16:13,  6.49s/it]
+
 10%|█         | 17/166 [01:40<16:16,  6.55s/it]
+
 11%|█         | 18/166 [01:46<15:56,  6.46s/it]
+
 11%|█▏        | 19/166 [01:52<15:24,  6.29s/it]
+
 12%|█▏        | 20/166 [02:03<18:32,  7.62s/it]
+
 13%|█▎        | 21/166 [02:10<18:07,  7.50s/it]
+
 13%|█▎        | 22/166 [02:18<17:52,  7.45s/it]
+
 14%|█▍        | 23/166 [02:25<17:50,  7.48s/it]
+
 14%|█▍        | 24/166 [02:32<16:59,  7.18s/it]
+
 15%|█▌        | 25/166 [02:39<16:53,  7.19s/it]
+
 16%|█▌        | 26/166 [02:46<16:40,  7.15s/it]
+
 16%|█▋        | 27/166 [02:53<16:43,  7.22s/it]
+
 17%|█▋        | 28/166 [02:59<15:46,  6.86s/it]
+
 17%|█▋        | 29/166 [03:07<16:35,  7.27s/it]
+
 18%|█▊        | 30/166 [03:14<16:11,  7.15s/it]
+
 19%|█▊        | 31/166 [03:22<16:09,  7.18s/it]
+
 19%|█▉        | 32/166 [03:29<16:22,  7.34s/it]
+
 20%|█▉        | 33/166 [03:37<16:18,  7.36s/it]
+
 20%|██        | 34/166 [03:43<15:34,  7.08s/it]
+
 21%|██        | 35/166 [03:51<16:01,  7.34s/it]
+
 22%|██▏       | 36/166 [03:58<15:57,  7.37s/it]
+
 22%|██▏       | 37/166 [04:05<15:18,  7.12s/it]
+
 23%|██▎       | 38/166 [04:14<16:05,  7.54s/it]
+
 23%|██▎       | 39/166 [04:20<15:32,  7.34s/it]
+
 24%|██▍       | 40/166 [04:28<15:40,  7.47s/it]
+
 25%|██▍       | 41/166 [04:35<15:04,  7.24s/it]
+
 25%|██▌       | 42/166 [04:42<14:37,  7.08s/it]
+
 26%|██▌       | 43/166 [04:48<14:06,  6.88s/it]
+
 27%|██▋       | 44/166 [04:57<15:00,  7.38s/it]
+
 27%|██▋       | 45/166 [05:05<15:18,  7.59s/it]
+
 28%|██▊       | 46/166 [05:15<16:41,  8.34s/it]
+
 28%|██▊       | 47/166 [05:23<16:28,  8.30s/it]
+
 29%|██▉       | 48/166 [05:30<15:37,  7.94s/it]
+
 30%|██▉       | 49/166 [05:36<14:16,  7.32s/it]
+
 30%|███       | 50/166 [05:44<14:51,  7.69s/it]
+
 31%|███       | 51/166 [05:52<14:35,  7.61s/it]
+
 31%|███▏      | 52/166 [06:00<14:28,  7.62s/it]
+
 32%|███▏      | 53/166 [06:07<14:05,  7.48s/it]
+
 33%|███▎      | 54/166 [06:15<14:13,  7.62s/it]
+
 33%|███▎      | 55/166 [06:21<13:09,  7.12s/it]
+
 34%|███▎      | 56/166 [06:27<12:32,  6.84s/it]
+
 34%|███▍      | 57/166 [06:35<13:08,  7.24s/it]
+
 35%|███▍      | 58/166 [06:41<12:07,  6.73s/it]
+
 36%|███▌      | 59/166 [06:48<12:12,  6.84s/it]
+
 36%|███▌      | 60/166 [06:55<12:15,  6.94s/it]
+
 37%|███▋      | 61/166 [07:04<13:17,  7.60s/it]
+
 37%|███▋      | 62/166 [07:11<13:01,  7.52s/it]
+
 38%|███▊      | 63/166 [07:19<12:46,  7.44s/it]
+
 39%|███▊      | 64/166 [07:28<13:28,  7.93s/it]
+
 39%|███▉      | 65/166 [07:36<13:37,  8.09s/it]
+
 40%|███▉      | 66/166 [07:43<12:52,  7.73s/it]
+
 40%|████      | 67/166 [07:50<12:21,  7.49s/it]
+
 41%|████      | 68/166 [07:55<11:04,  6.79s/it]
+
 42%|████▏     | 69/166 [08:02<11:00,  6.81s/it]
+
 42%|████▏     | 70/166 [08:10<11:28,  7.18s/it]
+
 43%|████▎     | 71/166 [08:18<11:42,  7.39s/it]
+
 43%|████▎     | 72/166 [08:24<11:09,  7.12s/it]
+
 44%|████▍     | 73/166 [08:31<10:42,  6.91s/it]
+
 45%|████▍     | 74/166 [08:37<10:19,  6.73s/it]
+
 45%|████▌     | 75/166 [08:43<10:01,  6.61s/it]
+
 46%|████▌     | 76/166 [08:51<10:14,  6.82s/it]
+
 46%|████▋     | 77/166 [08:56<09:39,  6.51s/it]
+
 47%|████▋     | 78/166 [09:04<09:48,  6.69s/it]
+
 48%|████▊     | 79/166 [09:12<10:15,  7.07s/it]
+
 48%|████▊     | 80/166 [09:20<10:36,  7.40s/it]
+
 49%|████▉     | 81/166 [09:28<10:41,  7.55s/it]
+
 49%|████▉     | 82/166 [09:35<10:18,  7.36s/it]
+
 50%|█████     | 83/166 [09:40<09:20,  6.76s/it]
+
 51%|█████     | 84/166 [09:46<09:06,  6.66s/it]
+
 51%|█████     | 85/166 [09:54<09:15,  6.86s/it]
+
 52%|█████▏    | 86/166 [10:02<09:43,  7.29s/it]
+
 52%|█████▏    | 87/166 [10:09<09:42,  7.38s/it]
+
 53%|█████▎    | 88/166 [10:16<09:19,  7.18s/it]
+
 54%|█████▎    | 89/166 [10:24<09:16,  7.23s/it]
+
 54%|█████▍    | 90/166 [10:30<08:53,  7.02s/it]
+
 55%|█████▍    | 91/166 [10:37<08:39,  6.93s/it]
+
 55%|█████▌    | 92/166 [10:43<08:12,  6.66s/it]
+
 56%|█████▌    | 93/166 [10:51<08:37,  7.09s/it]
+
 57%|█████▋    | 94/166 [11:01<09:36,  8.01s/it]
+
 57%|█████▋    | 95/166 [11:06<08:30,  7.18s/it]
+
 58%|█████▊    | 96/166 [11:13<08:09,  6.99s/it]
+
 58%|█████▊    | 97/166 [11:23<09:09,  7.97s/it]
+
 59%|█████▉    | 98/166 [11:33<09:41,  8.55s/it]
+
 60%|█████▉    | 99/166 [11:40<09:10,  8.22s/it]
+
 60%|██████    | 100/166 [11:47<08:22,  7.61s/it]
+
 61%|██████    | 101/166 [11:53<07:55,  7.31s/it]
+
 61%|██████▏   | 102/166 [12:02<08:17,  7.78s/it]
+
 62%|██████▏   | 103/166 [12:11<08:31,  8.12s/it]
+
 63%|██████▎   | 104/166 [12:18<08:07,  7.87s/it]
+
 63%|██████▎   | 105/166 [12:27<08:06,  7.98s/it]
+
 64%|██████▍   | 106/166 [12:36<08:31,  8.53s/it]
+
 64%|██████▍   | 107/166 [12:46<08:41,  8.83s/it]
+
 65%|██████▌   | 108/166 [12:51<07:33,  7.82s/it]
+
 66%|██████▌   | 109/166 [12:58<07:01,  7.39s/it]
+
 66%|██████▋   | 110/166 [13:05<06:54,  7.41s/it]
+
 67%|██████▋   | 111/166 [13:11<06:23,  6.98s/it]
+
 67%|██████▋   | 112/166 [13:19<06:22,  7.09s/it]
+
 68%|██████▊   | 113/166 [13:27<06:38,  7.53s/it]
+
 69%|██████▊   | 114/166 [13:35<06:30,  7.50s/it]
+
 69%|██████▉   | 115/166 [13:41<06:04,  7.15s/it]
+
 70%|██████▉   | 116/166 [13:47<05:44,  6.88s/it]
+
 70%|███████   | 117/166 [13:55<05:46,  7.07s/it]
+
 71%|███████   | 118/166 [14:02<05:50,  7.30s/it]
+
 72%|███████▏  | 119/166 [14:10<05:48,  7.42s/it]
+
 72%|███████▏  | 120/166 [14:18<05:52,  7.66s/it]
+
 73%|███████▎  | 121/166 [14:27<05:53,  7.86s/it]
+
 73%|███████▎  | 122/166 [14:34<05:43,  7.80s/it]
+
 74%|███████▍  | 123/166 [14:42<05:29,  7.65s/it]
+
 75%|███████▍  | 124/166 [14:48<05:10,  7.39s/it]
+
 75%|███████▌  | 125/166 [14:56<05:03,  7.40s/it]
+
 76%|███████▌  | 126/166 [15:04<05:08,  7.72s/it]
+
 77%|███████▋  | 127/166 [15:09<04:29,  6.90s/it]
+
 77%|███████▋  | 128/166 [15:19<04:58,  7.86s/it]
+
 78%|███████▊  | 129/166 [15:27<04:51,  7.87s/it]
+
 78%|███████▊  | 130/166 [15:34<04:32,  7.57s/it]
+
 79%|███████▉  | 131/166 [15:41<04:19,  7.41s/it]
+
 80%|███████▉  | 132/166 [15:49<04:18,  7.61s/it]
+
 80%|████████  | 133/166 [15:57<04:12,  7.66s/it]
+
 81%|████████  | 134/166 [16:04<03:54,  7.34s/it]
+
 81%|████████▏ | 135/166 [16:10<03:36,  6.98s/it]
+
 82%|████████▏ | 136/166 [16:20<04:00,  8.02s/it]
+
 83%|████████▎ | 137/166 [16:29<04:01,  8.31s/it]
+
 83%|████████▎ | 138/166 [16:39<04:07,  8.85s/it]
+
 84%|████████▎ | 139/166 [16:48<03:54,  8.68s/it]
+
 84%|████████▍ | 140/166 [16:54<03:27,  7.97s/it]
+
 85%|████████▍ | 141/166 [17:00<03:04,  7.36s/it]
+
 86%|████████▌ | 142/166 [17:07<02:55,  7.30s/it]
+
 86%|████████▌ | 143/166 [17:13<02:35,  6.76s/it]
+
 87%|████████▋ | 144/166 [17:21<02:42,  7.37s/it]
+
 87%|████████▋ | 145/166 [17:31<02:50,  8.10s/it]
+
 88%|████████▊ | 146/166 [17:41<02:49,  8.47s/it]
+
 89%|████████▊ | 147/166 [17:49<02:40,  8.47s/it]
+
 89%|████████▉ | 148/166 [17:57<02:32,  8.45s/it]
+
 90%|████████▉ | 149/166 [18:07<02:27,  8.67s/it]
+
 90%|█████████ | 150/166 [18:16<02:23,  8.97s/it]
+
 91%|█████████ | 151/166 [18:24<02:10,  8.68s/it]
+
 92%|█████████▏| 152/166 [18:30<01:49,  7.81s/it]
+
 92%|█████████▏| 153/166 [18:36<01:36,  7.41s/it]
+
 93%|█████████▎| 154/166 [18:44<01:28,  7.38s/it]
+
 93%|█████████▎| 155/166 [18:52<01:22,  7.50s/it]
+
 94%|█████████▍| 156/166 [18:57<01:10,  7.01s/it]
+
 95%|█████████▍| 157/166 [19:04<01:02,  6.92s/it]
+
 95%|█████████▌| 158/166 [19:10<00:52,  6.52s/it]
+
 96%|█████████▌| 159/166 [19:15<00:41,  5.99s/it]
+
 96%|█████████▋| 160/166 [19:20<00:34,  5.71s/it]
+
 97%|█████████▋| 161/166 [19:26<00:29,  5.82s/it]
+
 98%|█████████▊| 162/166 [19:33<00:24,  6.15s/it]
+
 98%|█████████▊| 163/166 [19:40<00:19,  6.45s/it]
+
 99%|█████████▉| 164/166 [19:46<00:12,  6.36s/it]
+
 99%|█████████▉| 165/166 [19:52<00:06,  6.23s/it]
+
100%|██████████| 166/166 [19:56<00:00,  5.63s/it]
                                                        
+
                                                 
{'eval_loss': 0.287109375, 'eval_wer': 8.482348391374007, 'eval_runtime': 1274.1311, 'eval_samples_per_second': 1.04, 'eval_steps_per_second': 0.13, 'epoch': 97.55}
+
 80%|████████  | 4000/5000 [13:55:23<3:03:08, 10.99s/it]
+
100%|██████████| 166/166 [21:08<00:00,  5.63s/it]
+
                                                 [INFO|trainer.py:2700] 2022-12-21 00:17:47,575 >> Saving model checkpoint to ./checkpoint-4000
+[INFO|configuration_utils.py:447] 2022-12-21 00:17:47,576 >> Configuration saved in ./checkpoint-4000/config.json
+[INFO|modeling_utils.py:1680] 2022-12-21 00:17:51,119 >> Model weights saved in ./checkpoint-4000/pytorch_model.bin
+[INFO|feature_extraction_utils.py:368] 2022-12-21 00:17:51,134 >> Feature extractor saved in ./checkpoint-4000/preprocessor_config.json
+[2022-12-21 00:17:51,134] [INFO] [logging.py:68:log_dist] [Rank 0] [Torch] Checkpoint global_step4048 is begin to save!
+/home/milan/hf_env/lib/python3.8/site-packages/torch/nn/modules/module.py:1428: UserWarning: Positional args are being deprecated, use kwargs instead. Refer to https://pytorch.org/docs/master/generated/torch.nn.Module.html#torch.nn.Module.state_dict for details.
+  warnings.warn(
+[2022-12-21 00:17:51,144] [INFO] [logging.py:68:log_dist] [Rank 0] Saving model checkpoint: ./checkpoint-4000/global_step4048/mp_rank_00_model_states.pt
+[2022-12-21 00:17:51,144] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving ./checkpoint-4000/global_step4048/mp_rank_00_model_states.pt...
+[2022-12-21 00:17:54,992] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved ./checkpoint-4000/global_step4048/mp_rank_00_model_states.pt.
+[2022-12-21 00:17:54,993] [INFO] [torch_checkpoint_engine.py:15:save] [Torch] Saving ./checkpoint-4000/global_step4048/zero_pp_rank_0_mp_rank_00_optim_states.pt...
+[2022-12-21 00:18:12,523] [INFO] [torch_checkpoint_engine.py:17:save] [Torch] Saved ./checkpoint-4000/global_step4048/zero_pp_rank_0_mp_rank_00_optim_states.pt.
+[2022-12-21 00:18:12,524] [INFO] [engine.py:3269:_save_zero_checkpoint] zero checkpoint saved ./checkpoint-4000/global_step4048/zero_pp_rank_0_mp_rank_00_optim_states.pt
+[2022-12-21 00:18:12,524] [INFO] [torch_checkpoint_engine.py:27:commit] [Torch] Checkpoint global_step4048 is ready now!
+[INFO|feature_extraction_utils.py:368] 2022-12-21 00:18:17,037 >> Feature extractor saved in ./preprocessor_config.json