Training in progress, step 2000

.gitattributes

@@ -37,3 +37,4 @@ community-events/huggan/assets/cyclegan.png filter=lfs diff=lfs merge=lfs -text
 community-events/huggan/assets/lightweight_gan_wandb.png filter=lfs diff=lfs merge=lfs -text
 community-events/huggan/assets/pix2pix_maps.png filter=lfs diff=lfs merge=lfs -text
 community-events/huggan/assets/wandb.png filter=lfs diff=lfs merge=lfs -text
+wandb/run-20231118_132020-qg88991p/run-qg88991p.wandb filter=lfs diff=lfs merge=lfs -text

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1809e1646847952ad2b1f09458b46175ade1f5805dd12dfae21dff86337b1bfd
+oid sha256:5debab932d724916b64afb47dcdffe070515ab976867ca2f47b2909b3955e612
 size 3219908024

wandb/run-20231118_132020-qg88991p/files/output.log

@@ -1145,3 +1145,1068 @@ Reading metadata...: 1it [00:00,  5.88it/s]
 
 
 
+
+
+
+
+
+
+
+
+
+  2%|████▎                                                                                                                                                                                                                                        | 1079/60000 [3:13:36<160:28:49,  9.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  2%|████▎                                                                                                                                                                                                                                        | 1099/60000 [3:16:51<154:50:54,  9.46s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  2%|████▍                                                                                                                                                                                                                                        | 1119/60000 [3:20:07<163:46:21, 10.01s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  2%|████▍                                                                                                                                                                                                                                        | 1139/60000 [3:23:25<161:44:53,  9.89s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  2%|████▌                                                                                                                                                                                                                                        | 1160/60000 [3:26:43<151:47:46,  9.29s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  2%|████▋                                                                                                                                                                                                                                        | 1180/60000 [3:29:56<157:42:37,  9.65s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  2%|████▋                                                                                                                                                                                                                                        | 1199/60000 [3:32:53<156:43:48,  9.60s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  2%|████▊                                                                                                                                                                                                                                        | 1220/60000 [3:36:19<153:50:38,  9.42s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  2%|████▉                                                                                                                                                                                                                                        | 1240/60000 [3:39:40<162:05:41,  9.93s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  2%|████▉                                                                                                                                                                                                                                        | 1259/60000 [3:42:40<154:56:28,  9.50s/it]
+
+
+
+
+
+
+
+
+
+  2%|█████                                                                                                                                                                                                                                        | 1268/60000 [3:44:08<153:51:32,  9.43s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+  2%|█████                                                                                                                                                                                                                                        | 1279/60000 [3:45:58<159:14:10,  9.76s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  2%|█████▏                                                                                                                                                                                                                                       | 1299/60000 [3:49:18<169:32:55, 10.40s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  2%|█████▏                                                                                                                                                                                                                                       | 1320/60000 [3:52:53<160:35:35,  9.85s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  2%|█████▎                                                                                                                                                                                                                                       | 1339/60000 [3:56:03<151:16:59,  9.28s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  2%|█████▎                                                                                                                                                                                                                                       | 1360/60000 [3:59:18<150:14:39,  9.22s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  2%|█████▍                                                                                                                                                                                                                                       | 1379/60000 [4:02:24<161:24:32,  9.91s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  2%|█████▌                                                                                                                                                                                                                                       | 1399/60000 [4:05:36<150:50:03,  9.27s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  2%|█████▌                                                                                                                                                                                                                                       | 1420/60000 [4:08:58<153:51:31,  9.46s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  2%|█████▋                                                                                                                                                                                                                                       | 1440/60000 [4:12:32<268:42:47, 16.52s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  2%|█████▊                                                                                                                                                                                                                                       | 1459/60000 [4:15:40<167:25:48, 10.30s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  2%|█████▊                                                                                                                                                                                                                                       | 1479/60000 [4:19:20<180:20:19, 11.09s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  2%|█████▉                                                                                                                                                                                                                                       | 1499/60000 [4:22:39<158:46:13,  9.77s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3%|██████                                                                                                                                                                                                                                       | 1520/60000 [4:26:11<158:51:45,  9.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3%|██████                                                                                                                                                                                                                                       | 1539/60000 [4:29:26<162:36:09, 10.01s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3%|██████▏                                                                                                                                                                                                                                      | 1560/60000 [4:33:03<173:55:15, 10.71s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3%|██████▏                                                                                                                                                                                                                                      | 1580/60000 [4:36:23<161:38:39,  9.96s/it]
+
+
+
+
+
+
+  3%|██████▎                                                                                                                                                                                                                                      | 1586/60000 [4:37:19<142:22:29,  8.77s/it]
+
+  3%|██████▎                                                                                                                                                                                                                                      | 1587/60000 [4:37:26<136:23:29,  8.41s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3%|██████▎                                                                                                                                                                                                                                      | 1600/60000 [4:39:40<166:18:31, 10.25s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3%|██████▍                                                                                                                                                                                                                                      | 1619/60000 [4:42:51<158:19:48,  9.76s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3%|██████▍                                                                                                                                                                                                                                      | 1640/60000 [4:46:34<179:40:16, 11.08s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3%|██████▌                                                                                                                                                                                                                                      | 1660/60000 [4:49:57<159:16:34,  9.83s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3%|██████▋                                                                                                                                                                                                                                      | 1680/60000 [4:53:13<159:55:32,  9.87s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3%|██████▋                                                                                                                                                                                                                                      | 1700/60000 [4:56:31<160:07:22,  9.89s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3%|██████▊                                                                                                                                                                                                                                      | 1720/60000 [4:59:48<160:30:27,  9.91s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3%|██████▊                                                                                                                                                                                                                                      | 1740/60000 [5:03:12<163:35:37, 10.11s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3%|██████▉                                                                                                                                                                                                                                      | 1759/60000 [5:06:50<184:16:28, 11.39s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3%|███████                                                                                                                                                                                                                                      | 1779/60000 [5:10:08<163:21:51, 10.10s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3%|███████                                                                                                                                                                                                                                      | 1799/60000 [5:14:02<163:45:05, 10.13s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3%|███████▏                                                                                                                                                                                                                                     | 1819/60000 [5:17:18<159:26:20,  9.87s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3%|███████▎                                                                                                                                                                                                                                     | 1839/60000 [5:20:39<159:09:01,  9.85s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3%|███████▎                                                                                                                                                                                                                                     | 1859/60000 [5:23:48<153:23:42,  9.50s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3%|███████▍                                                                                                                                                                                                                                     | 1879/60000 [5:27:22<159:48:10,  9.90s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3%|███████▌                                                                                                                                                                                                                                     | 1900/60000 [5:30:41<150:21:31,  9.32s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3%|███████▌                                                                                                                                                                                                                                     | 1919/60000 [5:33:49<156:00:28,  9.67s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3%|███████▋                                                                                                                                                                                                                                     | 1940/60000 [5:37:08<153:47:37,  9.54s/it]
+
+
+
+
+
+
+
+
+
+Reading metadata...: 1650it [00:00, 7956.90it/s]                                                                                                                                                                                                  | 1949/60000 [5:38:34<154:59:37,  9.61s/it]
+
+
+
+
+
+
+
+
+
+
+
+  3%|███████▋                                                                                                                                                                                                                                     | 1960/60000 [5:40:18<151:20:44,  9.39s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3%|███████▊                                                                                                                                                                                                                                     | 1980/60000 [5:43:36<152:42:22,  9.48s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3%|███████▉                                                                                                                                                                                                                                     | 1999/60000 [5:46:40<158:55:24,  9.86s/it]
+  3%|███████▉                                                                                                                                                                                                                                     | 2000/60000 [5:46:54<179:09:00, 11.12s/it][INFO|trainer.py:3173] 2023-11-18 19:07:17,798 >> ***** Running Evaluation *****
+[INFO|trainer.py:3177] 2023-11-18 19:07:17,799 >>   Num examples: Unknown
+[INFO|trainer.py:3178] 2023-11-18 19:07:17,799 >>   Batch size = 4
+Reading metadata...: 1704it [00:00, 9996.31it/s]
+
+[INFO|trainer_utils.py:759] 2023-11-18 19:07:18,883 >> The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: input_length, segment, accent, up_votes, age, path, client_id, down_votes, gender, locale. If input_length, segment, accent, up_votes, age, path, client_id, down_votes, gender, locale are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.
+  3%|███████▉                                                                                                                                                                                                                                     | 2000/60000 [5:57:35<179:09:00, 11.12s/it]
+  3%|███████▉                                                                                                                                                                                                                                     | 2000/60000 [5:57:35<179:09:00, 11.12s/it][INFO|trainer.py:2896] 2023-11-18 19:18:28,722 >> Saving model checkpoint to ./checkpoint-2000
+[INFO|configuration_utils.py:462] 2023-11-18 19:18:28,733 >> Configuration saved in ./checkpoint-2000/config.json
+[INFO|configuration_utils.py:568] 2023-11-18 19:18:28,740 >> Configuration saved in ./checkpoint-2000/generation_config.json
+[INFO|modeling_utils.py:2194] 2023-11-18 19:18:49,012 >> Model weights saved in ./checkpoint-2000/pytorch_model.bin
+[INFO|feature_extraction_utils.py:425] 2023-11-18 19:18:49,017 >> Feature extractor saved in ./checkpoint-2000/preprocessor_config.json
+[2023-11-18 19:18:49,048] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step2000 is about to be saved!
+[2023-11-18 19:18:49,076] [INFO] [logging.py:96:log_dist] [Rank 0] Saving model checkpoint: ./checkpoint-2000/global_step2000/mp_rank_00_model_states.pt
+[2023-11-18 19:18:49,077] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-2000/global_step2000/mp_rank_00_model_states.pt...
+[2023-11-18 19:18:58,661] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-2000/global_step2000/mp_rank_00_model_states.pt.
+[2023-11-18 19:18:58,678] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-2000/global_step2000/zero_pp_rank_0_mp_rank_00_optim_states.pt...
+[2023-11-18 19:19:20,153] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-2000/global_step2000/zero_pp_rank_0_mp_rank_00_optim_states.pt.
+[2023-11-18 19:19:20,168] [INFO] [engine.py:3417:_save_zero_checkpoint] zero checkpoint saved ./checkpoint-2000/global_step2000/zero_pp_rank_0_mp_rank_00_optim_states.pt
+[2023-11-18 19:19:20,168] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step2000 is ready now!
+[INFO|feature_extraction_utils.py:425] 2023-11-18 19:20:30,717 >> Feature extractor saved in ./preprocessor_config.json
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3%|███████▉                                                                                                                                                                                                                                     | 2020/60000 [6:03:46<170:10:04, 10.57s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3%|████████                                                                                                                                                                                                                                     | 2039/60000 [6:07:12<185:41:14, 11.53s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  3%|████████▏                                                                                                                                                                                                                                    | 2060/60000 [6:11:16<189:05:09, 11.75s/it]
+

wandb/run-20231118_132020-qg88991p/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 0.1306, "train/learning_rate": 2.997050847457627e-06, "train/epoch": 0.02, "train/global_step": 1060, "_timestamp": 1700317854.0811276, "_runtime": 11433.16603755951, "_step": 53, "eval/loss": 0.129150390625, "eval/wer": 9.498680738786279, "eval/runtime": 612.1544, "eval/samples_per_second": 2.784, "eval/steps_per_second": 0.696}
+{"train/loss": 0.1253, "train/learning_rate": 2.946305084745763e-06, "train/epoch": 0.03, "train/global_step": 2060, "_timestamp": 1700328700.533228, "_runtime": 22279.618137836456, "_step": 104, "eval/loss": 0.12255859375, "eval/wer": 9.150018846588766, "eval/runtime": 641.3242, "eval/samples_per_second": 2.657, "eval/steps_per_second": 0.664}