Training in progress, step 6000

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8924c508de593eda2c41d8422527b39bdf81e48ebbc1a04f8233470a2eefca2f
+oid sha256:13e8bfb6cc0a5778a11cf6475ff56b532497650ddb8b982d4a541b76fea9f445
 size 3219908024

wandb/run-20231118_132020-qg88991p/files/output.log

@@ -4354,3 +4354,2154 @@ Reading metadata...: 1it [00:00,  6.53it/s]
   7%|███████████████▉                                                                                                                                                                                                                            | 4060/60000 [12:20:57<155:35:39, 10.01s/it]
 
 
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  7%|████████████████                                                                                                                                                                                                                            | 4080/60000 [12:24:15<148:47:04,  9.58s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  7%|████████████████                                                                                                                                                                                                                            | 4099/60000 [12:27:10<142:22:51,  9.17s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  7%|████████████████▏                                                                                                                                                                                                                           | 4119/60000 [12:30:18<149:47:00,  9.65s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  7%|████████████████▎                                                                                                                                                                                                                           | 4139/60000 [12:33:21<142:13:05,  9.17s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  7%|████████████████▎                                                                                                                                                                                                                           | 4159/60000 [12:36:24<141:09:26,  9.10s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  7%|████████████████▍                                                                                                                                                                                                                           | 4180/60000 [12:39:38<141:22:34,  9.12s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  7%|████████████████▌                                                                                                                                                                                                                           | 4199/60000 [12:42:32<143:07:23,  9.23s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  7%|████████████████▌                                                                                                                                                                                                                           | 4220/60000 [12:45:49<140:44:27,  9.08s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  7%|████████████████▋                                                                                                                                                                                                                           | 4240/60000 [12:48:52<142:41:50,  9.21s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  7%|████████████████▊                                                                                                                                                                                                                           | 4260/60000 [12:51:57<141:40:28,  9.15s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  7%|████████████████▊                                                                                                                                                                                                                           | 4279/60000 [12:54:51<143:00:35,  9.24s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  7%|████████████████▉                                                                                                                                                                                                                           | 4300/60000 [12:58:09<146:41:35,  9.48s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  7%|████████████████▉                                                                                                                                                                                                                           | 4319/60000 [13:01:17<143:17:07,  9.26s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  7%|█████████████████                                                                                                                                                                                                                           | 4339/60000 [13:04:19<140:12:50,  9.07s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  7%|█████████████████▏                                                                                                                                                                                                                          | 4360/60000 [13:07:32<141:53:59,  9.18s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  7%|█████████████████▏                                                                                                                                                                                                                          | 4380/60000 [13:10:38<145:43:12,  9.43s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  7%|█████████████████▎                                                                                                                                                                                                                          | 4399/60000 [13:14:23<142:17:34,  9.21s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  7%|█████████████████▍                                                                                                                                                                                                                          | 4419/60000 [13:17:25<140:58:10,  9.13s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  7%|█████████████████▍                                                                                                                                                                                                                          | 4439/60000 [13:20:31<141:51:43,  9.19s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  7%|█████████████████▌                                                                                                                                                                                                                          | 4460/60000 [13:23:45<141:09:48,  9.15s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  7%|█████████████████▌                                                                                                                                                                                                                          | 4480/60000 [13:27:06<177:08:56, 11.49s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  8%|█████████████████▋                                                                                                                                                                                                                          | 4500/60000 [13:30:08<139:35:12,  9.05s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  8%|█████████████████▊                                                                                                                                                                                                                          | 4520/60000 [13:33:36<140:19:47,  9.11s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  8%|█████████████████▊                                                                                                                                                                                                                          | 4539/60000 [13:36:30<140:32:06,  9.12s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  8%|█████████████████▉                                                                                                                                                                                                                          | 4559/60000 [13:39:38<150:37:32,  9.78s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  8%|██████████████████                                                                                                                                                                                                                          | 4579/60000 [13:42:45<144:33:59,  9.39s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+  8%|██████████████████                                                                                                                                                                                                                          | 4592/60000 [13:44:38<123:38:09,  8.03s/it]
+
+  8%|██████████████████                                                                                                                                                                                                                          | 4593/60000 [13:44:44<112:47:52,  7.33s/it]
+
+
+
+
+
+
+
+  8%|██████████████████                                                                                                                                                                                                                          | 4600/60000 [13:45:48<138:21:37,  8.99s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  8%|██████████████████▏                                                                                                                                                                                                                         | 4619/60000 [13:48:41<138:11:57,  8.98s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  8%|██████████████████▏                                                                                                                                                                                                                         | 4639/60000 [13:51:53<144:05:49,  9.37s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  8%|██████████████████▎                                                                                                                                                                                                                         | 4660/60000 [13:55:10<142:38:56,  9.28s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  8%|██████████████████▍                                                                                                                                                                                                                         | 4680/60000 [13:58:16<142:21:32,  9.26s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  8%|██████████████████▍                                                                                                                                                                                                                         | 4700/60000 [14:01:20<139:31:05,  9.08s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  8%|██████████████████▌                                                                                                                                                                                                                         | 4720/60000 [14:04:23<140:58:48,  9.18s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  8%|██████████████████▋                                                                                                                                                                                                                         | 4740/60000 [14:07:47<142:14:29,  9.27s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  8%|██████████████████▋                                                                                                                                                                                                                         | 4760/60000 [14:10:50<141:40:34,  9.23s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  8%|██████████████████▊                                                                                                                                                                                                                         | 4780/60000 [14:13:53<139:52:31,  9.12s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  8%|██████████████████▉                                                                                                                                                                                                                         | 4800/60000 [14:16:55<138:27:32,  9.03s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  8%|██████████████████▉                                                                                                                                                                                                                         | 4820/60000 [14:19:59<141:38:49,  9.24s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  8%|███████████████████                                                                                                                                                                                                                         | 4840/60000 [14:23:13<154:09:08, 10.06s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  8%|███████████████████                                                                                                                                                                                                                         | 4859/60000 [14:26:14<141:10:51,  9.22s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  8%|███████████████████▏                                                                                                                                                                                                                        | 4880/60000 [14:29:29<141:33:20,  9.25s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  8%|███████████████████▎                                                                                                                                                                                                                        | 4899/60000 [14:32:25<138:47:05,  9.07s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  8%|███████████████████▎                                                                                                                                                                                                                        | 4919/60000 [14:35:34<142:20:01,  9.30s/it]
+
+Reading metadata...: 1it [00:00,  2.55it/s]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  8%|███████████████████▍                                                                                                                                                                                                                        | 4939/60000 [14:38:35<137:08:39,  8.97s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  8%|███████████████████▌                                                                                                                                                                                                                        | 4959/60000 [14:41:37<138:07:06,  9.03s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  8%|███████████████████▌                                                                                                                                                                                                                        | 4979/60000 [14:44:51<136:58:29,  8.96s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  8%|███████████████████▋                                                                                                                                                                                                                        | 4999/60000 [14:47:58<161:55:31, 10.60s/it]
+  8%|███████████████████▋                                                                                                                                                                                                                        | 5000/60000 [14:48:07<154:23:38, 10.11s/it][INFO|trainer.py:3173] 2023-11-19 04:08:31,335 >> ***** Running Evaluation *****
+[INFO|trainer.py:3177] 2023-11-19 04:08:31,336 >>   Num examples: Unknown
+[INFO|trainer.py:3178] 2023-11-19 04:08:31,336 >>   Batch size = 4
+Reading metadata...: 1704it [00:00, 10489.07it/s]
+[INFO|trainer_utils.py:759] 2023-11-19 04:08:32,633 >> The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: input_length, segment, accent, up_votes, age, path, client_id, down_votes, gender, locale. If input_length, segment, accent, up_votes, age, path, client_id, down_votes, gender, locale are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.
+{'eval_loss': 0.13427734375, 'eval_wer': 8.876743309460988, 'eval_runtime': 590.5226, 'eval_samples_per_second': 2.886, 'eval_steps_per_second': 0.721, 'epoch': 0.08}
+  8%|███████████████████▋                                                                                                                                                                                                                        | 5000/60000 [14:57:58<154:23:38, 10.11s/it][INFO|trainer.py:2896] 2023-11-19 04:18:48,757 >> Saving model checkpoint to ./checkpoint-5000
+[INFO|configuration_utils.py:462] 2023-11-19 04:18:48,768 >> Configuration saved in ./checkpoint-5000/config.json
+[INFO|configuration_utils.py:568] 2023-11-19 04:18:48,773 >> Configuration saved in ./checkpoint-5000/generation_config.json
+[2023-11-19 04:19:36,452] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step5000 is about to be saved!
+[2023-11-19 04:19:36,477] [INFO] [logging.py:96:log_dist] [Rank 0] Saving model checkpoint: ./checkpoint-5000/global_step5000/mp_rank_00_model_states.pt
+[2023-11-19 04:19:36,477] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-5000/global_step5000/mp_rank_00_model_states.pt...
+[INFO|modeling_utils.py:2194] 2023-11-19 04:19:36,434 >> Model weights saved in ./checkpoint-5000/pytorch_model.bin
+[INFO|feature_extraction_utils.py:425] 2023-11-19 04:19:36,438 >> Feature extractor saved in ./checkpoint-5000/preprocessor_config.json
+[2023-11-19 04:19:45,104] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-5000/global_step5000/mp_rank_00_model_states.pt.
+[2023-11-19 04:19:45,108] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-5000/global_step5000/zero_pp_rank_0_mp_rank_00_optim_states.pt...
+[2023-11-19 04:20:11,141] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-5000/global_step5000/zero_pp_rank_0_mp_rank_00_optim_states.pt.
+[2023-11-19 04:20:11,150] [INFO] [engine.py:3417:_save_zero_checkpoint] zero checkpoint saved ./checkpoint-5000/global_step5000/zero_pp_rank_0_mp_rank_00_optim_states.pt
+[2023-11-19 04:20:11,150] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step5000 is ready now!
+[INFO|feature_extraction_utils.py:425] 2023-11-19 04:21:08,374 >> Feature extractor saved in ./preprocessor_config.json
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  8%|███████████████████▋                                                                                                                                                                                                                        | 5020/60000 [15:03:55<151:26:59,  9.92s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  8%|███████████████████▊                                                                                                                                                                                                                        | 5039/60000 [15:06:53<141:57:34,  9.30s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  8%|███████████████████▉                                                                                                                                                                                                                        | 5060/60000 [15:10:10<144:48:47,  9.49s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  8%|███████████████████▉                                                                                                                                                                                                                        | 5079/60000 [15:13:10<142:38:34,  9.35s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  8%|████████████████████                                                                                                                                                                                                                        | 5099/60000 [15:16:17<138:15:38,  9.07s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  9%|████████████████████▏                                                                                                                                                                                                                       | 5119/60000 [15:19:49<186:27:32, 12.23s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  9%|████████████████████▏                                                                                                                                                                                                                       | 5140/60000 [15:23:05<140:57:48,  9.25s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  9%|████████████████████▎                                                                                                                                                                                                                       | 5159/60000 [15:26:01<141:07:57,  9.26s/it]
+
+
+
+
+
+Reading metadata...: 2165it [00:00, 13143.40it/s]                                                                                                                                                                                                | 5165/60000 [15:26:56<140:01:37,  9.19s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  9%|████████████████████▎                                                                                                                                                                                                                       | 5180/60000 [15:29:22<147:19:27,  9.67s/it]
+
+
+
+
+  9%|████████████████████▍                                                                                                                                                                                                                       | 5184/60000 [15:29:55<126:22:47,  8.30s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  9%|████████████████████▍                                                                                                                                                                                                                       | 5199/60000 [15:32:13<141:17:06,  9.28s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  9%|████████████████████▌                                                                                                                                                                                                                       | 5219/60000 [15:35:18<138:39:00,  9.11s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  9%|████████████████████▌                                                                                                                                                                                                                       | 5240/60000 [15:38:44<142:33:38,  9.37s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  9%|████████████████████▋                                                                                                                                                                                                                       | 5259/60000 [15:41:40<138:42:07,  9.12s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  9%|████████████████████▊                                                                                                                                                                                                                       | 5279/60000 [15:44:50<139:40:03,  9.19s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  9%|████████████████████▊                                                                                                                                                                                                                       | 5299/60000 [15:47:54<143:46:47,  9.46s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  9%|████████████████████▉                                                                                                                                                                                                                       | 5319/60000 [15:51:13<140:19:23,  9.24s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  9%|█████████████████████                                                                                                                                                                                                                       | 5340/60000 [15:55:01<143:07:48,  9.43s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  9%|█████████████████████                                                                                                                                                                                                                       | 5360/60000 [15:58:12<142:34:52,  9.39s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  9%|█████████████████████▏                                                                                                                                                                                                                      | 5380/60000 [16:01:16<138:08:35,  9.11s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  9%|█████████████████████▏                                                                                                                                                                                                                      | 5400/60000 [16:04:22<144:24:15,  9.52s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  9%|█████████████████████▎                                                                                                                                                                                                                      | 5420/60000 [16:07:50<157:42:27, 10.40s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  9%|█████████████████████▍                                                                                                                                                                                                                      | 5440/60000 [16:11:02<156:41:23, 10.34s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  9%|█████████████████████▍                                                                                                                                                                                                                      | 5460/60000 [16:14:06<140:40:02,  9.28s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  9%|█████████████████████▌                                                                                                                                                                                                                      | 5480/60000 [16:17:07<135:30:01,  8.95s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  9%|█████████████████████▋                                                                                                                                                                                                                      | 5500/60000 [16:20:12<139:29:05,  9.21s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  9%|█████████████████████▋                                                                                                                                                                                                                      | 5520/60000 [16:23:17<139:41:46,  9.23s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  9%|█████████████████████▊                                                                                                                                                                                                                      | 5539/60000 [16:26:15<138:57:22,  9.19s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  9%|█████████████████████▊                                                                                                                                                                                                                      | 5560/60000 [16:29:26<136:37:37,  9.03s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  9%|█████████████████████▉                                                                                                                                                                                                                      | 5580/60000 [16:32:26<135:34:52,  8.97s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  9%|██████████████████████                                                                                                                                                                                                                      | 5600/60000 [16:35:27<136:14:29,  9.02s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  9%|██████████████████████                                                                                                                                                                                                                      | 5619/60000 [16:38:24<144:47:43,  9.59s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  9%|██████████████████████▏                                                                                                                                                                                                                     | 5639/60000 [16:41:44<142:26:22,  9.43s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  9%|██████████████████████▎                                                                                                                                                                                                                     | 5660/60000 [16:44:55<136:47:25,  9.06s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  9%|██████████████████████▎                                                                                                                                                                                                                     | 5680/60000 [16:48:15<161:38:48, 10.71s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+  9%|██████████████████████▍                                                                                                                                                                                                                     | 5699/60000 [16:51:10<139:03:35,  9.22s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 10%|██████████████████████▍                                                                                                                                                                                                                     | 5720/60000 [16:54:27<135:50:07,  9.01s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 10%|██████████████████████▌                                                                                                                                                                                                                     | 5740/60000 [16:57:26<136:37:25,  9.06s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 10%|██████████████████████▋                                                                                                                                                                                                                     | 5760/60000 [17:00:30<139:16:32,  9.24s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 10%|██████████████████████▋                                                                                                                                                                                                                     | 5780/60000 [17:03:34<138:17:50,  9.18s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 10%|██████████████████████▊                                                                                                                                                                                                                     | 5800/60000 [17:06:42<138:04:03,  9.17s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 10%|██████████████████████▉                                                                                                                                                                                                                     | 5820/60000 [17:09:43<135:29:11,  9.00s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 10%|██████████████████████▉                                                                                                                                                                                                                     | 5839/60000 [17:12:37<136:53:55,  9.10s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 10%|███████████████████████                                                                                                                                                                                                                     | 5859/60000 [17:16:20<154:49:23, 10.29s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 10%|███████████████████████                                                                                                                                                                                                                     | 5879/60000 [17:19:28<145:28:07,  9.68s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 10%|███████████████████████▏                                                                                                                                                                                                                    | 5899/60000 [17:22:36<137:05:28,  9.12s/it]
+
+
+
+
+
+
+
+
+
+
+Reading metadata...: 1650it [00:00, 10583.89it/s]                                                                                                                                                                                                | 5910/60000 [17:24:15<135:54:39,  9.05s/it]
+
+
+
+
+
+
+
+
+
+ 10%|███████████████████████▎                                                                                                                                                                                                                    | 5919/60000 [17:25:40<138:56:39,  9.25s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 10%|███████████████████████▎                                                                                                                                                                                                                    | 5939/60000 [17:28:43<138:23:48,  9.22s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 10%|███████████████████████▍                                                                                                                                                                                                                    | 5959/60000 [17:31:44<134:51:49,  8.98s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 10%|███████████████████████▌                                                                                                                                                                                                                    | 5979/60000 [17:35:03<136:08:23,  9.07s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 10%|███████████████████████▌                                                                                                                                                                                                                    | 6000/60000 [17:38:28<137:45:52,  9.18s/it][INFO|trainer.py:3173] 2023-11-19 06:58:51,907 >> ***** Running Evaluation *****
+[INFO|trainer.py:3177] 2023-11-19 06:58:51,911 >>   Num examples: Unknown
+[INFO|trainer.py:3178] 2023-11-19 06:58:51,911 >>   Batch size = 4
+Reading metadata...: 1704it [00:00, 9946.01it/s]
+Reading metadata...: 1it [00:00,  6.69it/s]
+
+[INFO|trainer_utils.py:759] 2023-11-19 06:58:52,895 >> The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: input_length, segment, accent, up_votes, age, path, client_id, down_votes, gender, locale. If input_length, segment, accent, up_votes, age, path, client_id, down_votes, gender, locale are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.
+{'eval_loss': 0.1368408203125, 'eval_wer': 8.876743309460988, 'eval_runtime': 591.8673, 'eval_samples_per_second': 2.879, 'eval_steps_per_second': 0.72, 'epoch': 0.1}
+ 10%|███████████████████████▌                                                                                                                                                                                                                    | 6000/60000 [17:48:20<137:45:52,  9.18s/it][INFO|trainer.py:2896] 2023-11-19 07:09:16,613 >> Saving model checkpoint to ./checkpoint-6000
+[INFO|configuration_utils.py:462] 2023-11-19 07:09:16,627 >> Configuration saved in ./checkpoint-6000/config.json
+[INFO|configuration_utils.py:568] 2023-11-19 07:09:16,632 >> Configuration saved in ./checkpoint-6000/generation_config.json
+[2023-11-19 07:10:00,067] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step6000 is about to be saved!
+[2023-11-19 07:10:00,107] [INFO] [logging.py:96:log_dist] [Rank 0] Saving model checkpoint: ./checkpoint-6000/global_step6000/mp_rank_00_model_states.pt
+[2023-11-19 07:10:00,108] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-6000/global_step6000/mp_rank_00_model_states.pt...
+[INFO|modeling_utils.py:2194] 2023-11-19 07:10:00,022 >> Model weights saved in ./checkpoint-6000/pytorch_model.bin
+[INFO|feature_extraction_utils.py:425] 2023-11-19 07:10:00,031 >> Feature extractor saved in ./checkpoint-6000/preprocessor_config.json
+[2023-11-19 07:10:08,388] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-6000/global_step6000/mp_rank_00_model_states.pt.
+[2023-11-19 07:10:08,398] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-6000/global_step6000/zero_pp_rank_0_mp_rank_00_optim_states.pt...
+[2023-11-19 07:10:29,571] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-6000/global_step6000/zero_pp_rank_0_mp_rank_00_optim_states.pt.
+[2023-11-19 07:10:29,584] [INFO] [engine.py:3417:_save_zero_checkpoint] zero checkpoint saved ./checkpoint-6000/global_step6000/zero_pp_rank_0_mp_rank_00_optim_states.pt
+[2023-11-19 07:10:29,584] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step6000 is ready now!
+[INFO|feature_extraction_utils.py:425] 2023-11-19 07:11:26,721 >> Feature extractor saved in ./preprocessor_config.json
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 10%|███████████████████████▋                                                                                                                                                                                                                    | 6019/60000 [17:54:07<147:13:43,  9.82s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 10%|███████████████████████▊                                                                                                                                                                                                                    | 6039/60000 [17:57:18<140:10:10,  9.35s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 10%|███████████████████████▊                                                                                                                                                                                                                    | 6060/60000 [18:00:41<148:15:47,  9.90s/it]
+
+
+
+
+
+
+
+
+
+
+

wandb/run-20231118_132020-qg88991p/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 0.1151, "train/learning_rate": 2.844813559322034e-06, "train/epoch": 0.07, "train/global_step": 4060, "_timestamp": 1700350880.8220081, "_runtime": 44459.90691804886, "_step": 206, "eval/loss": 0.1268310546875, "eval/wer": 8.89558989822842, "eval/runtime": 729.8005, "eval/samples_per_second": 2.335, "eval/steps_per_second": 0.584}
+{"train/loss": 0.0959, "train/learning_rate": 2.7432711864406777e-06, "train/epoch": 0.1, "train/global_step": 6060, "_timestamp": 1700371265.4985242, "_runtime": 64844.58343410492, "_step": 308, "eval/loss": 0.1368408203125, "eval/wer": 8.876743309460988, "eval/runtime": 591.8673, "eval/samples_per_second": 2.879, "eval/steps_per_second": 0.72}

wandb/run-20231118_132020-qg88991p/run-qg88991p.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d48a9fc5fbdbdb154e3cf030b4e64719973b84fe5479fde0b847917c8fa252a4
-size 3370680
+oid sha256:45abb02b364f3abe2253240c15f9ce34b371ded1f82652adbb3b0ae2cfbbceb2
+size 5049554