Training in progress, step 32000

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:df95fbcfc0eb4bfed629e4280bbb789a37aa3cf5571ceb0ee20543d94a2cc062
+oid sha256:93366238e00d087254b8a44197afae96b22f77853d1e5b2aed180099196dc1a2
 size 3219908024

wandb/run-20231122_234140-blvl5h7j/files/output.log

@@ -3346,3 +3346,1073 @@ Reading metadata...: 1704it [00:00, 9424.36it/s]
 
 
 
+
+
+
+
+
+
+
+
+
+
+
+ 52%|██████████▎         | 31079/60000 [15:16:29<80:23:28, 10.01s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 52%|██████████▎         | 31099/60000 [15:19:46<78:37:11,  9.79s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 52%|██████████▎         | 31119/60000 [15:23:05<76:41:11,  9.56s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 52%|██████████▍         | 31140/60000 [15:26:29<77:42:41,  9.69s/it]
+
+
+
+
+
+
+Reading metadata...: 2165it [00:00, 6749.61it/s]<76:31:41,  9.55s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 52%|██████████▍         | 31159/60000 [15:29:32<77:20:47,  9.65s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 52%|██████████▍         | 31180/60000 [15:32:55<77:27:19,  9.68s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 52%|██████████▍         | 31199/60000 [15:36:08<80:50:31, 10.10s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 52%|██████████▍         | 31219/60000 [15:39:21<76:46:47,  9.60s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 52%|██████████▍         | 31239/60000 [15:42:33<76:31:21,  9.58s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 52%|██████████▍         | 31259/60000 [15:45:44<76:36:26,  9.60s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 52%|██████████▍         | 31279/60000 [15:49:12<84:00:36, 10.53s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 52%|██████████▍         | 31299/60000 [15:52:29<78:14:05,  9.81s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 52%|██████████▍         | 31319/60000 [15:55:44<75:27:27,  9.47s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 52%|██████████▍         | 31340/60000 [15:59:07<77:56:01,  9.79s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 52%|██████████▍         | 31360/60000 [16:03:13<97:38:32, 12.27s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 52%|██████████▍         | 31379/60000 [16:06:22<77:34:20,  9.76s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 52%|██████████▍         | 31399/60000 [16:09:37<76:35:12,  9.64s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 52%|██████████▍         | 31419/60000 [16:12:52<78:40:22,  9.91s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 52%|██████████▍         | 31440/60000 [16:16:16<77:01:35,  9.71s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 52%|██████████▍         | 31460/60000 [16:19:34<80:42:00, 10.18s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 52%|██████████▍         | 31479/60000 [16:22:36<75:20:09,  9.51s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 52%|██████████▍         | 31499/60000 [16:25:46<74:22:41,  9.39s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 53%|██████████▌         | 31520/60000 [16:29:07<75:48:58,  9.58s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 53%|██████████▌         | 31539/60000 [16:32:23<81:02:04, 10.25s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 53%|██████████▌         | 31559/60000 [16:35:43<75:24:15,  9.54s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 53%|██████████▌         | 31579/60000 [16:38:55<76:18:50,  9.67s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 53%|██████████▌         | 31600/60000 [16:42:46<75:49:11,  9.61s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 53%|██████████▌         | 31619/60000 [16:45:59<77:53:34,  9.88s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 53%|██████████▌         | 31639/60000 [16:49:17<80:36:13, 10.23s/it]
+
+
+
+ 53%|██████████▌         | 31642/60000 [16:49:43<69:23:13,  8.81s/it]
+
+ 53%|██████████▌         | 31643/60000 [16:49:49<62:55:40,  7.99s/it]
+
+
+
+
+
+
+
+Reading metadata...: 1650it [00:00, 4118.28it/s]<75:10:32,  9.55s/it]
+
+
+
+
+
+
+
+
+ 53%|██████████▌         | 31659/60000 [16:52:24<76:39:33,  9.74s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 53%|██████████▌         | 31679/60000 [16:55:36<75:08:29,  9.55s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 53%|██████████▌         | 31699/60000 [16:58:50<77:46:06,  9.89s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 53%|██████████▌         | 31719/60000 [17:02:07<75:27:43,  9.61s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 53%|██████████▌         | 31739/60000 [17:05:51<81:27:09, 10.38s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 53%|██████████▌         | 31759/60000 [17:09:05<75:06:48,  9.58s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 53%|██████████���         | 31779/60000 [17:12:19<76:15:47,  9.73s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 53%|██████████▌         | 31799/60000 [17:15:35<75:16:05,  9.61s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 53%|██████████▌         | 31819/60000 [17:18:52<76:15:16,  9.74s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 53%|██████████▌         | 31839/60000 [17:22:03<74:49:45,  9.57s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 53%|██████████▌         | 31859/60000 [17:25:14<75:52:47,  9.71s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 53%|██████████▋         | 31879/60000 [17:28:26<74:55:17,  9.59s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 53%|██████████▋         | 31899/60000 [17:31:38<76:16:29,  9.77s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 53%|██████████▋         | 31919/60000 [17:34:55<74:49:00,  9.59s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 53%|██████████▋         | 31939/60000 [17:38:07<74:52:52,  9.61s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 53%|██████████▋         | 31959/60000 [17:41:28<74:09:04,  9.52s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 53%|██████████▋         | 31980/60000 [17:44:54<76:17:54,  9.80s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 53%|██████████▋         | 31999/60000 [17:48:04<75:32:17,  9.71s/it]
+ 53%|██████████▋         | 32000/60000 [17:48:13<74:55:28,  9.63s/it][INFO|trainer.py:3173] 2023-11-23 17:30:00,790 >> ***** Running Evaluation *****
+[INFO|trainer.py:3177] 2023-11-23 17:30:00,791 >>   Num examples: Unknown
+[INFO|trainer.py:3178] 2023-11-23 17:30:00,791 >>   Batch size = 4
+Reading metadata...: 1704it [00:00, 5075.76it/s]
+[INFO|trainer_utils.py:759] 2023-11-23 17:30:02,321 >> The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: down_votes, up_votes, gender, locale, segment, client_id, accent, input_length, path, age. If down_votes, up_votes, gender, locale, segment, client_id, accent, input_length, path, age are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.
+{'eval_loss': 0.2012939453125, 'eval_wer': 8.254805880135695, 'eval_runtime': 610.9979, 'eval_samples_per_second': 2.789, 'eval_steps_per_second': 0.697, 'epoch': 0.53}
+ 53%|██████████▋         | 32000/60000 [17:58:24<74:55:28,  9.63s/it][INFO|trainer.py:2896] 2023-11-23 17:40:42,279 >> Saving model checkpoint to ./checkpoint-32000
+[INFO|configuration_utils.py:462] 2023-11-23 17:40:42,287 >> Configuration saved in ./checkpoint-32000/config.json
+[INFO|configuration_utils.py:568] 2023-11-23 17:40:42,299 >> Configuration saved in ./checkpoint-32000/generation_config.json
+[INFO|modeling_utils.py:2194] 2023-11-23 17:41:23,174 >> Model weights saved in ./checkpoint-32000/pytorch_model.bin
+[INFO|feature_extraction_utils.py:425] 2023-11-23 17:41:23,179 >> Feature extractor saved in ./checkpoint-32000/preprocessor_config.json
+[2023-11-23 17:41:23,209] [INFO] [logging.py:96:log_dist] [Rank 0] [Torch] Checkpoint global_step32000 is about to be saved!
+[2023-11-23 17:41:23,240] [INFO] [logging.py:96:log_dist] [Rank 0] Saving model checkpoint: ./checkpoint-32000/global_step32000/mp_rank_00_model_states.pt
+[2023-11-23 17:41:23,241] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-32000/global_step32000/mp_rank_00_model_states.pt...
+[2023-11-23 17:41:29,490] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-32000/global_step32000/mp_rank_00_model_states.pt.
+[2023-11-23 17:41:29,497] [INFO] [torch_checkpoint_engine.py:21:save] [Torch] Saving ./checkpoint-32000/global_step32000/zero_pp_rank_0_mp_rank_00_optim_states.pt...
+[2023-11-23 17:41:50,444] [INFO] [torch_checkpoint_engine.py:23:save] [Torch] Saved ./checkpoint-32000/global_step32000/zero_pp_rank_0_mp_rank_00_optim_states.pt.
+[2023-11-23 17:41:50,458] [INFO] [engine.py:3417:_save_zero_checkpoint] zero checkpoint saved ./checkpoint-32000/global_step32000/zero_pp_rank_0_mp_rank_00_optim_states.pt
+[2023-11-23 17:41:50,460] [INFO] [torch_checkpoint_engine.py:33:commit] [Torch] Checkpoint global_step32000 is ready now!
+[INFO|feature_extraction_utils.py:425] 2023-11-23 17:42:47,521 >> Feature extractor saved in ./preprocessor_config.json
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 53%|██████████▋         | 32019/60000 [18:04:12<79:28:54, 10.23s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 53%|██████████▋         | 32039/60000 [18:07:36<77:58:57, 10.04s/it]
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ 53%|██████████▋         | 32059/60000 [18:10:56<77:19:23,  9.96s/it]
+
+
+
+
+
+
+
+

wandb/run-20231122_234140-blvl5h7j/files/wandb-summary.json

@@ -1 +1 @@
-{"train/loss": 0.039, "train/learning_rate": 1.4740677966101696e-06, "train/epoch": 0.52, "train/global_step": 31060, "_timestamp": 1700744106.793268, "_runtime": 54805.90479707718, "_step": 155, "eval/loss": 0.202880859375, "eval/wer": 8.386732001507728, "eval/runtime": 595.6213, "eval/samples_per_second": 2.861, "eval/steps_per_second": 0.715}
+{"train/loss": 0.0376, "train/learning_rate": 1.423322033898305e-06, "train/epoch": 0.53, "train/global_step": 32060, "_timestamp": 1700754773.7870994, "_runtime": 65472.89862847328, "_step": 206, "eval/loss": 0.2012939453125, "eval/wer": 8.254805880135695, "eval/runtime": 610.9979, "eval/samples_per_second": 2.789, "eval/steps_per_second": 0.697}

wandb/run-20231122_234140-blvl5h7j/run-blvl5h7j.wandb

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e1d76afc306ea64fa0af8f18e568ce1cd446c18118aa650f13d61464fe0dfd0c
-size 2588702
+oid sha256:6689d206f4732a222c1725faa6095875ebfa4ae7a3f2f7bea00929c8ceeb7962
+size 3185782