Training in progress, step 100

.gitignore

@@ -1 +1 @@
-checkpoint-*/
+#checkpoint-*/

checkpoint-100/config.json

@@ -0,0 +1,42 @@
+{
+  "_name_or_path": "openai/whisper-small",
+  "activation_dropout": 0.0,
+  "activation_function": "gelu",
+  "architectures": [
+    "WhisperForConditionalGeneration"
+  ],
+  "attention_dropout": 0.0,
+  "begin_suppress_tokens": [
+    220,
+    50257
+  ],
+  "bos_token_id": 50257,
+  "d_model": 768,
+  "decoder_attention_heads": 12,
+  "decoder_ffn_dim": 3072,
+  "decoder_layerdrop": 0.0,
+  "decoder_layers": 12,
+  "decoder_start_token_id": 50258,
+  "dropout": 0.0,
+  "encoder_attention_heads": 12,
+  "encoder_ffn_dim": 3072,
+  "encoder_layerdrop": 0.0,
+  "encoder_layers": 12,
+  "eos_token_id": 50257,
+  "forced_decoder_ids": null,
+  "init_std": 0.02,
+  "is_encoder_decoder": true,
+  "max_length": 448,
+  "max_source_positions": 1500,
+  "max_target_positions": 448,
+  "model_type": "whisper",
+  "num_hidden_layers": 12,
+  "num_mel_bins": 80,
+  "pad_token_id": 50257,
+  "scale_embedding": false,
+  "suppress_tokens": [],
+  "torch_dtype": "float32",
+  "transformers_version": "4.26.0.dev0",
+  "use_cache": false,
+  "vocab_size": 51865
+}

checkpoint-100/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1002d1d1f52543b717c6981da336cb2b7d5de41f31e4ce2d17156a34b040ee6
+size 1934160645

checkpoint-100/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:946ddc68bfd507e62f2331943cb15a787bbdda6a4389be1fcfd25cff7522f970
+size 967102601

checkpoint-100/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3cb1efe7cf554d988385a8b3b6ee72bcb439a80ae12b6972729b5fd2605e1c6e
+size 14511

checkpoint-100/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc1d2baeb4b37bcdfe6e1f87922490255ada6e3a285d3888ea80d73c23b6b2a5
+size 557

checkpoint-100/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:27824508c9b835e63987aba694f6253319bfea6d8742d490d9270366f9e41f6c
+size 627

checkpoint-100/trainer_state.json

@@ -0,0 +1,86 @@
+{
+  "best_metric": 146.85230024213075,
+  "best_model_checkpoint": "./checkpoint-100",
+  "epoch": 2.376470588235294,
+  "global_step": 100,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.24,
+      "learning_rate": 2.666666666666667e-06,
+      "loss": 4.3134,
+      "step": 10
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 6e-06,
+      "loss": 2.9329,
+      "step": 20
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 9.333333333333334e-06,
+      "loss": 2.0584,
+      "step": 30
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 9.829787234042554e-06,
+      "loss": 1.566,
+      "step": 40
+    },
+    {
+      "epoch": 1.19,
+      "learning_rate": 9.617021276595745e-06,
+      "loss": 1.3777,
+      "step": 50
+    },
+    {
+      "epoch": 1.42,
+      "learning_rate": 9.404255319148937e-06,
+      "loss": 1.1469,
+      "step": 60
+    },
+    {
+      "epoch": 1.66,
+      "learning_rate": 9.191489361702128e-06,
+      "loss": 1.0638,
+      "step": 70
+    },
+    {
+      "epoch": 1.89,
+      "learning_rate": 8.97872340425532e-06,
+      "loss": 0.9974,
+      "step": 80
+    },
+    {
+      "epoch": 2.14,
+      "learning_rate": 8.765957446808512e-06,
+      "loss": 0.9615,
+      "step": 90
+    },
+    {
+      "epoch": 2.38,
+      "learning_rate": 8.553191489361703e-06,
+      "loss": 0.8262,
+      "step": 100
+    },
+    {
+      "epoch": 2.38,
+      "eval_cer": 149.33157314260887,
+      "eval_loss": 0.8188337683677673,
+      "eval_runtime": 459.9395,
+      "eval_samples_per_second": 1.113,
+      "eval_steps_per_second": 0.139,
+      "eval_wer": 146.85230024213075,
+      "step": 100
+    }
+  ],
+  "max_steps": 500,
+  "num_train_epochs": 12,
+  "total_flos": 1.86022149046272e+18,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-100/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5518cd575512706c66c89476ae9c64328c1bcac98ec06151bdc9afebf6c503fb
+size 3643

checkpoint-200/config.json

@@ -0,0 +1,42 @@
+{
+  "_name_or_path": "openai/whisper-small",
+  "activation_dropout": 0.0,
+  "activation_function": "gelu",
+  "architectures": [
+    "WhisperForConditionalGeneration"
+  ],
+  "attention_dropout": 0.0,
+  "begin_suppress_tokens": [
+    220,
+    50257
+  ],
+  "bos_token_id": 50257,
+  "d_model": 768,
+  "decoder_attention_heads": 12,
+  "decoder_ffn_dim": 3072,
+  "decoder_layerdrop": 0.0,
+  "decoder_layers": 12,
+  "decoder_start_token_id": 50258,
+  "dropout": 0.0,
+  "encoder_attention_heads": 12,
+  "encoder_ffn_dim": 3072,
+  "encoder_layerdrop": 0.0,
+  "encoder_layers": 12,
+  "eos_token_id": 50257,
+  "forced_decoder_ids": null,
+  "init_std": 0.02,
+  "is_encoder_decoder": true,
+  "max_length": 448,
+  "max_source_positions": 1500,
+  "max_target_positions": 448,
+  "model_type": "whisper",
+  "num_hidden_layers": 12,
+  "num_mel_bins": 80,
+  "pad_token_id": 50257,
+  "scale_embedding": false,
+  "suppress_tokens": [],
+  "torch_dtype": "float32",
+  "transformers_version": "4.26.0.dev0",
+  "use_cache": false,
+  "vocab_size": 51865
+}

checkpoint-200/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:138a6ddac36f3135f9ce799bc0aa0c6ee6e8510cecdd696448298fce2403ddb0
+size 1934160645

checkpoint-200/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d091c8707cb0763632c8b311f2d581a51bd141ec488b3c948362989bfac0bdda
+size 967102601

checkpoint-200/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b22328e86e2f01e3b161f44df6b51159ec6b4a940fe644d7490ef269503f585
+size 14511

checkpoint-200/scaler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:02ee58f3dc2c47aec3ec5dbf8581a593f6a668e46904253580e157d97de3e149
+size 557

checkpoint-200/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e82e796ecfbb8fa98125f306d5ff56f81ebaf5a863f8fefc032c86533b2fdf8f
+size 627

checkpoint-200/trainer_state.json

@@ -0,0 +1,156 @@
+{
+  "best_metric": 139.37651331719127,
+  "best_model_checkpoint": "./checkpoint-100",
+  "epoch": 2.377581120943953,
+  "global_step": 200,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.12,
+      "learning_rate": 3e-06,
+      "loss": 4.1495,
+      "step": 10
+    },
+    {
+      "epoch": 0.24,
+      "learning_rate": 6.333333333333333e-06,
+      "loss": 2.9287,
+      "step": 20
+    },
+    {
+      "epoch": 0.35,
+      "learning_rate": 9.666666666666667e-06,
+      "loss": 2.0462,
+      "step": 30
+    },
+    {
+      "epoch": 0.47,
+      "learning_rate": 9.666666666666667e-06,
+      "loss": 1.6138,
+      "step": 40
+    },
+    {
+      "epoch": 0.59,
+      "learning_rate": 9.296296296296296e-06,
+      "loss": 1.3862,
+      "step": 50
+    },
+    {
+      "epoch": 0.71,
+      "learning_rate": 8.925925925925927e-06,
+      "loss": 1.2604,
+      "step": 60
+    },
+    {
+      "epoch": 0.83,
+      "learning_rate": 8.555555555555556e-06,
+      "loss": 1.1436,
+      "step": 70
+    },
+    {
+      "epoch": 0.94,
+      "learning_rate": 8.185185185185187e-06,
+      "loss": 1.168,
+      "step": 80
+    },
+    {
+      "epoch": 1.07,
+      "learning_rate": 7.814814814814816e-06,
+      "loss": 1.1041,
+      "step": 90
+    },
+    {
+      "epoch": 1.19,
+      "learning_rate": 7.444444444444445e-06,
+      "loss": 0.9683,
+      "step": 100
+    },
+    {
+      "epoch": 1.19,
+      "eval_cer": 131.61659035460045,
+      "eval_loss": 0.8811978697776794,
+      "eval_runtime": 593.3196,
+      "eval_samples_per_second": 0.863,
+      "eval_steps_per_second": 0.431,
+      "eval_wer": 139.37651331719127,
+      "step": 100
+    },
+    {
+      "epoch": 1.31,
+      "learning_rate": 7.074074074074074e-06,
+      "loss": 0.909,
+      "step": 110
+    },
+    {
+      "epoch": 1.42,
+      "learning_rate": 6.703703703703704e-06,
+      "loss": 0.9213,
+      "step": 120
+    },
+    {
+      "epoch": 1.54,
+      "learning_rate": 6.333333333333333e-06,
+      "loss": 0.9092,
+      "step": 130
+    },
+    {
+      "epoch": 1.66,
+      "learning_rate": 5.962962962962963e-06,
+      "loss": 0.8481,
+      "step": 140
+    },
+    {
+      "epoch": 1.78,
+      "learning_rate": 5.5925925925925926e-06,
+      "loss": 0.8471,
+      "step": 150
+    },
+    {
+      "epoch": 1.9,
+      "learning_rate": 5.2222222222222226e-06,
+      "loss": 0.8504,
+      "step": 160
+    },
+    {
+      "epoch": 2.02,
+      "learning_rate": 4.851851851851852e-06,
+      "loss": 0.8264,
+      "step": 170
+    },
+    {
+      "epoch": 2.14,
+      "learning_rate": 4.481481481481482e-06,
+      "loss": 0.7236,
+      "step": 180
+    },
+    {
+      "epoch": 2.26,
+      "learning_rate": 4.111111111111111e-06,
+      "loss": 0.6898,
+      "step": 190
+    },
+    {
+      "epoch": 2.38,
+      "learning_rate": 3.740740740740741e-06,
+      "loss": 0.6848,
+      "step": 200
+    },
+    {
+      "epoch": 2.38,
+      "eval_cer": 151.33685371478225,
+      "eval_loss": 0.7542899250984192,
+      "eval_runtime": 551.6472,
+      "eval_samples_per_second": 0.928,
+      "eval_steps_per_second": 0.464,
+      "eval_wer": 145.9972760290557,
+      "step": 200
+    }
+  ],
+  "max_steps": 300,
+  "num_train_epochs": 4,
+  "total_flos": 1.86022149046272e+18,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-200/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bcf3a24325bb49c25270193816ed6035a253ca3ae300c31ffcec0afeb4229266
+size 3579

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3659cbd57caabfa6834081314e1044f720d4f82db5a36a341158bdc9fc0cf4f2
+oid sha256:946ddc68bfd507e62f2331943cb15a787bbdda6a4389be1fcfd25cff7522f970
 size 967102601

runs/Dec20_20-02-45_129-146-32-172/1671566571.0086486/events.out.tfevents.1671566571.129-146-32-172.146952.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ebfd8972b7d001f37f279924678679a975977b7be87a6d8f31555aad29e3833
+size 5862

runs/Dec20_20-02-45_129-146-32-172/events.out.tfevents.1671566571.129-146-32-172.146952.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1c8bad0c4902c1b9b166697cb4e7edeb7f56d7c5dec9f45409cd1c9c00f0c914
+size 4747

runs/Dec20_20-22-25_129-146-32-172/1671567753.387425/events.out.tfevents.1671567753.129-146-32-172.150576.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62a9e3c9153b155528dd9febed01a15398f438aafea941f78410a763a9be2cb2
+size 5862

runs/Dec20_20-22-25_129-146-32-172/events.out.tfevents.1671567753.129-146-32-172.150576.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:961a3b8618713dfc24c483c8f04cfa559969b9e3a329008c3b26019ce9fabdd4
+size 4903

runs/Dec20_20-28-50_129-146-32-172/1671568135.4799154/events.out.tfevents.1671568135.129-146-32-172.151517.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5843873d279e134418900972f42218231fa7cff59eac0ce951355225ec162fa1
+size 5862

runs/Dec20_20-28-50_129-146-32-172/events.out.tfevents.1671568135.129-146-32-172.151517.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:782028836de28c83d061b5870d65ffd6d0cfb2cf7e8386ba8180ee926b2c5967
+size 6185

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bcf3a24325bb49c25270193816ed6035a253ca3ae300c31ffcec0afeb4229266
-size 3579
+oid sha256:5518cd575512706c66c89476ae9c64328c1bcac98ec06151bdc9afebf6c503fb
+size 3643

whisper_small_ps_augmented.py

@@ -251,16 +251,16 @@ In the final step, we define all the parameters related to training. For more de
 
 training_args = Seq2SeqTrainingArguments(
     output_dir="./",
-    per_device_train_batch_size=2,
+    per_device_train_batch_size=16,
     # increase by 2x for every 2x decrease in batch size
-    gradient_accumulation_steps=16,
+    gradient_accumulation_steps=4,
     learning_rate=1e-5,
     warmup_steps=30,
     max_steps=500,
     gradient_checkpointing=True,
     fp16=True,
     evaluation_strategy="steps",
-    per_device_eval_batch_size=2,
+    per_device_eval_batch_size=8,
     predict_with_generate=True,
     generation_max_length=225,
     save_steps=100,
@@ -272,7 +272,8 @@ training_args = Seq2SeqTrainingArguments(
     greater_is_better=False,
     push_to_hub=True,
     #optim='adamw_bnb_8bit',  # 'adamw_bnb_8bit',
-    overwrite_output_dir="False"
+    overwrite_output_dir="False",
+    resume_from_checkpoint="True"
 )
 
 
@@ -298,7 +299,7 @@ trainer.train()
 kwargs = {
     "dataset_tags": "google/fleurs",
     "dataset": "google/fleurs",  # a 'pretty' name for the training dataset
-    "language": "ps_af",
+    "language": "ps",
     "model_name": "Whisper Small Pashto - Augmented",  # a 'pretty' name for your model
     "finetuned_from": "openai/whisper-small",
     "tasks": "automatic-speech-recognition",