Training in progress, step 200

fine-tune-whisper-streaming.ipynb

@@ -17221,7 +17221,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 53,
+   "execution_count": 60,
    "id": "0ae3e9af-97b7-4aa0-ae85-20b23b5bcb3a",
    "metadata": {},
    "outputs": [
@@ -17241,12 +17241,12 @@
     "    per_device_train_batch_size=64,\n",
     "    gradient_accumulation_steps=1,  # increase by 2x for every 2x decrease in batch size\n",
     "    learning_rate=1e-5,\n",
-    "    warmup_steps=100,\n",
+    "    warmup_steps=400,\n",
     "    max_steps=1000,\n",
     "    gradient_checkpointing=True,\n",
     "    fp16=True,\n",
     "    evaluation_strategy=\"steps\",\n",
-    "    per_device_eval_batch_size=8,\n",
+    "    per_device_eval_batch_size=32,\n",
     "    predict_with_generate=True,\n",
     "    generation_max_length=225,\n",
     "    save_steps=200,\n",
@@ -17308,7 +17308,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 55,
+   "execution_count": 61,
    "id": "d546d7fe-0543-479a-b708-2ebabec19493",
    "metadata": {},
    "outputs": [
@@ -17347,7 +17347,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 56,
+   "execution_count": 62,
    "id": "a1ccb9ed-cbc8-4419-91c0-651e9424b672",
    "metadata": {},
    "outputs": [
@@ -17393,7 +17393,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 57,
+   "execution_count": null,
    "id": "ee8b7b8e-1c9a-4d77-9137-1778a629e6de",
    "metadata": {
     "scrolled": false
@@ -17413,8 +17413,8 @@
       "  Gradient Accumulation steps = 1\n",
       "  Total optimization steps = 1000\n",
       "  Number of trainable parameters = 241734912\n",
-      "Reading metadata...: 6568it [00:00, 31531.18it/s]\n",
-      "Reading metadata...: 4709it [00:00, 27757.91it/s]\n",
+      "Reading metadata...: 6568it [00:00, 35448.85it/s]\n",
+      "Reading metadata...: 4709it [00:00, 28564.29it/s]\n",
       "The following columns in the training set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: input_length. If input_length are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.\n"
      ]
     },
@@ -17424,8 +17424,8 @@
        "\n",
        "    <div>\n",
        "      \n",
-       "      <progress value='1000' max='1000' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
-       "      [1000/1000 3:24:18, Epoch 6/9223372036854775807]\n",
+       "      <progress value='201' max='1000' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
+       "      [ 201/1000 22:02 < 1:28:28, 0.15 it/s, Epoch 1.05/9223372036854775807]\n",
        "    </div>\n",
        "    <table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
@@ -17433,40 +17433,9 @@
        "      <th>Step</th>\n",
        "      <th>Training Loss</th>\n",
        "      <th>Validation Loss</th>\n",
-       "      <th>Wer</th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
-       "    <tr>\n",
-       "      <td>200</td>\n",
-       "      <td>0.143800</td>\n",
-       "      <td>0.182196</td>\n",
-       "      <td>42.436029</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <td>400</td>\n",
-       "      <td>0.031500</td>\n",
-       "      <td>0.186944</td>\n",
-       "      <td>42.128966</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <td>600</td>\n",
-       "      <td>0.011300</td>\n",
-       "      <td>0.195321</td>\n",
-       "      <td>40.634596</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <td>800</td>\n",
-       "      <td>0.005300</td>\n",
-       "      <td>0.194970</td>\n",
-       "      <td>40.675537</td>\n",
-       "    </tr>\n",
-       "    <tr>\n",
-       "      <td>1000</td>\n",
-       "      <td>0.003500</td>\n",
-       "      <td>0.198375</td>\n",
-       "      <td>40.818833</td>\n",
-       "    </tr>\n",
        "  </tbody>\n",
        "</table><p>"
       ],
@@ -17481,115 +17450,14 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "Reading metadata...: 6568it [00:00, 36204.27it/s]\n",
-      "Reading metadata...: 4709it [00:00, 51489.81it/s]\n",
-      "***** Running Evaluation *****\n",
-      "  Num examples: Unknown\n",
-      "  Batch size = 8\n",
-      "Reading metadata...: 4709it [00:00, 28976.18it/s]\n",
-      "The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: input_length. If input_length are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.\n",
-      "Saving model checkpoint to ./checkpoint-200\n",
-      "Configuration saved in ./checkpoint-200/config.json\n",
-      "Model weights saved in ./checkpoint-200/pytorch_model.bin\n",
-      "Feature extractor saved in ./checkpoint-200/preprocessor_config.json\n",
-      "tokenizer config file saved in ./checkpoint-200/tokenizer_config.json\n",
-      "Special tokens file saved in ./checkpoint-200/special_tokens_map.json\n",
-      "added tokens file saved in ./checkpoint-200/added_tokens.json\n",
-      "Feature extractor saved in ./preprocessor_config.json\n",
-      "tokenizer config file saved in ./tokenizer_config.json\n",
-      "Special tokens file saved in ./special_tokens_map.json\n",
-      "added tokens file saved in ./added_tokens.json\n",
-      "Reading metadata...: 6568it [00:00, 38299.83it/s]\n",
-      "Reading metadata...: 4709it [00:00, 30779.86it/s]\n",
-      "***** Running Evaluation *****\n",
-      "  Num examples: Unknown\n",
-      "  Batch size = 8\n",
-      "Reading metadata...: 4709it [00:00, 26095.46it/s]\n",
-      "The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: input_length. If input_length are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.\n",
-      "Saving model checkpoint to ./checkpoint-400\n",
-      "Configuration saved in ./checkpoint-400/config.json\n",
-      "Model weights saved in ./checkpoint-400/pytorch_model.bin\n",
-      "Feature extractor saved in ./checkpoint-400/preprocessor_config.json\n",
-      "tokenizer config file saved in ./checkpoint-400/tokenizer_config.json\n",
-      "Special tokens file saved in ./checkpoint-400/special_tokens_map.json\n",
-      "added tokens file saved in ./checkpoint-400/added_tokens.json\n",
-      "Feature extractor saved in ./preprocessor_config.json\n",
-      "tokenizer config file saved in ./tokenizer_config.json\n",
-      "Special tokens file saved in ./special_tokens_map.json\n",
-      "added tokens file saved in ./added_tokens.json\n",
-      "Reading metadata...: 6568it [00:00, 41768.54it/s]\n",
-      "Reading metadata...: 4709it [00:00, 23263.16it/s]\n",
-      "Reading metadata...: 6568it [00:00, 38529.47it/s]\n",
-      "Reading metadata...: 4709it [00:00, 70740.82it/s]\n",
+      "Reading metadata...: 6568it [00:00, 74826.88it/s]\n",
+      "Reading metadata...: 4709it [00:00, 56808.90it/s]\n",
       "***** Running Evaluation *****\n",
       "  Num examples: Unknown\n",
-      "  Batch size = 8\n",
-      "Reading metadata...: 4709it [00:00, 77474.87it/s]\n",
-      "The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: input_length. If input_length are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.\n",
-      "Saving model checkpoint to ./checkpoint-600\n",
-      "Configuration saved in ./checkpoint-600/config.json\n",
-      "Model weights saved in ./checkpoint-600/pytorch_model.bin\n",
-      "Feature extractor saved in ./checkpoint-600/preprocessor_config.json\n",
-      "tokenizer config file saved in ./checkpoint-600/tokenizer_config.json\n",
-      "Special tokens file saved in ./checkpoint-600/special_tokens_map.json\n",
-      "added tokens file saved in ./checkpoint-600/added_tokens.json\n",
-      "Feature extractor saved in ./preprocessor_config.json\n",
-      "tokenizer config file saved in ./tokenizer_config.json\n",
-      "Special tokens file saved in ./special_tokens_map.json\n",
-      "added tokens file saved in ./added_tokens.json\n",
-      "Reading metadata...: 6568it [00:01, 5110.39it/s]\n",
-      "Reading metadata...: 4709it [00:01, 4212.71it/s]\n",
-      "***** Running Evaluation *****\n",
-      "  Num examples: Unknown\n",
-      "  Batch size = 8\n",
-      "Reading metadata...: 4709it [00:00, 24099.70it/s]\n",
-      "The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: input_length. If input_length are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.\n",
-      "Saving model checkpoint to ./checkpoint-800\n",
-      "Configuration saved in ./checkpoint-800/config.json\n",
-      "Model weights saved in ./checkpoint-800/pytorch_model.bin\n",
-      "Feature extractor saved in ./checkpoint-800/preprocessor_config.json\n",
-      "tokenizer config file saved in ./checkpoint-800/tokenizer_config.json\n",
-      "Special tokens file saved in ./checkpoint-800/special_tokens_map.json\n",
-      "added tokens file saved in ./checkpoint-800/added_tokens.json\n",
-      "Feature extractor saved in ./preprocessor_config.json\n",
-      "tokenizer config file saved in ./tokenizer_config.json\n",
-      "Special tokens file saved in ./special_tokens_map.json\n",
-      "added tokens file saved in ./added_tokens.json\n",
-      "Reading metadata...: 6568it [00:00, 37023.20it/s]\n",
-      "Reading metadata...: 4709it [00:00, 27973.51it/s]\n",
-      "***** Running Evaluation *****\n",
-      "  Num examples: Unknown\n",
-      "  Batch size = 8\n",
-      "Reading metadata...: 4709it [00:00, 75003.52it/s]\n",
-      "The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: input_length. If input_length are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.\n",
-      "Saving model checkpoint to ./checkpoint-1000\n",
-      "Configuration saved in ./checkpoint-1000/config.json\n",
-      "Model weights saved in ./checkpoint-1000/pytorch_model.bin\n",
-      "Feature extractor saved in ./checkpoint-1000/preprocessor_config.json\n",
-      "tokenizer config file saved in ./checkpoint-1000/tokenizer_config.json\n",
-      "Special tokens file saved in ./checkpoint-1000/special_tokens_map.json\n",
-      "added tokens file saved in ./checkpoint-1000/added_tokens.json\n",
-      "Feature extractor saved in ./preprocessor_config.json\n",
-      "tokenizer config file saved in ./tokenizer_config.json\n",
-      "Special tokens file saved in ./special_tokens_map.json\n",
-      "added tokens file saved in ./added_tokens.json\n",
-      "\n",
-      "\n",
-      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
-      "\n",
-      "\n",
-      "Loading best model from ./checkpoint-600 (score: 40.634595701125896).\n"
+      "  Batch size = 32\n",
+      "Reading metadata...: 4709it [00:00, 75164.22it/s]\n",
+      "The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: input_length. If input_length are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.\n"
      ]
-    },
-    {
-     "data": {
-      "text/plain": [
-       "TrainOutput(global_step=1000, training_loss=0.2253765360042453, metrics={'train_runtime': 12295.6043, 'train_samples_per_second': 5.205, 'train_steps_per_second': 0.081, 'total_flos': 1.837596395077632e+19, 'train_loss': 0.2253765360042453, 'epoch': 6.11})"
-      ]
-     },
-     "execution_count": 57,
-     "metadata": {},
-     "output_type": "execute_result"
     }
    ],
    "source": [
@@ -17618,7 +17486,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 40,
+   "execution_count": 58,
    "id": "6dd0e310-9b07-4133-ac14-2ed2d7524e22",
    "metadata": {},
    "outputs": [],
@@ -17644,7 +17512,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 41,
+   "execution_count": 59,
    "id": "95737cda-c5dd-4887-a4d0-dfcb0d61d977",
    "metadata": {},
    "outputs": [
@@ -17658,21 +17526,44 @@
       "Feature extractor saved in ./preprocessor_config.json\n",
       "tokenizer config file saved in ./tokenizer_config.json\n",
       "Special tokens file saved in ./special_tokens_map.json\n",
-      "added tokens file saved in ./added_tokens.json\n",
-      "Several commits (2) will be pushed upstream.\n",
-      "The progress bars may be unreliable.\n",
+      "added tokens file saved in ./added_tokens.json\n"
+     ]
+    },
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "0b88e4ea531d43cfa1236af181f8c616",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "Upload file runs/Dec12_02-00-57_129-213-26-143/events.out.tfevents.1670810506.129-213-26-143.742932.2: 100%|##…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "remote: Scanning LFS files for validity, may be slow...        \n",
+      "remote: LFS file scan complete.        \n",
+      "To https://huggingface.co/kimbochen/whisper-small-zh-tw\n",
+      "   6b0c554..7c46fec  main -> main\n",
+      "\n",
       "To https://huggingface.co/kimbochen/whisper-small-zh-tw\n",
-      "   61dfe27..a94bfc3  main -> main\n",
+      "   7c46fec..f850092  main -> main\n",
       "\n"
      ]
     },
     {
      "data": {
       "text/plain": [
-       "'https://huggingface.co/kimbochen/whisper-small-zh-tw/commit/a94bfc3cbb71b83e10525899df53ee0d4db96e32'"
+       "'https://huggingface.co/kimbochen/whisper-small-zh-tw/commit/7c46fec34e4a0b7f00520d6b3a631c660abd4bcb'"
       ]
      },
-     "execution_count": 41,
+     "execution_count": 59,
      "metadata": {},
      "output_type": "execute_result"
     }

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aaecf242a6944c2e9633600b31b6c65ef158093a1c1e3d9eb4f0499ddcb2dca1
+oid sha256:0cb37f4807695ef0cf30f794f76b317c5b79f755c467f435befb67f31df77787
 size 967102601

runs/Dec12_05-39-20_129-213-26-143/1670823583.842599/events.out.tfevents.1670823583.129-213-26-143.742932.5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1d5806263a1076590d79082c55193f82aba73d96ef13fd2e8e0834b98375f7bf
+size 5864

runs/Dec12_05-39-20_129-213-26-143/events.out.tfevents.1670823583.129-213-26-143.742932.4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:996919504c3e7191f68da639257628f842bc64f49df4b512988d59ebeaf27882
+size 5845

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9d97d92b4d57394382fb181e666c91764f7c97306ef64244dbb9caab670c3ac3
+oid sha256:0f28d2ccc5bfe83c89432546f340a509d32bcdb5ecc2c9ee24b9703b9d097b8d
 size 3579