Training in progress, step 200

.ipynb_checkpoints/fine-tune-whisper-streaming-checkpoint.ipynb

@@ -142,74 +142,21 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 3,
    "id": "a2787582-554f-44ce-9f38-4180a5ed6b44",
    "metadata": {},
-   "outputs": [
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "ecce3a630cdb4ebab217a88a0163b257",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Downloading builder script:   0%|          | 0.00/8.30k [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "b0141b068f944775867034bc494f88d7",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Downloading readme:   0%|          | 0.00/12.2k [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "9dd1f4ded47c4160b55f1bcedce2694f",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Downloading extra modules:   0%|          | 0.00/3.44k [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    },
-    {
-     "data": {
-      "application/vnd.jupyter.widget-view+json": {
-       "model_id": "a442da1e2a6b4271bae8ae0c655594b6",
-       "version_major": 2,
-       "version_minor": 0
-      },
-      "text/plain": [
-       "Downloading extra modules:   0%|          | 0.00/60.9k [00:00<?, ?B/s]"
-      ]
-     },
-     "metadata": {},
-     "output_type": "display_data"
-    }
-   ],
+   "outputs": [],
    "source": [
     "from datasets import IterableDatasetDict\n",
     "\n",
     "raw_datasets = IterableDatasetDict()\n",
     "\n",
-    "raw_datasets[\"train\"] = load_streaming_dataset(\"mozilla-foundation/common_voice_11_0\", \"zh-TW\", split=\"train\", use_auth_token=True)  # set split=\"train+validation\" for low-resource\n",
-    "raw_datasets[\"test\"] = load_streaming_dataset(\"mozilla-foundation/common_voice_11_0\", \"zh-TW\", split=\"test\", use_auth_token=True)"
+    "raw_datasets[\"train\"] = load_streaming_dataset(\"mozilla-foundation/common_voice_11_0\", \"zh-CN\", split=\"train\", use_auth_token=True)\n",
+    "raw_datasets[\"test\"] = load_streaming_dataset(\"mozilla-foundation/common_voice_11_0\", \"zh-CN\", split=\"test\", use_auth_token=True)\n",
+    "\n",
+    "# zh-TW is low resource\n",
+    "# raw_datasets[\"train\"] = load_streaming_dataset(\"mozilla-foundation/common_voice_11_0\", \"zh-TW\", split=\"train+validation\", use_auth_token=True)\n",
+    "# raw_datasets[\"test\"] = load_streaming_dataset(\"mozilla-foundation/common_voice_11_0\", \"zh-TW\", split=\"test\", use_auth_token=True)"
    ]
   },
   {
@@ -242,14 +189,16 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 4,
    "id": "77d9f0c5-8607-4642-a8ac-c3ab2e223ea6",
-   "metadata": {},
+   "metadata": {
+    "tags": []
+   },
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "0d0c17f582474beebea009f021515946",
+       "model_id": "48ef23eaa9fb4d6ca621fd252befca48",
        "version_major": 2,
        "version_minor": 0
       },
@@ -263,7 +212,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "9f48049fe65c4045ba74c6fac892945e",
+       "model_id": "3087512d9e6642cf8ed5e9b07405b9ef",
        "version_major": 2,
        "version_minor": 0
       },
@@ -277,7 +226,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "25615259dd364494bc5782b4e8231b05",
+       "model_id": "37a576b26aa04522b786a919b955fd88",
        "version_major": 2,
        "version_minor": 0
       },
@@ -291,7 +240,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "6867564094bf4c7d82d0046dccb173fe",
+       "model_id": "f62d110f392d495094520328b181a119",
        "version_major": 2,
        "version_minor": 0
       },
@@ -305,7 +254,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "2cb3be77451542868602317c4d7eff85",
+       "model_id": "b03cdae0d93f4eacbc32508d5320c583",
        "version_major": 2,
        "version_minor": 0
       },
@@ -319,7 +268,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "6dfc5dedce13459bbac6f2d695695ae0",
+       "model_id": "fceca071ff53478e98a6c6ceff6647df",
        "version_major": 2,
        "version_minor": 0
       },
@@ -333,7 +282,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "944cb945f9dd47178ab22d418aa2934b",
+       "model_id": "4647b098c1034018b007c86dd0e4ea53",
        "version_major": 2,
        "version_minor": 0
       },
@@ -369,7 +318,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 5,
    "id": "ab5a13b4-9bd4-4aa0-aef2-b3de9b762988",
    "metadata": {},
    "outputs": [
@@ -389,7 +338,7 @@
        " 'segment': Value(dtype='string', id=None)}"
       ]
      },
-     "execution_count": 4,
+     "execution_count": 5,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -415,7 +364,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 6,
    "id": "3ab6a724-3d1e-478b-a9e9-d2f85feb6c39",
    "metadata": {},
    "outputs": [],
@@ -435,7 +384,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 7,
    "id": "d041650e-1c48-4439-87b3-5b6f4a514107",
    "metadata": {},
    "outputs": [],
@@ -462,7 +411,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 8,
    "id": "c085911c-a10a-41ef-8874-306e0503e9bb",
    "metadata": {},
    "outputs": [],
@@ -498,7 +447,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 9,
    "id": "a37a7cdb-9013-427f-8de9-6a8d0e9dc684",
    "metadata": {},
    "outputs": [],
@@ -516,7 +465,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 10,
    "id": "1b145699-acfc-4b1d-93a2-a2ad3d62674c",
    "metadata": {},
    "outputs": [],
@@ -537,7 +486,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 11,
    "id": "01cb25ef-4bb0-4325-9461-f59198acadf6",
    "metadata": {},
    "outputs": [],
@@ -558,7 +507,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 12,
    "id": "333f7f6e-6053-4d3b-8924-c733c79b82ac",
    "metadata": {},
    "outputs": [],
@@ -628,7 +577,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 13,
    "id": "8326221e-ec13-4731-bb4e-51e5fc1486c5",
    "metadata": {},
    "outputs": [],
@@ -676,7 +625,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 13,
+   "execution_count": 14,
    "id": "fc834702-c0d3-4a96-b101-7b87be32bf42",
    "metadata": {},
    "outputs": [],
@@ -703,14 +652,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 14,
+   "execution_count": 15,
    "id": "b22b4011-f31f-4b57-b684-c52332f92890",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "bafc0b31fe9a4d239eedc348d5521dfc",
+       "model_id": "037e658a8e194212a068ba9eea85cf11",
        "version_major": 2,
        "version_minor": 0
       },
@@ -747,7 +696,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 15,
+   "execution_count": 16,
    "id": "a11d1bfc-9e28-460f-a287-72d8f7bc1acb",
    "metadata": {},
    "outputs": [],
@@ -780,10 +729,10 @@
   },
   {
    "cell_type": "markdown",
-   "id": "daf2a825-6d9f-4a23-b145-c37c0039075b",
+   "id": "e7ca118c",
    "metadata": {},
    "source": [
-    "### Load a Pre-Trained Checkpoint"
+    "### Load a Pre-Trained Checkpoint"
    ]
   },
   {
@@ -797,14 +746,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 16,
+   "execution_count": 17,
    "id": "5a10cc4b-07ec-4ebd-ac1d-7c601023594f",
    "metadata": {},
    "outputs": [
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "e1d5d79e596a416aa96bde21be6fb551",
+       "model_id": "d7c74c4e4fc3495393f4060c8d5e4bb2",
        "version_major": 2,
        "version_minor": 0
       },
@@ -818,7 +767,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "3d722a61d7a440479d0f5497a6200345",
+       "model_id": "77b037a9877a417d88d6f1bdd975e034",
        "version_major": 2,
        "version_minor": 0
       },
@@ -846,7 +795,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 17,
+   "execution_count": 18,
    "id": "62038ba3-88ed-4fce-84db-338f50dcd04f",
    "metadata": {},
    "outputs": [],
@@ -874,7 +823,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 18,
+   "execution_count": 19,
    "id": "0ae3e9af-97b7-4aa0-ae85-20b23b5bcb3a",
    "metadata": {},
    "outputs": [],
@@ -886,16 +835,16 @@
     "    per_device_train_batch_size=64,\n",
     "    gradient_accumulation_steps=1,  # increase by 2x for every 2x decrease in batch size\n",
     "    learning_rate=1e-5,\n",
-    "    warmup_steps=500,\n",
-    "    max_steps=5000,\n",
+    "    warmup_steps=200,\n",
+    "    max_steps=2000,\n",
     "    gradient_checkpointing=True,\n",
     "    fp16=True,\n",
     "    evaluation_strategy=\"steps\",\n",
     "    per_device_eval_batch_size=8,\n",
     "    predict_with_generate=True,\n",
     "    generation_max_length=225,\n",
-    "    save_steps=1000,\n",
-    "    eval_steps=1000,\n",
+    "    save_steps=400,\n",
+    "    eval_steps=200,\n",
     "    logging_steps=25,\n",
     "    report_to=[\"tensorboard\"],\n",
     "    load_best_model_at_end=True,\n",
@@ -924,7 +873,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 19,
+   "execution_count": 20,
    "id": "3ac16b62-b3c0-4c68-8f3d-9ecf471534b2",
    "metadata": {},
    "outputs": [],
@@ -953,7 +902,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": 21,
    "id": "d546d7fe-0543-479a-b708-2ebabec19493",
    "metadata": {},
    "outputs": [
@@ -992,7 +941,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": 22,
    "id": "a1ccb9ed-cbc8-4419-91c0-651e9424b672",
    "metadata": {},
    "outputs": [
@@ -1040,7 +989,9 @@
    "cell_type": "code",
    "execution_count": null,
    "id": "ee8b7b8e-1c9a-4d77-9137-1778a629e6de",
-   "metadata": {},
+   "metadata": {
+    "scrolled": false
+   },
    "outputs": [
     {
      "name": "stderr",
@@ -1049,14 +1000,14 @@
       "/home/ubuntu/.venv/lib/python3.8/site-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
       "  warnings.warn(\n",
       "***** Running training *****\n",
-      "  Num examples = 320000\n",
+      "  Num examples = 128000\n",
       "  Num Epochs = 9223372036854775807\n",
       "  Instantaneous batch size per device = 64\n",
       "  Total train batch size (w. parallel, distributed & accumulation) = 64\n",
       "  Gradient Accumulation steps = 1\n",
-      "  Total optimization steps = 5000\n",
+      "  Total optimization steps = 2000\n",
       "  Number of trainable parameters = 241734912\n",
-      "Reading metadata...: 6568it [00:00, 41540.60it/s]\n",
+      "Reading metadata...: 29056it [00:00, 64790.22it/s]\n",
       "The following columns in the training set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: input_length. If input_length are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.\n"
      ]
     },
@@ -1066,8 +1017,8 @@
        "\n",
        "    <div>\n",
        "      \n",
-       "      <progress value='35' max='5000' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
-       "      [  35/5000 03:29 < 8:46:02, 0.16 it/s, Epoch 0.01/9223372036854775807]\n",
+       "      <progress value='201' max='2000' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
+       "      [ 201/2000 22:53 < 3:26:59, 0.14 it/s, Epoch 0.10/9223372036854775807]\n",
        "    </div>\n",
        "    <table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
@@ -1087,6 +1038,17 @@
      },
      "metadata": {},
      "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "***** Running Evaluation *****\n",
+      "  Num examples: Unknown\n",
+      "  Batch size = 8\n",
+      "Reading metadata...: 10581it [00:00, 28979.41it/s]\n",
+      "The following columns in the evaluation set don't have a corresponding argument in `WhisperForConditionalGeneration.forward` and have been ignored: input_length. If input_length are not expected by `WhisperForConditionalGeneration.forward`,  you can safely ignore this message.\n"
+     ]
     }
    ],
    "source": [
@@ -1181,9 +1143,9 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": "wspsr",
    "language": "python",
-   "name": "python3"
+   "name": "wspsr"
   },
   "language_info": {
    "codemirror_mode": {

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9f194b437e01232adf390094500baf37653fed4bfba50c6514a2eb0cdcb74f5e
+oid sha256:706aa3238840d1df7b20ac7afc35ab74373d09f0af44c2bc541e341cc9b51771
 size 967102601

runs/Dec12_00-48-47_129-213-26-143/1670806154.5337327/events.out.tfevents.1670806154.129-213-26-143.128161.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:643669bcfb9cfe899fda5786ba8948b49d4519704908d55deef9c50c4a18078b
+size 5864

runs/Dec12_00-48-47_129-213-26-143/events.out.tfevents.1670806154.129-213-26-143.128161.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f260375523d8a81a246006bb8a79ba6533543f960b74a6267c68bd3525d8b722
+size 5526

runs/Dec12_01-54-52_129-213-26-143/1670810104.398764/events.out.tfevents.1670810104.129-213-26-143.742932.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d56cd00dda0c930668340dd283d136919ef21f16a0778d50c99995b33ea74128
+size 5864

runs/Dec12_01-54-52_129-213-26-143/events.out.tfevents.1670810104.129-213-26-143.742932.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8008ff5de93c11cc5c83ead75026474e3dab32f10c8815e6b71fb9ddd18dfda4
+size 4285

runs/Dec12_02-00-57_129-213-26-143/1670810506.3012445/events.out.tfevents.1670810506.129-213-26-143.742932.3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cec5b063f93ebdbdb62ebff706f75eff9fc3ddb6b3ff8614e8a1ed748daee8a7
+size 5864

runs/Dec12_02-00-57_129-213-26-143/events.out.tfevents.1670810506.129-213-26-143.742932.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:75a75228382f5ea305d02e4c38f7f7105c0c081b96f141bc517ad9f89f2eb234
+size 5844

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dea523f572893e3c4ca713d3731c8194b372c3af7337897cc7ca69fa8dc28498
+oid sha256:9d97d92b4d57394382fb181e666c91764f7c97306ef64244dbb9caab670c3ac3
 size 3579