working version, needs better readm

.ipynb_checkpoints/transcribe-checkpoint.ipynb

@@ -0,0 +1,913 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "speechbrain.utils.distributed - distributed_launch flag is disabled, this experiment will be executed without DDP.\n",
+      "speechbrain.lobes.models.huggingface_wav2vec - speechbrain.lobes.models.huggingface_wav2vec - wav2vec 2.0 feature extractor is frozen.\n",
+      "speechbrain.core - Beginning experiment!\n",
+      "speechbrain.core - Experiment folder: TunisianASR/semi_wavlm_large_tunisian_ctc/1234\n",
+      "speechbrain.pretrained.fetching - Fetch hyperparams.yaml: Using existing file/symlink in pretrained_models/asr-wav2vec2-commonvoice-fr/hyperparams.yaml.\n",
+      "speechbrain.pretrained.fetching - Fetch custom.py: Linking to local file in /home/salah/Code_Switched_Tunisian_Speech_Recognition/asr-wav2vec2-commonvoice-fr/custom.py.\n",
+      "speechbrain.lobes.models.huggingface_wav2vec - speechbrain.lobes.models.huggingface_wav2vec - wav2vec 2.0 is frozen.\n",
+      "speechbrain.pretrained.fetching - Fetch wav2vec2.ckpt: Using existing file/symlink in pretrained_models/asr-wav2vec2-commonvoice-fr/wav2vec2.ckpt.\n",
+      "speechbrain.pretrained.fetching - Fetch asr.ckpt: Using existing file/symlink in pretrained_models/asr-wav2vec2-commonvoice-fr/asr.ckpt.\n",
+      "speechbrain.pretrained.fetching - Fetch tokenizer.ckpt: Using existing file/symlink in pretrained_models/asr-wav2vec2-commonvoice-fr/tokenizer.ckpt.\n",
+      "speechbrain.utils.parameter_transfer - Loading pretrained files for: wav2vec2, asr, tokenizer\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Some weights of the model checkpoint at wav2vec2-large-lv60/ were not used when initializing Wav2Vec2Model: ['project_hid.bias', 'project_q.bias', 'project_hid.weight', 'quantizer.codevectors', 'quantizer.weight_proj.weight', 'quantizer.weight_proj.bias', 'project_q.weight']\n",
+      "- This IS expected if you are initializing Wav2Vec2Model from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
+      "- This IS NOT expected if you are initializing Wav2Vec2Model from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "speechbrain.lobes.models.huggingface_wav2vec - speechbrain.lobes.models.huggingface_wav2vec - wav2vec 2.0 feature extractor is frozen.\n",
+      "speechbrain.core - Info: auto_mix_prec arg from hparam file is used\n",
+      "speechbrain.core - Info: ckpt_interval_minutes arg from hparam file is used\n",
+      "speechbrain.core - 314.4M trainable parameters in ASRCV\n",
+      "speechbrain.utils.checkpoints - Loading a checkpoint from EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00\n",
+      "speechbrain.core - Info: auto_mix_prec arg from hparam file is used\n",
+      "speechbrain.core - Info: ckpt_interval_minutes arg from hparam file is used\n",
+      "speechbrain.core - 314.4M trainable parameters in ASR\n",
+      "speechbrain.utils.checkpoints - Loading a checkpoint from TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import sys\n",
+    "import torch\n",
+    "import logging\n",
+    "import speechbrain as sb\n",
+    "from speechbrain.utils.distributed import run_on_main\n",
+    "from hyperpyyaml import load_hyperpyyaml\n",
+    "from pathlib import Path\n",
+    "import torchaudio.transforms as T\n",
+    "from cv_train import ASRCV\n",
+    "import torchaudio\n",
+    "import numpy as np\n",
+    "import kenlm\n",
+    "from pyctcdecode import build_ctcdecoder\n",
+    "import re\n",
+    "from torch.nn.utils.rnn import pad_sequence\n",
+    "import torch.optim as optim\n",
+    "import torch.nn as nn\n",
+    "\n",
+    "\n",
+    "# Commented out IPython magic to ensure Python compatibility.\n",
+    "hparams_file, run_opts, overrides = sb.parse_arguments([\"TunisianASR/train_semi.yaml\"])\n",
+    "\n",
+    "# If distributed_launch=True then\n",
+    "# create ddp_group with the right communication protocol\n",
+    "sb.utils.distributed.ddp_init_group(run_opts)\n",
+    "\n",
+    "with open(hparams_file) as fin:\n",
+    "    hparams = load_hyperpyyaml(fin, overrides)\n",
+    "\n",
+    "# Create experiment directory\n",
+    "sb.create_experiment_directory(\n",
+    "    experiment_directory=hparams[\"output_folder\"],\n",
+    "    hyperparams_to_save=hparams_file,\n",
+    "    overrides=overrides,\n",
+    ")\n",
+    "# Dataset prep (parsing Librispeech)\n",
+    "\n",
+    "def dataio_prepare(hparams):\n",
+    "    \"\"\"This function prepares the datasets to be used in the brain class.\n",
+    "    It also defines the data processing pipeline through user-defined functions.\"\"\"\n",
+    "\n",
+    "    # 1. Define datasets\n",
+    "    data_folder = hparams[\"data_folder\"]\n",
+    "\n",
+    "    train_data = sb.dataio.dataset.DynamicItemDataset.from_csv(\n",
+    "        csv_path=hparams[\"train_csv\"], replacements={\"data_root\": data_folder},\n",
+    "    )\n",
+    "\n",
+    "    if hparams[\"sorting\"] == \"ascending\":\n",
+    "        # we sort training data to speed up training and get better results.\n",
+    "        train_data = train_data.filtered_sorted(\n",
+    "            sort_key=\"duration\",\n",
+    "            key_max_value={\"duration\": hparams[\"avoid_if_longer_than\"]},\n",
+    "        )\n",
+    "        # when sorting do not shuffle in dataloader ! otherwise is pointless\n",
+    "        hparams[\"dataloader_options\"][\"shuffle\"] = False\n",
+    "\n",
+    "    elif hparams[\"sorting\"] == \"descending\":\n",
+    "        train_data = train_data.filtered_sorted(\n",
+    "            sort_key=\"duration\",\n",
+    "            reverse=True,\n",
+    "            key_max_value={\"duration\": hparams[\"avoid_if_longer_than\"]},\n",
+    "        )\n",
+    "        # when sorting do not shuffle in dataloader ! otherwise is pointless\n",
+    "        hparams[\"dataloader_options\"][\"shuffle\"] = False\n",
+    "\n",
+    "    elif hparams[\"sorting\"] == \"random\":\n",
+    "        pass\n",
+    "\n",
+    "    else:\n",
+    "        raise NotImplementedError(\n",
+    "            \"sorting must be random, ascending or descending\"\n",
+    "        )\n",
+    "\n",
+    "    valid_data = sb.dataio.dataset.DynamicItemDataset.from_csv(\n",
+    "        csv_path=hparams[\"valid_csv\"], replacements={\"data_root\": data_folder},\n",
+    "    )\n",
+    "    # We also sort the validation data so it is faster to validate\n",
+    "    valid_data = valid_data.filtered_sorted(sort_key=\"duration\")\n",
+    "    test_datasets = {}\n",
+    "    for csv_file in hparams[\"test_csv\"]:\n",
+    "        name = Path(csv_file).stem\n",
+    "        test_datasets[name] = sb.dataio.dataset.DynamicItemDataset.from_csv(\n",
+    "            csv_path=csv_file, replacements={\"data_root\": data_folder}\n",
+    "        )\n",
+    "        test_datasets[name] = test_datasets[name].filtered_sorted(\n",
+    "            sort_key=\"duration\"\n",
+    "        )\n",
+    "\n",
+    "    datasets = [train_data, valid_data] + [i for k, i in test_datasets.items()]\n",
+    "\n",
+    "\n",
+    "    # 2. Define audio pipeline:\n",
+    "    @sb.utils.data_pipeline.takes(\"wav\")\n",
+    "    @sb.utils.data_pipeline.provides(\"sig\")\n",
+    "    def audio_pipeline(wav):\n",
+    "        info = torchaudio.info(wav)\n",
+    "        sig = sb.dataio.dataio.read_audio(wav)\n",
+    "        if len(sig.shape)>1 :\n",
+    "            sig = torch.mean(sig, dim=1)\n",
+    "        resampled = torchaudio.transforms.Resample(\n",
+    "            info.sample_rate, hparams[\"sample_rate\"],\n",
+    "        )(sig)\n",
+    "        return resampled\n",
+    "\n",
+    "    sb.dataio.dataset.add_dynamic_item(datasets, audio_pipeline)\n",
+    "    label_encoder = sb.dataio.encoder.CTCTextEncoder()\n",
+    "\n",
+    "    # 3. Define text pipeline:\n",
+    "    @sb.utils.data_pipeline.takes(\"wrd\")\n",
+    "    @sb.utils.data_pipeline.provides(\n",
+    "        \"wrd\", \"char_list\", \"tokens_list\", \"tokens\"\n",
+    "    )\n",
+    "    def text_pipeline(wrd):\n",
+    "        yield wrd\n",
+    "        char_list = list(wrd)\n",
+    "        yield char_list\n",
+    "        tokens_list = label_encoder.encode_sequence(char_list)\n",
+    "        yield tokens_list\n",
+    "        tokens = torch.LongTensor(tokens_list)\n",
+    "        yield tokens\n",
+    "\n",
+    "    sb.dataio.dataset.add_dynamic_item(datasets, text_pipeline)\n",
+    "    lab_enc_file = os.path.join(hparams[\"save_folder\"], \"label_encoder.txt\")\n",
+    "    special_labels = {\n",
+    "        \"blank_label\": hparams[\"blank_index\"],\n",
+    "        \"unk_label\": hparams[\"unk_index\"]\n",
+    "    }\n",
+    "    label_encoder.load_or_create(\n",
+    "        path=lab_enc_file,\n",
+    "        from_didatasets=[train_data],\n",
+    "        output_key=\"char_list\",\n",
+    "        special_labels=special_labels,\n",
+    "        sequence_input=True,\n",
+    "    )\n",
+    "\n",
+    "    # 4. Set output:\n",
+    "    sb.dataio.dataset.set_output_keys(\n",
+    "        datasets, [\"id\", \"sig\", \"wrd\", \"char_list\", \"tokens\"],\n",
+    "    )\n",
+    "    return train_data, valid_data,test_datasets, label_encoder\n",
+    "\n",
+    "class ASR(sb.core.Brain):\n",
+    "    def compute_forward(self, batch, stage):\n",
+    "        \"\"\"Forward computations from the waveform batches to the output probabilities.\"\"\"\n",
+    "\n",
+    "        batch = batch.to(self.device)\n",
+    "        wavs, wav_lens = batch.sig\n",
+    "        wavs, wav_lens = wavs.to(self.device), wav_lens.to(self.device)\n",
+    "\n",
+    "        if stage == sb.Stage.TRAIN:\n",
+    "            if hasattr(self.hparams, \"augmentation\"):\n",
+    "                wavs = self.hparams.augmentation(wavs, wav_lens)\n",
+    "\n",
+    "        # Forward pass\n",
+    "        feats = self.modules.wav2vec2(wavs, wav_lens)\n",
+    "        x = self.modules.enc(feats)\n",
+    "        logits = self.modules.ctc_lin(x)\n",
+    "        p_ctc = self.hparams.log_softmax(logits)\n",
+    "\n",
+    "        return p_ctc, wav_lens\n",
+    "\n",
+    "    def custom_encode(self,wavs,wav_lens) :\n",
+    "        wavs = wavs.to(\"cpu\")\n",
+    "        if(wav_lens is not None): wav_lens.to(self.device)\n",
+    "\n",
+    "        feats = self.modules.wav2vec2(wavs, wav_lens)\n",
+    "        x = self.modules.enc(feats)\n",
+    "        logits = self.modules.ctc_lin(x)\n",
+    "        p_ctc = self.hparams.log_softmax(logits)\n",
+    "\n",
+    "        return feats,p_ctc\n",
+    "\n",
+    "\n",
+    "\n",
+    "    def compute_objectives(self, predictions, batch, stage):\n",
+    "        \"\"\"Computes the loss (CTC) given predictions and targets.\"\"\"\n",
+    "\n",
+    "        p_ctc, wav_lens = predictions\n",
+    "\n",
+    "        ids = batch.id\n",
+    "        tokens, tokens_lens = batch.tokens\n",
+    "\n",
+    "        loss = self.hparams.ctc_cost(p_ctc, tokens, wav_lens, tokens_lens)\n",
+    "\n",
+    "        if stage != sb.Stage.TRAIN:\n",
+    "            predicted_tokens = sb.decoders.ctc_greedy_decode(\n",
+    "                p_ctc, wav_lens, blank_id=self.hparams.blank_index\n",
+    "            )\n",
+    "            # Decode token terms to words\n",
+    "            if self.hparams.use_language_modelling:\n",
+    "                predicted_words = []\n",
+    "                for logs in p_ctc:\n",
+    "                    text = decoder.decode(logs.detach().cpu().numpy())\n",
+    "                    predicted_words.append(text.split(\" \"))\n",
+    "            else:\n",
+    "                predicted_words = [\n",
+    "                    \"\".join(self.tokenizer.decode_ndim(utt_seq)).split(\" \")\n",
+    "                    for utt_seq in predicted_tokens\n",
+    "                ]\n",
+    "            # Convert indices to words\n",
+    "            target_words = [wrd.split(\" \") for wrd in batch.wrd]\n",
+    "\n",
+    "            self.wer_metric.append(ids, predicted_words, target_words)\n",
+    "            self.cer_metric.append(ids, predicted_words, target_words)\n",
+    "\n",
+    "        return loss\n",
+    "\n",
+    "    def fit_batch(self, batch):\n",
+    "        \"\"\"Train the parameters given a single batch in input\"\"\"\n",
+    "        should_step = self.step % self.grad_accumulation_factor == 0\n",
+    "        # Managing automatic mixed precision\n",
+    "        # TOFIX: CTC fine-tuning currently is unstable\n",
+    "        # This is certainly due to CTC being done in fp16 instead of fp32\n",
+    "        if self.auto_mix_prec:\n",
+    "            with torch.cuda.amp.autocast():\n",
+    "                with self.no_sync():\n",
+    "                    outputs = self.compute_forward(batch, sb.Stage.TRAIN)\n",
+    "                loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)\n",
+    "            with self.no_sync(not should_step):\n",
+    "                self.scaler.scale(\n",
+    "                    loss / self.grad_accumulation_factor\n",
+    "                ).backward()\n",
+    "            if should_step:\n",
+    "\n",
+    "                if not self.hparams.wav2vec2.freeze:\n",
+    "                    self.scaler.unscale_(self.wav2vec_optimizer)\n",
+    "                self.scaler.unscale_(self.model_optimizer)\n",
+    "                if self.check_gradients(loss):\n",
+    "                    if not self.hparams.wav2vec2.freeze:\n",
+    "                        if self.optimizer_step >= self.hparams.warmup_steps:\n",
+    "                            self.scaler.step(self.wav2vec_optimizer)\n",
+    "                    self.scaler.step(self.model_optimizer)\n",
+    "                self.scaler.update()\n",
+    "                self.zero_grad()\n",
+    "                self.optimizer_step += 1\n",
+    "        else:\n",
+    "            # This is mandatory because HF models have a weird behavior with DDP\n",
+    "            # on the forward pass\n",
+    "            with self.no_sync():\n",
+    "                outputs = self.compute_forward(batch, sb.Stage.TRAIN)\n",
+    "\n",
+    "            loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)\n",
+    "\n",
+    "            with self.no_sync(not should_step):\n",
+    "                (loss / self.grad_accumulation_factor).backward()\n",
+    "            if should_step:\n",
+    "                if self.check_gradients(loss):\n",
+    "                    if not self.hparams.wav2vec2.freeze:\n",
+    "                        if self.optimizer_step >= self.hparams.warmup_steps:\n",
+    "                            self.wav2vec_optimizer.step()\n",
+    "                    self.model_optimizer.step()\n",
+    "                self.zero_grad()\n",
+    "                self.optimizer_step += 1\n",
+    "\n",
+    "        self.on_fit_batch_end(batch, outputs, loss, should_step)\n",
+    "        return loss.detach().cpu()\n",
+    "\n",
+    "    def evaluate_batch(self, batch, stage):\n",
+    "        \"\"\"Computations needed for validation/test batches\"\"\"\n",
+    "        predictions = self.compute_forward(batch, stage=stage)\n",
+    "        with torch.no_grad():\n",
+    "            loss = self.compute_objectives(predictions, batch, stage=stage)\n",
+    "        return loss.detach()\n",
+    "\n",
+    "    def on_stage_start(self, stage, epoch):\n",
+    "        \"\"\"Gets called at the beginning of each epoch\"\"\"\n",
+    "        if stage != sb.Stage.TRAIN:\n",
+    "            self.cer_metric = self.hparams.cer_computer()\n",
+    "            self.wer_metric = self.hparams.error_rate_computer()\n",
+    "\n",
+    "    def on_stage_end(self, stage, stage_loss, epoch):\n",
+    "        \"\"\"Gets called at the end of an epoch.\"\"\"\n",
+    "        # Compute/store important stats\n",
+    "        stage_stats = {\"loss\": stage_loss}\n",
+    "        if stage == sb.Stage.TRAIN:\n",
+    "            self.train_stats = stage_stats\n",
+    "        else:\n",
+    "            stage_stats[\"CER\"] = self.cer_metric.summarize(\"error_rate\")\n",
+    "            stage_stats[\"WER\"] = self.wer_metric.summarize(\"error_rate\")\n",
+    "\n",
+    "        # Perform end-of-iteration things, like annealing, logging, etc.\n",
+    "        if stage == sb.Stage.VALID:\n",
+    "            old_lr_model, new_lr_model = self.hparams.lr_annealing_model(\n",
+    "                stage_stats[\"loss\"]\n",
+    "            )\n",
+    "            old_lr_wav2vec, new_lr_wav2vec = self.hparams.lr_annealing_wav2vec(\n",
+    "                stage_stats[\"loss\"]\n",
+    "            )\n",
+    "            sb.nnet.schedulers.update_learning_rate(\n",
+    "                self.model_optimizer, new_lr_model\n",
+    "            )\n",
+    "            if not self.hparams.wav2vec2.freeze:\n",
+    "                sb.nnet.schedulers.update_learning_rate(\n",
+    "                    self.wav2vec_optimizer, new_lr_wav2vec\n",
+    "                )\n",
+    "            self.hparams.train_logger.log_stats(\n",
+    "                stats_meta={\n",
+    "                    \"epoch\": epoch,\n",
+    "                    \"lr_model\": old_lr_model,\n",
+    "                    \"lr_wav2vec\": old_lr_wav2vec,\n",
+    "                },\n",
+    "                train_stats=self.train_stats,\n",
+    "                valid_stats=stage_stats,\n",
+    "            )\n",
+    "            self.checkpointer.save_and_keep_only(\n",
+    "                meta={\"WER\": stage_stats[\"WER\"]}, min_keys=[\"WER\"],\n",
+    "            )\n",
+    "        elif stage == sb.Stage.TEST:\n",
+    "            self.hparams.train_logger.log_stats(\n",
+    "                stats_meta={\"Epoch loaded\": self.hparams.epoch_counter.current},\n",
+    "                test_stats=stage_stats,\n",
+    "            )\n",
+    "            with open(self.hparams.wer_file, \"w\") as w:\n",
+    "                self.wer_metric.write_stats(w)\n",
+    "\n",
+    "    def init_optimizers(self):\n",
+    "        \"Initializes the wav2vec2 optimizer and model optimizer\"\n",
+    "\n",
+    "        # If the wav2vec encoder is unfrozen, we create the optimizer\n",
+    "        if not self.hparams.wav2vec2.freeze:\n",
+    "            self.wav2vec_optimizer = self.hparams.wav2vec_opt_class(\n",
+    "                self.modules.wav2vec2.parameters()\n",
+    "            )\n",
+    "            if self.checkpointer is not None:\n",
+    "                self.checkpointer.add_recoverable(\n",
+    "                    \"wav2vec_opt\", self.wav2vec_optimizer\n",
+    "                )\n",
+    "\n",
+    "        self.model_optimizer = self.hparams.model_opt_class(\n",
+    "            self.hparams.model.parameters()\n",
+    "        )\n",
+    "\n",
+    "        if self.checkpointer is not None:\n",
+    "            self.checkpointer.add_recoverable(\"modelopt\", self.model_optimizer)\n",
+    "\n",
+    "    def zero_grad(self, set_to_none=False):\n",
+    "        if not self.hparams.wav2vec2.freeze:\n",
+    "            self.wav2vec_optimizer.zero_grad(set_to_none)\n",
+    "        self.model_optimizer.zero_grad(set_to_none)\n",
+    "\n",
+    "\n",
+    "from speechbrain.pretrained import EncoderASR,EncoderDecoderASR\n",
+    "french_asr_model = EncoderASR.from_hparams(source=\"asr-wav2vec2-commonvoice-fr\", savedir=\"pretrained_models/asr-wav2vec2-commonvoice-fr\").cuda()\n",
+    "french_asr_model.to(\"cpu\")\n",
+    "cvhparams_file, cvrun_opts, cvoverrides = sb.parse_arguments([\"EnglishCV/train_en_with_wav2vec.yaml\"])\n",
+    "with open(cvhparams_file) as cvfin:\n",
+    "    cvhparams = load_hyperpyyaml(cvfin, cvoverrides)\n",
+    "english_asr_model = ASRCV(\n",
+    "        modules=cvhparams[\"modules\"],\n",
+    "        hparams=cvhparams,\n",
+    "        run_opts=cvrun_opts,\n",
+    "        checkpointer=cvhparams[\"checkpointer\"],\n",
+    "    )\n",
+    "english_asr_model.modules.to(\"cpu\")\n",
+    "english_asr_model.checkpointer.recover_if_possible()\n",
+    "asr_brain = ASR(\n",
+    "    modules=hparams[\"modules\"],\n",
+    "    hparams=hparams,\n",
+    "    run_opts=run_opts,\n",
+    "    checkpointer=hparams[\"checkpointer\"],\n",
+    ")\n",
+    "asr_brain.modules.to(\"cpu\")\n",
+    "asr_brain.checkpointer.recover_if_possible()\n",
+    "asr_brain.modules.eval()\n",
+    "english_asr_model.modules.eval()\n",
+    "french_asr_model.mods.eval()\n",
+    "asr_brain.modules.to(\"cpu\")\n",
+    "\n",
+    "# Commented out IPython magic to ensure Python compatibility.\n",
+    "# %ls\n",
+    "\n",
+    "#UTILS FUNCTIOJNS\n",
+    "def get_size_dimensions(arr):\n",
+    "    size_dimensions = []\n",
+    "    while isinstance(arr, list):\n",
+    "        size_dimensions.append(len(arr))\n",
+    "        arr = arr[0]\n",
+    "    return size_dimensions\n",
+    "\n",
+    "def scale_array(batch,n):\n",
+    "    scaled_batch = []\n",
+    "\n",
+    "    for array in batch:\n",
+    "        if(n < len(array)): raise ValueError(\"Cannot scale Array down\")\n",
+    "\n",
+    "        repeat = round(n/len(array))+1\n",
+    "        scaled_length_array= []\n",
+    "\n",
+    "        for i in array:\n",
+    "            for j in range(repeat) :\n",
+    "                if(len(scaled_length_array) == n): break\n",
+    "                scaled_length_array.append(i)\n",
+    "\n",
+    "        scaled_batch.append(scaled_length_array)\n",
+    "\n",
+    "    return torch.tensor(scaled_batch)\n",
+    "\n",
+    "\n",
+    "def load_paths(wavs_path):\n",
+    "    waveforms = []\n",
+    "    for path in wavs_path :\n",
+    "        waveform, _ = torchaudio.load(path)\n",
+    "        waveforms.append(waveform.squeeze(0))\n",
+    "    # normalize array length to the bigger arrays by pading with 0's\n",
+    "    padded_arrays = pad_sequence(waveforms, batch_first=True)\n",
+    "    return torch.tensor(padded_arrays)\n",
+    "\n",
+    "\n",
+    "\n",
+    "device = 'cuda'\n",
+    "verbose = 0\n",
+    "#FLOW LEVEL FUNCTIONS\n",
+    "def merge_strategy(embeddings1, embeddings2, embeddings3,post1, post2,post3):\n",
+    "\n",
+    "\n",
+    "    post1 = post1.to(device)\n",
+    "    post2 = post2.to(device)\n",
+    "    post3 = post3.to(device)\n",
+    "    embeddings1 = embeddings1.to(device)\n",
+    "    embeddings2 = embeddings2.to(device)\n",
+    "    embeddings3 = embeddings3.to(device)\n",
+    "\n",
+    "    posteriograms_merged = torch.cat((post1,post2,post3),dim=2)\n",
+    "    embeddings_merged = torch.cat((embeddings1,embeddings2,embeddings3),dim=2)\n",
+    "\n",
+    "    if(verbose !=0):\n",
+    "      print('MERGED POST ',posteriograms_merged.shape)\n",
+    "      print('MERGED emb ',embeddings_merged.shape)\n",
+    "\n",
+    "    return torch.cat((posteriograms_merged,embeddings_merged),dim=2).to(device)\n",
+    "\n",
+    "def decode(model,wavs,wav_lens):\n",
+    "\n",
+    "    with torch.no_grad():\n",
+    "        wav_lens = wav_lens.to(model.device)\n",
+    "        encoder_out = model.encode_batch(wavs, wav_lens)\n",
+    "        predictions = model.decoding_function(encoder_out, wav_lens)\n",
+    "        return predictions\n",
+    "\n",
+    "def middle_layer(batch, lens):\n",
+    "\n",
+    "    tn_embeddings, tn_posteriogram = asr_brain.custom_encode(batch,None)\n",
+    "\n",
+    "    fr_embeddings = french_asr_model.mods.encoder.wav2vec2(batch)\n",
+    "    fr_posteriogram =french_asr_model.encode_batch(batch,lens)\n",
+    "    en_embeddings = english_asr_model.modules.wav2vec2(batch, lens)\n",
+    "    x = english_asr_model.modules.enc(en_embeddings)\n",
+    "    en_posteriogram = english_asr_model.modules.ctc_lin(x)\n",
+    "    #scores, en_posteriogram = english_asr_model.mods.decoder(en_embeddings ,lens)\n",
+    "    if(verbose !=0):\n",
+    "      print('[EMBEDDINGS] FR:',fr_embeddings.shape, \"EN:\",en_embeddings.shape, \"TN:\", tn_embeddings.shape)\n",
+    "      print('[POSTERIOGRAM] FR:',fr_posteriogram.shape, \"EN:\",en_posteriogram.shape,\"TN:\",tn_posteriogram.shape)\n",
+    "\n",
+    "\n",
+    "    bilangual_sample = merge_strategy(fr_embeddings,en_embeddings,tn_embeddings,fr_posteriogram,en_posteriogram,tn_posteriogram)\n",
+    "    return bilangual_sample\n",
+    "\n",
+    "class Mixer(sb.core.Brain):\n",
+    "\n",
+    "    def compute_forward(self, batch, stage):\n",
+    "        \"\"\"Forward computations from the waveform batches to the output probabilities.\"\"\"\n",
+    "        wavs, wav_lens = batch.sig\n",
+    "        wavs, wav_lens = wavs.to(self.device), wav_lens.to(self.device)\n",
+    "\n",
+    "        if stage == sb.Stage.TRAIN:\n",
+    "            if hasattr(self.hparams, \"augmentation\"):\n",
+    "                wavs = self.hparams.augmentation(wavs, wav_lens)\n",
+    "\n",
+    "        multi_langual_feats = middle_layer(wavs, wav_lens)\n",
+    "        multi_langual_feats= multi_langual_feats.to(device)\n",
+    "        feats, _ = self.modules.enc(multi_langual_feats)\n",
+    "        logits = self.modules.ctc_lin(feats)\n",
+    "        p_ctc = self.hparams.log_softmax(logits)\n",
+    "        \n",
+    "        if stage!= sb.Stage.TRAIN:\n",
+    "            p_tokens = sb.decoders.ctc_greedy_decode(\n",
+    "                p_ctc, wav_lens, blank_id=self.hparams.blank_index\n",
+    "            )\n",
+    "        else : \n",
+    "            p_tokens = None\n",
+    "        return p_ctc, wav_lens, p_tokens\n",
+    "    \n",
+    "    \n",
+    "    def treat_wav(self,sig):\n",
+    "        multi_langual_feats = middle_layer(sig.to(\"cpu\"), torch.tensor([1]).to(\"cpu\"))\n",
+    "        multi_langual_feats= multi_langual_feats.to(device)\n",
+    "        feats, _ = self.modules.enc(multi_langual_feats)\n",
+    "        logits = self.modules.ctc_lin(feats)\n",
+    "        p_ctc = self.hparams.log_softmax(logits)\n",
+    "        predicted_words =[]\n",
+    "        for logs in p_ctc:\n",
+    "            text = decoder.decode(logs.detach().cpu().numpy())\n",
+    "            predicted_words.append(text.split(\" \"))\n",
+    "        return \" \".join(predicted_words[0])\n",
+    "        \n",
+    "\n",
+    "    def compute_objectives(self, predictions, batch, stage):\n",
+    "        \"\"\"Computes the loss (CTC) given predictions and targets.\"\"\"\n",
+    "\n",
+    "        p_ctc, wav_lens , predicted_tokens= predictions\n",
+    "\n",
+    "        ids = batch.id\n",
+    "        tokens, tokens_lens = batch.tokens\n",
+    "\n",
+    "        loss = self.hparams.ctc_cost(p_ctc, tokens, wav_lens, tokens_lens)\n",
+    "\n",
+    "\n",
+    "        if stage == sb.Stage.VALID:\n",
+    "            predicted_words = [\n",
+    "                \"\".join(self.tokenizer.decode_ndim(utt_seq)).split(\" \")\n",
+    "                for utt_seq in predicted_tokens\n",
+    "            ]\n",
+    "            target_words = [wrd.split(\" \") for wrd in batch.wrd]\n",
+    "            self.wer_metric.append(ids, predicted_words, target_words)\n",
+    "            self.cer_metric.append(ids, predicted_words, target_words)\n",
+    "        if stage ==sb.Stage.TEST : \n",
+    "            if self.hparams.language_modelling:\n",
+    "                predicted_words = []\n",
+    "                for logs in p_ctc:\n",
+    "                    text = decoder.decode(logs.detach().cpu().numpy())\n",
+    "                    predicted_words.append(text.split(\" \"))\n",
+    "            else : \n",
+    "                predicted_words = [\n",
+    "                    \"\".join(self.tokenizer.decode_ndim(utt_seq)).split(\" \")\n",
+    "                    for utt_seq in predicted_tokens\n",
+    "                ]\n",
+    "\n",
+    "            target_words = [wrd.split(\" \") for wrd in batch.wrd]\n",
+    "            self.wer_metric.append(ids, predicted_words, target_words)\n",
+    "            self.cer_metric.append(ids, predicted_words, target_words)\n",
+    "\n",
+    "        return loss\n",
+    "\n",
+    "    def fit_batch(self, batch):\n",
+    "        \"\"\"Train the parameters given a single batch in input\"\"\"\n",
+    "        should_step = self.step % self.grad_accumulation_factor == 0\n",
+    "        # Managing automatic mixed precision\n",
+    "        # TOFIX: CTC fine-tuning currently is unstable\n",
+    "        # This is certainly due to CTC being done in fp16 instead of fp32\n",
+    "        if self.auto_mix_prec:\n",
+    "            with torch.cuda.amp.autocast():\n",
+    "                with self.no_sync():\n",
+    "                    outputs = self.compute_forward(batch, sb.Stage.TRAIN)\n",
+    "                loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)\n",
+    "            with self.no_sync(not should_step):\n",
+    "                self.scaler.scale(\n",
+    "                    loss / self.grad_accumulation_factor\n",
+    "                ).backward()\n",
+    "            if should_step:\n",
+    "\n",
+    "\n",
+    "                self.scaler.unscale_(self.model_optimizer)\n",
+    "                if self.check_gradients(loss):\n",
+    "                    self.scaler.step(self.model_optimizer)\n",
+    "                self.scaler.update()\n",
+    "                self.zero_grad()\n",
+    "                self.optimizer_step += 1\n",
+    "        else:\n",
+    "            # This is mandatory because HF models have a weird behavior with DDP\n",
+    "            # on the forward pass\n",
+    "            with self.no_sync():\n",
+    "                outputs = self.compute_forward(batch, sb.Stage.TRAIN)\n",
+    "\n",
+    "            loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)\n",
+    "\n",
+    "            with self.no_sync(not should_step):\n",
+    "                (loss / self.grad_accumulation_factor).backward()\n",
+    "            if should_step:\n",
+    "                if self.check_gradients(loss):\n",
+    "                    self.model_optimizer.step()\n",
+    "                self.zero_grad()\n",
+    "                self.optimizer_step += 1\n",
+    "\n",
+    "        self.on_fit_batch_end(batch, outputs, loss, should_step)\n",
+    "        return loss.detach().cpu()\n",
+    "\n",
+    "    def evaluate_batch(self, batch, stage):\n",
+    "        \"\"\"Computations needed for validation/test batches\"\"\"\n",
+    "        predictions = self.compute_forward(batch, stage=stage)\n",
+    "        with torch.no_grad():\n",
+    "            loss = self.compute_objectives(predictions, batch, stage=stage)\n",
+    "        return loss.detach()\n",
+    "\n",
+    "    def on_stage_start(self, stage, epoch):\n",
+    "        \"\"\"Gets called at the beginning of each epoch\"\"\"\n",
+    "        if stage != sb.Stage.TRAIN:\n",
+    "            self.cer_metric = self.hparams.cer_computer()\n",
+    "            self.wer_metric = self.hparams.error_rate_computer()\n",
+    "\n",
+    "    def on_stage_end(self, stage, stage_loss, epoch):\n",
+    "        \"\"\"Gets called at the end of an epoch.\"\"\"\n",
+    "        # Compute/store important stats\n",
+    "        stage_stats = {\"loss\": stage_loss}\n",
+    "        if stage == sb.Stage.TRAIN:\n",
+    "            self.train_stats = stage_stats\n",
+    "        else:\n",
+    "            stage_stats[\"CER\"] = self.cer_metric.summarize(\"error_rate\")\n",
+    "            stage_stats[\"WER\"] = self.wer_metric.summarize(\"error_rate\")\n",
+    "\n",
+    "        # Perform end-of-iteration things, like annealing, logging, etc.\n",
+    "        if stage == sb.Stage.VALID:\n",
+    "            old_lr_model, new_lr_model = self.hparams.lr_annealing_model(\n",
+    "                stage_stats[\"loss\"]\n",
+    "            )\n",
+    "            sb.nnet.schedulers.update_learning_rate(\n",
+    "                self.model_optimizer, new_lr_model\n",
+    "            )\n",
+    "            self.hparams.train_logger.log_stats(\n",
+    "                stats_meta={\n",
+    "                    \"epoch\": epoch,\n",
+    "                    \"lr_model\": old_lr_model,\n",
+    "                },\n",
+    "                train_stats=self.train_stats,\n",
+    "                valid_stats=stage_stats,\n",
+    "            )\n",
+    "            self.checkpointer.save_and_keep_only(\n",
+    "                meta={\"WER\": stage_stats[\"WER\"]}, min_keys=[\"WER\"],\n",
+    "            )\n",
+    "        elif stage == sb.Stage.TEST:\n",
+    "            self.hparams.train_logger.log_stats(\n",
+    "                stats_meta={\"Epoch loaded\": self.hparams.epoch_counter.current},\n",
+    "                test_stats=stage_stats,\n",
+    "            )\n",
+    "            with open(self.hparams.wer_file, \"w\") as w:\n",
+    "                self.wer_metric.write_stats(w)\n",
+    "\n",
+    "    def init_optimizers(self):\n",
+    "\n",
+    "        self.model_optimizer = self.hparams.model_opt_class(\n",
+    "            self.hparams.model.parameters()\n",
+    "        )\n",
+    "\n",
+    "        if self.checkpointer is not None:\n",
+    "            self.checkpointer.add_recoverable(\"modelopt\", self.model_optimizer)\n",
+    "\n",
+    "    def zero_grad(self, set_to_none=False):\n",
+    "\n",
+    "        self.model_optimizer.zero_grad(set_to_none)\n",
+    "\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "speechbrain.utils.distributed - distributed_launch flag is disabled, this experiment will be executed without DDP.\n",
+      "speechbrain.core - Beginning experiment!\n",
+      "speechbrain.core - Experiment folder: results/non_semi_final_stac\n",
+      "speechbrain.dataio.encoder - Load called, but CTCTextEncoder is not empty. Loaded data will overwrite everything. This is normal if there is e.g. an unk label defined at init.\n",
+      "pyctcdecode.decoder - Using arpa instead of binary LM file, decoder instantiation might be slow.\n",
+      "pyctcdecode.alphabet - Alphabet determined to be of regular style.\n",
+      "pyctcdecode.alphabet - Unigrams and labels don't seem to agree.\n",
+      "speechbrain.core - Info: auto_mix_prec arg from hparam file is used\n",
+      "speechbrain.core - 60.1M trainable parameters in Mixer\n",
+      "pyctcdecode.decoder - Using arpa instead of binary LM file, decoder instantiation might be slow.\n",
+      "pyctcdecode.alphabet - Alphabet determined to be of regular style.\n",
+      "pyctcdecode.alphabet - Unigrams and labels don't seem to agree.\n",
+      "speechbrain.utils.checkpoints - Loading a checkpoint from TunisianASR/semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "<ipython-input-26-84a6e2d9fce8>:119: GradioDeprecationWarning: `optional` parameter is deprecated, and it has no effect\n",
+      "  inputs=[gr.Audio(source=\"microphone\", type='filepath', label = \"record\", optional = True),\n",
+      "<ipython-input-26-84a6e2d9fce8>:120: GradioDeprecationWarning: `optional` parameter is deprecated, and it has no effect\n",
+      "  gr.Audio(source=\"upload\", type='filepath', label=\"filein\", optional=True)]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Running on local URL:  http://127.0.0.1:7860\n",
+      "\n",
+      "To create a public link, set `share=True` in `launch()`.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div><iframe src=\"http://127.0.0.1:7860/\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/salah/anaconda3/envs/salah/lib/python3.8/site-packages/gradio/processing_utils.py:188: UserWarning: Trying to convert audio automatically from int32 to 16-bit int format.\n",
+      "  warnings.warn(warning.format(data.dtype))\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "tensor([[ 0.0000,  0.0000,  0.0000,  ...,  0.0075, -0.0042, -0.0031]])\n",
+      "tensor([[ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  3.0518e-05,\n",
+      "         -3.0518e-05,  0.0000e+00]])\n"
+     ]
+    }
+   ],
+   "source": [
+    "hparams_file, run_opts, overrides = sb.parse_arguments([\"cs.yaml\"])\n",
+    "\n",
+    "# If distributed_launch=True then\n",
+    "# create ddp_group with the right communication protocol\n",
+    "sb.utils.distributed.ddp_init_group(run_opts)\n",
+    "\n",
+    "with open(hparams_file) as fin:\n",
+    "    hparams = load_hyperpyyaml(fin, overrides)\n",
+    "\n",
+    "# Create experiment directory\n",
+    "sb.create_experiment_directory(\n",
+    "    experiment_directory=hparams[\"output_folder\"],\n",
+    "    hyperparams_to_save=hparams_file,\n",
+    "    overrides=overrides,\n",
+    ")\n",
+    "def read_labels_file(labels_file):\n",
+    "    with open(labels_file, \"r\",encoding=\"utf-8\") as lf:\n",
+    "        lines = lf.read().splitlines()\n",
+    "        division = \"===\"\n",
+    "        numbers = {}\n",
+    "        for line in lines :\n",
+    "            if division in line :\n",
+    "                break\n",
+    "            string, number = line.split(\"=>\")\n",
+    "            number = int(number)\n",
+    "            string = string[1:-2]\n",
+    "            numbers[number] = string\n",
+    "        return [numbers[x] for x in range(len(numbers))]\n",
+    "\n",
+    "label_encoder = sb.dataio.encoder.CTCTextEncoder()\n",
+    "\n",
+    "lab_enc_file = os.path.join(hparams[\"save_folder\"], \"label_encoder.txt\")\n",
+    "special_labels = {\n",
+    "    \"blank_label\": hparams[\"blank_index\"],\n",
+    "    \"unk_label\": hparams[\"unk_index\"]\n",
+    "}\n",
+    "label_encoder.load_or_create(\n",
+    "    path=lab_enc_file,\n",
+    "    from_didatasets=[[]],\n",
+    "    output_key=\"char_list\",\n",
+    "    special_labels=special_labels,\n",
+    "    sequence_input=True,\n",
+    ")\n",
+    "\n",
+    "\n",
+    "labels = read_labels_file(os.path.join(hparams[\"save_folder\"], \"label_encoder.txt\"))\n",
+    "labels = [\"\"] + labels[1:-1] + [\"1\"] \n",
+    "if hparams[\"language_modelling\"]:\n",
+    "    decoder = build_ctcdecoder(\n",
+    "        labels,\n",
+    "        kenlm_model_path=hparams[\"ngram_lm_path\"],  # either .arpa or .bin file\n",
+    "        alpha=0.5,  # tuned on a val set\n",
+    "        beta=1,  # tuned on a val set\n",
+    "    )\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "mixer = Mixer(\n",
+    "    modules=hparams[\"modules\"],\n",
+    "    hparams=hparams,\n",
+    "    run_opts=run_opts,\n",
+    "    checkpointer=hparams[\"checkpointer\"],\n",
+    ")\n",
+    "mixer.tokenizer = label_encoder\n",
+    "\n",
+    "\n",
+    "label_encoder = sb.dataio.encoder.CTCTextEncoder()\n",
+    "\n",
+    "\n",
+    "# We dynamicaly add the tokenizer to our brain class.\n",
+    "# NB: This tokenizer corresponds to the one used for the LM!!\n",
+    "\n",
+    "decoder = build_ctcdecoder(\n",
+    "    labels,\n",
+    "    kenlm_model_path= \"arpas/everything.arpa\",  # either .arpa or .bin file\n",
+    "    alpha=0.5,  # tuned on a val set\n",
+    "    beta=1,  # tuned on a val set\n",
+    ")\n",
+    "\n",
+    "run_opts[\"device\"]=\"cpu\"\n",
+    "\n",
+    "\n",
+    "device = \"cpu\"\n",
+    "mixer.device= \"cpu\"\n",
+    "mixer.modules.to(\"cpu\")\n",
+    "\n",
+    "from enum import Enum, auto\n",
+    "class Stage(Enum):\n",
+    "    TRAIN = auto()\n",
+    "    VALID = auto()\n",
+    "    TEST = auto()\n",
+    "\n",
+    "asr_brain.on_evaluate_start()\n",
+    "asr_brain.modules.eval()\n",
+    "\n",
+    "\n",
+    "import gradio as gr\n",
+    "\n",
+    "def treat_wav_file(file_mic,file_upload ,asr=mixer, device=\"cpu\") :\n",
+    "    if (file_mic is not None) and (file_upload is not None):\n",
+    "        warn_output = \"WARNING: You've uploaded an audio file and used the microphone. The recorded file from the microphone will be used and the uploaded audio will be discarded.\\n\"\n",
+    "        wav = file_mic\n",
+    "    elif (file_mic is None) and (file_upload is None):\n",
+    "        return \"ERROR: You have to either use the microphone or upload an audio file\"\n",
+    "    elif file_mic is not None:\n",
+    "        wav = file_mic\n",
+    "    else:\n",
+    "        wav = file_upload\n",
+    "    sig, sr = torchaudio.load(wav)\n",
+    "    tensor_wav = sig.to(device)\n",
+    "    resampled = torchaudio.functional.resample( tensor_wav, sr, 16000)\n",
+    "    sentence = asr.treat_wav(resampled)\n",
+    "    return sentence\n",
+    "\n",
+    "gr.Interface(\n",
+    "    fn=treat_wav_file, \n",
+    "    inputs=[gr.Audio(source=\"microphone\", type='filepath', label = \"record\", optional = True), \n",
+    "            gr.Audio(source=\"upload\", type='filepath', label=\"filein\", optional=True)]\n",
+    "    ,outputs=\"text\").launch(share= False, debug = True)\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

EnglishCV/train_en_with_wav2vec.yaml

@@ -7,13 +7,13 @@
 # Seed needs to be set at top of yaml, before objects with parameters are made
 seed: 1234
 __set_seed: !!python/object/apply:torch.manual_seed [!ref <seed>]
-output_folder: !ref results/wav2vec2_ctc_en/<seed>
+output_folder: !ref EnglishCV/results/wav2vec2_ctc_en/<seed>
 wer_file: !ref <output_folder>/wer.txt
 save_folder: !ref <output_folder>/save
 train_log: !ref <output_folder>/train_log.txt
 
 # URL for the biggest Fairseq english wav2vec2 model.
-wav2vec2_hub: facebook/wav2vec2-large-lv60
+wav2vec2_hub: wav2vec2-large-lv60/
 wav2vec2_folder: !ref <save_folder>/wav2vec2_checkpoint
 
 # Data files
@@ -109,7 +109,7 @@ enc: !new:speechbrain.nnet.containers.Sequential
     activation3: !new:torch.nn.LeakyReLU
 
 wav2vec2: !new:speechbrain.lobes.models.huggingface_wav2vec.HuggingFaceWav2Vec2
-    source: /gpfsscratch/rech/nou/uzn19yk/wav2vec2-large-lv60/
+    source: wav2vec2-large-lv60/
     output_norm: True
     freeze: !ref <freeze_wav2vec>
     freeze_feature_extractor: !ref <freeze_feature_extractor>

EnglishCV/train_with_wav2vec.py

@@ -37,7 +37,7 @@ logger = logging.getLogger(__name__)
 
 
 # Define training procedure
-class ASR(sb.core.Brain):
+class ASRCV(sb.core.Brain):
     def compute_forward(self, batch, stage):
         """Forward computations from the waveform batches to the output probabilities."""
 
@@ -360,7 +360,7 @@ if __name__ == "__main__":
     train_data, valid_data, test_data = dataio_prepare(hparams, tokenizer)
 
     # Trainer initialization
-    asr_brain = ASR(
+    asr_brain = ASRCV(
         modules=hparams["modules"],
         hparams=hparams,
         run_opts=run_opts,

TunisianASR/results/14epoch_tunisian/1234/env.log

@@ -0,0 +1,347 @@
+SpeechBrain system description
+==============================
+Python version:
+3.8.5 (default, Sep  4 2020, 07:30:14) 
+[GCC 7.3.0]
+==============================
+Installed Python packages:
+absl-py==1.2.0
+aiohttp==3.8.1
+aiosignal==1.2.0
+alabaster==0.7.12
+anaconda-client==1.7.2
+anaconda-navigator==1.10.0
+anaconda-project==0.8.3
+antlr4-python3-runtime==4.9.3
+appdirs==1.4.4
+argh==0.26.2
+argon2-cffi @ file:///tmp/build/80754af9/argon2-cffi_1596828493937/work
+asn1crypto @ file:///tmp/build/80754af9/asn1crypto_1596577642040/work
+astroid @ file:///tmp/build/80754af9/astroid_1592495912941/work
+astropy==4.0.2
+async-generator==1.10
+async-timeout==4.0.2
+atomicwrites==1.4.0
+attrs @ file:///tmp/build/80754af9/attrs_1604765588209/work
+audioread==2.1.9
+autopep8 @ file:///tmp/build/80754af9/autopep8_1596578164842/work
+Babel @ file:///tmp/build/80754af9/babel_1605108370292/work
+backcall==0.2.0
+backports.functools-lru-cache==1.6.1
+backports.shutil-get-terminal-size==1.0.0
+backports.tempfile==1.0
+backports.weakref==1.0.post1
+beautifulsoup4 @ file:///tmp/build/80754af9/beautifulsoup4_1601924105527/work
+bitarray @ file:///tmp/build/80754af9/bitarray_1605065113847/work
+bkcharts==0.2
+black==22.12.0
+bleach @ file:///tmp/build/80754af9/bleach_1600439572647/work
+bokeh @ file:///tmp/build/80754af9/bokeh_1603297833684/work
+boto==2.49.0
+boto3==1.28.43
+botocore==1.31.43
+Bottleneck==1.3.2
+bpemb==0.3.4
+brotlipy==0.7.0
+cachetools==5.2.0
+certifi==2020.6.20
+cffi @ file:///tmp/build/80754af9/cffi_1600699146221/work
+chardet==3.0.4
+charset-normalizer==2.0.12
+click==8.1.3
+cloudpickle @ file:///tmp/build/80754af9/cloudpickle_1598884132938/work
+clyent==1.2.2
+colorama @ file:///tmp/build/80754af9/colorama_1603211150991/work
+coloredlogs==15.0.1
+conda==4.9.2
+conda-build==3.20.5
+conda-package-handling @ file:///tmp/build/80754af9/conda-package-handling_1603018141399/work
+conda-verify==3.4.2
+conllu==4.5.3
+contextlib2==0.6.0.post1
+cryptography @ file:///tmp/build/80754af9/cryptography_1601046815590/work
+cycler==0.10.0
+Cython @ file:///tmp/build/80754af9/cython_1594831566883/work
+cytoolz==0.11.0
+dask @ file:///tmp/build/80754af9/dask-core_1602083700509/work
+datasets==1.18.3
+decorator==4.4.2
+defusedxml==0.6.0
+Deprecated==1.2.14
+diff-match-patch @ file:///tmp/build/80754af9/diff-match-patch_1594828741838/work
+dill==0.3.4
+distributed @ file:///tmp/build/80754af9/distributed_1605066520644/work
+docutils==0.16
+easyocr==1.2.1
+einops==0.3.0
+entrypoints==0.3
+et-xmlfile==1.0.1
+farasapy==0.0.14
+fastcache==1.1.0
+ffmpeg-python==0.2.0
+filelock==3.0.12
+flair==0.12.2
+flake8 @ file:///tmp/build/80754af9/flake8_1601911421857/work
+Flask==1.1.2
+flatbuffers==22.9.24
+frozenlist==1.3.0
+fsspec==2022.3.0
+ftfy==6.1.1
+future==0.18.2
+gdown==4.4.0
+gensim==4.1.2
+gevent @ file:///tmp/build/80754af9/gevent_1601397537062/work
+glob2==0.7
+gmpy2==2.0.8
+google-auth==2.12.0
+google-auth-oauthlib==0.4.6
+greenlet @ file:///tmp/build/80754af9/greenlet_1600874013538/work
+grpcio==1.49.1
+h5py==2.10.0
+HeapDict==1.0.1
+html5lib @ file:///tmp/build/80754af9/html5lib_1593446221756/work
+huggingface-hub==0.16.4
+humanfriendly==10.0
+hyperopt==0.2.7
+idna @ file:///tmp/build/80754af9/idna_1593446292537/work
+imageio @ file:///tmp/build/80754af9/imageio_1594161405741/work
+imagesize==1.2.0
+imhist==0.0.4
+importlib-metadata==5.0.0
+imWatermark==0.0.2
+iniconfig @ file:///tmp/build/80754af9/iniconfig_1602780191262/work
+install==1.3.5
+intervaltree @ file:///tmp/build/80754af9/intervaltree_1598376443606/work
+invisible-watermark==0.1.5
+ipykernel @ file:///tmp/build/80754af9/ipykernel_1596207638929/work/dist/ipykernel-5.3.4-py3-none-any.whl
+ipython @ file:///tmp/build/80754af9/ipython_1604101197014/work
+ipython-genutils==0.2.0
+ipywidgets @ file:///tmp/build/80754af9/ipywidgets_1601490159889/work
+isort @ file:///tmp/build/80754af9/isort_1602603989581/work
+itsdangerous==1.1.0
+Janome==0.5.0
+jdcal==1.4.1
+jedi @ file:///tmp/build/80754af9/jedi_1592841866100/work
+jeepney @ file:///tmp/build/80754af9/jeepney_1605069705079/work
+Jinja2==2.11.2
+jiwer==2.3.0
+jmespath==1.0.1
+joblib @ file:///tmp/build/80754af9/joblib_1601912903842/work
+json5==0.9.5
+jsonschema @ file:///tmp/build/80754af9/jsonschema_1602607155483/work
+jupyter==1.0.0
+jupyter-client @ file:///tmp/build/80754af9/jupyter_client_1601311786391/work
+jupyter-console @ file:///tmp/build/80754af9/jupyter_console_1598884538475/work
+jupyter-core==4.6.3
+jupyterlab==2.2.6
+jupyterlab-pygments @ file:///tmp/build/80754af9/jupyterlab_pygments_1601490720602/work
+jupyterlab-server @ file:///tmp/build/80754af9/jupyterlab_server_1594164409481/work
+keyring @ file:///tmp/build/80754af9/keyring_1601490835422/work
+kiwisolver @ file:///tmp/build/80754af9/kiwisolver_1604014535162/work
+langdetect==1.0.9
+lazy-object-proxy==1.4.3
+libarchive-c==2.9
+librosa==0.9.1
+llvmlite==0.34.0
+locket==0.2.0
+lxml @ file:///tmp/build/80754af9/lxml_1603216285000/work
+Markdown==3.4.1
+MarkupSafe==1.1.1
+matplotlib @ file:///tmp/build/80754af9/matplotlib-base_1603378225747/work
+mccabe==0.6.1
+mido==1.2.10
+mistune==0.8.4
+mkl-fft==1.2.0
+mkl-random==1.1.1
+mkl-service==2.3.0
+mock==4.0.2
+more-itertools @ file:///tmp/build/80754af9/more-itertools_1605111547926/work
+mpld3==0.3
+mpmath==1.1.0
+msgpack==1.0.0
+multidict==6.0.2
+multipledispatch==0.6.0
+multiprocess==0.70.12.2
+mypy-extensions==0.4.3
+navigator-updater==0.2.1
+nbclient @ file:///tmp/build/80754af9/nbclient_1602783176460/work
+nbconvert @ file:///tmp/build/80754af9/nbconvert_1601914830498/work
+nbformat @ file:///tmp/build/80754af9/nbformat_1602783287752/work
+nest-asyncio @ file:///tmp/build/80754af9/nest-asyncio_1605115881283/work
+networkx @ file:///tmp/build/80754af9/networkx_1598376031484/work
+nltk @ file:///tmp/build/80754af9/nltk_1592496090529/work
+nose==1.3.7
+notebook @ file:///tmp/build/80754af9/notebook_1601501575118/work
+numba @ file:///tmp/build/80754af9/numba_1600100669015/work
+numexpr==2.7.1
+numpy @ file:///tmp/build/80754af9/numpy_and_numpy_base_1603570489231/work
+numpydoc @ file:///tmp/build/80754af9/numpydoc_1605117425582/work
+oauthlib==3.2.1
+olefile==0.46
+omegaconf==2.2.3
+onnx==1.12.0
+onnxruntime==1.12.1
+opencv-python==4.4.0.46
+openpyxl @ file:///tmp/build/80754af9/openpyxl_1598113097404/work
+packaging==20.9
+pandas @ file:///tmp/build/80754af9/pandas_1602088120436/work
+pandocfilters @ file:///tmp/build/80754af9/pandocfilters_1605120460739/work
+parso==0.7.0
+partd==1.1.0
+path @ file:///tmp/build/80754af9/path_1598376507494/work
+pathlib2==2.3.5
+pathspec==0.10.3
+pathtools==0.1.2
+patsy==0.5.1
+pep8==1.7.1
+pexpect==4.8.0
+pickleshare==0.7.5
+Pillow @ file:///tmp/build/80754af9/pillow_1603822255246/work
+pkginfo==1.6.1
+platformdirs==2.6.0
+pluggy==0.13.1
+ply==3.11
+pooch==1.6.0
+pptree==3.1
+pretty-midi==0.2.9
+prometheus-client==0.8.0
+prompt-toolkit @ file:///tmp/build/80754af9/prompt-toolkit_1602688806899/work
+protobuf==3.19.6
+psutil @ file:///tmp/build/80754af9/psutil_1598370257551/work
+ptyprocess==0.6.0
+py @ file:///tmp/build/80754af9/py_1593446248552/work
+py-espeak-ng==0.1.8
+py4j==0.10.9.7
+PyArabic==0.6.15
+pyarrow==7.0.0
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycodestyle==2.6.0
+pycosat==0.6.3
+pycparser @ file:///tmp/build/80754af9/pycparser_1594388511720/work
+pycurl==7.43.0.6
+pyDeprecate==0.3.1
+pydocstyle @ file:///tmp/build/80754af9/pydocstyle_1598885001695/work
+pyflakes==2.2.0
+Pygments @ file:///tmp/build/80754af9/pygments_1604103097372/work
+pylint @ file:///tmp/build/80754af9/pylint_1598623985952/work
+pyodbc===4.0.0-unsupported
+pyOpenSSL @ file:///tmp/build/80754af9/pyopenssl_1594392929924/work
+pyparsing==2.4.7
+pyrsistent @ file:///tmp/build/80754af9/pyrsistent_1600141720057/work
+PySocks==1.7.1
+pytest==0.0.0
+python-bidi==0.4.2
+python-crfsuite==0.9.7
+python-dateutil==2.8.1
+python-jsonrpc-server @ file:///tmp/build/80754af9/python-jsonrpc-server_1600278539111/work
+python-language-server @ file:///tmp/build/80754af9/python-language-server_1600454544709/work
+python-Levenshtein==0.12.2
+pytorch-lightning==1.4.2
+pytorch-revgrad==0.2.0
+pytz==2020.1
+PyWavelets @ file:///tmp/build/80754af9/pywavelets_1601658317819/work
+pyxdg @ file:///tmp/build/80754af9/pyxdg_1603822279816/work
+PyYAML==5.3.1
+pyzmq==19.0.2
+QDarkStyle==2.8.1
+QtAwesome @ file:///tmp/build/80754af9/qtawesome_1602272867890/work
+qtconsole @ file:///tmp/build/80754af9/qtconsole_1600870028330/work
+QtPy==1.9.0
+regex @ file:///tmp/build/80754af9/regex_1602786672676/work
+requests @ file:///tmp/build/80754af9/requests_1592841827918/work
+requests-oauthlib==1.3.1
+resampy==0.2.2
+rope @ file:///tmp/build/80754af9/rope_1602264064449/work
+rsa==4.9
+Rtree==0.9.4
+ruamel-yaml==0.15.87
+s3transfer==0.6.2
+sacremoses==0.0.49
+safetensors==0.3.3
+scikit-image==0.17.2
+scikit-learn @ file:///tmp/build/80754af9/scikit-learn_1598376899566/work
+scipy @ file:///tmp/build/80754af9/scipy_1597686649129/work
+seaborn @ file:///tmp/build/80754af9/seaborn_1600553570093/work
+SecretStorage==3.1.2
+segtok==1.5.11
+Send2Trash==1.5.0
+sentencepiece==0.1.97
+simplegeneric==0.8.1
+singledispatch @ file:///tmp/build/80754af9/singledispatch_1602523705405/work
+sip==4.19.13
+six @ file:///tmp/build/80754af9/six_1605205327372/work
+smart-open==5.2.1
+snowballstemmer==2.0.0
+sortedcollections==1.2.1
+sortedcontainers==2.2.2
+SoundFile==0.10.3.post1
+soupsieve==2.0.1
+sphfile==1.0.3
+Sphinx @ file:///tmp/build/80754af9/sphinx_1597428793432/work
+sphinxcontrib-applehelp==1.0.2
+sphinxcontrib-devhelp==1.0.2
+sphinxcontrib-htmlhelp==1.0.3
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.3
+sphinxcontrib-serializinghtml==1.1.4
+sphinxcontrib-websupport @ file:///tmp/build/80754af9/sphinxcontrib-websupport_1597081412696/work
+spyder @ file:///tmp/build/80754af9/spyder_1599056981321/work
+spyder-kernels @ file:///tmp/build/80754af9/spyder-kernels_1599056754858/work
+SQLAlchemy @ file:///tmp/build/80754af9/sqlalchemy_1603397987316/work
+sqlitedict==2.1.0
+statsmodels @ file:///tmp/build/80754af9/statsmodels_1602280205159/work
+sympy @ file:///tmp/build/80754af9/sympy_1605119542615/work
+tables==3.6.1
+tabulate==0.9.0
+tblib @ file:///tmp/build/80754af9/tblib_1597928476713/work
+tensorboard==2.10.1
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.1
+terminado==0.9.1
+testpath==0.4.4
+threadpoolctl @ file:///tmp/tmp9twdgx9k/threadpoolctl-2.1.0-py3-none-any.whl
+tifffile==2020.10.1
+tkseem==0.0.3
+tokenizers==0.13.3
+toml @ file:///tmp/build/80754af9/toml_1592853716807/work
+tomli==2.0.1
+toolz @ file:///tmp/build/80754af9/toolz_1601054250827/work
+torch==1.11.0
+torchaudio==0.11.0
+torchmetrics==0.6.0
+torchvision==0.8.2
+tornado==6.0.4
+tqdm==4.64.0
+traitlets @ file:///tmp/build/80754af9/traitlets_1602787416690/work
+transformer-smaller-training-vocab==0.3.1
+transformers==4.33.1
+typing-extensions==4.4.0
+ujson @ file:///tmp/build/80754af9/ujson_1602523317881/work
+unicodecsv==0.14.1
+urllib3 @ file:///tmp/build/80754af9/urllib3_1603305693037/work
+watchdog @ file:///tmp/build/80754af9/watchdog_1593447344699/work
+wcwidth @ file:///tmp/build/80754af9/wcwidth_1593447189090/work
+webencodings==0.5.1
+Werkzeug==1.0.1
+widgetsnbextension==3.5.1
+Wikipedia-API==0.6.0
+wrapt==1.11.2
+wurlitzer @ file:///tmp/build/80754af9/wurlitzer_1594753850195/work
+xlrd==1.2.0
+XlsxWriter @ file:///tmp/build/80754af9/xlsxwriter_1602692860603/work
+xlwt==1.3.0
+xmltodict==0.12.0
+xxhash==3.0.0
+yapf @ file:///tmp/build/80754af9/yapf_1593528177422/work
+yarl==1.7.2
+zict==2.0.0
+zipp @ file:///tmp/build/80754af9/zipp_1604001098328/work
+zope.event==4.5.0
+zope.interface @ file:///tmp/build/80754af9/zope.interface_1602002420968/work
+==============================
+Git revision:
+8a51838
+==============================
+CUDA version:
+11.7

TunisianASR/{semi_wavlm_large_tunisian_ctc → results/14epoch_tunisian}/1234/hyperparams.yaml

@@ -1,5 +1,5 @@
-# Generated 2023-09-08 from:
-# /gpfsdsstore/projects/rech/nou/uzn19yk/switched_code_tunisian/train/tunisian_asr/hparams/train_semi.yaml
+# Generated 2023-09-20 from:
+# /home/salah/Code_Switched_Tunisian_Speech_Recognition/TunisianASR/semi_trained.yaml
 # yamllint disable
 # ################################
 # Model: wav2vec2 + DNN + CTC
@@ -10,13 +10,13 @@
 # Seed needs to be set at top of yaml, before objects with parameters are made
 seed: 1234
 __set_seed: !!python/object/apply:torch.manual_seed [1234]
-output_folder: results/semi_wavlm_large_tunisian_ctc/1234
-wer_file: results/semi_wavlm_large_tunisian_ctc/1234/wer.txt
-save_folder: results/semi_wavlm_large_tunisian_ctc/1234/save
-train_log: results/semi_wavlm_large_tunisian_ctc/1234/train_log.txt
+output_folder: TunisianASR/results/14epoch_tunisian/1234/
+wer_file: TunisianASR/results/14epoch_tunisian/1234//wer.txt
+save_folder: TunisianASR/results/14epoch_tunisian/1234//save
+train_log: TunisianASR/results/14epoch_tunisian/1234//train_log.txt
 
 # URL for the biggest LeBenchmark wav2vec french.
-wav2vec2_folder: results/semi_wavlm_large_tunisian_ctc/1234/save/wav2vec2_checkpoint
+wav2vec2_folder: TunisianASR/results/14epoch_tunisian/1234//save/wav2vec2_checkpoint
 
 # Data files
 data_folder: /gpfsscratch/rech/nou/uzn19yk/tunisian_junk  # e.g, /localscratch/cv-corpus-5.1-2020-06-22/fr
@@ -25,7 +25,7 @@ dev_tsv_file: /gpfsscratch/rech/nou/uzn19yk/tunisian_junk/dev.tsv # Standard Com
 test_tsv_file: /gpfsscratch/rech/nou/uzn19yk/tunisian_junk/test.tsv # Standard CommonVoice .tsv files
 accented_letters: true
 language: fr # use 'it' for Italian, 'rw' for Kinyarwanda, 'en' for english
-train_csv: /gpfsscratch/rech/nou/uzn19yk/tunisian_csvs/good_final/train_enhanced.csv
+train_csv: /gpfsscratch/rech/nou/uzn19yk/tunisian_csvs/good_final/train.csv
 valid_csv: /gpfsscratch/rech/nou/uzn19yk/tunisian_csvs/good_final/dev.csv
 test_csv:
 - /gpfsscratch/rech/nou/uzn19yk/tunisian_csvs/full_annotation_test.csv
@@ -44,7 +44,7 @@ avoid_if_shorter_than: 1.2
 
 
 # Training parameters
-number_of_epochs: 12
+number_of_epochs: 14
 lr: 1.0
 lr_wav2vec: 0.0001
 sorting: ascending
@@ -91,7 +91,7 @@ unk_index: 1
 #
 epoch_counter: &id007 !new:speechbrain.utils.epoch_loop.EpochCounter
 
-  limit: 12
+  limit: 14
 
 augmentation: !new:speechbrain.lobes.augment.TimeDomainSpecAugment
   sample_rate: 16000
@@ -120,11 +120,11 @@ enc: &id002 !new:speechbrain.nnet.containers.Sequential
   activation3: !new:torch.nn.LeakyReLU
 
 wav2vec2: &id001 !new:speechbrain.lobes.models.huggingface_wav2vec.HuggingFaceWav2Vec2
-  source: /gpfsstore/rech/nou/uzn19yk/wavlm/
+  source: wavlm-large/
   output_norm: false
   freeze: false
   freeze_feature_extractor: true
-  save_path: results/semi_wavlm_large_tunisian_ctc/1234/save/wav2vec2_checkpoint
+  save_path: TunisianASR/results/14epoch_tunisian/1234//save/wav2vec2_checkpoint
 
 #####
 # Uncomment this block if you prefer to use a Fairseq pretrained model instead
@@ -178,7 +178,7 @@ lr_annealing_wav2vec: &id006 !new:speechbrain.nnet.schedulers.NewBobScheduler
   patient: 0
 
 checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer
-  checkpoints_dir: results/semi_wavlm_large_tunisian_ctc/1234/save
+  checkpoints_dir: TunisianASR/results/14epoch_tunisian/1234//save
   recoverables:
     wav2vec2: *id001
     model: *id004
@@ -186,7 +186,7 @@ checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer
     scheduler_wav2vec: *id006
     counter: *id007
 train_logger: !new:speechbrain.utils.train_logger.FileTrainLogger
-  save_file: results/semi_wavlm_large_tunisian_ctc/1234/train_log.txt
+  save_file: TunisianASR/results/14epoch_tunisian/1234//train_log.txt
 
 error_rate_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats
 

TunisianASR/results/14epoch_tunisian/1234/log.txt

@@ -0,0 +1,359 @@
+2023-09-20 16:23:38,106 - speechbrain.core - INFO - Beginning experiment!
+2023-09-20 16:23:38,106 - speechbrain.core - INFO - Experiment folder: TunisianASR/results/14epoch_tunisian/1234/
+2023-09-20 16:23:39,287 - speechbrain.utils.superpowers - DEBUG - absl-py==1.2.0
+aiohttp==3.8.1
+aiosignal==1.2.0
+alabaster==0.7.12
+anaconda-client==1.7.2
+anaconda-navigator==1.10.0
+anaconda-project==0.8.3
+antlr4-python3-runtime==4.9.3
+appdirs==1.4.4
+argh==0.26.2
+argon2-cffi @ file:///tmp/build/80754af9/argon2-cffi_1596828493937/work
+asn1crypto @ file:///tmp/build/80754af9/asn1crypto_1596577642040/work
+astroid @ file:///tmp/build/80754af9/astroid_1592495912941/work
+astropy==4.0.2
+async-generator==1.10
+async-timeout==4.0.2
+atomicwrites==1.4.0
+attrs @ file:///tmp/build/80754af9/attrs_1604765588209/work
+audioread==2.1.9
+autopep8 @ file:///tmp/build/80754af9/autopep8_1596578164842/work
+Babel @ file:///tmp/build/80754af9/babel_1605108370292/work
+backcall==0.2.0
+backports.functools-lru-cache==1.6.1
+backports.shutil-get-terminal-size==1.0.0
+backports.tempfile==1.0
+backports.weakref==1.0.post1
+beautifulsoup4 @ file:///tmp/build/80754af9/beautifulsoup4_1601924105527/work
+bitarray @ file:///tmp/build/80754af9/bitarray_1605065113847/work
+bkcharts==0.2
+black==22.12.0
+bleach @ file:///tmp/build/80754af9/bleach_1600439572647/work
+bokeh @ file:///tmp/build/80754af9/bokeh_1603297833684/work
+boto==2.49.0
+boto3==1.28.43
+botocore==1.31.43
+Bottleneck==1.3.2
+bpemb==0.3.4
+brotlipy==0.7.0
+cachetools==5.2.0
+certifi==2020.6.20
+cffi @ file:///tmp/build/80754af9/cffi_1600699146221/work
+chardet==3.0.4
+charset-normalizer==2.0.12
+click==8.1.3
+cloudpickle @ file:///tmp/build/80754af9/cloudpickle_1598884132938/work
+clyent==1.2.2
+colorama @ file:///tmp/build/80754af9/colorama_1603211150991/work
+coloredlogs==15.0.1
+conda==4.9.2
+conda-build==3.20.5
+conda-package-handling @ file:///tmp/build/80754af9/conda-package-handling_1603018141399/work
+conda-verify==3.4.2
+conllu==4.5.3
+contextlib2==0.6.0.post1
+cryptography @ file:///tmp/build/80754af9/cryptography_1601046815590/work
+cycler==0.10.0
+Cython @ file:///tmp/build/80754af9/cython_1594831566883/work
+cytoolz==0.11.0
+dask @ file:///tmp/build/80754af9/dask-core_1602083700509/work
+datasets==1.18.3
+decorator==4.4.2
+defusedxml==0.6.0
+Deprecated==1.2.14
+diff-match-patch @ file:///tmp/build/80754af9/diff-match-patch_1594828741838/work
+dill==0.3.4
+distributed @ file:///tmp/build/80754af9/distributed_1605066520644/work
+docutils==0.16
+easyocr==1.2.1
+einops==0.3.0
+entrypoints==0.3
+et-xmlfile==1.0.1
+farasapy==0.0.14
+fastcache==1.1.0
+ffmpeg-python==0.2.0
+filelock==3.0.12
+flair==0.12.2
+flake8 @ file:///tmp/build/80754af9/flake8_1601911421857/work
+Flask==1.1.2
+flatbuffers==22.9.24
+frozenlist==1.3.0
+fsspec==2022.3.0
+ftfy==6.1.1
+future==0.18.2
+gdown==4.4.0
+gensim==4.1.2
+gevent @ file:///tmp/build/80754af9/gevent_1601397537062/work
+glob2==0.7
+gmpy2==2.0.8
+google-auth==2.12.0
+google-auth-oauthlib==0.4.6
+greenlet @ file:///tmp/build/80754af9/greenlet_1600874013538/work
+grpcio==1.49.1
+h5py==2.10.0
+HeapDict==1.0.1
+html5lib @ file:///tmp/build/80754af9/html5lib_1593446221756/work
+huggingface-hub==0.16.4
+humanfriendly==10.0
+hyperopt==0.2.7
+idna @ file:///tmp/build/80754af9/idna_1593446292537/work
+imageio @ file:///tmp/build/80754af9/imageio_1594161405741/work
+imagesize==1.2.0
+imhist==0.0.4
+importlib-metadata==5.0.0
+imWatermark==0.0.2
+iniconfig @ file:///tmp/build/80754af9/iniconfig_1602780191262/work
+install==1.3.5
+intervaltree @ file:///tmp/build/80754af9/intervaltree_1598376443606/work
+invisible-watermark==0.1.5
+ipykernel @ file:///tmp/build/80754af9/ipykernel_1596207638929/work/dist/ipykernel-5.3.4-py3-none-any.whl
+ipython @ file:///tmp/build/80754af9/ipython_1604101197014/work
+ipython-genutils==0.2.0
+ipywidgets @ file:///tmp/build/80754af9/ipywidgets_1601490159889/work
+isort @ file:///tmp/build/80754af9/isort_1602603989581/work
+itsdangerous==1.1.0
+Janome==0.5.0
+jdcal==1.4.1
+jedi @ file:///tmp/build/80754af9/jedi_1592841866100/work
+jeepney @ file:///tmp/build/80754af9/jeepney_1605069705079/work
+Jinja2==2.11.2
+jiwer==2.3.0
+jmespath==1.0.1
+joblib @ file:///tmp/build/80754af9/joblib_1601912903842/work
+json5==0.9.5
+jsonschema @ file:///tmp/build/80754af9/jsonschema_1602607155483/work
+jupyter==1.0.0
+jupyter-client @ file:///tmp/build/80754af9/jupyter_client_1601311786391/work
+jupyter-console @ file:///tmp/build/80754af9/jupyter_console_1598884538475/work
+jupyter-core==4.6.3
+jupyterlab==2.2.6
+jupyterlab-pygments @ file:///tmp/build/80754af9/jupyterlab_pygments_1601490720602/work
+jupyterlab-server @ file:///tmp/build/80754af9/jupyterlab_server_1594164409481/work
+keyring @ file:///tmp/build/80754af9/keyring_1601490835422/work
+kiwisolver @ file:///tmp/build/80754af9/kiwisolver_1604014535162/work
+langdetect==1.0.9
+lazy-object-proxy==1.4.3
+libarchive-c==2.9
+librosa==0.9.1
+llvmlite==0.34.0
+locket==0.2.0
+lxml @ file:///tmp/build/80754af9/lxml_1603216285000/work
+Markdown==3.4.1
+MarkupSafe==1.1.1
+matplotlib @ file:///tmp/build/80754af9/matplotlib-base_1603378225747/work
+mccabe==0.6.1
+mido==1.2.10
+mistune==0.8.4
+mkl-fft==1.2.0
+mkl-random==1.1.1
+mkl-service==2.3.0
+mock==4.0.2
+more-itertools @ file:///tmp/build/80754af9/more-itertools_1605111547926/work
+mpld3==0.3
+mpmath==1.1.0
+msgpack==1.0.0
+multidict==6.0.2
+multipledispatch==0.6.0
+multiprocess==0.70.12.2
+mypy-extensions==0.4.3
+navigator-updater==0.2.1
+nbclient @ file:///tmp/build/80754af9/nbclient_1602783176460/work
+nbconvert @ file:///tmp/build/80754af9/nbconvert_1601914830498/work
+nbformat @ file:///tmp/build/80754af9/nbformat_1602783287752/work
+nest-asyncio @ file:///tmp/build/80754af9/nest-asyncio_1605115881283/work
+networkx @ file:///tmp/build/80754af9/networkx_1598376031484/work
+nltk @ file:///tmp/build/80754af9/nltk_1592496090529/work
+nose==1.3.7
+notebook @ file:///tmp/build/80754af9/notebook_1601501575118/work
+numba @ file:///tmp/build/80754af9/numba_1600100669015/work
+numexpr==2.7.1
+numpy @ file:///tmp/build/80754af9/numpy_and_numpy_base_1603570489231/work
+numpydoc @ file:///tmp/build/80754af9/numpydoc_1605117425582/work
+oauthlib==3.2.1
+olefile==0.46
+omegaconf==2.2.3
+onnx==1.12.0
+onnxruntime==1.12.1
+opencv-python==4.4.0.46
+openpyxl @ file:///tmp/build/80754af9/openpyxl_1598113097404/work
+packaging==20.9
+pandas @ file:///tmp/build/80754af9/pandas_1602088120436/work
+pandocfilters @ file:///tmp/build/80754af9/pandocfilters_1605120460739/work
+parso==0.7.0
+partd==1.1.0
+path @ file:///tmp/build/80754af9/path_1598376507494/work
+pathlib2==2.3.5
+pathspec==0.10.3
+pathtools==0.1.2
+patsy==0.5.1
+pep8==1.7.1
+pexpect==4.8.0
+pickleshare==0.7.5
+Pillow @ file:///tmp/build/80754af9/pillow_1603822255246/work
+pkginfo==1.6.1
+platformdirs==2.6.0
+pluggy==0.13.1
+ply==3.11
+pooch==1.6.0
+pptree==3.1
+pretty-midi==0.2.9
+prometheus-client==0.8.0
+prompt-toolkit @ file:///tmp/build/80754af9/prompt-toolkit_1602688806899/work
+protobuf==3.19.6
+psutil @ file:///tmp/build/80754af9/psutil_1598370257551/work
+ptyprocess==0.6.0
+py @ file:///tmp/build/80754af9/py_1593446248552/work
+py-espeak-ng==0.1.8
+py4j==0.10.9.7
+PyArabic==0.6.15
+pyarrow==7.0.0
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycodestyle==2.6.0
+pycosat==0.6.3
+pycparser @ file:///tmp/build/80754af9/pycparser_1594388511720/work
+pycurl==7.43.0.6
+pyDeprecate==0.3.1
+pydocstyle @ file:///tmp/build/80754af9/pydocstyle_1598885001695/work
+pyflakes==2.2.0
+Pygments @ file:///tmp/build/80754af9/pygments_1604103097372/work
+pylint @ file:///tmp/build/80754af9/pylint_1598623985952/work
+pyodbc===4.0.0-unsupported
+pyOpenSSL @ file:///tmp/build/80754af9/pyopenssl_1594392929924/work
+pyparsing==2.4.7
+pyrsistent @ file:///tmp/build/80754af9/pyrsistent_1600141720057/work
+PySocks==1.7.1
+pytest==0.0.0
+python-bidi==0.4.2
+python-crfsuite==0.9.7
+python-dateutil==2.8.1
+python-jsonrpc-server @ file:///tmp/build/80754af9/python-jsonrpc-server_1600278539111/work
+python-language-server @ file:///tmp/build/80754af9/python-language-server_1600454544709/work
+python-Levenshtein==0.12.2
+pytorch-lightning==1.4.2
+pytorch-revgrad==0.2.0
+pytz==2020.1
+PyWavelets @ file:///tmp/build/80754af9/pywavelets_1601658317819/work
+pyxdg @ file:///tmp/build/80754af9/pyxdg_1603822279816/work
+PyYAML==5.3.1
+pyzmq==19.0.2
+QDarkStyle==2.8.1
+QtAwesome @ file:///tmp/build/80754af9/qtawesome_1602272867890/work
+qtconsole @ file:///tmp/build/80754af9/qtconsole_1600870028330/work
+QtPy==1.9.0
+regex @ file:///tmp/build/80754af9/regex_1602786672676/work
+requests @ file:///tmp/build/80754af9/requests_1592841827918/work
+requests-oauthlib==1.3.1
+resampy==0.2.2
+rope @ file:///tmp/build/80754af9/rope_1602264064449/work
+rsa==4.9
+Rtree==0.9.4
+ruamel-yaml==0.15.87
+s3transfer==0.6.2
+sacremoses==0.0.49
+safetensors==0.3.3
+scikit-image==0.17.2
+scikit-learn @ file:///tmp/build/80754af9/scikit-learn_1598376899566/work
+scipy @ file:///tmp/build/80754af9/scipy_1597686649129/work
+seaborn @ file:///tmp/build/80754af9/seaborn_1600553570093/work
+SecretStorage==3.1.2
+segtok==1.5.11
+Send2Trash==1.5.0
+sentencepiece==0.1.97
+simplegeneric==0.8.1
+singledispatch @ file:///tmp/build/80754af9/singledispatch_1602523705405/work
+sip==4.19.13
+six @ file:///tmp/build/80754af9/six_1605205327372/work
+smart-open==5.2.1
+snowballstemmer==2.0.0
+sortedcollections==1.2.1
+sortedcontainers==2.2.2
+SoundFile==0.10.3.post1
+soupsieve==2.0.1
+sphfile==1.0.3
+Sphinx @ file:///tmp/build/80754af9/sphinx_1597428793432/work
+sphinxcontrib-applehelp==1.0.2
+sphinxcontrib-devhelp==1.0.2
+sphinxcontrib-htmlhelp==1.0.3
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.3
+sphinxcontrib-serializinghtml==1.1.4
+sphinxcontrib-websupport @ file:///tmp/build/80754af9/sphinxcontrib-websupport_1597081412696/work
+spyder @ file:///tmp/build/80754af9/spyder_1599056981321/work
+spyder-kernels @ file:///tmp/build/80754af9/spyder-kernels_1599056754858/work
+SQLAlchemy @ file:///tmp/build/80754af9/sqlalchemy_1603397987316/work
+sqlitedict==2.1.0
+statsmodels @ file:///tmp/build/80754af9/statsmodels_1602280205159/work
+sympy @ file:///tmp/build/80754af9/sympy_1605119542615/work
+tables==3.6.1
+tabulate==0.9.0
+tblib @ file:///tmp/build/80754af9/tblib_1597928476713/work
+tensorboard==2.10.1
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.1
+terminado==0.9.1
+testpath==0.4.4
+threadpoolctl @ file:///tmp/tmp9twdgx9k/threadpoolctl-2.1.0-py3-none-any.whl
+tifffile==2020.10.1
+tkseem==0.0.3
+tokenizers==0.13.3
+toml @ file:///tmp/build/80754af9/toml_1592853716807/work
+tomli==2.0.1
+toolz @ file:///tmp/build/80754af9/toolz_1601054250827/work
+torch==1.11.0
+torchaudio==0.11.0
+torchmetrics==0.6.0
+torchvision==0.8.2
+tornado==6.0.4
+tqdm==4.64.0
+traitlets @ file:///tmp/build/80754af9/traitlets_1602787416690/work
+transformer-smaller-training-vocab==0.3.1
+transformers==4.33.1
+typing-extensions==4.4.0
+ujson @ file:///tmp/build/80754af9/ujson_1602523317881/work
+unicodecsv==0.14.1
+urllib3 @ file:///tmp/build/80754af9/urllib3_1603305693037/work
+watchdog @ file:///tmp/build/80754af9/watchdog_1593447344699/work
+wcwidth @ file:///tmp/build/80754af9/wcwidth_1593447189090/work
+webencodings==0.5.1
+Werkzeug==1.0.1
+widgetsnbextension==3.5.1
+Wikipedia-API==0.6.0
+wrapt==1.11.2
+wurlitzer @ file:///tmp/build/80754af9/wurlitzer_1594753850195/work
+xlrd==1.2.0
+XlsxWriter @ file:///tmp/build/80754af9/xlsxwriter_1602692860603/work
+xlwt==1.3.0
+xmltodict==0.12.0
+xxhash==3.0.0
+yapf @ file:///tmp/build/80754af9/yapf_1593528177422/work
+yarl==1.7.2
+zict==2.0.0
+zipp @ file:///tmp/build/80754af9/zipp_1604001098328/work
+zope.event==4.5.0
+zope.interface @ file:///tmp/build/80754af9/zope.interface_1602002420968/work
+
+
+2023-09-20 16:23:39,866 - speechbrain.utils.superpowers - DEBUG - 8a51838
+
+
+2023-09-20 16:23:39,869 - speechbrain.pretrained.fetching - INFO - Fetch hyperparams.yaml: Using existing file/symlink in pretrained_models/asr-wav2vec2-commonvoice-fr/hyperparams.yaml.
+2023-09-20 16:23:39,871 - speechbrain.pretrained.fetching - INFO - Fetch custom.py: Linking to local file in /home/salah/Code_Switched_Tunisian_Speech_Recognition/asr-wav2vec2-commonvoice-fr/custom.py.
+2023-09-20 16:23:47,958 - speechbrain.lobes.models.huggingface_wav2vec - WARNING - speechbrain.lobes.models.huggingface_wav2vec - wav2vec 2.0 is frozen.
+2023-09-20 16:23:48,018 - speechbrain.utils.parameter_transfer - DEBUG - Collecting files (or symlinks) for pretraining in pretrained_models/asr-wav2vec2-commonvoice-fr.
+2023-09-20 16:23:48,023 - speechbrain.pretrained.fetching - INFO - Fetch wav2vec2.ckpt: Using existing file/symlink in pretrained_models/asr-wav2vec2-commonvoice-fr/wav2vec2.ckpt.
+2023-09-20 16:23:48,025 - speechbrain.pretrained.fetching - INFO - Fetch asr.ckpt: Using existing file/symlink in pretrained_models/asr-wav2vec2-commonvoice-fr/asr.ckpt.
+2023-09-20 16:23:48,028 - speechbrain.pretrained.fetching - INFO - Fetch tokenizer.ckpt: Using existing file/symlink in pretrained_models/asr-wav2vec2-commonvoice-fr/tokenizer.ckpt.
+2023-09-20 16:23:48,029 - speechbrain.utils.parameter_transfer - INFO - Loading pretrained files for: wav2vec2, asr, tokenizer
+2023-09-20 16:23:56,361 - speechbrain.lobes.models.huggingface_wav2vec - WARNING - speechbrain.lobes.models.huggingface_wav2vec - wav2vec 2.0 feature extractor is frozen.
+2023-09-20 16:23:56,366 - speechbrain.core - INFO - Info: auto_mix_prec arg from hparam file is used
+2023-09-20 16:23:56,366 - speechbrain.core - INFO - Info: ckpt_interval_minutes arg from hparam file is used
+2023-09-20 16:23:56,529 - speechbrain.core - INFO - 314.4M trainable parameters in ASRCV
+2023-09-20 16:23:57,316 - speechbrain.utils.checkpoints - INFO - Loading a checkpoint from EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00
+2023-09-20 16:23:59,928 - speechbrain.core - INFO - Info: auto_mix_prec arg from hparam file is used
+2023-09-20 16:23:59,940 - speechbrain.core - INFO - Info: ckpt_interval_minutes arg from hparam file is used
+2023-09-20 16:24:00,139 - speechbrain.core - INFO - 314.4M trainable parameters in ASR
+2023-09-20 16:24:00,967 - speechbrain.utils.checkpoints - INFO - Loading a checkpoint from TunisianASR/results/14epoch_tunisian/1234/save/CKPT+2023-08-03+01-38-38+00
+2023-09-20 16:24:49,007 - speechbrain.utils.distributed - INFO - distributed_launch flag is disabled, this experiment will be executed without DDP.

TunisianASR/{semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00 → results/14epoch_tunisian/1234/save/CKPT+2023-08-03+01-38-38+00}/CKPT.yaml

@@ -1,4 +1,4 @@
 # yamllint disable
-WER: 27.83210816487267
+WER: 26.88369650826989
 end-of-epoch: true
-unixtime: 1693868963.5220973
+unixtime: 1691019518.289327

TunisianASR/{semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00 → results/14epoch_tunisian/1234/save/CKPT+2023-08-03+01-38-38+00}/brain.ckpt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3947a24e8dff5a14299b9cf2fe66ffb4d738cb88717de7f0cf7e8547a76e9776
+oid sha256:c991c52635ebf5f1d342ff11f149ab3000260e4b08bf1b4356e5134002a60feb
 size 51

TunisianASR/{semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00 → results/14epoch_tunisian/1234/save/CKPT+2023-08-03+01-38-38+00}/counter.ckpt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6b51d431df5d7f141cbececcf79edf3dd861c3b4069f0b11661a3eefacbba918
+oid sha256:8527a891e224136950ff32ca212b45bc93f69fbb801c3b1ebedac52775f99e61
 size 2

TunisianASR/{semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00 → results/14epoch_tunisian/1234/save/CKPT+2023-08-03+01-38-38+00}/dataloader-TRAIN.ckpt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b363886c229e536bd3c84e0c3e89312d70e00422578e076a62df1b45c9390793
+oid sha256:646ecafa8b16fbb513bf9ddc56ba5e34c8818c0c8a7858871698ef9d15ddea68
 size 5

TunisianASR/{semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00 → results/14epoch_tunisian/1234/save/CKPT+2023-08-03+01-38-38+00}/model.ckpt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bc1dbeca1e1f1340b08d8ebea6e492f474708dddbbe8cabbcdde5ee9660704f2
+oid sha256:7cb0cabac5780ffeb4b9850d30e8cd10f748896ddb04aa963d29512463f9b65c
 size 12814446

TunisianASR/{semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00 → results/14epoch_tunisian/1234/save/CKPT+2023-08-03+01-38-38+00}/modelopt.ckpt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3af1791eb9a5bfbfc087d2c10b94634df24cad3ac503ce9ba280a3ecc4737781
-size 25575663
+oid sha256:20f6aecbdbc179aeac4a305431e9f4d17a3436a4aa8426d20f066d6c99c7b449
+size 25575599

TunisianASR/{semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00 → results/14epoch_tunisian/1234/save/CKPT+2023-08-03+01-38-38+00}/scheduler_model.ckpt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c275ab9245b440d1586f72058d9edaac1a2fb3e7a52712aa9a9ad022b99a1c0d
+oid sha256:8a25325a19660b044c1edc3405e8a298702a93f2f569b0548f8066bb50e8e3c8
 size 639

TunisianASR/{semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00 → results/14epoch_tunisian/1234/save/CKPT+2023-08-03+01-38-38+00}/scheduler_wav2vec.ckpt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a88187f7882dc3e10c108f1b7abfbd819285b34bded4e88e91c4ff699c1bb5d2
+oid sha256:b238edab7c400d8ad289eb44c8912bc0d1d2144f2ac59e48b9ab736dc4ef5f79
 size 643

TunisianASR/{semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00 → results/14epoch_tunisian/1234/save/CKPT+2023-08-03+01-38-38+00}/wav2vec2.ckpt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:788267bd25ef37623715fa21a975090e5e316fff05971375cd3f62e5160f0743
+oid sha256:f9f5cd05dd7941f51ce7f19acd406b3eb562de4bc4c6ed818f709563a8308e8d
 size 1262005979

TunisianASR/{semi_wavlm_large_tunisian_ctc/1234/save/CKPT+2023-09-05+01-09-23+00 → results/14epoch_tunisian/1234/save/CKPT+2023-08-03+01-38-38+00}/wav2vec_opt.ckpt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:efa967fdd8067be7d88c18cd197980c9c91f344a3dff2b2518b8381c49f28b1e
+oid sha256:8d191b5b044b5dd0414bafb8c762083c4d344e3c81de807ef6a092dba4a383dd
 size 2490361859

TunisianASR/results/14epoch_tunisian/1234/train_with_wav2vec.py

@@ -0,0 +1,399 @@
+#!/usr/bin/env python3
+import sys
+import torch
+import logging
+import speechbrain as sb
+from pathlib import Path
+import os
+import torchaudio
+from hyperpyyaml import load_hyperpyyaml
+from speechbrain.tokenizers.SentencePiece import SentencePiece
+from speechbrain.utils.data_utils import undo_padding
+from speechbrain.utils.distributed import run_on_main
+
+"""Recipe for training a sequence-to-sequence ASR system with CommonVoice.
+The system employs a wav2vec2 encoder and a CTC decoder.
+Decoding is performed with greedy decoding (will be extended to beam search).
+
+To run this recipe, do the following:
+> python train_with_wav2vec2.py hparams/train_with_wav2vec2.yaml
+
+With the default hyperparameters, the system employs a pretrained wav2vec2 encoder.
+The wav2vec2 model is pretrained following the model given in the hprams file.
+It may be dependent on the language.
+
+The neural network is trained with CTC on sub-word units estimated with
+Byte Pairwise Encoding (BPE).
+
+The experiment file is flexible enough to support a large variety of
+different systems. By properly changing the parameter files, you can try
+different encoders, decoders, tokens (e.g, characters instead of BPE),
+training languages (all CommonVoice languages), and many
+other possible variations.
+
+Authors
+ * Titouan Parcollet 2021
+"""
+
+logger = logging.getLogger(__name__)
+
+
+# Define training procedure
+class ASR(sb.core.Brain):
+    def compute_forward(self, batch, stage):
+        """Forward computations from the waveform batches to the output probabilities."""
+
+        batch = batch.to(self.device)
+        wavs, wav_lens = batch.sig
+        wavs, wav_lens = wavs.to(self.device), wav_lens.to(self.device)
+        if stage == sb.Stage.TRAIN:
+            if hasattr(self.hparams, "augmentation"):
+                wavs = self.hparams.augmentation(wavs, wav_lens)
+
+        # Forward pass
+        feats = self.modules.wav2vec2(wavs, wav_lens)
+        x = self.modules.enc(feats)
+        logits = self.modules.ctc_lin(x)
+        p_ctc = self.hparams.log_softmax(logits)
+
+        return p_ctc, wav_lens
+
+    def compute_objectives(self, predictions, batch, stage):
+        """Computes the loss (CTC) given predictions and targets."""
+
+        p_ctc, wav_lens = predictions
+
+        ids = batch.id
+        tokens, tokens_lens = batch.tokens
+
+        loss = self.hparams.ctc_cost(p_ctc, tokens, wav_lens, tokens_lens)
+
+        if stage != sb.Stage.TRAIN:
+            predicted_tokens = sb.decoders.ctc_greedy_decode(
+                p_ctc, wav_lens, blank_id=self.hparams.blank_index
+            )
+            # Decode token terms to words
+            if self.hparams.use_language_modelling:
+                predicted_words = []
+                for logs in p_ctc:
+                    text = decoder.decode(logs.detach().cpu().numpy())
+                    predicted_words.append(text.split(" "))
+            else:
+                predicted_words = [
+                    "".join(self.tokenizer.decode_ndim(utt_seq)).split(" ")
+                    for utt_seq in predicted_tokens
+                ]
+            # Convert indices to words
+            target_words = [wrd.split(" ") for wrd in batch.wrd]
+
+            self.wer_metric.append(ids, predicted_words, target_words)
+            self.cer_metric.append(ids, predicted_words, target_words)
+
+        return loss
+
+    def fit_batch(self, batch):
+        """Train the parameters given a single batch in input"""
+        should_step = self.step % self.grad_accumulation_factor == 0
+        # Managing automatic mixed precision
+        # TOFIX: CTC fine-tuning currently is unstable
+        # This is certainly due to CTC being done in fp16 instead of fp32
+        if self.auto_mix_prec:
+            with torch.cuda.amp.autocast():
+                with self.no_sync():
+                    outputs = self.compute_forward(batch, sb.Stage.TRAIN)
+                loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)
+            with self.no_sync(not should_step):
+                self.scaler.scale(
+                    loss / self.grad_accumulation_factor
+                ).backward()
+            if should_step:
+
+                if not self.hparams.wav2vec2.freeze:
+                    self.scaler.unscale_(self.wav2vec_optimizer)
+                self.scaler.unscale_(self.model_optimizer)
+                if self.check_gradients(loss):
+                    if not self.hparams.wav2vec2.freeze:
+                        if self.optimizer_step >= self.hparams.warmup_steps:
+                            self.scaler.step(self.wav2vec_optimizer)
+                    self.scaler.step(self.model_optimizer)
+                self.scaler.update()
+                self.zero_grad()
+                self.optimizer_step += 1
+        else:
+            # This is mandatory because HF models have a weird behavior with DDP
+            # on the forward pass
+            with self.no_sync():
+                outputs = self.compute_forward(batch, sb.Stage.TRAIN)
+
+            loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)
+
+            with self.no_sync(not should_step):
+                (loss / self.grad_accumulation_factor).backward()
+            if should_step:
+                if self.check_gradients(loss):
+                    if not self.hparams.wav2vec2.freeze:
+                        if self.optimizer_step >= self.hparams.warmup_steps:
+                            self.wav2vec_optimizer.step()
+                    self.model_optimizer.step()
+                self.zero_grad()
+                self.optimizer_step += 1
+
+        self.on_fit_batch_end(batch, outputs, loss, should_step)
+        return loss.detach().cpu()
+
+    def evaluate_batch(self, batch, stage):
+        """Computations needed for validation/test batches"""
+        predictions = self.compute_forward(batch, stage=stage)
+        with torch.no_grad():
+            loss = self.compute_objectives(predictions, batch, stage=stage)
+        return loss.detach()
+
+    def on_stage_start(self, stage, epoch):
+        """Gets called at the beginning of each epoch"""
+        if stage != sb.Stage.TRAIN:
+            self.cer_metric = self.hparams.cer_computer()
+            self.wer_metric = self.hparams.error_rate_computer()
+
+    def on_stage_end(self, stage, stage_loss, epoch):
+        """Gets called at the end of an epoch."""
+        # Compute/store important stats
+        stage_stats = {"loss": stage_loss}
+        if stage == sb.Stage.TRAIN:
+            self.train_stats = stage_stats
+        else:
+            stage_stats["CER"] = self.cer_metric.summarize("error_rate")
+            stage_stats["WER"] = self.wer_metric.summarize("error_rate")
+
+        # Perform end-of-iteration things, like annealing, logging, etc.
+        if stage == sb.Stage.VALID:
+            old_lr_model, new_lr_model = self.hparams.lr_annealing_model(
+                stage_stats["loss"]
+            )
+            old_lr_wav2vec, new_lr_wav2vec = self.hparams.lr_annealing_wav2vec(
+                stage_stats["loss"]
+            )
+            sb.nnet.schedulers.update_learning_rate(
+                self.model_optimizer, new_lr_model
+            )
+            if not self.hparams.wav2vec2.freeze:
+                sb.nnet.schedulers.update_learning_rate(
+                    self.wav2vec_optimizer, new_lr_wav2vec
+                )
+            self.hparams.train_logger.log_stats(
+                stats_meta={
+                    "epoch": epoch,
+                    "lr_model": old_lr_model,
+                    "lr_wav2vec": old_lr_wav2vec,
+                },
+                train_stats=self.train_stats,
+                valid_stats=stage_stats,
+            )
+            self.checkpointer.save_and_keep_only(
+                meta={"WER": stage_stats["WER"]}, min_keys=["WER"],
+            )
+        elif stage == sb.Stage.TEST:
+            self.hparams.train_logger.log_stats(
+                stats_meta={"Epoch loaded": self.hparams.epoch_counter.current},
+                test_stats=stage_stats,
+            )
+            with open(self.hparams.wer_file, "w") as w:
+                self.wer_metric.write_stats(w)
+
+    def init_optimizers(self):
+        "Initializes the wav2vec2 optimizer and model optimizer"
+
+        # If the wav2vec encoder is unfrozen, we create the optimizer
+        if not self.hparams.wav2vec2.freeze:
+            self.wav2vec_optimizer = self.hparams.wav2vec_opt_class(
+                self.modules.wav2vec2.parameters()
+            )
+            if self.checkpointer is not None:
+                self.checkpointer.add_recoverable(
+                    "wav2vec_opt", self.wav2vec_optimizer
+                )
+
+        self.model_optimizer = self.hparams.model_opt_class(
+            self.hparams.model.parameters()
+        )
+
+        if self.checkpointer is not None:
+            self.checkpointer.add_recoverable("modelopt", self.model_optimizer)
+
+    def zero_grad(self, set_to_none=False):
+        if not self.hparams.wav2vec2.freeze:
+            self.wav2vec_optimizer.zero_grad(set_to_none)
+        self.model_optimizer.zero_grad(set_to_none)
+
+
+# Define custom data procedure
+def dataio_prepare(hparams):
+    """This function prepares the datasets to be used in the brain class.
+    It also defines the data processing pipeline through user-defined functions."""
+
+    # 1. Define datasets
+    data_folder = hparams["data_folder"]
+
+    train_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
+        csv_path=hparams["train_csv"], replacements={"data_root": data_folder},
+    )
+
+    if hparams["sorting"] == "ascending":
+        # we sort training data to speed up training and get better results.
+        train_data = train_data.filtered_sorted(
+            sort_key="duration",
+            key_max_value={"duration": hparams["avoid_if_longer_than"]},
+        )
+        # when sorting do not shuffle in dataloader ! otherwise is pointless
+        hparams["dataloader_options"]["shuffle"] = False
+
+    elif hparams["sorting"] == "descending":
+        train_data = train_data.filtered_sorted(
+            sort_key="duration",
+            reverse=True,
+            key_max_value={"duration": hparams["avoid_if_longer_than"]},
+        )
+        # when sorting do not shuffle in dataloader ! otherwise is pointless
+        hparams["dataloader_options"]["shuffle"] = False
+
+    elif hparams["sorting"] == "random":
+        pass
+
+    else:
+        raise NotImplementedError(
+            "sorting must be random, ascending or descending"
+        )
+
+    valid_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
+        csv_path=hparams["valid_csv"], replacements={"data_root": data_folder},
+    )
+    # We also sort the validation data so it is faster to validate
+    valid_data = valid_data.filtered_sorted(sort_key="duration")
+    test_datasets = {}
+    for csv_file in hparams["test_csv"]:
+        name = Path(csv_file).stem
+        test_datasets[name] = sb.dataio.dataset.DynamicItemDataset.from_csv(
+            csv_path=csv_file, replacements={"data_root": data_folder}
+        )
+        test_datasets[name] = test_datasets[name].filtered_sorted(
+            sort_key="duration"
+        )
+
+    datasets = [train_data, valid_data] + [i for k, i in test_datasets.items()]
+
+
+    # 2. Define audio pipeline:
+    @sb.utils.data_pipeline.takes("wav")
+    @sb.utils.data_pipeline.provides("sig")
+    def audio_pipeline(wav):
+        info = torchaudio.info(wav)
+        sig = sb.dataio.dataio.read_audio(wav)
+        resampled = torchaudio.transforms.Resample(
+            info.sample_rate, hparams["sample_rate"],
+        )(sig)
+        return resampled
+
+    sb.dataio.dataset.add_dynamic_item(datasets, audio_pipeline)
+    label_encoder = sb.dataio.encoder.CTCTextEncoder()
+
+    # 3. Define text pipeline:
+    @sb.utils.data_pipeline.takes("wrd")
+    @sb.utils.data_pipeline.provides(
+        "wrd", "char_list", "tokens_list", "tokens"
+    )
+    def text_pipeline(wrd):
+        yield wrd
+        char_list = list(wrd)
+        yield char_list
+        tokens_list = label_encoder.encode_sequence(char_list)
+        yield tokens_list
+        tokens = torch.LongTensor(tokens_list)
+        yield tokens
+
+    sb.dataio.dataset.add_dynamic_item(datasets, text_pipeline)
+    lab_enc_file = os.path.join(hparams["save_folder"], "label_encoder.txt")
+    special_labels = {
+        "blank_label": hparams["blank_index"],
+        "unk_label": hparams["unk_index"]
+    }
+    label_encoder.load_or_create(
+        path=lab_enc_file,
+        from_didatasets=[train_data],
+        output_key="char_list",
+        special_labels=special_labels,
+        sequence_input=True,
+    )
+
+    # 4. Set output:
+    sb.dataio.dataset.set_output_keys(
+        datasets, ["id", "sig", "wrd", "char_list", "tokens"],
+    )
+    return train_data, valid_data,test_datasets, label_encoder
+
+
+if __name__ == "__main__":
+
+    # Load hyperparameters file with command-line overrides
+    hparams_file, run_opts, overrides = sb.parse_arguments(sys.argv[1:])
+    with open(hparams_file) as fin:
+        hparams = load_hyperpyyaml(fin, overrides)
+
+    # If --distributed_launch then
+    # create ddp_group with the right communication protocol
+    sb.utils.distributed.ddp_init_group(run_opts)
+
+
+    # Create experiment directory
+    sb.create_experiment_directory(
+        experiment_directory=hparams["output_folder"],
+        hyperparams_to_save=hparams_file,
+        overrides=overrides,
+    )
+
+    # Due to DDP, we do the preparation ONLY on the main python process
+    # Defining tokenizer and loading it
+    # Create the datasets objects as well as tokenization and encoding :-D
+    train_data, valid_data, test_datasets, label_encoder = dataio_prepare(hparams)
+    if hparams["use_language_modelling"]:
+        print("using langauge_modeeling")
+        from pyctcdecode import build_ctcdecoder
+        ind2lab = label_encoder.ind2lab
+        print(ind2lab)
+        labels = [ind2lab[x] for x in range(len(ind2lab))]
+        labels = [""] + labels[1:-1] + ["1"] 
+        # Replace the <blank> token with a blank character, needed for PyCTCdecode
+        print(labels)
+        decoder = build_ctcdecoder(
+            labels,
+            kenlm_model_path=hparams["ngram_lm_path"],  # .arpa or .bin
+            alpha=0.5,  # Default by KenLM
+            beta=1.0,  # Default by KenLM
+        )
+    # Trainer initialization
+    asr_brain = ASR(
+        modules=hparams["modules"],
+        hparams=hparams,
+        run_opts=run_opts,
+        checkpointer=hparams["checkpointer"],
+    )
+
+    # Adding objects to trainer.
+    asr_brain.tokenizer = label_encoder
+
+    # Training
+    asr_brain.fit(
+        asr_brain.hparams.epoch_counter,
+        train_data,
+        valid_data,
+        train_loader_kwargs=hparams["dataloader_options"],
+        valid_loader_kwargs=hparams["test_dataloader_options"],
+    )
+
+    # Test
+    for k in test_datasets.keys():  # keys are test_clean, test_other etc
+        asr_brain.hparams.wer_file = os.path.join(
+            hparams["output_folder"], "wer_{}.txt".format(k)
+        )
+        asr_brain.evaluate(
+            test_datasets[k], test_loader_kwargs=hparams["test_dataloader_options"]
+        )
+

TunisianASR/results/14epoch_tunisian/<seed>/copy_of_wavlm_tun.py

@@ -0,0 +1,761 @@
+# -*- coding: utf-8 -*-
+
+import os
+import sys
+import torch
+import logging
+import speechbrain as sb
+from speechbrain.utils.distributed import run_on_main
+from hyperpyyaml import load_hyperpyyaml
+from pathlib import Path
+import torchaudio.transforms as T
+import torchaudio
+import numpy as np
+import kenlm
+from pyctcdecode import build_ctcdecoder
+import re
+
+# Commented out IPython magic to ensure Python compatibility.
+# %cd /content/drive/MyDrive/tunisian_corpora/tunisian_without_wavlm
+
+hparams_file, run_opts, overrides = sb.parse_arguments(["semi_supervised_test_tunisian.yaml"])
+
+# If distributed_launch=True then
+# create ddp_group with the right communication protocol
+sb.utils.distributed.ddp_init_group(run_opts)
+
+with open(hparams_file) as fin:
+    hparams = load_hyperpyyaml(fin, overrides)
+
+# Create experiment directory
+sb.create_experiment_directory(
+    experiment_directory=hparams["output_folder"],
+    hyperparams_to_save=hparams_file,
+    overrides=overrides,
+)
+"""
+def read_labels_file(labels_file):
+    with open(labels_file, "r",encoding="utf-8") as lf:
+        lines = lf.read().splitlines()
+        division = "==="
+        numbers = {}
+        for line in lines :
+            if division in line :
+                break
+            string, number = line.split("=>")
+            number = int(number)
+            string = string[1:-2]
+            numbers[number] = string
+        return [numbers[x] for x in range(len(numbers))]
+
+labels = read_labels_file(os.path.join(hparams["save_folder"], "label_encoder.txt"))
+labels = [""] + labels[1:-1] + ["1"]
+
+# Dataset prep (parsing Librispeech)
+"""
+
+def dataio_prepare(hparams):
+    """This function prepares the datasets to be used in the brain class.
+    It also defines the data processing pipeline through user-defined functions."""
+
+    # 1. Define datasets
+    data_folder = hparams["data_folder"]
+
+    train_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
+        csv_path=hparams["train_csv"], replacements={"data_root": data_folder},
+    )
+
+    if hparams["sorting"] == "ascending":
+        # we sort training data to speed up training and get better results.
+        train_data = train_data.filtered_sorted(
+            sort_key="duration",
+            key_max_value={"duration": hparams["avoid_if_longer_than"]},
+        )
+        # when sorting do not shuffle in dataloader ! otherwise is pointless
+        hparams["dataloader_options"]["shuffle"] = False
+
+    elif hparams["sorting"] == "descending":
+        train_data = train_data.filtered_sorted(
+            sort_key="duration",
+            reverse=True,
+            key_max_value={"duration": hparams["avoid_if_longer_than"]},
+        )
+        # when sorting do not shuffle in dataloader ! otherwise is pointless
+        hparams["dataloader_options"]["shuffle"] = False
+
+    elif hparams["sorting"] == "random":
+        pass
+
+    else:
+        raise NotImplementedError(
+            "sorting must be random, ascending or descending"
+        )
+
+    valid_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
+        csv_path=hparams["valid_csv"], replacements={"data_root": data_folder},
+    )
+    # We also sort the validation data so it is faster to validate
+    valid_data = valid_data.filtered_sorted(sort_key="duration")
+    test_datasets = {}
+    for csv_file in hparams["test_csv"]:
+        name = Path(csv_file).stem
+        test_datasets[name] = sb.dataio.dataset.DynamicItemDataset.from_csv(
+            csv_path=csv_file, replacements={"data_root": data_folder}
+        )
+        test_datasets[name] = test_datasets[name].filtered_sorted(
+            sort_key="duration"
+        )
+
+    datasets = [train_data, valid_data] + [i for k, i in test_datasets.items()]
+
+
+    # 2. Define audio pipeline:
+    @sb.utils.data_pipeline.takes("wav")
+    @sb.utils.data_pipeline.provides("sig")
+    def audio_pipeline(wav):
+        info = torchaudio.info(wav)
+        sig = sb.dataio.dataio.read_audio(wav)
+        if len(sig.shape)>1 :
+            sig = torch.mean(sig, dim=1)
+        resampled = torchaudio.transforms.Resample(
+            info.sample_rate, hparams["sample_rate"],
+        )(sig)
+        return resampled
+
+    sb.dataio.dataset.add_dynamic_item(datasets, audio_pipeline)
+    label_encoder = sb.dataio.encoder.CTCTextEncoder()
+
+    # 3. Define text pipeline:
+    @sb.utils.data_pipeline.takes("wrd")
+    @sb.utils.data_pipeline.provides(
+        "wrd", "char_list", "tokens_list", "tokens"
+    )
+    def text_pipeline(wrd):
+        yield wrd
+        char_list = list(wrd)
+        yield char_list
+        tokens_list = label_encoder.encode_sequence(char_list)
+        yield tokens_list
+        tokens = torch.LongTensor(tokens_list)
+        yield tokens
+
+    sb.dataio.dataset.add_dynamic_item(datasets, text_pipeline)
+    lab_enc_file = os.path.join(hparams["save_folder"], "label_encoder.txt")
+    special_labels = {
+        "blank_label": hparams["blank_index"],
+        "unk_label": hparams["unk_index"]
+    }
+    label_encoder.load_or_create(
+        path=lab_enc_file,
+        from_didatasets=[train_data],
+        output_key="char_list",
+        special_labels=special_labels,
+        sequence_input=True,
+    )
+
+    # 4. Set output:
+    sb.dataio.dataset.set_output_keys(
+        datasets, ["id", "sig", "wrd", "char_list", "tokens"],
+    )
+    return train_data, valid_data,test_datasets, label_encoder
+
+class ASR(sb.core.Brain):
+    def compute_forward(self, batch, stage):
+        """Forward computations from the waveform batches to the output probabilities."""
+
+        batch = batch.to(self.device)
+        wavs, wav_lens = batch.sig
+        wavs, wav_lens = wavs.to(self.device), wav_lens.to(self.device)
+
+        if stage == sb.Stage.TRAIN:
+            if hasattr(self.hparams, "augmentation"):
+                wavs = self.hparams.augmentation(wavs, wav_lens)
+
+        # Forward pass
+        feats = self.modules.wav2vec2(wavs, wav_lens)
+        x = self.modules.enc(feats)
+        logits = self.modules.ctc_lin(x)
+        p_ctc = self.hparams.log_softmax(logits)
+
+        return p_ctc, wav_lens
+
+    def custom_encode(self,wavs,wav_lens) :
+        wavs = wavs.to(self.device)
+        if(wav_lens is not None): wav_lens.to(self.device)
+
+        feats = self.modules.wav2vec2(wavs, wav_lens)
+        x = self.modules.enc(feats)
+        logits = self.modules.ctc_lin(x)
+        p_ctc = self.hparams.log_softmax(logits)
+
+        return feats,p_ctc
+
+
+
+    def compute_objectives(self, predictions, batch, stage):
+        """Computes the loss (CTC) given predictions and targets."""
+
+        p_ctc, wav_lens = predictions
+
+        ids = batch.id
+        tokens, tokens_lens = batch.tokens
+
+        loss = self.hparams.ctc_cost(p_ctc, tokens, wav_lens, tokens_lens)
+
+        if stage != sb.Stage.TRAIN:
+            predicted_tokens = sb.decoders.ctc_greedy_decode(
+                p_ctc, wav_lens, blank_id=self.hparams.blank_index
+            )
+            # Decode token terms to words
+            if self.hparams.use_language_modelling:
+                predicted_words = []
+                for logs in p_ctc:
+                    text = decoder.decode(logs.detach().cpu().numpy())
+                    predicted_words.append(text.split(" "))
+            else:
+                predicted_words = [
+                    "".join(self.tokenizer.decode_ndim(utt_seq)).split(" ")
+                    for utt_seq in predicted_tokens
+                ]
+            # Convert indices to words
+            target_words = [wrd.split(" ") for wrd in batch.wrd]
+
+            self.wer_metric.append(ids, predicted_words, target_words)
+            self.cer_metric.append(ids, predicted_words, target_words)
+
+        return loss
+
+    def fit_batch(self, batch):
+        """Train the parameters given a single batch in input"""
+        should_step = self.step % self.grad_accumulation_factor == 0
+        # Managing automatic mixed precision
+        # TOFIX: CTC fine-tuning currently is unstable
+        # This is certainly due to CTC being done in fp16 instead of fp32
+        if self.auto_mix_prec:
+            with torch.cuda.amp.autocast():
+                with self.no_sync():
+                    outputs = self.compute_forward(batch, sb.Stage.TRAIN)
+                loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)
+            with self.no_sync(not should_step):
+                self.scaler.scale(
+                    loss / self.grad_accumulation_factor
+                ).backward()
+            if should_step:
+
+                if not self.hparams.wav2vec2.freeze:
+                    self.scaler.unscale_(self.wav2vec_optimizer)
+                self.scaler.unscale_(self.model_optimizer)
+                if self.check_gradients(loss):
+                    if not self.hparams.wav2vec2.freeze:
+                        if self.optimizer_step >= self.hparams.warmup_steps:
+                            self.scaler.step(self.wav2vec_optimizer)
+                    self.scaler.step(self.model_optimizer)
+                self.scaler.update()
+                self.zero_grad()
+                self.optimizer_step += 1
+        else:
+            # This is mandatory because HF models have a weird behavior with DDP
+            # on the forward pass
+            with self.no_sync():
+                outputs = self.compute_forward(batch, sb.Stage.TRAIN)
+
+            loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)
+
+            with self.no_sync(not should_step):
+                (loss / self.grad_accumulation_factor).backward()
+            if should_step:
+                if self.check_gradients(loss):
+                    if not self.hparams.wav2vec2.freeze:
+                        if self.optimizer_step >= self.hparams.warmup_steps:
+                            self.wav2vec_optimizer.step()
+                    self.model_optimizer.step()
+                self.zero_grad()
+                self.optimizer_step += 1
+
+        self.on_fit_batch_end(batch, outputs, loss, should_step)
+        return loss.detach().cpu()
+
+    def evaluate_batch(self, batch, stage):
+        """Computations needed for validation/test batches"""
+        predictions = self.compute_forward(batch, stage=stage)
+        with torch.no_grad():
+            loss = self.compute_objectives(predictions, batch, stage=stage)
+        return loss.detach()
+
+    def on_stage_start(self, stage, epoch):
+        """Gets called at the beginning of each epoch"""
+        if stage != sb.Stage.TRAIN:
+            self.cer_metric = self.hparams.cer_computer()
+            self.wer_metric = self.hparams.error_rate_computer()
+
+    def on_stage_end(self, stage, stage_loss, epoch):
+        """Gets called at the end of an epoch."""
+        # Compute/store important stats
+        stage_stats = {"loss": stage_loss}
+        if stage == sb.Stage.TRAIN:
+            self.train_stats = stage_stats
+        else:
+            stage_stats["CER"] = self.cer_metric.summarize("error_rate")
+            stage_stats["WER"] = self.wer_metric.summarize("error_rate")
+
+        # Perform end-of-iteration things, like annealing, logging, etc.
+        if stage == sb.Stage.VALID:
+            old_lr_model, new_lr_model = self.hparams.lr_annealing_model(
+                stage_stats["loss"]
+            )
+            old_lr_wav2vec, new_lr_wav2vec = self.hparams.lr_annealing_wav2vec(
+                stage_stats["loss"]
+            )
+            sb.nnet.schedulers.update_learning_rate(
+                self.model_optimizer, new_lr_model
+            )
+            if not self.hparams.wav2vec2.freeze:
+                sb.nnet.schedulers.update_learning_rate(
+                    self.wav2vec_optimizer, new_lr_wav2vec
+                )
+            self.hparams.train_logger.log_stats(
+                stats_meta={
+                    "epoch": epoch,
+                    "lr_model": old_lr_model,
+                    "lr_wav2vec": old_lr_wav2vec,
+                },
+                train_stats=self.train_stats,
+                valid_stats=stage_stats,
+            )
+            self.checkpointer.save_and_keep_only(
+                meta={"WER": stage_stats["WER"]}, min_keys=["WER"],
+            )
+        elif stage == sb.Stage.TEST:
+            self.hparams.train_logger.log_stats(
+                stats_meta={"Epoch loaded": self.hparams.epoch_counter.current},
+                test_stats=stage_stats,
+            )
+            with open(self.hparams.wer_file, "w") as w:
+                self.wer_metric.write_stats(w)
+
+    def init_optimizers(self):
+        "Initializes the wav2vec2 optimizer and model optimizer"
+
+        # If the wav2vec encoder is unfrozen, we create the optimizer
+        if not self.hparams.wav2vec2.freeze:
+            self.wav2vec_optimizer = self.hparams.wav2vec_opt_class(
+                self.modules.wav2vec2.parameters()
+            )
+            if self.checkpointer is not None:
+                self.checkpointer.add_recoverable(
+                    "wav2vec_opt", self.wav2vec_optimizer
+                )
+
+        self.model_optimizer = self.hparams.model_opt_class(
+            self.hparams.model.parameters()
+        )
+
+        if self.checkpointer is not None:
+            self.checkpointer.add_recoverable("modelopt", self.model_optimizer)
+
+    def zero_grad(self, set_to_none=False):
+        if not self.hparams.wav2vec2.freeze:
+            self.wav2vec_optimizer.zero_grad(set_to_none)
+        self.model_optimizer.zero_grad(set_to_none)
+
+
+"""
+label_encoder = sb.dataio.encoder.CTCTextEncoder()
+
+train_data, valid_data, test_datasets, label_encoder = dataio_prepare(
+        hparams
+    )
+
+
+# We dynamicaly add the tokenizer to our brain class.
+# NB: This tokenizer corresponds to the one used for the LM!!
+decoder = build_ctcdecoder(
+    labels,
+    kenlm_model_path="/content/drive/MyDrive/tunisian_corpora/tunisian_without_wavlm/lm_data/arpas/indomain.arpa",  # either .arpa or .bin file
+    alpha=0.5,  # tuned on a val set
+    beta=1,  # tuned on a val set
+)
+"""
+from speechbrain.pretrained import EncoderASR,EncoderDecoderASR
+french_asr_model = EncoderASR.from_hparams(source="speechbrain/asr-wav2vec2-commonvoice-fr", savedir="pretrained_models/asr-wav2vec2-commonvoice-fr").cuda()
+french_asr_model.mods.eval()
+#french_asr_model = "r"
+
+english_asr_model = EncoderDecoderASR.from_hparams(source="speechbrain/asr-wav2vec2-commonvoice-en", savedir="pretrained_models/asr-wav2vec2-commonvoice-en/").cuda()
+english_asr_model.mods.eval()
+
+asr_brain = ASR(
+    modules=hparams["modules"],
+    hparams=hparams,
+    run_opts=run_opts,
+    checkpointer=hparams["checkpointer"],
+)
+asr_brain.checkpointer.recover_if_possible()
+asr_brain.modules.eval()
+"""
+asr_brain.tokenizer = label_encoder
+
+# Testing
+real = True
+if real :
+    for k in test_datasets.keys():  # keys are test_clean, test_other etc
+        asr_brain.hparams.wer_file = os.path.join(
+            hparams["output_folder"], "wer_{}.txt".format(k)
+        )
+        asr_brain.evaluate(
+            test_datasets[k], test_loader_kwargs=hparams["dataloader_options"]
+        )
+"""
+
+"""
+from torch.nn.utils.rnn import pad_sequence
+def load_paths(wavs_path):
+    waveforms = []
+    for path in wavs_path :
+        waveform, _ = torchaudio.load(path)
+        waveforms.append(waveform.squeeze(0))
+    # normalize array length to the bigger arrays by pading with 0's
+    padded_arrays = pad_sequence(waveforms, batch_first=True)
+    return torch.tensor(padded_arrays)
+
+waveform = load_paths(["/content/drive/MyDrive/tunisian_corpora/tunisian_without_wavlm/samples/Salah10.wav","/content/drive/MyDrive/tunisian_corpora/tunisian_without_wavlm/samples/Salah10.wav"])
+embeddings, posteriogram = asr_brain.custom_encode(waveform,None)
+print(embeddings.shape)
+print(posteriogram.shape)
+"""
+
+from speechbrain.pretrained import EncoderASR,EncoderDecoderASR
+import torchaudio
+import  speechbrain as sb
+import torch
+from torch.nn.utils.rnn import pad_sequence
+import torch
+import speechbrain as sb
+import numpy as np
+import torch.optim as optim
+import torch.nn as nn
+
+# Commented out IPython magic to ensure Python compatibility.
+# %ls
+
+#UTILS FUNCTIOJNS
+def get_size_dimensions(arr):
+    size_dimensions = []
+    while isinstance(arr, list):
+        size_dimensions.append(len(arr))
+        arr = arr[0]
+    return size_dimensions
+
+def scale_array(batch,n):
+    scaled_batch = []
+
+    for array in batch:
+        if(n < len(array)): raise ValueError("Cannot scale Array down")
+
+        repeat = round(n/len(array))+1
+        scaled_length_array= []
+
+        for i in array:
+            for j in range(repeat) :
+                if(len(scaled_length_array) == n): break
+                scaled_length_array.append(i)
+
+        scaled_batch.append(scaled_length_array)
+
+    return torch.tensor(scaled_batch)
+
+
+def load_paths(wavs_path):
+    waveforms = []
+    for path in wavs_path :
+        waveform, _ = torchaudio.load(path)
+        waveforms.append(waveform.squeeze(0))
+    # normalize array length to the bigger arrays by pading with 0's
+    padded_arrays = pad_sequence(waveforms, batch_first=True)
+    return torch.tensor(padded_arrays)
+
+
+
+def word_to_vec(input_string):
+    mapping= {'a': 1, 'b': 2, 'c': 3, 'd': 4, 'e': 5, 'f': 6, 'g': 7, 'h': 8, 'i': 9, 'j': 10, 'k': 11, 'l': 12, 'm': 13, 'n': 14, 'o': 15, 'p': 16, 'q': 17, 'r': 18, 's': 19, 't': 20, 'u': 21, 'v': 22, 'w': 23, 'x': 24, 'y': 25, 'z': 26, 'ا': 27, 'ب': 28, 'ت': 29, 'ث': 30, 'ج': 31, 'ح': 32, 'خ': 33, 'د': 34, 'ذ': 35, 'ر': 36, 'ز': 37, 'س': 38, 'ش': 39, 'ص': 40, 'ض': 41, 'ط': 42, 'ظ': 43, 'ع': 44, 'غ': 45, 'ف': 46, 'ق': 47, 'ك': 48, 'ل': 49, 'م': 50, 'ن': 51, 'ه': 52, 'و': 53, 'ي': 54,' ':55}
+
+    numbers = [mapping[word] for word in input_string if word in mapping]
+    return numbers
+
+device = 'cuda'
+verbose = 0
+#FLOW LEVEL FUNCTIONS
+def merge_strategy(embeddings1, embeddings2, embeddings3,post1, post2,post3):
+
+
+    post1 = post1.to(device)
+    post2 = post2.to(device)
+    post3 = post3.to(device)
+    embeddings1 = embeddings1.to(device)
+    embeddings2 = embeddings2.to(device)
+    embeddings3 = embeddings3.to(device)
+
+    posteriograms_merged = torch.cat((post1,post2,post3),dim=2)
+    embeddings_merged = torch.cat((embeddings1,embeddings2,embeddings3),dim=2)
+
+    if(verbose !=0):
+      print('MERGED POST ',posteriograms_merged.shape)
+      print('MERGED emb ',embeddings_merged.shape)
+
+    return torch.cat((posteriograms_merged,embeddings_merged),dim=2).to(device)
+
+def decode(model,wavs,wav_lens):
+
+    with torch.no_grad():
+        wav_lens = wav_lens.to(model.device)
+        encoder_out = model.encode_batch(wavs, wav_lens)
+        predictions = model.decoding_function(encoder_out, wav_lens)
+        return predictions
+
+def middle_layer(batch, lens):
+
+    tn_embeddings, tn_posteriogram = asr_brain.custom_encode(batch,None)
+
+    fr_embeddings = french_asr_model.mods.encoder.wav2vec2(batch)
+    fr_posteriogram =french_asr_model.encode_batch(batch,lens)
+
+    en_embeddings = english_asr_model.encode_batch(batch, lens)
+    #scores, en_posteriogram = english_asr_model.mods.decoder(en_embeddings ,lens)
+    en_posteriogram = en_embeddings
+
+    if(verbose !=0):
+      print('[EMBEDDINGS] FR:',fr_embeddings.shape, "EN:",en_embeddings.shape, "TN:", tn_embeddings.shape)
+      print('[POSTERIOGRAM] FR:',fr_posteriogram.shape, "EN:",en_posteriogram.shape,"TN:",tn_posteriogram.shape)
+
+
+    bilangual_sample = merge_strategy(fr_embeddings,en_embeddings,tn_embeddings,fr_posteriogram,en_posteriogram,tn_posteriogram)
+    return bilangual_sample
+
+class Mixer(sb.core.Brain):
+
+    def compute_forward(self, batch, stage):
+        """Forward computations from the waveform batches to the output probabilities."""
+        wavs, wav_lens = batch.sig
+        wavs, wav_lens = wavs.to(self.device), wav_lens.to(self.device)
+
+        if stage == sb.Stage.TRAIN:
+            if hasattr(self.hparams, "augmentation"):
+                wavs = self.hparams.augmentation(wavs, wav_lens)
+
+        multi_langual_feats = middle_layer(wavs, wav_lens)
+        multi_langual_feats= multi_langual_feats.to(device)
+        feats, _ = self.modules.enc(multi_langual_feats)
+        logits = self.modules.ctc_lin(feats)
+        p_ctc = self.hparams.log_softmax(logits)
+        
+        if stage!= sb.Stage.TRAIN:
+            p_tokens = sb.decoders.ctc_greedy_decode(
+                p_ctc, wav_lens, blank_id=self.hparams.blank_index
+            )
+        else : 
+            p_tokens = None
+        return p_ctc, wav_lens, p_tokens
+
+    def compute_objectives(self, predictions, batch, stage):
+        """Computes the loss (CTC) given predictions and targets."""
+
+        p_ctc, wav_lens , predicted_tokens= predictions
+
+        ids = batch.id
+        tokens, tokens_lens = batch.tokens
+
+        loss = self.hparams.ctc_cost(p_ctc, tokens, wav_lens, tokens_lens)
+
+
+        if stage != sb.Stage.TRAIN:
+            predicted_words = [
+                "".join(self.tokenizer.decode_ndim(utt_seq)).split(" ")
+                for utt_seq in predicted_tokens
+            ]
+            target_words = [wrd.split(" ") for wrd in batch.wrd]
+            self.wer_metric.append(ids, predicted_words, target_words)
+            self.cer_metric.append(ids, predicted_words, target_words)
+
+        return loss
+
+    def fit_batch(self, batch):
+        """Train the parameters given a single batch in input"""
+        should_step = self.step % self.grad_accumulation_factor == 0
+        # Managing automatic mixed precision
+        # TOFIX: CTC fine-tuning currently is unstable
+        # This is certainly due to CTC being done in fp16 instead of fp32
+        if self.auto_mix_prec:
+            with torch.cuda.amp.autocast():
+                with self.no_sync():
+                    outputs = self.compute_forward(batch, sb.Stage.TRAIN)
+                loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)
+            with self.no_sync(not should_step):
+                self.scaler.scale(
+                    loss / self.grad_accumulation_factor
+                ).backward()
+            if should_step:
+
+
+                self.scaler.unscale_(self.model_optimizer)
+                if self.check_gradients(loss):
+                    self.scaler.step(self.model_optimizer)
+                self.scaler.update()
+                self.zero_grad()
+                self.optimizer_step += 1
+        else:
+            # This is mandatory because HF models have a weird behavior with DDP
+            # on the forward pass
+            with self.no_sync():
+                outputs = self.compute_forward(batch, sb.Stage.TRAIN)
+
+            loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)
+
+            with self.no_sync(not should_step):
+                (loss / self.grad_accumulation_factor).backward()
+            if should_step:
+                if self.check_gradients(loss):
+                    self.model_optimizer.step()
+                self.zero_grad()
+                self.optimizer_step += 1
+
+        self.on_fit_batch_end(batch, outputs, loss, should_step)
+        return loss.detach().cpu()
+
+    def evaluate_batch(self, batch, stage):
+        """Computations needed for validation/test batches"""
+        predictions = self.compute_forward(batch, stage=stage)
+        with torch.no_grad():
+            loss = self.compute_objectives(predictions, batch, stage=stage)
+        return loss.detach()
+
+    def on_stage_start(self, stage, epoch):
+        """Gets called at the beginning of each epoch"""
+        if stage != sb.Stage.TRAIN:
+            self.cer_metric = self.hparams.cer_computer()
+            self.wer_metric = self.hparams.error_rate_computer()
+
+    def on_stage_end(self, stage, stage_loss, epoch):
+        """Gets called at the end of an epoch."""
+        # Compute/store important stats
+        stage_stats = {"loss": stage_loss}
+        if stage == sb.Stage.TRAIN:
+            self.train_stats = stage_stats
+        else:
+            stage_stats["CER"] = self.cer_metric.summarize("error_rate")
+            stage_stats["WER"] = self.wer_metric.summarize("error_rate")
+
+        # Perform end-of-iteration things, like annealing, logging, etc.
+        if stage == sb.Stage.VALID:
+            old_lr_model, new_lr_model = self.hparams.lr_annealing_model(
+                stage_stats["loss"]
+            )
+            sb.nnet.schedulers.update_learning_rate(
+                self.model_optimizer, new_lr_model
+            )
+            self.hparams.train_logger.log_stats(
+                stats_meta={
+                    "epoch": epoch,
+                    "lr_model": old_lr_model,
+                },
+                train_stats=self.train_stats,
+                valid_stats=stage_stats,
+            )
+            self.checkpointer.save_and_keep_only(
+                meta={"WER": stage_stats["WER"]}, min_keys=["WER"],
+            )
+        elif stage == sb.Stage.TEST:
+            self.hparams.train_logger.log_stats(
+                stats_meta={"Epoch loaded": self.hparams.epoch_counter.current},
+                test_stats=stage_stats,
+            )
+            with open(self.hparams.wer_file, "w") as w:
+                self.wer_metric.write_stats(w)
+
+    def init_optimizers(self):
+
+        self.model_optimizer = self.hparams.model_opt_class(
+            self.hparams.model.parameters()
+        )
+
+        if self.checkpointer is not None:
+            self.checkpointer.add_recoverable("modelopt", self.model_optimizer)
+
+    def zero_grad(self, set_to_none=False):
+
+        self.model_optimizer.zero_grad(set_to_none)
+
+
+hparams_file, run_opts, overrides = sb.parse_arguments([sys.argv[1]])
+
+# If distributed_launch=True then
+# create ddp_group with the right communication protocol
+sb.utils.distributed.ddp_init_group(run_opts)
+
+with open(hparams_file) as fin:
+    hparams = load_hyperpyyaml(fin, overrides)
+
+# Create experiment directory
+sb.create_experiment_directory(
+    experiment_directory=hparams["output_folder"],
+    hyperparams_to_save=hparams_file,
+    overrides=overrides,
+)
+"""
+def read_labels_file(labels_file):
+    with open(labels_file, "r",encoding="utf-8") as lf:
+        lines = lf.read().splitlines()
+        division = "==="
+        numbers = {}
+        for line in lines :
+            if division in line :
+                break
+            string, number = line.split("=>")
+            number = int(number)
+            string = string[1:-2]
+            numbers[number] = string
+        return [numbers[x] for x in range(len(numbers))]
+labels = read_labels_file(os.path.join(hparams["save_folder"], "label_encoder.txt"))
+labels = [""] + labels[1:-1] + ["1"]
+
+label_encoder = sb.dataio.encoder.CTCTextEncoder()
+"""
+train_data, valid_data, test_datasets, label_encoder = dataio_prepare(
+        hparams
+    )
+
+
+
+
+"""
+decoder = build_ctcdecoder(
+    labels,
+    kenlm_model_path="/content/drive/MyDrive/tunisian_corpora/tunisian_without_wavlm/lm_data/arpas/indomain.arpa",  # either .arpa or .bin file
+    alpha=0.5,  # tuned on a val set
+    beta=1,  # tuned on a val set
+)
+"""
+mixer = Mixer(
+    modules=hparams["modules"],
+    hparams=hparams,
+    run_opts=run_opts,
+    checkpointer=hparams["checkpointer"],
+)
+mixer.tokenizer = label_encoder
+
+
+mixer.fit(
+    mixer.hparams.epoch_counter,
+    train_data,
+    valid_data,
+    train_loader_kwargs=hparams["dataloader_options"],
+    valid_loader_kwargs=hparams["test_dataloader_options"],
+)
+
+for k in test_datasets.keys():  # keys are test_clean, test_other etc
+    mixer.hparams.wer_file = os.path.join(
+        hparams["output_folder"], "wer_{}.txt".format(k)
+    )
+    mixer.evaluate(
+        test_datasets[k], test_loader_kwargs=hparams["test_dataloader_options"]
+        )
+

TunisianASR/results/14epoch_tunisian/<seed>/ctc_lin.py

@@ -0,0 +1,756 @@
+# -*- coding: utf-8 -*-
+
+import os
+import sys
+import torch
+import logging
+import speechbrain as sb
+from speechbrain.utils.distributed import run_on_main
+from hyperpyyaml import load_hyperpyyaml
+from pathlib import Path
+import torchaudio.transforms as T
+from cv_train import ASRCV
+import torchaudio
+import numpy as np
+import kenlm
+from pyctcdecode import build_ctcdecoder
+import re
+
+# Commented out IPython magic to ensure Python compatibility.
+# %cd /content/drive/MyDrive/tunisian_corpora/tunisian_without_wavlm
+#hparams_file, run_opts, overrides = sb.parse_arguments(["/gpfsstore/rech/nou/uzn19yk/switched_code_tunisian/train/tunisian_asr/hparams/train_semi.yaml"])
+hparams_file, run_opts, overrides = sb.parse_arguments(["semi_supervised_test_tunisian.yaml"])
+
+# If distributed_launch=True then
+# create ddp_group with the right communication protocol
+sb.utils.distributed.ddp_init_group(run_opts)
+
+with open(hparams_file) as fin:
+    hparams = load_hyperpyyaml(fin, overrides)
+
+# Create experiment directory
+sb.create_experiment_directory(
+    experiment_directory=hparams["output_folder"],
+    hyperparams_to_save=hparams_file,
+    overrides=overrides,
+)
+# Dataset prep (parsing Librispeech)
+
+def dataio_prepare(hparams):
+    """This function prepares the datasets to be used in the brain class.
+    It also defines the data processing pipeline through user-defined functions."""
+
+    # 1. Define datasets
+    data_folder = hparams["data_folder"]
+
+    train_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
+        csv_path=hparams["train_csv"], replacements={"data_root": data_folder},
+    )
+
+    if hparams["sorting"] == "ascending":
+        # we sort training data to speed up training and get better results.
+        train_data = train_data.filtered_sorted(
+            sort_key="duration",
+            key_max_value={"duration": hparams["avoid_if_longer_than"]},
+        )
+        # when sorting do not shuffle in dataloader ! otherwise is pointless
+        hparams["dataloader_options"]["shuffle"] = False
+
+    elif hparams["sorting"] == "descending":
+        train_data = train_data.filtered_sorted(
+            sort_key="duration",
+            reverse=True,
+            key_max_value={"duration": hparams["avoid_if_longer_than"]},
+        )
+        # when sorting do not shuffle in dataloader ! otherwise is pointless
+        hparams["dataloader_options"]["shuffle"] = False
+
+    elif hparams["sorting"] == "random":
+        pass
+
+    else:
+        raise NotImplementedError(
+            "sorting must be random, ascending or descending"
+        )
+
+    valid_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
+        csv_path=hparams["valid_csv"], replacements={"data_root": data_folder},
+    )
+    # We also sort the validation data so it is faster to validate
+    valid_data = valid_data.filtered_sorted(sort_key="duration")
+    test_datasets = {}
+    for csv_file in hparams["test_csv"]:
+        name = Path(csv_file).stem
+        test_datasets[name] = sb.dataio.dataset.DynamicItemDataset.from_csv(
+            csv_path=csv_file, replacements={"data_root": data_folder}
+        )
+        test_datasets[name] = test_datasets[name].filtered_sorted(
+            sort_key="duration"
+        )
+
+    datasets = [train_data, valid_data] + [i for k, i in test_datasets.items()]
+
+
+    # 2. Define audio pipeline:
+    @sb.utils.data_pipeline.takes("wav")
+    @sb.utils.data_pipeline.provides("sig")
+    def audio_pipeline(wav):
+        info = torchaudio.info(wav)
+        sig = sb.dataio.dataio.read_audio(wav)
+        if len(sig.shape)>1 :
+            sig = torch.mean(sig, dim=1)
+        resampled = torchaudio.transforms.Resample(
+            info.sample_rate, hparams["sample_rate"],
+        )(sig)
+        return resampled
+
+    sb.dataio.dataset.add_dynamic_item(datasets, audio_pipeline)
+    label_encoder = sb.dataio.encoder.CTCTextEncoder()
+
+    # 3. Define text pipeline:
+    @sb.utils.data_pipeline.takes("wrd")
+    @sb.utils.data_pipeline.provides(
+        "wrd", "char_list", "tokens_list", "tokens"
+    )
+    def text_pipeline(wrd):
+        yield wrd
+        char_list = list(wrd)
+        yield char_list
+        tokens_list = label_encoder.encode_sequence(char_list)
+        yield tokens_list
+        tokens = torch.LongTensor(tokens_list)
+        yield tokens
+
+    sb.dataio.dataset.add_dynamic_item(datasets, text_pipeline)
+    lab_enc_file = os.path.join(hparams["save_folder"], "label_encoder.txt")
+    special_labels = {
+        "blank_label": hparams["blank_index"],
+        "unk_label": hparams["unk_index"]
+    }
+    label_encoder.load_or_create(
+        path=lab_enc_file,
+        from_didatasets=[train_data],
+        output_key="char_list",
+        special_labels=special_labels,
+        sequence_input=True,
+    )
+
+    # 4. Set output:
+    sb.dataio.dataset.set_output_keys(
+        datasets, ["id", "sig", "wrd", "char_list", "tokens"],
+    )
+    return train_data, valid_data,test_datasets, label_encoder
+
+class ASR(sb.core.Brain):
+    def compute_forward(self, batch, stage):
+        """Forward computations from the waveform batches to the output probabilities."""
+
+        batch = batch.to(self.device)
+        wavs, wav_lens = batch.sig
+        wavs, wav_lens = wavs.to(self.device), wav_lens.to(self.device)
+
+        if stage == sb.Stage.TRAIN:
+            if hasattr(self.hparams, "augmentation"):
+                wavs = self.hparams.augmentation(wavs, wav_lens)
+
+        # Forward pass
+        feats = self.modules.wav2vec2(wavs, wav_lens)
+        x = self.modules.enc(feats)
+        logits = self.modules.ctc_lin(x)
+        p_ctc = self.hparams.log_softmax(logits)
+
+        return p_ctc, wav_lens
+
+    def custom_encode(self,wavs,wav_lens) :
+        wavs = wavs.to(self.device)
+        if(wav_lens is not None): wav_lens.to(self.device)
+
+        feats = self.modules.wav2vec2(wavs, wav_lens)
+        x = self.modules.enc(feats)
+        logits = self.modules.ctc_lin(x)
+        p_ctc = self.hparams.log_softmax(logits)
+
+        return feats,p_ctc
+
+
+
+    def compute_objectives(self, predictions, batch, stage):
+        """Computes the loss (CTC) given predictions and targets."""
+
+        p_ctc, wav_lens = predictions
+
+        ids = batch.id
+        tokens, tokens_lens = batch.tokens
+
+        loss = self.hparams.ctc_cost(p_ctc, tokens, wav_lens, tokens_lens)
+
+        if stage != sb.Stage.TRAIN:
+            predicted_tokens = sb.decoders.ctc_greedy_decode(
+                p_ctc, wav_lens, blank_id=self.hparams.blank_index
+            )
+            # Decode token terms to words
+            if self.hparams.use_language_modelling:
+                predicted_words = []
+                for logs in p_ctc:
+                    text = decoder.decode(logs.detach().cpu().numpy())
+                    predicted_words.append(text.split(" "))
+            else:
+                predicted_words = [
+                    "".join(self.tokenizer.decode_ndim(utt_seq)).split(" ")
+                    for utt_seq in predicted_tokens
+                ]
+            # Convert indices to words
+            target_words = [wrd.split(" ") for wrd in batch.wrd]
+
+            self.wer_metric.append(ids, predicted_words, target_words)
+            self.cer_metric.append(ids, predicted_words, target_words)
+
+        return loss
+
+    def fit_batch(self, batch):
+        """Train the parameters given a single batch in input"""
+        should_step = self.step % self.grad_accumulation_factor == 0
+        # Managing automatic mixed precision
+        # TOFIX: CTC fine-tuning currently is unstable
+        # This is certainly due to CTC being done in fp16 instead of fp32
+        if self.auto_mix_prec:
+            with torch.cuda.amp.autocast():
+                with self.no_sync():
+                    outputs = self.compute_forward(batch, sb.Stage.TRAIN)
+                loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)
+            with self.no_sync(not should_step):
+                self.scaler.scale(
+                    loss / self.grad_accumulation_factor
+                ).backward()
+            if should_step:
+
+                if not self.hparams.wav2vec2.freeze:
+                    self.scaler.unscale_(self.wav2vec_optimizer)
+                self.scaler.unscale_(self.model_optimizer)
+                if self.check_gradients(loss):
+                    if not self.hparams.wav2vec2.freeze:
+                        if self.optimizer_step >= self.hparams.warmup_steps:
+                            self.scaler.step(self.wav2vec_optimizer)
+                    self.scaler.step(self.model_optimizer)
+                self.scaler.update()
+                self.zero_grad()
+                self.optimizer_step += 1
+        else:
+            # This is mandatory because HF models have a weird behavior with DDP
+            # on the forward pass
+            with self.no_sync():
+                outputs = self.compute_forward(batch, sb.Stage.TRAIN)
+
+            loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)
+
+            with self.no_sync(not should_step):
+                (loss / self.grad_accumulation_factor).backward()
+            if should_step:
+                if self.check_gradients(loss):
+                    if not self.hparams.wav2vec2.freeze:
+                        if self.optimizer_step >= self.hparams.warmup_steps:
+                            self.wav2vec_optimizer.step()
+                    self.model_optimizer.step()
+                self.zero_grad()
+                self.optimizer_step += 1
+
+        self.on_fit_batch_end(batch, outputs, loss, should_step)
+        return loss.detach().cpu()
+
+    def evaluate_batch(self, batch, stage):
+        """Computations needed for validation/test batches"""
+        predictions = self.compute_forward(batch, stage=stage)
+        with torch.no_grad():
+            loss = self.compute_objectives(predictions, batch, stage=stage)
+        return loss.detach()
+
+    def on_stage_start(self, stage, epoch):
+        """Gets called at the beginning of each epoch"""
+        if stage != sb.Stage.TRAIN:
+            self.cer_metric = self.hparams.cer_computer()
+            self.wer_metric = self.hparams.error_rate_computer()
+
+    def on_stage_end(self, stage, stage_loss, epoch):
+        """Gets called at the end of an epoch."""
+        # Compute/store important stats
+        stage_stats = {"loss": stage_loss}
+        if stage == sb.Stage.TRAIN:
+            self.train_stats = stage_stats
+        else:
+            stage_stats["CER"] = self.cer_metric.summarize("error_rate")
+            stage_stats["WER"] = self.wer_metric.summarize("error_rate")
+
+        # Perform end-of-iteration things, like annealing, logging, etc.
+        if stage == sb.Stage.VALID:
+            old_lr_model, new_lr_model = self.hparams.lr_annealing_model(
+                stage_stats["loss"]
+            )
+            old_lr_wav2vec, new_lr_wav2vec = self.hparams.lr_annealing_wav2vec(
+                stage_stats["loss"]
+            )
+            sb.nnet.schedulers.update_learning_rate(
+                self.model_optimizer, new_lr_model
+            )
+            if not self.hparams.wav2vec2.freeze:
+                sb.nnet.schedulers.update_learning_rate(
+                    self.wav2vec_optimizer, new_lr_wav2vec
+                )
+            self.hparams.train_logger.log_stats(
+                stats_meta={
+                    "epoch": epoch,
+                    "lr_model": old_lr_model,
+                    "lr_wav2vec": old_lr_wav2vec,
+                },
+                train_stats=self.train_stats,
+                valid_stats=stage_stats,
+            )
+            self.checkpointer.save_and_keep_only(
+                meta={"WER": stage_stats["WER"]}, min_keys=["WER"],
+            )
+        elif stage == sb.Stage.TEST:
+            self.hparams.train_logger.log_stats(
+                stats_meta={"Epoch loaded": self.hparams.epoch_counter.current},
+                test_stats=stage_stats,
+            )
+            with open(self.hparams.wer_file, "w") as w:
+                self.wer_metric.write_stats(w)
+
+    def init_optimizers(self):
+        "Initializes the wav2vec2 optimizer and model optimizer"
+
+        # If the wav2vec encoder is unfrozen, we create the optimizer
+        if not self.hparams.wav2vec2.freeze:
+            self.wav2vec_optimizer = self.hparams.wav2vec_opt_class(
+                self.modules.wav2vec2.parameters()
+            )
+            if self.checkpointer is not None:
+                self.checkpointer.add_recoverable(
+                    "wav2vec_opt", self.wav2vec_optimizer
+                )
+
+        self.model_optimizer = self.hparams.model_opt_class(
+            self.hparams.model.parameters()
+        )
+
+        if self.checkpointer is not None:
+            self.checkpointer.add_recoverable("modelopt", self.model_optimizer)
+
+    def zero_grad(self, set_to_none=False):
+        if not self.hparams.wav2vec2.freeze:
+            self.wav2vec_optimizer.zero_grad(set_to_none)
+        self.model_optimizer.zero_grad(set_to_none)
+
+
+"""
+label_encoder = sb.dataio.encoder.CTCTextEncoder()
+
+train_data, valid_data, test_datasets, label_encoder = dataio_prepare(
+        hparams
+    )
+
+
+# We dynamicaly add the tokenizer to our brain class.
+# NB: This tokenizer corresponds to the one used for the LM!!
+"""
+from speechbrain.pretrained import EncoderASR,EncoderDecoderASR
+french_asr_model = EncoderASR.from_hparams(source="speechbrain/asr-wav2vec2-commonvoice-fr", savedir="pretrained_models/asr-wav2vec2-commonvoice-fr").cuda()
+#french_asr_model = "r"
+
+cvhparams_file, cvrun_opts, cvoverrides = sb.parse_arguments(["en_cv.yaml"])
+with open(cvhparams_file) as cvfin:
+    cvhparams = load_hyperpyyaml(cvfin, cvoverrides)
+english_asr_model = ASRCV(
+        modules=cvhparams["modules"],
+        hparams=cvhparams,
+        run_opts=cvrun_opts,
+        checkpointer=cvhparams["checkpointer"],
+    )
+english_asr_model.checkpointer.recover_if_possible()
+asr_brain = ASR(
+    modules=hparams["modules"],
+    hparams=hparams,
+    run_opts=run_opts,
+    checkpointer=hparams["checkpointer"],
+)
+asr_brain.checkpointer.recover_if_possible()
+asr_brain.modules.eval()
+english_asr_model.modules.eval()
+french_asr_model.mods.eval()
+"""
+asr_brain.tokenizer = label_encoder
+
+# Testing
+real = True
+if real :
+    for k in test_datasets.keys():  # keys are test_clean, test_other etc
+        asr_brain.hparams.wer_file = os.path.join(
+            hparams["output_folder"], "wer_{}.txt".format(k)
+        )
+        asr_brain.evaluate(
+            test_datasets[k], test_loader_kwargs=hparams["dataloader_options"]
+        )
+"""
+
+"""
+from torch.nn.utils.rnn import pad_sequence
+def load_paths(wavs_path):
+    waveforms = []
+    for path in wavs_path :
+        waveform, _ = torchaudio.load(path)
+        waveforms.append(waveform.squeeze(0))
+    # normalize array length to the bigger arrays by pading with 0's
+    padded_arrays = pad_sequence(waveforms, batch_first=True)
+    return torch.tensor(padded_arrays)
+
+waveform = load_paths(["/content/drive/MyDrive/tunisian_corpora/tunisian_without_wavlm/samples/Salah10.wav","/content/drive/MyDrive/tunisian_corpora/tunisian_without_wavlm/samples/Salah10.wav"])
+embeddings, posteriogram = asr_brain.custom_encode(waveform,None)
+print(embeddings.shape)
+print(posteriogram.shape)
+"""
+
+from speechbrain.pretrained import EncoderASR,EncoderDecoderASR
+import torchaudio
+import  speechbrain as sb
+import torch
+from torch.nn.utils.rnn import pad_sequence
+import torch
+import speechbrain as sb
+import numpy as np
+import torch.optim as optim
+import torch.nn as nn
+
+# Commented out IPython magic to ensure Python compatibility.
+# %ls
+
+#UTILS FUNCTIOJNS
+def get_size_dimensions(arr):
+    size_dimensions = []
+    while isinstance(arr, list):
+        size_dimensions.append(len(arr))
+        arr = arr[0]
+    return size_dimensions
+
+def scale_array(batch,n):
+    scaled_batch = []
+
+    for array in batch:
+        if(n < len(array)): raise ValueError("Cannot scale Array down")
+
+        repeat = round(n/len(array))+1
+        scaled_length_array= []
+
+        for i in array:
+            for j in range(repeat) :
+                if(len(scaled_length_array) == n): break
+                scaled_length_array.append(i)
+
+        scaled_batch.append(scaled_length_array)
+
+    return torch.tensor(scaled_batch)
+
+
+def load_paths(wavs_path):
+    waveforms = []
+    for path in wavs_path :
+        waveform, _ = torchaudio.load(path)
+        waveforms.append(waveform.squeeze(0))
+    # normalize array length to the bigger arrays by pading with 0's
+    padded_arrays = pad_sequence(waveforms, batch_first=True)
+    return torch.tensor(padded_arrays)
+
+
+
+def word_to_vec(input_string):
+    mapping= {'a': 1, 'b': 2, 'c': 3, 'd': 4, 'e': 5, 'f': 6, 'g': 7, 'h': 8, 'i': 9, 'j': 10, 'k': 11, 'l': 12, 'm': 13, 'n': 14, 'o': 15, 'p': 16, 'q': 17, 'r': 18, 's': 19, 't': 20, 'u': 21, 'v': 22, 'w': 23, 'x': 24, 'y': 25, 'z': 26, 'ا': 27, 'ب': 28, 'ت': 29, 'ث': 30, 'ج': 31, 'ح': 32, 'خ': 33, 'د': 34, 'ذ': 35, 'ر': 36, 'ز': 37, 'س': 38, 'ش': 39, 'ص': 40, 'ض': 41, 'ط': 42, 'ظ': 43, 'ع': 44, 'غ': 45, 'ف': 46, 'ق': 47, 'ك': 48, 'ل': 49, 'م': 50, 'ن': 51, 'ه': 52, 'و': 53, 'ي': 54,' ':55}
+
+    numbers = [mapping[word] for word in input_string if word in mapping]
+    return numbers
+
+device = 'cuda'
+verbose = 0
+#FLOW LEVEL FUNCTIONS
+def merge_strategy(embeddings1, embeddings2, embeddings3,post1, post2,post3):
+
+
+    post1 = post1.to(device)
+    post2 = post2.to(device)
+    post3 = post3.to(device)
+    embeddings1 = embeddings1.to(device)
+    embeddings2 = embeddings2.to(device)
+    embeddings3 = embeddings3.to(device)
+
+    posteriograms_merged = torch.cat((post1,post2,post3),dim=2)
+    embeddings_merged = torch.cat((embeddings1,embeddings2,embeddings3),dim=2)
+
+    if(verbose !=0):
+      print('MERGED POST ',posteriograms_merged.shape)
+      print('MERGED emb ',embeddings_merged.shape)
+
+    return torch.cat((posteriograms_merged,embeddings_merged),dim=2).to(device)
+
+def decode(model,wavs,wav_lens):
+
+    with torch.no_grad():
+        wav_lens = wav_lens.to(model.device)
+        encoder_out = model.encode_batch(wavs, wav_lens)
+        predictions = model.decoding_function(encoder_out, wav_lens)
+        return predictions
+
+def middle_layer(batch, lens):
+
+    tn_embeddings, tn_posteriogram = asr_brain.custom_encode(batch,None)
+
+    fr_embeddings = french_asr_model.mods.encoder.wav2vec2(batch)
+    fr_posteriogram =french_asr_model.encode_batch(batch,lens)
+    en_embeddings = english_asr_model.modules.wav2vec2(batch, lens)
+    x = english_asr_model.modules.enc(en_embeddings)
+    en_posteriogram = english_asr_model.modules.ctc_lin(x)
+    #scores, en_posteriogram = english_asr_model.mods.decoder(en_embeddings ,lens)
+    if(verbose !=0):
+      print('[EMBEDDINGS] FR:',fr_embeddings.shape, "EN:",en_embeddings.shape, "TN:", tn_embeddings.shape)
+      print('[POSTERIOGRAM] FR:',fr_posteriogram.shape, "EN:",en_posteriogram.shape,"TN:",tn_posteriogram.shape)
+
+
+    bilangual_sample = merge_strategy(fr_embeddings,en_embeddings,tn_embeddings,fr_posteriogram,en_posteriogram,tn_posteriogram)
+    return bilangual_sample
+
+class Mixer(sb.core.Brain):
+
+    def compute_forward(self, batch, stage):
+        """Forward computations from the waveform batches to the output probabilities."""
+        wavs, wav_lens = batch.sig
+        wavs, wav_lens = wavs.to(self.device), wav_lens.to(self.device)
+
+        if stage == sb.Stage.TRAIN:
+            if hasattr(self.hparams, "augmentation"):
+                wavs = self.hparams.augmentation(wavs, wav_lens)
+
+        multi_langual_feats = middle_layer(wavs, wav_lens)
+        multi_langual_feats= multi_langual_feats.to(device)
+        feats, _ = self.modules.enc(multi_langual_feats)
+        logits = self.modules.ctc_lin(feats)
+        p_ctc = self.hparams.log_softmax(logits)
+        
+        if stage!= sb.Stage.TRAIN:
+            p_tokens = sb.decoders.ctc_greedy_decode(
+                p_ctc, wav_lens, blank_id=self.hparams.blank_index
+            )
+        else : 
+            p_tokens = None
+        return p_ctc, wav_lens, p_tokens
+
+    def compute_objectives(self, predictions, batch, stage):
+        """Computes the loss (CTC) given predictions and targets."""
+
+        p_ctc, wav_lens , predicted_tokens= predictions
+
+        ids = batch.id
+        tokens, tokens_lens = batch.tokens
+
+        loss = self.hparams.ctc_cost(p_ctc, tokens, wav_lens, tokens_lens)
+
+
+        if stage == sb.Stage.VALID:
+            predicted_words = [
+                "".join(self.tokenizer.decode_ndim(utt_seq)).split(" ")
+                for utt_seq in predicted_tokens
+            ]
+            target_words = [wrd.split(" ") for wrd in batch.wrd]
+            self.wer_metric.append(ids, predicted_words, target_words)
+            self.cer_metric.append(ids, predicted_words, target_words)
+        if stage ==sb.Stage.TEST : 
+            if self.hparams.language_modelling:
+                predicted_words = []
+                for logs in p_ctc:
+                    text = decoder.decode(logs.detach().cpu().numpy())
+                    predicted_words.append(text.split(" "))
+            else : 
+                predicted_words = [
+                    "".join(self.tokenizer.decode_ndim(utt_seq)).split(" ")
+                    for utt_seq in predicted_tokens
+                ]
+
+            target_words = [wrd.split(" ") for wrd in batch.wrd]
+            self.wer_metric.append(ids, predicted_words, target_words)
+            self.cer_metric.append(ids, predicted_words, target_words)
+
+        return loss
+
+    def fit_batch(self, batch):
+        """Train the parameters given a single batch in input"""
+        should_step = self.step % self.grad_accumulation_factor == 0
+        # Managing automatic mixed precision
+        # TOFIX: CTC fine-tuning currently is unstable
+        # This is certainly due to CTC being done in fp16 instead of fp32
+        if self.auto_mix_prec:
+            with torch.cuda.amp.autocast():
+                with self.no_sync():
+                    outputs = self.compute_forward(batch, sb.Stage.TRAIN)
+                loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)
+            with self.no_sync(not should_step):
+                self.scaler.scale(
+                    loss / self.grad_accumulation_factor
+                ).backward()
+            if should_step:
+
+
+                self.scaler.unscale_(self.model_optimizer)
+                if self.check_gradients(loss):
+                    self.scaler.step(self.model_optimizer)
+                self.scaler.update()
+                self.zero_grad()
+                self.optimizer_step += 1
+        else:
+            # This is mandatory because HF models have a weird behavior with DDP
+            # on the forward pass
+            with self.no_sync():
+                outputs = self.compute_forward(batch, sb.Stage.TRAIN)
+
+            loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)
+
+            with self.no_sync(not should_step):
+                (loss / self.grad_accumulation_factor).backward()
+            if should_step:
+                if self.check_gradients(loss):
+                    self.model_optimizer.step()
+                self.zero_grad()
+                self.optimizer_step += 1
+
+        self.on_fit_batch_end(batch, outputs, loss, should_step)
+        return loss.detach().cpu()
+
+    def evaluate_batch(self, batch, stage):
+        """Computations needed for validation/test batches"""
+        predictions = self.compute_forward(batch, stage=stage)
+        with torch.no_grad():
+            loss = self.compute_objectives(predictions, batch, stage=stage)
+        return loss.detach()
+
+    def on_stage_start(self, stage, epoch):
+        """Gets called at the beginning of each epoch"""
+        if stage != sb.Stage.TRAIN:
+            self.cer_metric = self.hparams.cer_computer()
+            self.wer_metric = self.hparams.error_rate_computer()
+
+    def on_stage_end(self, stage, stage_loss, epoch):
+        """Gets called at the end of an epoch."""
+        # Compute/store important stats
+        stage_stats = {"loss": stage_loss}
+        if stage == sb.Stage.TRAIN:
+            self.train_stats = stage_stats
+        else:
+            stage_stats["CER"] = self.cer_metric.summarize("error_rate")
+            stage_stats["WER"] = self.wer_metric.summarize("error_rate")
+
+        # Perform end-of-iteration things, like annealing, logging, etc.
+        if stage == sb.Stage.VALID:
+            old_lr_model, new_lr_model = self.hparams.lr_annealing_model(
+                stage_stats["loss"]
+            )
+            sb.nnet.schedulers.update_learning_rate(
+                self.model_optimizer, new_lr_model
+            )
+            self.hparams.train_logger.log_stats(
+                stats_meta={
+                    "epoch": epoch,
+                    "lr_model": old_lr_model,
+                },
+                train_stats=self.train_stats,
+                valid_stats=stage_stats,
+            )
+            self.checkpointer.save_and_keep_only(
+                meta={"WER": stage_stats["WER"]}, min_keys=["WER"],
+            )
+        elif stage == sb.Stage.TEST:
+            self.hparams.train_logger.log_stats(
+                stats_meta={"Epoch loaded": self.hparams.epoch_counter.current},
+                test_stats=stage_stats,
+            )
+            with open(self.hparams.wer_file, "w") as w:
+                self.wer_metric.write_stats(w)
+
+    def init_optimizers(self):
+
+        self.model_optimizer = self.hparams.model_opt_class(
+            self.hparams.model.parameters()
+        )
+
+        if self.checkpointer is not None:
+            self.checkpointer.add_recoverable("modelopt", self.model_optimizer)
+
+    def zero_grad(self, set_to_none=False):
+
+        self.model_optimizer.zero_grad(set_to_none)
+
+
+hparams_file, run_opts, overrides = sb.parse_arguments(sys.argv[1:])
+
+# If distributed_launch=True then
+# create ddp_group with the right communication protocol
+sb.utils.distributed.ddp_init_group(run_opts)
+
+with open(hparams_file) as fin:
+    hparams = load_hyperpyyaml(fin, overrides)
+
+# Create experiment directory
+sb.create_experiment_directory(
+    experiment_directory=hparams["output_folder"],
+    hyperparams_to_save=hparams_file,
+    overrides=overrides,
+)
+def read_labels_file(labels_file):
+    with open(labels_file, "r",encoding="utf-8") as lf:
+        lines = lf.read().splitlines()
+        division = "==="
+        numbers = {}
+        for line in lines :
+            if division in line :
+                break
+            string, number = line.split("=>")
+            number = int(number)
+            string = string[1:-2]
+            numbers[number] = string
+        return [numbers[x] for x in range(len(numbers))]
+train_data, valid_data, test_datasets, label_encoder = dataio_prepare(
+        hparams
+    )
+
+
+labels = read_labels_file(os.path.join(hparams["save_folder"], "label_encoder.txt"))
+labels = [""] + labels[1:-1] + ["1"] 
+if hparams["language_modelling"]:
+    decoder = build_ctcdecoder(
+        labels,
+        kenlm_model_path=hparams["ngram_lm_path"],  # either .arpa or .bin file
+        alpha=0.5,  # tuned on a val set
+        beta=1,  # tuned on a val set
+    )
+
+
+
+
+mixer = Mixer(
+    modules=hparams["modules"],
+    hparams=hparams,
+    run_opts=run_opts,
+    checkpointer=hparams["checkpointer"],
+)
+mixer.tokenizer = label_encoder
+
+
+mixer.fit(
+    mixer.hparams.epoch_counter,
+    train_data,
+    valid_data,
+    train_loader_kwargs=hparams["dataloader_options"],
+    valid_loader_kwargs=hparams["test_dataloader_options"],
+)
+print(test_datasets.keys())
+for k in test_datasets.keys():  # keys are test_clean, test_other etc
+    mixer.hparams.wer_file = os.path.join(
+        hparams["output_folder"], "wer_{}.txt".format(k)
+    )
+    mixer.evaluate(
+        test_datasets[k], test_loader_kwargs=hparams["test_dataloader_options"]
+        )
+

TunisianASR/results/14epoch_tunisian/<seed>/env.log

@@ -0,0 +1,97 @@
+SpeechBrain system description
+==============================
+Python version:
+3.9.12 | packaged by conda-forge | (main, Mar 24 2022, 23:22:55) 
+[GCC 10.3.0]
+==============================
+Installed Python packages:
+aiohttp==3.8.5
+aiosignal==1.3.1
+async-timeout==4.0.3
+attrs==23.1.0
+audioread==3.0.0
+certifi==2023.7.22
+cffi==1.15.1
+charset-normalizer==3.2.0
+click==8.1.7
+cmake==3.27.2
+datasets==2.14.4
+decorator==5.1.1
+dill==0.3.7
+exceptiongroup==1.1.3
+filelock==3.12.3
+frozenlist==1.4.0
+fsspec==2023.6.0
+huggingface-hub==0.16.4
+HyperPyYAML==1.2.1
+hypothesis==6.82.7
+idna==3.4
+Jinja2==3.1.2
+jiwer==3.0.3
+joblib==1.3.2
+kenlm @ https://github.com/kpu/kenlm/archive/master.zip#sha256=4d002dcde70b52d519cafff4dc0008696c40cff1c9184a531b40c7b45905be6b
+lazy_loader==0.3
+librosa==0.10.1
+lit==16.0.6
+llvmlite==0.40.1
+MarkupSafe==2.1.3
+mpmath==1.3.0
+msgpack==1.0.5
+multidict==6.0.4
+multiprocess==0.70.15
+networkx==3.1
+numba==0.57.1
+numpy==1.24.4
+nvidia-cublas-cu11==11.10.3.66
+nvidia-cuda-cupti-cu11==11.7.101
+nvidia-cuda-nvrtc-cu11==11.7.99
+nvidia-cuda-runtime-cu11==11.7.99
+nvidia-cudnn-cu11==8.5.0.96
+nvidia-cufft-cu11==10.9.0.58
+nvidia-curand-cu11==10.2.10.91
+nvidia-cusolver-cu11==11.4.0.1
+nvidia-cusparse-cu11==11.7.4.91
+nvidia-nccl-cu11==2.14.3
+nvidia-nvtx-cu11==11.7.91
+packaging==23.1
+pandas==2.0.3
+platformdirs==3.10.0
+pooch==1.7.0
+pyarrow==13.0.0
+pycparser==2.21
+pyctcdecode==0.5.0
+pygtrie==2.5.0
+python-dateutil==2.8.2
+pytz==2023.3
+PyYAML==6.0.1
+rapidfuzz==3.2.0
+regex==2023.8.8
+requests==2.31.0
+ruamel.yaml==0.17.28
+ruamel.yaml.clib==0.2.7
+safetensors==0.3.3
+scikit-learn==1.3.0
+scipy==1.11.2
+sentencepiece==0.1.99
+six==1.16.0
+sortedcontainers==2.4.0
+soundfile==0.12.1
+soxr==0.3.6
+speechbrain==0.5.15
+sympy==1.12
+threadpoolctl==3.2.0
+tokenizers==0.13.3
+torch==2.0.1
+torchaudio==2.0.2
+tqdm==4.66.1
+transformers==4.32.1
+triton==2.0.0
+typing_extensions==4.7.1
+tzdata==2023.3
+urllib3==2.0.4
+xxhash==3.3.0
+yarl==1.9.2
+==============================
+Could not get git revision==============================
+CUDA version:
+11.7

TunisianASR/results/14epoch_tunisian/<seed>/hyperparams.yaml

@@ -0,0 +1,200 @@
+# Generated 2023-09-08 from:
+# /gpfsssd/scratch/rech/nou/uzn19yk/switched_data/semi_supervised_test_tunisian.yaml
+# yamllint disable
+# ################################
+# Model: wav2vec2 + DNN + CTC
+# Augmentation: SpecAugment
+# Authors: Titouan Parcollet 2021
+# ################################
+
+# Seed needs to be set at top of yaml, before objects with parameters are made
+seed: 1234
+__set_seed: !!python/object/apply:torch.manual_seed [1234]
+output_folder: 
+  /gpfsstore/rech/nou/uzn19yk/switched_code_tunisian/train/tunisian_asr/results/14epoch_tunisian/<seed>
+wer_file: 
+  /gpfsstore/rech/nou/uzn19yk/switched_code_tunisian/train/tunisian_asr/results/14epoch_tunisian/1234/wer.txt
+save_folder: 
+  /gpfsstore/rech/nou/uzn19yk/switched_code_tunisian/train/tunisian_asr/results/14epoch_tunisian/1234/save
+train_log: 
+  /gpfsstore/rech/nou/uzn19yk/switched_code_tunisian/train/tunisian_asr/results/14epoch_tunisian/1234/train_log.txt
+
+# URL for the biggest LeBenchmark wav2vec french.
+wav2vec2_folder: 
+  /gpfsstore/rech/nou/uzn19yk/switched_code_tunisian/train/tunisian_asr/results/14epoch_tunisian/1234/save/wav2vec2_checkpoint
+
+# Data files
+data_folder: /gpfsscratch/rech/nou/uzn19yk/tunisian_junk  # e.g, /localscratch/cv-corpus-5.1-2020-06-22/fr
+train_tsv_file: /gpfsscratch/rech/nou/uzn19yk/tunisian_junk/train.tsv # Standard CommonVoice .tsv files
+dev_tsv_file: /gpfsscratch/rech/nou/uzn19yk/tunisian_junk/dev.tsv # Standard CommonVoice .tsv files
+test_tsv_file: /gpfsscratch/rech/nou/uzn19yk/tunisian_junk/test.tsv # Standard CommonVoice .tsv files
+accented_letters: true
+language: fr # use 'it' for Italian, 'rw' for Kinyarwanda, 'en' for english
+train_csv: /gpfsscratch/rech/nou/uzn19yk/tunisian_csvs/good_final/train.csv
+valid_csv: /gpfsscratch/rech/nou/uzn19yk/tunisian_csvs/good_final/dev.csv
+test_csv:
+- /gpfsscratch/rech/nou/uzn19yk/tunisian_semi/unlabeled.csv
+
+skip_prep: true # Skip data preparation
+
+use_language_modelling: true
+ngram_lm_path: arpas/indomain.arpa
+
+# We remove utterance slonger than 10s in the train/dev/test sets as
+# longer sentences certainly correspond to "open microphones".
+avoid_if_longer_than: 10.0
+avoid_if_shorter_than: 1.2
+
+
+# Training parameters
+number_of_epochs: 14
+lr: 1.0
+lr_wav2vec: 0.0001
+sorting: ascending
+auto_mix_prec: false
+sample_rate: 16000
+ckpt_interval_minutes: 30 # save checkpoint every N min
+
+# With data_parallel batch_size is split into N jobs
+# With DDP batch_size is multiplied by N jobs
+# Must be 6 per GPU to fit 16GB of VRAM
+batch_size: 10
+test_batch_size: 4
+
+dataloader_options:
+  batch_size: 10
+  num_workers: 6
+test_dataloader_options:
+  batch_size: 4
+  num_workers: 6
+
+# BPE parameters
+token_type: char  # ["unigram", "bpe", "char"]
+character_coverage: 1.0
+
+# Model parameters
+# activation: !name:torch.nn.LeakyReLU
+wav2vec_output_dim: 1024
+dnn_neurons: 1024
+freeze_wav2vec: false
+freeze_feature_extractor: true
+dropout: 0.15
+warmup_steps: 500 # The wav2vec 2 model isn't updated for this amount of steps
+
+# Outputs
+output_neurons: 40  # BPE size, index(blank/eos/bos) = 0
+
+# Decoding parameters
+# Be sure that the bos and eos index match with the BPEs ones
+blank_index: 0
+unk_index: 1
+
+#
+# Functions and classes
+#
+epoch_counter: &id007 !new:speechbrain.utils.epoch_loop.EpochCounter
+
+  limit: 14
+
+augmentation: !new:speechbrain.lobes.augment.TimeDomainSpecAugment
+  sample_rate: 16000
+  speeds: [95, 100, 105]
+
+enc: &id002 !new:speechbrain.nnet.containers.Sequential
+  input_shape: [null, null, 1024]
+  linear1: !name:speechbrain.nnet.linear.Linear
+    n_neurons: 1024
+    bias: true
+  bn1: !name:speechbrain.nnet.normalization.BatchNorm1d
+  activation: !new:torch.nn.LeakyReLU
+  drop: !new:torch.nn.Dropout
+    p: 0.15
+  linear2: !name:speechbrain.nnet.linear.Linear
+    n_neurons: 1024
+    bias: true
+  bn2: !name:speechbrain.nnet.normalization.BatchNorm1d
+  activation2: !new:torch.nn.LeakyReLU
+  drop2: !new:torch.nn.Dropout
+    p: 0.15
+  linear3: !name:speechbrain.nnet.linear.Linear
+    n_neurons: 1024
+    bias: true
+  bn3: !name:speechbrain.nnet.normalization.BatchNorm1d
+  activation3: !new:torch.nn.LeakyReLU
+
+wav2vec2: &id001 !new:speechbrain.lobes.models.huggingface_wav2vec.HuggingFaceWav2Vec2
+  source: /gpfsstore/rech/nou/uzn19yk/wavlm/
+  output_norm: false
+  freeze: false
+  freeze_feature_extractor: true
+  save_path: 
+    /gpfsstore/rech/nou/uzn19yk/switched_code_tunisian/train/tunisian_asr/results/14epoch_tunisian/1234/save/wav2vec2_checkpoint
+
+#####
+# Uncomment this block if you prefer to use a Fairseq pretrained model instead
+# of a HuggingFace one. Here, we provide an URL that is obtained from the
+# Fairseq github for the multilingual XLSR.
+#
+#wav2vec2_url: https://dl.fbaipublicfiles.com/fairseq/wav2vec/xlsr_53_56k.pt
+#wav2vec2: !new:speechbrain.lobes.models.fairseq_wav2vec.FairseqWav2Vec2
+#    pretrained_path: !ref <wav2vec2_url>
+#    output_norm: True
+#    freeze: False
+#    save_path: !ref <save_folder>/wav2vec2_checkpoint/model.pt
+#####
+
+
+ctc_lin: &id003 !new:speechbrain.nnet.linear.Linear
+
+  input_size: 1024
+  n_neurons: 40
+
+log_softmax: !new:speechbrain.nnet.activations.Softmax
+  apply_log: true
+
+ctc_cost: !name:speechbrain.nnet.losses.ctc_loss
+  blank_index: 0
+
+modules:
+  wav2vec2: *id001
+  enc: *id002
+  ctc_lin: *id003
+model: &id004 !new:torch.nn.ModuleList
+- [*id002, *id003]
+model_opt_class: !name:torch.optim.Adadelta
+  lr: 1.0
+  rho: 0.95
+  eps: 1.e-8
+
+wav2vec_opt_class: !name:torch.optim.Adam
+  lr: 0.0001
+
+lr_annealing_model: &id005 !new:speechbrain.nnet.schedulers.NewBobScheduler
+  initial_value: 1.0
+  improvement_threshold: 0.0025
+  annealing_factor: 0.8
+  patient: 0
+
+lr_annealing_wav2vec: &id006 !new:speechbrain.nnet.schedulers.NewBobScheduler
+  initial_value: 0.0001
+  improvement_threshold: 0.0025
+  annealing_factor: 0.9
+  patient: 0
+
+checkpointer: !new:speechbrain.utils.checkpoints.Checkpointer
+  checkpoints_dir: 
+    /gpfsstore/rech/nou/uzn19yk/switched_code_tunisian/train/tunisian_asr/results/14epoch_tunisian/1234/save
+  recoverables:
+    wav2vec2: *id001
+    model: *id004
+    scheduler_model: *id005
+    scheduler_wav2vec: *id006
+    counter: *id007
+train_logger: !new:speechbrain.utils.train_logger.FileTrainLogger
+  save_file: 
+    /gpfsstore/rech/nou/uzn19yk/switched_code_tunisian/train/tunisian_asr/results/14epoch_tunisian/1234/train_log.txt
+
+error_rate_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats
+
+cer_computer: !name:speechbrain.utils.metric_stats.ErrorRateStats
+  split_tokens: true

TunisianASR/{train_semi.yaml → semi_trained.yaml}

@@ -7,7 +7,7 @@
 # Seed needs to be set at top of yaml, before objects with parameters are made
 seed: 1234
 __set_seed: !!python/object/apply:torch.manual_seed [!ref <seed>]
-output_folder: !ref semi_wavlm_large_tunisian_ctc/<seed>
+output_folder:  TunisianASR/results/14epoch_tunisian/1234/
 wer_file: !ref <output_folder>/wer.txt
 save_folder: !ref <output_folder>/save
 train_log: !ref <output_folder>/train_log.txt
@@ -16,19 +16,23 @@ train_log: !ref <output_folder>/train_log.txt
 wav2vec2_folder: !ref <save_folder>/wav2vec2_checkpoint
 
 # Data files
-data_folder: /path/to/data  # e.g, /localscratch/cv-corpus-5.1-2020-06-22/fr
+data_folder: /gpfsscratch/rech/nou/uzn19yk/tunisian_junk  # e.g, /localscratch/cv-corpus-5.1-2020-06-22/fr
 train_tsv_file: !ref <data_folder>/train.tsv  # Standard CommonVoice .tsv files
 dev_tsv_file: !ref <data_folder>/dev.tsv  # Standard CommonVoice .tsv files
 test_tsv_file: !ref <data_folder>/test.tsv  # Standard CommonVoice .tsv files
 accented_letters: True
 language: fr # use 'it' for Italian, 'rw' for Kinyarwanda, 'en' for english
+train_csv: /gpfsscratch/rech/nou/uzn19yk/tunisian_csvs/good_final/train.csv
+valid_csv: /gpfsscratch/rech/nou/uzn19yk/tunisian_csvs/good_final/dev.csv
 test_csv: 
-    - /path/to/test_data
+    - /gpfsscratch/rech/nou/uzn19yk/tunisian_csvs/full_annotation_test.csv
+    - /gpfsscratch/rech/nou/uzn19yk/tunisian_csvs/good_final/iwslt_test.csv
+    - /gpfsscratch/rech/nou/uzn19yk/tunisian_csvs/good_final/taric_test.csv
 
 skip_prep: True # Skip data preparation
 
 use_language_modelling: True
-ngram_lm_path: outdomain.arpa
+ngram_lm_path: arpas/outdomain.arpa
 
 # We remove utterance slonger than 10s in the train/dev/test sets as
 # longer sentences certainly correspond to "open microphones".
@@ -37,7 +41,7 @@ avoid_if_shorter_than: 1.2
 
 
 # Training parameters
-number_of_epochs: 12
+number_of_epochs: 14
 lr: 1.0
 lr_wav2vec: 0.0001
 sorting: ascending
@@ -112,12 +116,25 @@ enc: !new:speechbrain.nnet.containers.Sequential
     activation3: !new:torch.nn.LeakyReLU
 
 wav2vec2: !new:speechbrain.lobes.models.huggingface_wav2vec.HuggingFaceWav2Vec2
-    source: /gpfsstore/rech/nou/uzn19yk/wavlm/
+    source: wavlm-large/
     output_norm: False
     freeze: !ref <freeze_wav2vec>
     freeze_feature_extractor: !ref <freeze_feature_extractor>
     save_path: !ref <wav2vec2_folder>
 
+#####
+# Uncomment this block if you prefer to use a Fairseq pretrained model instead
+# of a HuggingFace one. Here, we provide an URL that is obtained from the
+# Fairseq github for the multilingual XLSR.
+#
+#wav2vec2_url: https://dl.fbaipublicfiles.com/fairseq/wav2vec/xlsr_53_56k.pt
+#wav2vec2: !new:speechbrain.lobes.models.fairseq_wav2vec.FairseqWav2Vec2
+#    pretrained_path: !ref <wav2vec2_url>
+#    output_norm: True
+#    freeze: False
+#    save_path: !ref <save_folder>/wav2vec2_checkpoint/model.pt
+#####
+
 
 ctc_lin: !new:speechbrain.nnet.linear.Linear
     input_size: !ref <dnn_neurons>

app.py

@@ -0,0 +1,768 @@
+import os
+import sys
+import torch
+import logging
+import speechbrain as sb
+from speechbrain.utils.distributed import run_on_main
+from hyperpyyaml import load_hyperpyyaml
+from pathlib import Path
+import torchaudio.transforms as T
+from cv_train import ASRCV
+import torchaudio
+import numpy as np
+import kenlm
+from pyctcdecode import build_ctcdecoder
+import re
+from torch.nn.utils.rnn import pad_sequence
+import torch.optim as optim
+import torch.nn as nn
+
+
+# Commented out IPython magic to ensure Python compatibility.
+hparams_file, run_opts, overrides = sb.parse_arguments(["TunisianASR/semi_trained.yaml"])
+
+# If distributed_launch=True then
+# create ddp_group with the right communication protocol
+sb.utils.distributed.ddp_init_group(run_opts)
+
+with open(hparams_file) as fin:
+    hparams = load_hyperpyyaml(fin, overrides)
+
+# Create experiment directory
+sb.create_experiment_directory(
+    experiment_directory=hparams["output_folder"],
+    hyperparams_to_save=hparams_file,
+    overrides=overrides,
+)
+# Dataset prep (parsing Librispeech)
+
+def dataio_prepare(hparams):
+    """This function prepares the datasets to be used in the brain class.
+    It also defines the data processing pipeline through user-defined functions."""
+
+    # 1. Define datasets
+    data_folder = hparams["data_folder"]
+
+    train_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
+        csv_path=hparams["train_csv"], replacements={"data_root": data_folder},
+    )
+
+    if hparams["sorting"] == "ascending":
+        # we sort training data to speed up training and get better results.
+        train_data = train_data.filtered_sorted(
+            sort_key="duration",
+            key_max_value={"duration": hparams["avoid_if_longer_than"]},
+        )
+        # when sorting do not shuffle in dataloader ! otherwise is pointless
+        hparams["dataloader_options"]["shuffle"] = False
+
+    elif hparams["sorting"] == "descending":
+        train_data = train_data.filtered_sorted(
+            sort_key="duration",
+            reverse=True,
+            key_max_value={"duration": hparams["avoid_if_longer_than"]},
+        )
+        # when sorting do not shuffle in dataloader ! otherwise is pointless
+        hparams["dataloader_options"]["shuffle"] = False
+
+    elif hparams["sorting"] == "random":
+        pass
+
+    else:
+        raise NotImplementedError(
+            "sorting must be random, ascending or descending"
+        )
+
+    valid_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
+        csv_path=hparams["valid_csv"], replacements={"data_root": data_folder},
+    )
+    # We also sort the validation data so it is faster to validate
+    valid_data = valid_data.filtered_sorted(sort_key="duration")
+    test_datasets = {}
+    for csv_file in hparams["test_csv"]:
+        name = Path(csv_file).stem
+        test_datasets[name] = sb.dataio.dataset.DynamicItemDataset.from_csv(
+            csv_path=csv_file, replacements={"data_root": data_folder}
+        )
+        test_datasets[name] = test_datasets[name].filtered_sorted(
+            sort_key="duration"
+        )
+
+    datasets = [train_data, valid_data] + [i for k, i in test_datasets.items()]
+
+
+    # 2. Define audio pipeline:
+    @sb.utils.data_pipeline.takes("wav")
+    @sb.utils.data_pipeline.provides("sig")
+    def audio_pipeline(wav):
+        info = torchaudio.info(wav)
+        sig = sb.dataio.dataio.read_audio(wav)
+        if len(sig.shape)>1 :
+            sig = torch.mean(sig, dim=1)
+        resampled = torchaudio.transforms.Resample(
+            info.sample_rate, hparams["sample_rate"],
+        )(sig)
+        return resampled
+
+    sb.dataio.dataset.add_dynamic_item(datasets, audio_pipeline)
+    label_encoder = sb.dataio.encoder.CTCTextEncoder()
+
+    # 3. Define text pipeline:
+    @sb.utils.data_pipeline.takes("wrd")
+    @sb.utils.data_pipeline.provides(
+        "wrd", "char_list", "tokens_list", "tokens"
+    )
+    def text_pipeline(wrd):
+        yield wrd
+        char_list = list(wrd)
+        yield char_list
+        tokens_list = label_encoder.encode_sequence(char_list)
+        yield tokens_list
+        tokens = torch.LongTensor(tokens_list)
+        yield tokens
+
+    sb.dataio.dataset.add_dynamic_item(datasets, text_pipeline)
+    lab_enc_file = os.path.join(hparams["save_folder"], "label_encoder.txt")
+    special_labels = {
+        "blank_label": hparams["blank_index"],
+        "unk_label": hparams["unk_index"]
+    }
+    label_encoder.load_or_create(
+        path=lab_enc_file,
+        from_didatasets=[train_data],
+        output_key="char_list",
+        special_labels=special_labels,
+        sequence_input=True,
+    )
+
+    # 4. Set output:
+    sb.dataio.dataset.set_output_keys(
+        datasets, ["id", "sig", "wrd", "char_list", "tokens"],
+    )
+    return train_data, valid_data,test_datasets, label_encoder
+
+class ASR(sb.core.Brain):
+    def compute_forward(self, batch, stage):
+        """Forward computations from the waveform batches to the output probabilities."""
+
+        batch = batch.to(self.device)
+        wavs, wav_lens = batch.sig
+        wavs, wav_lens = wavs.to(self.device), wav_lens.to(self.device)
+
+        if stage == sb.Stage.TRAIN:
+            if hasattr(self.hparams, "augmentation"):
+                wavs = self.hparams.augmentation(wavs, wav_lens)
+
+        # Forward pass
+        feats = self.modules.wav2vec2(wavs, wav_lens)
+        x = self.modules.enc(feats)
+        logits = self.modules.ctc_lin(x)
+        p_ctc = self.hparams.log_softmax(logits)
+
+        return p_ctc, wav_lens
+
+    def custom_encode(self,wavs,wav_lens) :
+        wavs = wavs.to("cpu")
+        if(wav_lens is not None): wav_lens.to(self.device)
+
+        feats = self.modules.wav2vec2(wavs, wav_lens)
+        x = self.modules.enc(feats)
+        logits = self.modules.ctc_lin(x)
+        p_ctc = self.hparams.log_softmax(logits)
+
+        return feats,p_ctc
+
+
+
+    def compute_objectives(self, predictions, batch, stage):
+        """Computes the loss (CTC) given predictions and targets."""
+
+        p_ctc, wav_lens = predictions
+
+        ids = batch.id
+        tokens, tokens_lens = batch.tokens
+
+        loss = self.hparams.ctc_cost(p_ctc, tokens, wav_lens, tokens_lens)
+
+        if stage != sb.Stage.TRAIN:
+            predicted_tokens = sb.decoders.ctc_greedy_decode(
+                p_ctc, wav_lens, blank_id=self.hparams.blank_index
+            )
+            # Decode token terms to words
+            if self.hparams.use_language_modelling:
+                predicted_words = []
+                for logs in p_ctc:
+                    text = decoder.decode(logs.detach().cpu().numpy())
+                    predicted_words.append(text.split(" "))
+            else:
+                predicted_words = [
+                    "".join(self.tokenizer.decode_ndim(utt_seq)).split(" ")
+                    for utt_seq in predicted_tokens
+                ]
+            # Convert indices to words
+            target_words = [wrd.split(" ") for wrd in batch.wrd]
+
+            self.wer_metric.append(ids, predicted_words, target_words)
+            self.cer_metric.append(ids, predicted_words, target_words)
+
+        return loss
+
+    def fit_batch(self, batch):
+        """Train the parameters given a single batch in input"""
+        should_step = self.step % self.grad_accumulation_factor == 0
+        # Managing automatic mixed precision
+        # TOFIX: CTC fine-tuning currently is unstable
+        # This is certainly due to CTC being done in fp16 instead of fp32
+        if self.auto_mix_prec:
+            with torch.cuda.amp.autocast():
+                with self.no_sync():
+                    outputs = self.compute_forward(batch, sb.Stage.TRAIN)
+                loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)
+            with self.no_sync(not should_step):
+                self.scaler.scale(
+                    loss / self.grad_accumulation_factor
+                ).backward()
+            if should_step:
+
+                if not self.hparams.wav2vec2.freeze:
+                    self.scaler.unscale_(self.wav2vec_optimizer)
+                self.scaler.unscale_(self.model_optimizer)
+                if self.check_gradients(loss):
+                    if not self.hparams.wav2vec2.freeze:
+                        if self.optimizer_step >= self.hparams.warmup_steps:
+                            self.scaler.step(self.wav2vec_optimizer)
+                    self.scaler.step(self.model_optimizer)
+                self.scaler.update()
+                self.zero_grad()
+                self.optimizer_step += 1
+        else:
+            # This is mandatory because HF models have a weird behavior with DDP
+            # on the forward pass
+            with self.no_sync():
+                outputs = self.compute_forward(batch, sb.Stage.TRAIN)
+
+            loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)
+
+            with self.no_sync(not should_step):
+                (loss / self.grad_accumulation_factor).backward()
+            if should_step:
+                if self.check_gradients(loss):
+                    if not self.hparams.wav2vec2.freeze:
+                        if self.optimizer_step >= self.hparams.warmup_steps:
+                            self.wav2vec_optimizer.step()
+                    self.model_optimizer.step()
+                self.zero_grad()
+                self.optimizer_step += 1
+
+        self.on_fit_batch_end(batch, outputs, loss, should_step)
+        return loss.detach().cpu()
+
+    def evaluate_batch(self, batch, stage):
+        """Computations needed for validation/test batches"""
+        predictions = self.compute_forward(batch, stage=stage)
+        with torch.no_grad():
+            loss = self.compute_objectives(predictions, batch, stage=stage)
+        return loss.detach()
+
+    def on_stage_start(self, stage, epoch):
+        """Gets called at the beginning of each epoch"""
+        if stage != sb.Stage.TRAIN:
+            self.cer_metric = self.hparams.cer_computer()
+            self.wer_metric = self.hparams.error_rate_computer()
+
+    def on_stage_end(self, stage, stage_loss, epoch):
+        """Gets called at the end of an epoch."""
+        # Compute/store important stats
+        stage_stats = {"loss": stage_loss}
+        if stage == sb.Stage.TRAIN:
+            self.train_stats = stage_stats
+        else:
+            stage_stats["CER"] = self.cer_metric.summarize("error_rate")
+            stage_stats["WER"] = self.wer_metric.summarize("error_rate")
+
+        # Perform end-of-iteration things, like annealing, logging, etc.
+        if stage == sb.Stage.VALID:
+            old_lr_model, new_lr_model = self.hparams.lr_annealing_model(
+                stage_stats["loss"]
+            )
+            old_lr_wav2vec, new_lr_wav2vec = self.hparams.lr_annealing_wav2vec(
+                stage_stats["loss"]
+            )
+            sb.nnet.schedulers.update_learning_rate(
+                self.model_optimizer, new_lr_model
+            )
+            if not self.hparams.wav2vec2.freeze:
+                sb.nnet.schedulers.update_learning_rate(
+                    self.wav2vec_optimizer, new_lr_wav2vec
+                )
+            self.hparams.train_logger.log_stats(
+                stats_meta={
+                    "epoch": epoch,
+                    "lr_model": old_lr_model,
+                    "lr_wav2vec": old_lr_wav2vec,
+                },
+                train_stats=self.train_stats,
+                valid_stats=stage_stats,
+            )
+            self.checkpointer.save_and_keep_only(
+                meta={"WER": stage_stats["WER"]}, min_keys=["WER"],
+            )
+        elif stage == sb.Stage.TEST:
+            self.hparams.train_logger.log_stats(
+                stats_meta={"Epoch loaded": self.hparams.epoch_counter.current},
+                test_stats=stage_stats,
+            )
+            with open(self.hparams.wer_file, "w") as w:
+                self.wer_metric.write_stats(w)
+
+    def init_optimizers(self):
+        "Initializes the wav2vec2 optimizer and model optimizer"
+
+        # If the wav2vec encoder is unfrozen, we create the optimizer
+        if not self.hparams.wav2vec2.freeze:
+            self.wav2vec_optimizer = self.hparams.wav2vec_opt_class(
+                self.modules.wav2vec2.parameters()
+            )
+            if self.checkpointer is not None:
+                self.checkpointer.add_recoverable(
+                    "wav2vec_opt", self.wav2vec_optimizer
+                )
+
+        self.model_optimizer = self.hparams.model_opt_class(
+            self.hparams.model.parameters()
+        )
+
+        if self.checkpointer is not None:
+            self.checkpointer.add_recoverable("modelopt", self.model_optimizer)
+
+    def zero_grad(self, set_to_none=False):
+        if not self.hparams.wav2vec2.freeze:
+            self.wav2vec_optimizer.zero_grad(set_to_none)
+        self.model_optimizer.zero_grad(set_to_none)
+
+
+from speechbrain.pretrained import EncoderASR,EncoderDecoderASR
+french_asr_model = EncoderASR.from_hparams(source="asr-wav2vec2-commonvoice-fr", savedir="pretrained_models/asr-wav2vec2-commonvoice-fr").cuda()
+french_asr_model.to("cpu")
+cvhparams_file, cvrun_opts, cvoverrides = sb.parse_arguments(["EnglishCV/train_en_with_wav2vec.yaml"])
+with open(cvhparams_file) as cvfin:
+    cvhparams = load_hyperpyyaml(cvfin, cvoverrides)
+english_asr_model = ASRCV(
+        modules=cvhparams["modules"],
+        hparams=cvhparams,
+        run_opts=cvrun_opts,
+        checkpointer=cvhparams["checkpointer"],
+    )
+english_asr_model.modules.to("cpu")
+english_asr_model.checkpointer.recover_if_possible()
+print("moving to tunisian model")
+asr_brain = ASR(
+    modules=hparams["modules"],
+    hparams=hparams,
+    run_opts=run_opts,
+    checkpointer=hparams["checkpointer"],
+)
+asr_brain.modules.to("cpu")
+asr_brain.checkpointer.recover_if_possible()
+asr_brain.modules.eval()
+english_asr_model.modules.eval()
+french_asr_model.mods.eval()
+asr_brain.modules.to("cpu")
+
+# Commented out IPython magic to ensure Python compatibility.
+# %ls
+
+#UTILS FUNCTIOJNS
+def get_size_dimensions(arr):
+    size_dimensions = []
+    while isinstance(arr, list):
+        size_dimensions.append(len(arr))
+        arr = arr[0]
+    return size_dimensions
+
+def scale_array(batch,n):
+    scaled_batch = []
+
+    for array in batch:
+        if(n < len(array)): raise ValueError("Cannot scale Array down")
+
+        repeat = round(n/len(array))+1
+        scaled_length_array= []
+
+        for i in array:
+            for j in range(repeat) :
+                if(len(scaled_length_array) == n): break
+                scaled_length_array.append(i)
+
+        scaled_batch.append(scaled_length_array)
+
+    return torch.tensor(scaled_batch)
+
+
+def load_paths(wavs_path):
+    waveforms = []
+    for path in wavs_path :
+        waveform, _ = torchaudio.load(path)
+        waveforms.append(waveform.squeeze(0))
+    # normalize array length to the bigger arrays by pading with 0's
+    padded_arrays = pad_sequence(waveforms, batch_first=True)
+    return torch.tensor(padded_arrays)
+
+
+
+device = 'cuda'
+verbose = 0
+#FLOW LEVEL FUNCTIONS
+def merge_strategy(embeddings1, embeddings2, embeddings3,post1, post2,post3):
+
+
+    post1 = post1.to(device)
+    post2 = post2.to(device)
+    post3 = post3.to(device)
+    embeddings1 = embeddings1.to(device)
+    embeddings2 = embeddings2.to(device)
+    embeddings3 = embeddings3.to(device)
+
+    posteriograms_merged = torch.cat((post1,post2,post3),dim=2)
+    embeddings_merged = torch.cat((embeddings1,embeddings2,embeddings3),dim=2)
+
+    if(verbose !=0):
+      print('MERGED POST ',posteriograms_merged.shape)
+      print('MERGED emb ',embeddings_merged.shape)
+
+    return torch.cat((posteriograms_merged,embeddings_merged),dim=2).to(device)
+
+def decode(model,wavs,wav_lens):
+
+    with torch.no_grad():
+        wav_lens = wav_lens.to(model.device)
+        encoder_out = model.encode_batch(wavs, wav_lens)
+        predictions = model.decoding_function(encoder_out, wav_lens)
+        return predictions
+
+def middle_layer(batch, lens):
+
+    tn_embeddings, tn_posteriogram = asr_brain.custom_encode(batch,None)
+
+    fr_embeddings = french_asr_model.mods.encoder.wav2vec2(batch)
+    fr_posteriogram =french_asr_model.encode_batch(batch,lens)
+    en_embeddings = english_asr_model.modules.wav2vec2(batch, lens)
+    x = english_asr_model.modules.enc(en_embeddings)
+    en_posteriogram = english_asr_model.modules.ctc_lin(x)
+    #scores, en_posteriogram = english_asr_model.mods.decoder(en_embeddings ,lens)
+    if(verbose !=0):
+      print('[EMBEDDINGS] FR:',fr_embeddings.shape, "EN:",en_embeddings.shape, "TN:", tn_embeddings.shape)
+      print('[POSTERIOGRAM] FR:',fr_posteriogram.shape, "EN:",en_posteriogram.shape,"TN:",tn_posteriogram.shape)
+
+
+    bilangual_sample = merge_strategy(fr_embeddings,en_embeddings,tn_embeddings,fr_posteriogram,en_posteriogram,tn_posteriogram)
+    return bilangual_sample
+
+class Mixer(sb.core.Brain):
+
+    def compute_forward(self, batch, stage):
+        """Forward computations from the waveform batches to the output probabilities."""
+        wavs, wav_lens = batch.sig
+        wavs, wav_lens = wavs.to(self.device), wav_lens.to(self.device)
+
+        if stage == sb.Stage.TRAIN:
+            if hasattr(self.hparams, "augmentation"):
+                wavs = self.hparams.augmentation(wavs, wav_lens)
+
+        multi_langual_feats = middle_layer(wavs, wav_lens)
+        multi_langual_feats= multi_langual_feats.to(device)
+        feats, _ = self.modules.enc(multi_langual_feats)
+        logits = self.modules.ctc_lin(feats)
+        p_ctc = self.hparams.log_softmax(logits)
+        
+        if stage!= sb.Stage.TRAIN:
+            p_tokens = sb.decoders.ctc_greedy_decode(
+                p_ctc, wav_lens, blank_id=self.hparams.blank_index
+            )
+        else : 
+            p_tokens = None
+        return p_ctc, wav_lens, p_tokens
+    
+    
+    def treat_wav(self,sig):
+        multi_langual_feats = middle_layer(sig.to("cpu"), torch.tensor([1]).to("cpu"))
+        multi_langual_feats= multi_langual_feats.to(device)
+        feats, _ = self.modules.enc(multi_langual_feats)
+        logits = self.modules.ctc_lin(feats)
+        p_ctc = self.hparams.log_softmax(logits)
+        predicted_words =[]
+        for logs in p_ctc:
+            text = decoder.decode(logs.detach().cpu().numpy())
+            predicted_words.append(text.split(" "))
+        return " ".join(predicted_words[0])
+        
+
+    def compute_objectives(self, predictions, batch, stage):
+        """Computes the loss (CTC) given predictions and targets."""
+
+        p_ctc, wav_lens , predicted_tokens= predictions
+
+        ids = batch.id
+        tokens, tokens_lens = batch.tokens
+
+        loss = self.hparams.ctc_cost(p_ctc, tokens, wav_lens, tokens_lens)
+
+
+        if stage == sb.Stage.VALID:
+            predicted_words = [
+                "".join(self.tokenizer.decode_ndim(utt_seq)).split(" ")
+                for utt_seq in predicted_tokens
+            ]
+            target_words = [wrd.split(" ") for wrd in batch.wrd]
+            self.wer_metric.append(ids, predicted_words, target_words)
+            self.cer_metric.append(ids, predicted_words, target_words)
+        if stage ==sb.Stage.TEST : 
+            if self.hparams.language_modelling:
+                predicted_words = []
+                for logs in p_ctc:
+                    text = decoder.decode(logs.detach().cpu().numpy())
+                    predicted_words.append(text.split(" "))
+            else : 
+                predicted_words = [
+                    "".join(self.tokenizer.decode_ndim(utt_seq)).split(" ")
+                    for utt_seq in predicted_tokens
+                ]
+
+            target_words = [wrd.split(" ") for wrd in batch.wrd]
+            self.wer_metric.append(ids, predicted_words, target_words)
+            self.cer_metric.append(ids, predicted_words, target_words)
+
+        return loss
+
+    def fit_batch(self, batch):
+        """Train the parameters given a single batch in input"""
+        should_step = self.step % self.grad_accumulation_factor == 0
+        # Managing automatic mixed precision
+        # TOFIX: CTC fine-tuning currently is unstable
+        # This is certainly due to CTC being done in fp16 instead of fp32
+        if self.auto_mix_prec:
+            with torch.cuda.amp.autocast():
+                with self.no_sync():
+                    outputs = self.compute_forward(batch, sb.Stage.TRAIN)
+                loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)
+            with self.no_sync(not should_step):
+                self.scaler.scale(
+                    loss / self.grad_accumulation_factor
+                ).backward()
+            if should_step:
+
+
+                self.scaler.unscale_(self.model_optimizer)
+                if self.check_gradients(loss):
+                    self.scaler.step(self.model_optimizer)
+                self.scaler.update()
+                self.zero_grad()
+                self.optimizer_step += 1
+        else:
+            # This is mandatory because HF models have a weird behavior with DDP
+            # on the forward pass
+            with self.no_sync():
+                outputs = self.compute_forward(batch, sb.Stage.TRAIN)
+
+            loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)
+
+            with self.no_sync(not should_step):
+                (loss / self.grad_accumulation_factor).backward()
+            if should_step:
+                if self.check_gradients(loss):
+                    self.model_optimizer.step()
+                self.zero_grad()
+                self.optimizer_step += 1
+
+        self.on_fit_batch_end(batch, outputs, loss, should_step)
+        return loss.detach().cpu()
+
+    def evaluate_batch(self, batch, stage):
+        """Computations needed for validation/test batches"""
+        predictions = self.compute_forward(batch, stage=stage)
+        with torch.no_grad():
+            loss = self.compute_objectives(predictions, batch, stage=stage)
+        return loss.detach()
+
+    def on_stage_start(self, stage, epoch):
+        """Gets called at the beginning of each epoch"""
+        if stage != sb.Stage.TRAIN:
+            self.cer_metric = self.hparams.cer_computer()
+            self.wer_metric = self.hparams.error_rate_computer()
+
+    def on_stage_end(self, stage, stage_loss, epoch):
+        """Gets called at the end of an epoch."""
+        # Compute/store important stats
+        stage_stats = {"loss": stage_loss}
+        if stage == sb.Stage.TRAIN:
+            self.train_stats = stage_stats
+        else:
+            stage_stats["CER"] = self.cer_metric.summarize("error_rate")
+            stage_stats["WER"] = self.wer_metric.summarize("error_rate")
+
+        # Perform end-of-iteration things, like annealing, logging, etc.
+        if stage == sb.Stage.VALID:
+            old_lr_model, new_lr_model = self.hparams.lr_annealing_model(
+                stage_stats["loss"]
+            )
+            sb.nnet.schedulers.update_learning_rate(
+                self.model_optimizer, new_lr_model
+            )
+            self.hparams.train_logger.log_stats(
+                stats_meta={
+                    "epoch": epoch,
+                    "lr_model": old_lr_model,
+                },
+                train_stats=self.train_stats,
+                valid_stats=stage_stats,
+            )
+            self.checkpointer.save_and_keep_only(
+                meta={"WER": stage_stats["WER"]}, min_keys=["WER"],
+            )
+        elif stage == sb.Stage.TEST:
+            self.hparams.train_logger.log_stats(
+                stats_meta={"Epoch loaded": self.hparams.epoch_counter.current},
+                test_stats=stage_stats,
+            )
+            with open(self.hparams.wer_file, "w") as w:
+                self.wer_metric.write_stats(w)
+
+    def init_optimizers(self):
+
+        self.model_optimizer = self.hparams.model_opt_class(
+            self.hparams.model.parameters()
+        )
+
+        if self.checkpointer is not None:
+            self.checkpointer.add_recoverable("modelopt", self.model_optimizer)
+
+    def zero_grad(self, set_to_none=False):
+
+        self.model_optimizer.zero_grad(set_to_none)
+
+
+
+
+hparams_file, run_opts, overrides = sb.parse_arguments(["cs.yaml"])
+
+# If distributed_launch=True then
+# create ddp_group with the right communication protocol
+sb.utils.distributed.ddp_init_group(run_opts)
+
+with open(hparams_file) as fin:
+    hparams = load_hyperpyyaml(fin, overrides)
+
+# Create experiment directory
+sb.create_experiment_directory(
+    experiment_directory=hparams["output_folder"],
+    hyperparams_to_save=hparams_file,
+    overrides=overrides,
+)
+def read_labels_file(labels_file):
+    with open(labels_file, "r",encoding="utf-8") as lf:
+        lines = lf.read().splitlines()
+        division = "==="
+        numbers = {}
+        for line in lines :
+            if division in line :
+                break
+            string, number = line.split("=>")
+            number = int(number)
+            string = string[1:-2]
+            numbers[number] = string
+        return [numbers[x] for x in range(len(numbers))]
+
+label_encoder = sb.dataio.encoder.CTCTextEncoder()
+
+lab_enc_file = os.path.join(hparams["save_folder"], "label_encoder.txt")
+special_labels = {
+    "blank_label": hparams["blank_index"],
+    "unk_label": hparams["unk_index"]
+}
+label_encoder.load_or_create(
+    path=lab_enc_file,
+    from_didatasets=[[]],
+    output_key="char_list",
+    special_labels=special_labels,
+    sequence_input=True,
+)
+
+
+labels = read_labels_file(os.path.join(hparams["save_folder"], "label_encoder.txt"))
+labels = [""] + labels[1:-1] + ["1"]
+if hparams["language_modelling"]:
+    decoder = build_ctcdecoder(
+        labels,
+        kenlm_model_path=hparams["ngram_lm_path"],  # either .arpa or .bin file
+        alpha=0.5,  # tuned on a val set
+        beta=1,  # tuned on a val set
+    )
+
+
+
+
+mixer = Mixer(
+    modules=hparams["modules"],
+    hparams=hparams,
+    run_opts=run_opts,
+    checkpointer=hparams["checkpointer"],
+)
+mixer.tokenizer = label_encoder
+mixer.checkpointer.recover_if_possible()
+mixer.modules.eval()
+
+
+label_encoder = sb.dataio.encoder.CTCTextEncoder()
+
+
+# We dynamicaly add the tokenizer to our brain class.
+# NB: This tokenizer corresponds to the one used for the LM!!
+
+decoder = build_ctcdecoder(
+    labels,
+    kenlm_model_path= "arpas/everything.arpa",  # either .arpa or .bin file
+    alpha=0.5,  # tuned on a val set
+    beta=1,  # tuned on a val set
+)
+
+run_opts["device"]="cpu"
+
+
+device = "cpu"
+mixer.device= "cpu"
+mixer.modules.to("cpu")
+
+from enum import Enum, auto
+class Stage(Enum):
+    TRAIN = auto()
+    VALID = auto()
+    TEST = auto()
+
+asr_brain.on_evaluate_start()
+asr_brain.modules.eval()
+
+
+import gradio as gr
+
+def treat_wav_file(file_mic,file_upload ,asr=mixer, device="cpu") :
+    if (file_mic is not None) and (file_upload is not None):
+        warn_output = "WARNING: You've uploaded an audio file and used the microphone. The recorded file from the microphone will be used and the uploaded audio will be discarded.\n"
+        wav = file_mic
+    elif (file_mic is None) and (file_upload is None):
+        return "ERROR: You have to either use the microphone or upload an audio file"
+    elif file_mic is not None:
+        wav = file_mic
+    else:
+        wav = file_upload
+    sig, sr = torchaudio.load(wav)
+    tensor_wav = sig.to(device)
+    resampled = torchaudio.functional.resample( tensor_wav, sr, 16000)
+    sentence = asr.treat_wav(resampled)
+    return sentence
+
+gr.Interface(
+    fn=treat_wav_file,
+    inputs=[gr.Audio(source="microphone", type='filepath', label = "record", optional = True),
+            gr.Audio(source="upload", type='filepath', label="filein", optional=True)]
+    ,outputs="text").launch()
+

TunisianASR/outdomain.arpa → arpas/pluslanguages_everything.arpa

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:24654c1d236bb1bd367125131c847c4a734e69914eda71a6786964c20440d8fe
-size 324243244
+oid sha256:7a2a593a09bb07c74ac6f1aadbb4c0a6540edb53b9234f3da1aafa9e6d99f983
+size 354742106

asr-wav2vec2-commonvoice-fr/hyperparams.yaml

@@ -5,7 +5,7 @@
 # ################################
 
 sample_rate: 16000
-wav2vec2_hub: /gpfsscratch/rech/nou/uzn19yk/wav2vec2-FR-7K-large
+wav2vec2_hub: wav2vec2-FR-7K-large/
 
 # BPE parameters
 token_type: unigram  # ["unigram", "bpe", "char"]

cs.yaml

@@ -43,7 +43,7 @@ sorting: ascending
 auto_mix_prec: False
 sample_rate: 16000
 language_modelling: True
-ngram_lm_path: /gpfsstore/rech/nou/uzn19yk/switched_code_tunisian/train/tunisian_asr/arpas/pluslanguages_everything.arpa
+ngram_lm_path: arpas/pluslanguages_everything.arpa 
 
 # With data_parallel batch_size is split into N jobs
 # With DDP batch_size is multiplied by N jobs

cv_train.py

@@ -0,0 +1,388 @@
+#!/usr/bin/env python3
+import sys
+import torch
+import logging
+import speechbrain as sb
+import torchaudio
+from hyperpyyaml import load_hyperpyyaml
+from speechbrain.tokenizers.SentencePiece import SentencePiece
+from speechbrain.utils.data_utils import undo_padding
+from speechbrain.utils.distributed import run_on_main
+
+"""Recipe for training a sequence-to-sequence ASR system with CommonVoice.
+The system employs a wav2vec2 encoder and a CTC decoder.
+Decoding is performed with greedy decoding (will be extended to beam search).
+
+To run this recipe, do the following:
+> python train_with_wav2vec2.py hparams/train_with_wav2vec2.yaml
+
+With the default hyperparameters, the system employs a pretrained wav2vec2 encoder.
+The wav2vec2 model is pretrained following the model given in the hprams file.
+It may be dependent on the language.
+
+The neural network is trained with CTC on sub-word units estimated with
+Byte Pairwise Encoding (BPE).
+
+The experiment file is flexible enough to support a large variety of
+different systems. By properly changing the parameter files, you can try
+different encoders, decoders, tokens (e.g, characters instead of BPE),
+training languages (all CommonVoice languages), and many
+other possible variations.
+
+Authors
+ * Titouan Parcollet 2021
+"""
+
+logger = logging.getLogger(__name__)
+
+
+# Define training procedure
+class ASRCV(sb.core.Brain):
+    def compute_forward(self, batch, stage):
+        """Forward computations from the waveform batches to the output probabilities."""
+
+        batch = batch.to(self.device)
+        wavs, wav_lens = batch.sig
+        tokens_bos, _ = batch.tokens_bos
+        wavs, wav_lens = wavs.to(self.device), wav_lens.to(self.device)
+
+        if stage == sb.Stage.TRAIN:
+            if hasattr(self.hparams, "augmentation"):
+                wavs = self.hparams.augmentation(wavs, wav_lens)
+
+        # Forward pass
+        feats = self.modules.wav2vec2(wavs, wav_lens)
+        x = self.modules.enc(feats)
+        logits = self.modules.ctc_lin(x)
+        p_ctc = self.hparams.log_softmax(logits)
+
+        return p_ctc, wav_lens
+
+    def compute_objectives(self, predictions, batch, stage):
+        """Computes the loss (CTC) given predictions and targets."""
+
+        p_ctc, wav_lens = predictions
+
+        ids = batch.id
+        tokens_eos, tokens_eos_lens = batch.tokens_eos
+        tokens, tokens_lens = batch.tokens
+
+        loss = self.hparams.ctc_cost(p_ctc, tokens, wav_lens, tokens_lens)
+
+        if stage != sb.Stage.TRAIN:
+            # Decode token terms to words
+            sequence = sb.decoders.ctc_greedy_decode(
+                p_ctc, wav_lens, blank_id=self.hparams.blank_index
+            )
+
+            predicted_words = self.tokenizer(sequence, task="decode_from_list")
+
+            # Convert indices to words
+            target_words = undo_padding(tokens, tokens_lens)
+            target_words = self.tokenizer(target_words, task="decode_from_list")
+
+            self.wer_metric.append(ids, predicted_words, target_words)
+            self.cer_metric.append(ids, predicted_words, target_words)
+
+        return loss
+
+    def fit_batch(self, batch):
+        """Train the parameters given a single batch in input"""
+        should_step = self.step % self.grad_accumulation_factor == 0
+        # Managing automatic mixed precision
+        # TOFIX: CTC fine-tuning currently is unstable
+        # This is certainly due to CTC being done in fp16 instead of fp32
+        if self.auto_mix_prec:
+            with torch.cuda.amp.autocast():
+                with self.no_sync():
+                    outputs = self.compute_forward(batch, sb.Stage.TRAIN)
+                loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)
+            with self.no_sync(not should_step):
+                self.scaler.scale(
+                    loss / self.grad_accumulation_factor
+                ).backward()
+            if should_step:
+
+                if not self.hparams.wav2vec2.freeze:
+                    self.scaler.unscale_(self.wav2vec_optimizer)
+                self.scaler.unscale_(self.model_optimizer)
+                if self.check_gradients(loss):
+                    if not self.hparams.wav2vec2.freeze:
+                        if self.optimizer_step >= self.hparams.warmup_steps:
+                            self.scaler.step(self.wav2vec_optimizer)
+                    self.scaler.step(self.model_optimizer)
+                self.scaler.update()
+                self.zero_grad()
+                self.optimizer_step += 1
+        else:
+            # This is mandatory because HF models have a weird behavior with DDP
+            # on the forward pass
+            with self.no_sync():
+                outputs = self.compute_forward(batch, sb.Stage.TRAIN)
+
+            loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)
+
+            with self.no_sync(not should_step):
+                (loss / self.grad_accumulation_factor).backward()
+            if should_step:
+                if self.check_gradients(loss):
+                    if not self.hparams.wav2vec2.freeze:
+                        if self.optimizer_step >= self.hparams.warmup_steps:
+                            self.wav2vec_optimizer.step()
+                    self.model_optimizer.step()
+                self.zero_grad()
+                self.optimizer_step += 1
+
+        self.on_fit_batch_end(batch, outputs, loss, should_step)
+        return loss.detach().cpu()
+
+    def evaluate_batch(self, batch, stage):
+        """Computations needed for validation/test batches"""
+        predictions = self.compute_forward(batch, stage=stage)
+        with torch.no_grad():
+            loss = self.compute_objectives(predictions, batch, stage=stage)
+        return loss.detach()
+
+    def on_stage_start(self, stage, epoch):
+        """Gets called at the beginning of each epoch"""
+        if stage != sb.Stage.TRAIN:
+            self.cer_metric = self.hparams.cer_computer()
+            self.wer_metric = self.hparams.error_rate_computer()
+
+    def on_stage_end(self, stage, stage_loss, epoch):
+        """Gets called at the end of an epoch."""
+        # Compute/store important stats
+        stage_stats = {"loss": stage_loss}
+        if stage == sb.Stage.TRAIN:
+            self.train_stats = stage_stats
+        else:
+            stage_stats["CER"] = self.cer_metric.summarize("error_rate")
+            stage_stats["WER"] = self.wer_metric.summarize("error_rate")
+
+        # Perform end-of-iteration things, like annealing, logging, etc.
+        if stage == sb.Stage.VALID:
+            old_lr_model, new_lr_model = self.hparams.lr_annealing_model(
+                stage_stats["loss"]
+            )
+            old_lr_wav2vec, new_lr_wav2vec = self.hparams.lr_annealing_wav2vec(
+                stage_stats["loss"]
+            )
+            sb.nnet.schedulers.update_learning_rate(
+                self.model_optimizer, new_lr_model
+            )
+            if not self.hparams.wav2vec2.freeze:
+                sb.nnet.schedulers.update_learning_rate(
+                    self.wav2vec_optimizer, new_lr_wav2vec
+                )
+            self.hparams.train_logger.log_stats(
+                stats_meta={
+                    "epoch": epoch,
+                    "lr_model": old_lr_model,
+                    "lr_wav2vec": old_lr_wav2vec,
+                },
+                train_stats=self.train_stats,
+                valid_stats=stage_stats,
+            )
+            self.checkpointer.save_and_keep_only(
+                meta={"WER": stage_stats["WER"]}, min_keys=["WER"],
+            )
+        elif stage == sb.Stage.TEST:
+            self.hparams.train_logger.log_stats(
+                stats_meta={"Epoch loaded": self.hparams.epoch_counter.current},
+                test_stats=stage_stats,
+            )
+            with open(self.hparams.wer_file, "w") as w:
+                self.wer_metric.write_stats(w)
+
+    def init_optimizers(self):
+        "Initializes the wav2vec2 optimizer and model optimizer"
+
+        # If the wav2vec encoder is unfrozen, we create the optimizer
+        if not self.hparams.wav2vec2.freeze:
+            self.wav2vec_optimizer = self.hparams.wav2vec_opt_class(
+                self.modules.wav2vec2.parameters()
+            )
+            if self.checkpointer is not None:
+                self.checkpointer.add_recoverable(
+                    "wav2vec_opt", self.wav2vec_optimizer
+                )
+
+        self.model_optimizer = self.hparams.model_opt_class(
+            self.hparams.model.parameters()
+        )
+
+        if self.checkpointer is not None:
+            self.checkpointer.add_recoverable("modelopt", self.model_optimizer)
+
+    def zero_grad(self, set_to_none=False):
+        if not self.hparams.wav2vec2.freeze:
+            self.wav2vec_optimizer.zero_grad(set_to_none)
+        self.model_optimizer.zero_grad(set_to_none)
+
+
+# Define custom data procedure
+def dataio_prepare(hparams, tokenizer):
+    """This function prepares the datasets to be used in the brain class.
+    It also defines the data processing pipeline through user-defined functions."""
+
+    # 1. Define datasets
+    data_folder = hparams["data_folder"]
+
+    train_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
+        csv_path=hparams["train_csv"], replacements={"data_root": data_folder},
+    )
+
+    if hparams["sorting"] == "ascending":
+        # we sort training data to speed up training and get better results.
+        train_data = train_data.filtered_sorted(
+            sort_key="duration",
+            key_max_value={"duration": hparams["avoid_if_longer_than"]},
+        )
+        # when sorting do not shuffle in dataloader ! otherwise is pointless
+        hparams["dataloader_options"]["shuffle"] = False
+
+    elif hparams["sorting"] == "descending":
+        train_data = train_data.filtered_sorted(
+            sort_key="duration",
+            reverse=True,
+            key_max_value={"duration": hparams["avoid_if_longer_than"]},
+        )
+        # when sorting do not shuffle in dataloader ! otherwise is pointless
+        hparams["dataloader_options"]["shuffle"] = False
+
+    elif hparams["sorting"] == "random":
+        pass
+
+    else:
+        raise NotImplementedError(
+            "sorting must be random, ascending or descending"
+        )
+
+    valid_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
+        csv_path=hparams["valid_csv"], replacements={"data_root": data_folder},
+    )
+    # We also sort the validation data so it is faster to validate
+    valid_data = valid_data.filtered_sorted(sort_key="duration")
+
+    test_data = sb.dataio.dataset.DynamicItemDataset.from_csv(
+        csv_path=hparams["test_csv"], replacements={"data_root": data_folder},
+    )
+
+    # We also sort the validation data so it is faster to validate
+    test_data = test_data.filtered_sorted(sort_key="duration")
+
+    datasets = [train_data, valid_data, test_data]
+
+    # 2. Define audio pipeline:
+    @sb.utils.data_pipeline.takes("wav")
+    @sb.utils.data_pipeline.provides("sig")
+    def audio_pipeline(wav):
+        info = torchaudio.info(wav)
+        sig = sb.dataio.dataio.read_audio(wav)
+        resampled = torchaudio.transforms.Resample(
+            info.sample_rate, hparams["sample_rate"],
+        )(sig)
+        return resampled
+
+    sb.dataio.dataset.add_dynamic_item(datasets, audio_pipeline)
+
+    # 3. Define text pipeline:
+    @sb.utils.data_pipeline.takes("wrd")
+    @sb.utils.data_pipeline.provides(
+        "tokens_list", "tokens_bos", "tokens_eos", "tokens"
+    )
+    def text_pipeline(wrd):
+        tokens_list = tokenizer.sp.encode_as_ids(wrd)
+        yield tokens_list
+        tokens_bos = torch.LongTensor([hparams["bos_index"]] + (tokens_list))
+        yield tokens_bos
+        tokens_eos = torch.LongTensor(tokens_list + [hparams["eos_index"]])
+        yield tokens_eos
+        tokens = torch.LongTensor(tokens_list)
+        yield tokens
+
+    sb.dataio.dataset.add_dynamic_item(datasets, text_pipeline)
+
+    # 4. Set output:
+    sb.dataio.dataset.set_output_keys(
+        datasets, ["id", "sig", "tokens_bos", "tokens_eos", "tokens"],
+    )
+    return train_data, valid_data, test_data
+
+
+if __name__ == "__main__":
+
+    # Load hyperparameters file with command-line overrides
+    hparams_file, run_opts, overrides = sb.parse_arguments(sys.argv[1:])
+    with open(hparams_file) as fin:
+        hparams = load_hyperpyyaml(fin, overrides)
+
+    # If --distributed_launch then
+    # create ddp_group with the right communication protocol
+    sb.utils.distributed.ddp_init_group(run_opts)
+
+    # Dataset preparation (parsing CommonVoice)
+    from common_voice_prepare import prepare_common_voice  # noqa
+
+    # Create experiment directory
+    sb.create_experiment_directory(
+        experiment_directory=hparams["output_folder"],
+        hyperparams_to_save=hparams_file,
+        overrides=overrides,
+    )
+
+    # Due to DDP, we do the preparation ONLY on the main python process
+    run_on_main(
+        prepare_common_voice,
+        kwargs={
+            "data_folder": hparams["data_folder"],
+            "save_folder": hparams["save_folder"],
+            "train_tsv_file": hparams["train_tsv_file"],
+            "dev_tsv_file": hparams["dev_tsv_file"],
+            "test_tsv_file": hparams["test_tsv_file"],
+            "accented_letters": hparams["accented_letters"],
+            "language": hparams["language"],
+            "skip_prep": hparams["skip_prep"],
+        },
+    )
+
+    # Defining tokenizer and loading it
+    tokenizer = SentencePiece(
+        model_dir=hparams["save_folder"],
+        vocab_size=hparams["output_neurons"],
+        annotation_train=hparams["train_csv"],
+        annotation_read="wrd",
+        model_type=hparams["token_type"],
+        character_coverage=hparams["character_coverage"],
+    )
+
+    # Create the datasets objects as well as tokenization and encoding :-D
+    train_data, valid_data, test_data = dataio_prepare(hparams, tokenizer)
+
+    # Trainer initialization
+    asr_brain = ASRCV(
+        modules=hparams["modules"],
+        hparams=hparams,
+        run_opts=run_opts,
+        checkpointer=hparams["checkpointer"],
+    )
+
+    # Adding objects to trainer.
+    asr_brain.tokenizer = tokenizer
+
+    # Training
+    asr_brain.fit(
+        asr_brain.hparams.epoch_counter,
+        train_data,
+        valid_data,
+        train_loader_kwargs=hparams["dataloader_options"],
+        valid_loader_kwargs=hparams["test_dataloader_options"],
+    )
+
+    # Test
+    asr_brain.hparams.wer_file = hparams["output_folder"] + "/wer_test.txt"
+    asr_brain.evaluate(
+        test_data,
+        min_key="WER",
+        test_loader_kwargs=hparams["test_dataloader_options"],
+    )

pretrained_models/asr-wav2vec2-commonvoice-fr/asr.ckpt

@@ -0,0 +1 @@
+/home/salah/Code_Switched_Tunisian_Speech_Recognition/asr-wav2vec2-commonvoice-fr/asr.ckpt

pretrained_models/asr-wav2vec2-commonvoice-fr/custom.py

@@ -0,0 +1 @@
+/home/salah/Code_Switched_Tunisian_Speech_Recognition/asr-wav2vec2-commonvoice-fr/custom.py

pretrained_models/asr-wav2vec2-commonvoice-fr/hyperparams.yaml

@@ -0,0 +1 @@
+/home/salah/Code_Switched_Tunisian_Speech_Recognition/asr-wav2vec2-commonvoice-fr/hyperparams.yaml

pretrained_models/asr-wav2vec2-commonvoice-fr/tokenizer.ckpt

@@ -0,0 +1 @@
+/home/salah/Code_Switched_Tunisian_Speech_Recognition/asr-wav2vec2-commonvoice-fr/tokenizer.ckpt

pretrained_models/asr-wav2vec2-commonvoice-fr/wav2vec2.ckpt

@@ -0,0 +1 @@
+/home/salah/Code_Switched_Tunisian_Speech_Recognition/asr-wav2vec2-commonvoice-fr/wav2vec2.ckpt

requirements.txt

@@ -0,0 +1,17 @@
+huggingface_hub>=0.7.0
+gradio
+https://github.com/kpu/kenlm/archive/master.zip
+hyperpyyaml>=0.0.1
+joblib>=0.14.1
+numpy>=1.17.0
+packaging
+pre-commit>=2.3.0
+scipy>=1.4.1, <1.9
+sentencepiece>=0.1.91
+SoundFile; sys_platform == 'win32'
+torch>=1.13.0
+torchaudio>=0.9.0
+tqdm>=4.42.0
+transformers
+speechbrain
+pyctcdecode

results/non_semi_final_stac/env.log

@@ -0,0 +1,347 @@
+SpeechBrain system description
+==============================
+Python version:
+3.8.5 (default, Sep  4 2020, 07:30:14) 
+[GCC 7.3.0]
+==============================
+Installed Python packages:
+absl-py==1.2.0
+aiohttp==3.8.1
+aiosignal==1.2.0
+alabaster==0.7.12
+anaconda-client==1.7.2
+anaconda-navigator==1.10.0
+anaconda-project==0.8.3
+antlr4-python3-runtime==4.9.3
+appdirs==1.4.4
+argh==0.26.2
+argon2-cffi @ file:///tmp/build/80754af9/argon2-cffi_1596828493937/work
+asn1crypto @ file:///tmp/build/80754af9/asn1crypto_1596577642040/work
+astroid @ file:///tmp/build/80754af9/astroid_1592495912941/work
+astropy==4.0.2
+async-generator==1.10
+async-timeout==4.0.2
+atomicwrites==1.4.0
+attrs @ file:///tmp/build/80754af9/attrs_1604765588209/work
+audioread==2.1.9
+autopep8 @ file:///tmp/build/80754af9/autopep8_1596578164842/work
+Babel @ file:///tmp/build/80754af9/babel_1605108370292/work
+backcall==0.2.0
+backports.functools-lru-cache==1.6.1
+backports.shutil-get-terminal-size==1.0.0
+backports.tempfile==1.0
+backports.weakref==1.0.post1
+beautifulsoup4 @ file:///tmp/build/80754af9/beautifulsoup4_1601924105527/work
+bitarray @ file:///tmp/build/80754af9/bitarray_1605065113847/work
+bkcharts==0.2
+black==22.12.0
+bleach @ file:///tmp/build/80754af9/bleach_1600439572647/work
+bokeh @ file:///tmp/build/80754af9/bokeh_1603297833684/work
+boto==2.49.0
+boto3==1.28.43
+botocore==1.31.43
+Bottleneck==1.3.2
+bpemb==0.3.4
+brotlipy==0.7.0
+cachetools==5.2.0
+certifi==2020.6.20
+cffi @ file:///tmp/build/80754af9/cffi_1600699146221/work
+chardet==3.0.4
+charset-normalizer==2.0.12
+click==8.1.3
+cloudpickle @ file:///tmp/build/80754af9/cloudpickle_1598884132938/work
+clyent==1.2.2
+colorama @ file:///tmp/build/80754af9/colorama_1603211150991/work
+coloredlogs==15.0.1
+conda==4.9.2
+conda-build==3.20.5
+conda-package-handling @ file:///tmp/build/80754af9/conda-package-handling_1603018141399/work
+conda-verify==3.4.2
+conllu==4.5.3
+contextlib2==0.6.0.post1
+cryptography @ file:///tmp/build/80754af9/cryptography_1601046815590/work
+cycler==0.10.0
+Cython @ file:///tmp/build/80754af9/cython_1594831566883/work
+cytoolz==0.11.0
+dask @ file:///tmp/build/80754af9/dask-core_1602083700509/work
+datasets==1.18.3
+decorator==4.4.2
+defusedxml==0.6.0
+Deprecated==1.2.14
+diff-match-patch @ file:///tmp/build/80754af9/diff-match-patch_1594828741838/work
+dill==0.3.4
+distributed @ file:///tmp/build/80754af9/distributed_1605066520644/work
+docutils==0.16
+easyocr==1.2.1
+einops==0.3.0
+entrypoints==0.3
+et-xmlfile==1.0.1
+farasapy==0.0.14
+fastcache==1.1.0
+ffmpeg-python==0.2.0
+filelock==3.0.12
+flair==0.12.2
+flake8 @ file:///tmp/build/80754af9/flake8_1601911421857/work
+Flask==1.1.2
+flatbuffers==22.9.24
+frozenlist==1.3.0
+fsspec==2022.3.0
+ftfy==6.1.1
+future==0.18.2
+gdown==4.4.0
+gensim==4.1.2
+gevent @ file:///tmp/build/80754af9/gevent_1601397537062/work
+glob2==0.7
+gmpy2==2.0.8
+google-auth==2.12.0
+google-auth-oauthlib==0.4.6
+greenlet @ file:///tmp/build/80754af9/greenlet_1600874013538/work
+grpcio==1.49.1
+h5py==2.10.0
+HeapDict==1.0.1
+html5lib @ file:///tmp/build/80754af9/html5lib_1593446221756/work
+huggingface-hub==0.16.4
+humanfriendly==10.0
+hyperopt==0.2.7
+idna @ file:///tmp/build/80754af9/idna_1593446292537/work
+imageio @ file:///tmp/build/80754af9/imageio_1594161405741/work
+imagesize==1.2.0
+imhist==0.0.4
+importlib-metadata==5.0.0
+imWatermark==0.0.2
+iniconfig @ file:///tmp/build/80754af9/iniconfig_1602780191262/work
+install==1.3.5
+intervaltree @ file:///tmp/build/80754af9/intervaltree_1598376443606/work
+invisible-watermark==0.1.5
+ipykernel @ file:///tmp/build/80754af9/ipykernel_1596207638929/work/dist/ipykernel-5.3.4-py3-none-any.whl
+ipython @ file:///tmp/build/80754af9/ipython_1604101197014/work
+ipython-genutils==0.2.0
+ipywidgets @ file:///tmp/build/80754af9/ipywidgets_1601490159889/work
+isort @ file:///tmp/build/80754af9/isort_1602603989581/work
+itsdangerous==1.1.0
+Janome==0.5.0
+jdcal==1.4.1
+jedi @ file:///tmp/build/80754af9/jedi_1592841866100/work
+jeepney @ file:///tmp/build/80754af9/jeepney_1605069705079/work
+Jinja2==2.11.2
+jiwer==2.3.0
+jmespath==1.0.1
+joblib @ file:///tmp/build/80754af9/joblib_1601912903842/work
+json5==0.9.5
+jsonschema @ file:///tmp/build/80754af9/jsonschema_1602607155483/work
+jupyter==1.0.0
+jupyter-client @ file:///tmp/build/80754af9/jupyter_client_1601311786391/work
+jupyter-console @ file:///tmp/build/80754af9/jupyter_console_1598884538475/work
+jupyter-core==4.6.3
+jupyterlab==2.2.6
+jupyterlab-pygments @ file:///tmp/build/80754af9/jupyterlab_pygments_1601490720602/work
+jupyterlab-server @ file:///tmp/build/80754af9/jupyterlab_server_1594164409481/work
+keyring @ file:///tmp/build/80754af9/keyring_1601490835422/work
+kiwisolver @ file:///tmp/build/80754af9/kiwisolver_1604014535162/work
+langdetect==1.0.9
+lazy-object-proxy==1.4.3
+libarchive-c==2.9
+librosa==0.9.1
+llvmlite==0.34.0
+locket==0.2.0
+lxml @ file:///tmp/build/80754af9/lxml_1603216285000/work
+Markdown==3.4.1
+MarkupSafe==1.1.1
+matplotlib @ file:///tmp/build/80754af9/matplotlib-base_1603378225747/work
+mccabe==0.6.1
+mido==1.2.10
+mistune==0.8.4
+mkl-fft==1.2.0
+mkl-random==1.1.1
+mkl-service==2.3.0
+mock==4.0.2
+more-itertools @ file:///tmp/build/80754af9/more-itertools_1605111547926/work
+mpld3==0.3
+mpmath==1.1.0
+msgpack==1.0.0
+multidict==6.0.2
+multipledispatch==0.6.0
+multiprocess==0.70.12.2
+mypy-extensions==0.4.3
+navigator-updater==0.2.1
+nbclient @ file:///tmp/build/80754af9/nbclient_1602783176460/work
+nbconvert @ file:///tmp/build/80754af9/nbconvert_1601914830498/work
+nbformat @ file:///tmp/build/80754af9/nbformat_1602783287752/work
+nest-asyncio @ file:///tmp/build/80754af9/nest-asyncio_1605115881283/work
+networkx @ file:///tmp/build/80754af9/networkx_1598376031484/work
+nltk @ file:///tmp/build/80754af9/nltk_1592496090529/work
+nose==1.3.7
+notebook @ file:///tmp/build/80754af9/notebook_1601501575118/work
+numba @ file:///tmp/build/80754af9/numba_1600100669015/work
+numexpr==2.7.1
+numpy @ file:///tmp/build/80754af9/numpy_and_numpy_base_1603570489231/work
+numpydoc @ file:///tmp/build/80754af9/numpydoc_1605117425582/work
+oauthlib==3.2.1
+olefile==0.46
+omegaconf==2.2.3
+onnx==1.12.0
+onnxruntime==1.12.1
+opencv-python==4.4.0.46
+openpyxl @ file:///tmp/build/80754af9/openpyxl_1598113097404/work
+packaging==20.9
+pandas @ file:///tmp/build/80754af9/pandas_1602088120436/work
+pandocfilters @ file:///tmp/build/80754af9/pandocfilters_1605120460739/work
+parso==0.7.0
+partd==1.1.0
+path @ file:///tmp/build/80754af9/path_1598376507494/work
+pathlib2==2.3.5
+pathspec==0.10.3
+pathtools==0.1.2
+patsy==0.5.1
+pep8==1.7.1
+pexpect==4.8.0
+pickleshare==0.7.5
+Pillow @ file:///tmp/build/80754af9/pillow_1603822255246/work
+pkginfo==1.6.1
+platformdirs==2.6.0
+pluggy==0.13.1
+ply==3.11
+pooch==1.6.0
+pptree==3.1
+pretty-midi==0.2.9
+prometheus-client==0.8.0
+prompt-toolkit @ file:///tmp/build/80754af9/prompt-toolkit_1602688806899/work
+protobuf==3.19.6
+psutil @ file:///tmp/build/80754af9/psutil_1598370257551/work
+ptyprocess==0.6.0
+py @ file:///tmp/build/80754af9/py_1593446248552/work
+py-espeak-ng==0.1.8
+py4j==0.10.9.7
+PyArabic==0.6.15
+pyarrow==7.0.0
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycodestyle==2.6.0
+pycosat==0.6.3
+pycparser @ file:///tmp/build/80754af9/pycparser_1594388511720/work
+pycurl==7.43.0.6
+pyDeprecate==0.3.1
+pydocstyle @ file:///tmp/build/80754af9/pydocstyle_1598885001695/work
+pyflakes==2.2.0
+Pygments @ file:///tmp/build/80754af9/pygments_1604103097372/work
+pylint @ file:///tmp/build/80754af9/pylint_1598623985952/work
+pyodbc===4.0.0-unsupported
+pyOpenSSL @ file:///tmp/build/80754af9/pyopenssl_1594392929924/work
+pyparsing==2.4.7
+pyrsistent @ file:///tmp/build/80754af9/pyrsistent_1600141720057/work
+PySocks==1.7.1
+pytest==0.0.0
+python-bidi==0.4.2
+python-crfsuite==0.9.7
+python-dateutil==2.8.1
+python-jsonrpc-server @ file:///tmp/build/80754af9/python-jsonrpc-server_1600278539111/work
+python-language-server @ file:///tmp/build/80754af9/python-language-server_1600454544709/work
+python-Levenshtein==0.12.2
+pytorch-lightning==1.4.2
+pytorch-revgrad==0.2.0
+pytz==2020.1
+PyWavelets @ file:///tmp/build/80754af9/pywavelets_1601658317819/work
+pyxdg @ file:///tmp/build/80754af9/pyxdg_1603822279816/work
+PyYAML==5.3.1
+pyzmq==19.0.2
+QDarkStyle==2.8.1
+QtAwesome @ file:///tmp/build/80754af9/qtawesome_1602272867890/work
+qtconsole @ file:///tmp/build/80754af9/qtconsole_1600870028330/work
+QtPy==1.9.0
+regex @ file:///tmp/build/80754af9/regex_1602786672676/work
+requests @ file:///tmp/build/80754af9/requests_1592841827918/work
+requests-oauthlib==1.3.1
+resampy==0.2.2
+rope @ file:///tmp/build/80754af9/rope_1602264064449/work
+rsa==4.9
+Rtree==0.9.4
+ruamel-yaml==0.15.87
+s3transfer==0.6.2
+sacremoses==0.0.49
+safetensors==0.3.3
+scikit-image==0.17.2
+scikit-learn @ file:///tmp/build/80754af9/scikit-learn_1598376899566/work
+scipy @ file:///tmp/build/80754af9/scipy_1597686649129/work
+seaborn @ file:///tmp/build/80754af9/seaborn_1600553570093/work
+SecretStorage==3.1.2
+segtok==1.5.11
+Send2Trash==1.5.0
+sentencepiece==0.1.97
+simplegeneric==0.8.1
+singledispatch @ file:///tmp/build/80754af9/singledispatch_1602523705405/work
+sip==4.19.13
+six @ file:///tmp/build/80754af9/six_1605205327372/work
+smart-open==5.2.1
+snowballstemmer==2.0.0
+sortedcollections==1.2.1
+sortedcontainers==2.2.2
+SoundFile==0.10.3.post1
+soupsieve==2.0.1
+sphfile==1.0.3
+Sphinx @ file:///tmp/build/80754af9/sphinx_1597428793432/work
+sphinxcontrib-applehelp==1.0.2
+sphinxcontrib-devhelp==1.0.2
+sphinxcontrib-htmlhelp==1.0.3
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.3
+sphinxcontrib-serializinghtml==1.1.4
+sphinxcontrib-websupport @ file:///tmp/build/80754af9/sphinxcontrib-websupport_1597081412696/work
+spyder @ file:///tmp/build/80754af9/spyder_1599056981321/work
+spyder-kernels @ file:///tmp/build/80754af9/spyder-kernels_1599056754858/work
+SQLAlchemy @ file:///tmp/build/80754af9/sqlalchemy_1603397987316/work
+sqlitedict==2.1.0
+statsmodels @ file:///tmp/build/80754af9/statsmodels_1602280205159/work
+sympy @ file:///tmp/build/80754af9/sympy_1605119542615/work
+tables==3.6.1
+tabulate==0.9.0
+tblib @ file:///tmp/build/80754af9/tblib_1597928476713/work
+tensorboard==2.10.1
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.1
+terminado==0.9.1
+testpath==0.4.4
+threadpoolctl @ file:///tmp/tmp9twdgx9k/threadpoolctl-2.1.0-py3-none-any.whl
+tifffile==2020.10.1
+tkseem==0.0.3
+tokenizers==0.13.3
+toml @ file:///tmp/build/80754af9/toml_1592853716807/work
+tomli==2.0.1
+toolz @ file:///tmp/build/80754af9/toolz_1601054250827/work
+torch==1.11.0
+torchaudio==0.11.0
+torchmetrics==0.6.0
+torchvision==0.8.2
+tornado==6.0.4
+tqdm==4.64.0
+traitlets @ file:///tmp/build/80754af9/traitlets_1602787416690/work
+transformer-smaller-training-vocab==0.3.1
+transformers==4.33.1
+typing-extensions==4.4.0
+ujson @ file:///tmp/build/80754af9/ujson_1602523317881/work
+unicodecsv==0.14.1
+urllib3 @ file:///tmp/build/80754af9/urllib3_1603305693037/work
+watchdog @ file:///tmp/build/80754af9/watchdog_1593447344699/work
+wcwidth @ file:///tmp/build/80754af9/wcwidth_1593447189090/work
+webencodings==0.5.1
+Werkzeug==1.0.1
+widgetsnbextension==3.5.1
+Wikipedia-API==0.6.0
+wrapt==1.11.2
+wurlitzer @ file:///tmp/build/80754af9/wurlitzer_1594753850195/work
+xlrd==1.2.0
+XlsxWriter @ file:///tmp/build/80754af9/xlsxwriter_1602692860603/work
+xlwt==1.3.0
+xmltodict==0.12.0
+xxhash==3.0.0
+yapf @ file:///tmp/build/80754af9/yapf_1593528177422/work
+yarl==1.7.2
+zict==2.0.0
+zipp @ file:///tmp/build/80754af9/zipp_1604001098328/work
+zope.event==4.5.0
+zope.interface @ file:///tmp/build/80754af9/zope.interface_1602002420968/work
+==============================
+Git revision:
+8a51838
+==============================
+CUDA version:
+11.7

results/non_semi_final_stac/hyperparams.yaml

@@ -1,5 +1,5 @@
-# Generated 2023-09-08 from:
-# /gpfsssd/scratch/rech/nou/uzn19yk/switched_data/stac.yaml
+# Generated 2023-09-20 from:
+# /home/salah/Code_Switched_Tunisian_Speech_Recognition/cs.yaml
 # yamllint disable
 # Generated 2023-08-03 from:
 # /home/salah/new_tunisian_model/hparams/train_tunisian_withwavlm.yaml
@@ -46,8 +46,7 @@ sorting: ascending
 auto_mix_prec: false
 sample_rate: 16000
 language_modelling: true
-ngram_lm_path: 
-  /gpfsstore/rech/nou/uzn19yk/switched_code_tunisian/train/tunisian_asr/arpas/pluslanguages_everything.arpa
+ngram_lm_path: arpas/pluslanguages_everything.arpa
 
 # With data_parallel batch_size is split into N jobs
 # With DDP batch_size is multiplied by N jobs

transcribe.ipynb

@@ -0,0 +1,915 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "speechbrain.utils.distributed - distributed_launch flag is disabled, this experiment will be executed without DDP.\n",
+      "speechbrain.lobes.models.huggingface_wav2vec - speechbrain.lobes.models.huggingface_wav2vec - wav2vec 2.0 feature extractor is frozen.\n",
+      "speechbrain.core - Beginning experiment!\n",
+      "speechbrain.core - Experiment folder: TunisianASR/results/14epoch_tunisian/1234/\n",
+      "speechbrain.pretrained.fetching - Fetch hyperparams.yaml: Using existing file/symlink in pretrained_models/asr-wav2vec2-commonvoice-fr/hyperparams.yaml.\n",
+      "speechbrain.pretrained.fetching - Fetch custom.py: Linking to local file in /home/salah/Code_Switched_Tunisian_Speech_Recognition/asr-wav2vec2-commonvoice-fr/custom.py.\n",
+      "speechbrain.lobes.models.huggingface_wav2vec - speechbrain.lobes.models.huggingface_wav2vec - wav2vec 2.0 is frozen.\n",
+      "speechbrain.pretrained.fetching - Fetch wav2vec2.ckpt: Using existing file/symlink in pretrained_models/asr-wav2vec2-commonvoice-fr/wav2vec2.ckpt.\n",
+      "speechbrain.pretrained.fetching - Fetch asr.ckpt: Using existing file/symlink in pretrained_models/asr-wav2vec2-commonvoice-fr/asr.ckpt.\n",
+      "speechbrain.pretrained.fetching - Fetch tokenizer.ckpt: Using existing file/symlink in pretrained_models/asr-wav2vec2-commonvoice-fr/tokenizer.ckpt.\n",
+      "speechbrain.utils.parameter_transfer - Loading pretrained files for: wav2vec2, asr, tokenizer\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Some weights of the model checkpoint at wav2vec2-large-lv60/ were not used when initializing Wav2Vec2Model: ['project_hid.bias', 'project_q.bias', 'project_hid.weight', 'quantizer.codevectors', 'quantizer.weight_proj.weight', 'quantizer.weight_proj.bias', 'project_q.weight']\n",
+      "- This IS expected if you are initializing Wav2Vec2Model from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
+      "- This IS NOT expected if you are initializing Wav2Vec2Model from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "speechbrain.lobes.models.huggingface_wav2vec - speechbrain.lobes.models.huggingface_wav2vec - wav2vec 2.0 feature extractor is frozen.\n",
+      "speechbrain.core - Info: auto_mix_prec arg from hparam file is used\n",
+      "speechbrain.core - Info: ckpt_interval_minutes arg from hparam file is used\n",
+      "speechbrain.core - 314.4M trainable parameters in ASRCV\n",
+      "speechbrain.utils.checkpoints - Loading a checkpoint from EnglishCV/results/wav2vec2_ctc_en/1234/save/CKPT+2023-09-06+22-56-31+00\n",
+      "moving to tunisian model\n",
+      "speechbrain.core - Info: auto_mix_prec arg from hparam file is used\n",
+      "speechbrain.core - Info: ckpt_interval_minutes arg from hparam file is used\n",
+      "speechbrain.core - 314.4M trainable parameters in ASR\n",
+      "speechbrain.utils.checkpoints - Loading a checkpoint from TunisianASR/results/14epoch_tunisian/1234/save/CKPT+2023-08-03+01-38-38+00\n"
+     ]
+    }
+   ],
+   "source": [
+    "import os\n",
+    "import sys\n",
+    "import torch\n",
+    "import logging\n",
+    "import speechbrain as sb\n",
+    "from speechbrain.utils.distributed import run_on_main\n",
+    "from hyperpyyaml import load_hyperpyyaml\n",
+    "from pathlib import Path\n",
+    "import torchaudio.transforms as T\n",
+    "from cv_train import ASRCV\n",
+    "import torchaudio\n",
+    "import numpy as np\n",
+    "import kenlm\n",
+    "from pyctcdecode import build_ctcdecoder\n",
+    "import re\n",
+    "from torch.nn.utils.rnn import pad_sequence\n",
+    "import torch.optim as optim\n",
+    "import torch.nn as nn\n",
+    "\n",
+    "\n",
+    "# Commented out IPython magic to ensure Python compatibility.\n",
+    "hparams_file, run_opts, overrides = sb.parse_arguments([\"TunisianASR/semi_trained.yaml\"])\n",
+    "\n",
+    "# If distributed_launch=True then\n",
+    "# create ddp_group with the right communication protocol\n",
+    "sb.utils.distributed.ddp_init_group(run_opts)\n",
+    "\n",
+    "with open(hparams_file) as fin:\n",
+    "    hparams = load_hyperpyyaml(fin, overrides)\n",
+    "\n",
+    "# Create experiment directory\n",
+    "sb.create_experiment_directory(\n",
+    "    experiment_directory=hparams[\"output_folder\"],\n",
+    "    hyperparams_to_save=hparams_file,\n",
+    "    overrides=overrides,\n",
+    ")\n",
+    "# Dataset prep (parsing Librispeech)\n",
+    "\n",
+    "def dataio_prepare(hparams):\n",
+    "    \"\"\"This function prepares the datasets to be used in the brain class.\n",
+    "    It also defines the data processing pipeline through user-defined functions.\"\"\"\n",
+    "\n",
+    "    # 1. Define datasets\n",
+    "    data_folder = hparams[\"data_folder\"]\n",
+    "\n",
+    "    train_data = sb.dataio.dataset.DynamicItemDataset.from_csv(\n",
+    "        csv_path=hparams[\"train_csv\"], replacements={\"data_root\": data_folder},\n",
+    "    )\n",
+    "\n",
+    "    if hparams[\"sorting\"] == \"ascending\":\n",
+    "        # we sort training data to speed up training and get better results.\n",
+    "        train_data = train_data.filtered_sorted(\n",
+    "            sort_key=\"duration\",\n",
+    "            key_max_value={\"duration\": hparams[\"avoid_if_longer_than\"]},\n",
+    "        )\n",
+    "        # when sorting do not shuffle in dataloader ! otherwise is pointless\n",
+    "        hparams[\"dataloader_options\"][\"shuffle\"] = False\n",
+    "\n",
+    "    elif hparams[\"sorting\"] == \"descending\":\n",
+    "        train_data = train_data.filtered_sorted(\n",
+    "            sort_key=\"duration\",\n",
+    "            reverse=True,\n",
+    "            key_max_value={\"duration\": hparams[\"avoid_if_longer_than\"]},\n",
+    "        )\n",
+    "        # when sorting do not shuffle in dataloader ! otherwise is pointless\n",
+    "        hparams[\"dataloader_options\"][\"shuffle\"] = False\n",
+    "\n",
+    "    elif hparams[\"sorting\"] == \"random\":\n",
+    "        pass\n",
+    "\n",
+    "    else:\n",
+    "        raise NotImplementedError(\n",
+    "            \"sorting must be random, ascending or descending\"\n",
+    "        )\n",
+    "\n",
+    "    valid_data = sb.dataio.dataset.DynamicItemDataset.from_csv(\n",
+    "        csv_path=hparams[\"valid_csv\"], replacements={\"data_root\": data_folder},\n",
+    "    )\n",
+    "    # We also sort the validation data so it is faster to validate\n",
+    "    valid_data = valid_data.filtered_sorted(sort_key=\"duration\")\n",
+    "    test_datasets = {}\n",
+    "    for csv_file in hparams[\"test_csv\"]:\n",
+    "        name = Path(csv_file).stem\n",
+    "        test_datasets[name] = sb.dataio.dataset.DynamicItemDataset.from_csv(\n",
+    "            csv_path=csv_file, replacements={\"data_root\": data_folder}\n",
+    "        )\n",
+    "        test_datasets[name] = test_datasets[name].filtered_sorted(\n",
+    "            sort_key=\"duration\"\n",
+    "        )\n",
+    "\n",
+    "    datasets = [train_data, valid_data] + [i for k, i in test_datasets.items()]\n",
+    "\n",
+    "\n",
+    "    # 2. Define audio pipeline:\n",
+    "    @sb.utils.data_pipeline.takes(\"wav\")\n",
+    "    @sb.utils.data_pipeline.provides(\"sig\")\n",
+    "    def audio_pipeline(wav):\n",
+    "        info = torchaudio.info(wav)\n",
+    "        sig = sb.dataio.dataio.read_audio(wav)\n",
+    "        if len(sig.shape)>1 :\n",
+    "            sig = torch.mean(sig, dim=1)\n",
+    "        resampled = torchaudio.transforms.Resample(\n",
+    "            info.sample_rate, hparams[\"sample_rate\"],\n",
+    "        )(sig)\n",
+    "        return resampled\n",
+    "\n",
+    "    sb.dataio.dataset.add_dynamic_item(datasets, audio_pipeline)\n",
+    "    label_encoder = sb.dataio.encoder.CTCTextEncoder()\n",
+    "\n",
+    "    # 3. Define text pipeline:\n",
+    "    @sb.utils.data_pipeline.takes(\"wrd\")\n",
+    "    @sb.utils.data_pipeline.provides(\n",
+    "        \"wrd\", \"char_list\", \"tokens_list\", \"tokens\"\n",
+    "    )\n",
+    "    def text_pipeline(wrd):\n",
+    "        yield wrd\n",
+    "        char_list = list(wrd)\n",
+    "        yield char_list\n",
+    "        tokens_list = label_encoder.encode_sequence(char_list)\n",
+    "        yield tokens_list\n",
+    "        tokens = torch.LongTensor(tokens_list)\n",
+    "        yield tokens\n",
+    "\n",
+    "    sb.dataio.dataset.add_dynamic_item(datasets, text_pipeline)\n",
+    "    lab_enc_file = os.path.join(hparams[\"save_folder\"], \"label_encoder.txt\")\n",
+    "    special_labels = {\n",
+    "        \"blank_label\": hparams[\"blank_index\"],\n",
+    "        \"unk_label\": hparams[\"unk_index\"]\n",
+    "    }\n",
+    "    label_encoder.load_or_create(\n",
+    "        path=lab_enc_file,\n",
+    "        from_didatasets=[train_data],\n",
+    "        output_key=\"char_list\",\n",
+    "        special_labels=special_labels,\n",
+    "        sequence_input=True,\n",
+    "    )\n",
+    "\n",
+    "    # 4. Set output:\n",
+    "    sb.dataio.dataset.set_output_keys(\n",
+    "        datasets, [\"id\", \"sig\", \"wrd\", \"char_list\", \"tokens\"],\n",
+    "    )\n",
+    "    return train_data, valid_data,test_datasets, label_encoder\n",
+    "\n",
+    "class ASR(sb.core.Brain):\n",
+    "    def compute_forward(self, batch, stage):\n",
+    "        \"\"\"Forward computations from the waveform batches to the output probabilities.\"\"\"\n",
+    "\n",
+    "        batch = batch.to(self.device)\n",
+    "        wavs, wav_lens = batch.sig\n",
+    "        wavs, wav_lens = wavs.to(self.device), wav_lens.to(self.device)\n",
+    "\n",
+    "        if stage == sb.Stage.TRAIN:\n",
+    "            if hasattr(self.hparams, \"augmentation\"):\n",
+    "                wavs = self.hparams.augmentation(wavs, wav_lens)\n",
+    "\n",
+    "        # Forward pass\n",
+    "        feats = self.modules.wav2vec2(wavs, wav_lens)\n",
+    "        x = self.modules.enc(feats)\n",
+    "        logits = self.modules.ctc_lin(x)\n",
+    "        p_ctc = self.hparams.log_softmax(logits)\n",
+    "\n",
+    "        return p_ctc, wav_lens\n",
+    "\n",
+    "    def custom_encode(self,wavs,wav_lens) :\n",
+    "        wavs = wavs.to(\"cpu\")\n",
+    "        if(wav_lens is not None): wav_lens.to(self.device)\n",
+    "\n",
+    "        feats = self.modules.wav2vec2(wavs, wav_lens)\n",
+    "        x = self.modules.enc(feats)\n",
+    "        logits = self.modules.ctc_lin(x)\n",
+    "        p_ctc = self.hparams.log_softmax(logits)\n",
+    "\n",
+    "        return feats,p_ctc\n",
+    "\n",
+    "\n",
+    "\n",
+    "    def compute_objectives(self, predictions, batch, stage):\n",
+    "        \"\"\"Computes the loss (CTC) given predictions and targets.\"\"\"\n",
+    "\n",
+    "        p_ctc, wav_lens = predictions\n",
+    "\n",
+    "        ids = batch.id\n",
+    "        tokens, tokens_lens = batch.tokens\n",
+    "\n",
+    "        loss = self.hparams.ctc_cost(p_ctc, tokens, wav_lens, tokens_lens)\n",
+    "\n",
+    "        if stage != sb.Stage.TRAIN:\n",
+    "            predicted_tokens = sb.decoders.ctc_greedy_decode(\n",
+    "                p_ctc, wav_lens, blank_id=self.hparams.blank_index\n",
+    "            )\n",
+    "            # Decode token terms to words\n",
+    "            if self.hparams.use_language_modelling:\n",
+    "                predicted_words = []\n",
+    "                for logs in p_ctc:\n",
+    "                    text = decoder.decode(logs.detach().cpu().numpy())\n",
+    "                    predicted_words.append(text.split(\" \"))\n",
+    "            else:\n",
+    "                predicted_words = [\n",
+    "                    \"\".join(self.tokenizer.decode_ndim(utt_seq)).split(\" \")\n",
+    "                    for utt_seq in predicted_tokens\n",
+    "                ]\n",
+    "            # Convert indices to words\n",
+    "            target_words = [wrd.split(\" \") for wrd in batch.wrd]\n",
+    "\n",
+    "            self.wer_metric.append(ids, predicted_words, target_words)\n",
+    "            self.cer_metric.append(ids, predicted_words, target_words)\n",
+    "\n",
+    "        return loss\n",
+    "\n",
+    "    def fit_batch(self, batch):\n",
+    "        \"\"\"Train the parameters given a single batch in input\"\"\"\n",
+    "        should_step = self.step % self.grad_accumulation_factor == 0\n",
+    "        # Managing automatic mixed precision\n",
+    "        # TOFIX: CTC fine-tuning currently is unstable\n",
+    "        # This is certainly due to CTC being done in fp16 instead of fp32\n",
+    "        if self.auto_mix_prec:\n",
+    "            with torch.cuda.amp.autocast():\n",
+    "                with self.no_sync():\n",
+    "                    outputs = self.compute_forward(batch, sb.Stage.TRAIN)\n",
+    "                loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)\n",
+    "            with self.no_sync(not should_step):\n",
+    "                self.scaler.scale(\n",
+    "                    loss / self.grad_accumulation_factor\n",
+    "                ).backward()\n",
+    "            if should_step:\n",
+    "\n",
+    "                if not self.hparams.wav2vec2.freeze:\n",
+    "                    self.scaler.unscale_(self.wav2vec_optimizer)\n",
+    "                self.scaler.unscale_(self.model_optimizer)\n",
+    "                if self.check_gradients(loss):\n",
+    "                    if not self.hparams.wav2vec2.freeze:\n",
+    "                        if self.optimizer_step >= self.hparams.warmup_steps:\n",
+    "                            self.scaler.step(self.wav2vec_optimizer)\n",
+    "                    self.scaler.step(self.model_optimizer)\n",
+    "                self.scaler.update()\n",
+    "                self.zero_grad()\n",
+    "                self.optimizer_step += 1\n",
+    "        else:\n",
+    "            # This is mandatory because HF models have a weird behavior with DDP\n",
+    "            # on the forward pass\n",
+    "            with self.no_sync():\n",
+    "                outputs = self.compute_forward(batch, sb.Stage.TRAIN)\n",
+    "\n",
+    "            loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)\n",
+    "\n",
+    "            with self.no_sync(not should_step):\n",
+    "                (loss / self.grad_accumulation_factor).backward()\n",
+    "            if should_step:\n",
+    "                if self.check_gradients(loss):\n",
+    "                    if not self.hparams.wav2vec2.freeze:\n",
+    "                        if self.optimizer_step >= self.hparams.warmup_steps:\n",
+    "                            self.wav2vec_optimizer.step()\n",
+    "                    self.model_optimizer.step()\n",
+    "                self.zero_grad()\n",
+    "                self.optimizer_step += 1\n",
+    "\n",
+    "        self.on_fit_batch_end(batch, outputs, loss, should_step)\n",
+    "        return loss.detach().cpu()\n",
+    "\n",
+    "    def evaluate_batch(self, batch, stage):\n",
+    "        \"\"\"Computations needed for validation/test batches\"\"\"\n",
+    "        predictions = self.compute_forward(batch, stage=stage)\n",
+    "        with torch.no_grad():\n",
+    "            loss = self.compute_objectives(predictions, batch, stage=stage)\n",
+    "        return loss.detach()\n",
+    "\n",
+    "    def on_stage_start(self, stage, epoch):\n",
+    "        \"\"\"Gets called at the beginning of each epoch\"\"\"\n",
+    "        if stage != sb.Stage.TRAIN:\n",
+    "            self.cer_metric = self.hparams.cer_computer()\n",
+    "            self.wer_metric = self.hparams.error_rate_computer()\n",
+    "\n",
+    "    def on_stage_end(self, stage, stage_loss, epoch):\n",
+    "        \"\"\"Gets called at the end of an epoch.\"\"\"\n",
+    "        # Compute/store important stats\n",
+    "        stage_stats = {\"loss\": stage_loss}\n",
+    "        if stage == sb.Stage.TRAIN:\n",
+    "            self.train_stats = stage_stats\n",
+    "        else:\n",
+    "            stage_stats[\"CER\"] = self.cer_metric.summarize(\"error_rate\")\n",
+    "            stage_stats[\"WER\"] = self.wer_metric.summarize(\"error_rate\")\n",
+    "\n",
+    "        # Perform end-of-iteration things, like annealing, logging, etc.\n",
+    "        if stage == sb.Stage.VALID:\n",
+    "            old_lr_model, new_lr_model = self.hparams.lr_annealing_model(\n",
+    "                stage_stats[\"loss\"]\n",
+    "            )\n",
+    "            old_lr_wav2vec, new_lr_wav2vec = self.hparams.lr_annealing_wav2vec(\n",
+    "                stage_stats[\"loss\"]\n",
+    "            )\n",
+    "            sb.nnet.schedulers.update_learning_rate(\n",
+    "                self.model_optimizer, new_lr_model\n",
+    "            )\n",
+    "            if not self.hparams.wav2vec2.freeze:\n",
+    "                sb.nnet.schedulers.update_learning_rate(\n",
+    "                    self.wav2vec_optimizer, new_lr_wav2vec\n",
+    "                )\n",
+    "            self.hparams.train_logger.log_stats(\n",
+    "                stats_meta={\n",
+    "                    \"epoch\": epoch,\n",
+    "                    \"lr_model\": old_lr_model,\n",
+    "                    \"lr_wav2vec\": old_lr_wav2vec,\n",
+    "                },\n",
+    "                train_stats=self.train_stats,\n",
+    "                valid_stats=stage_stats,\n",
+    "            )\n",
+    "            self.checkpointer.save_and_keep_only(\n",
+    "                meta={\"WER\": stage_stats[\"WER\"]}, min_keys=[\"WER\"],\n",
+    "            )\n",
+    "        elif stage == sb.Stage.TEST:\n",
+    "            self.hparams.train_logger.log_stats(\n",
+    "                stats_meta={\"Epoch loaded\": self.hparams.epoch_counter.current},\n",
+    "                test_stats=stage_stats,\n",
+    "            )\n",
+    "            with open(self.hparams.wer_file, \"w\") as w:\n",
+    "                self.wer_metric.write_stats(w)\n",
+    "\n",
+    "    def init_optimizers(self):\n",
+    "        \"Initializes the wav2vec2 optimizer and model optimizer\"\n",
+    "\n",
+    "        # If the wav2vec encoder is unfrozen, we create the optimizer\n",
+    "        if not self.hparams.wav2vec2.freeze:\n",
+    "            self.wav2vec_optimizer = self.hparams.wav2vec_opt_class(\n",
+    "                self.modules.wav2vec2.parameters()\n",
+    "            )\n",
+    "            if self.checkpointer is not None:\n",
+    "                self.checkpointer.add_recoverable(\n",
+    "                    \"wav2vec_opt\", self.wav2vec_optimizer\n",
+    "                )\n",
+    "\n",
+    "        self.model_optimizer = self.hparams.model_opt_class(\n",
+    "            self.hparams.model.parameters()\n",
+    "        )\n",
+    "\n",
+    "        if self.checkpointer is not None:\n",
+    "            self.checkpointer.add_recoverable(\"modelopt\", self.model_optimizer)\n",
+    "\n",
+    "    def zero_grad(self, set_to_none=False):\n",
+    "        if not self.hparams.wav2vec2.freeze:\n",
+    "            self.wav2vec_optimizer.zero_grad(set_to_none)\n",
+    "        self.model_optimizer.zero_grad(set_to_none)\n",
+    "\n",
+    "\n",
+    "from speechbrain.pretrained import EncoderASR,EncoderDecoderASR\n",
+    "french_asr_model = EncoderASR.from_hparams(source=\"asr-wav2vec2-commonvoice-fr\", savedir=\"pretrained_models/asr-wav2vec2-commonvoice-fr\").cuda()\n",
+    "french_asr_model.to(\"cpu\")\n",
+    "cvhparams_file, cvrun_opts, cvoverrides = sb.parse_arguments([\"EnglishCV/train_en_with_wav2vec.yaml\"])\n",
+    "with open(cvhparams_file) as cvfin:\n",
+    "    cvhparams = load_hyperpyyaml(cvfin, cvoverrides)\n",
+    "english_asr_model = ASRCV(\n",
+    "        modules=cvhparams[\"modules\"],\n",
+    "        hparams=cvhparams,\n",
+    "        run_opts=cvrun_opts,\n",
+    "        checkpointer=cvhparams[\"checkpointer\"],\n",
+    "    )\n",
+    "english_asr_model.modules.to(\"cpu\")\n",
+    "english_asr_model.checkpointer.recover_if_possible()\n",
+    "print(\"moving to tunisian model\")\n",
+    "asr_brain = ASR(\n",
+    "    modules=hparams[\"modules\"],\n",
+    "    hparams=hparams,\n",
+    "    run_opts=run_opts,\n",
+    "    checkpointer=hparams[\"checkpointer\"],\n",
+    ")\n",
+    "asr_brain.modules.to(\"cpu\")\n",
+    "asr_brain.checkpointer.recover_if_possible()\n",
+    "asr_brain.modules.eval()\n",
+    "english_asr_model.modules.eval()\n",
+    "french_asr_model.mods.eval()\n",
+    "asr_brain.modules.to(\"cpu\")\n",
+    "\n",
+    "# Commented out IPython magic to ensure Python compatibility.\n",
+    "# %ls\n",
+    "\n",
+    "#UTILS FUNCTIOJNS\n",
+    "def get_size_dimensions(arr):\n",
+    "    size_dimensions = []\n",
+    "    while isinstance(arr, list):\n",
+    "        size_dimensions.append(len(arr))\n",
+    "        arr = arr[0]\n",
+    "    return size_dimensions\n",
+    "\n",
+    "def scale_array(batch,n):\n",
+    "    scaled_batch = []\n",
+    "\n",
+    "    for array in batch:\n",
+    "        if(n < len(array)): raise ValueError(\"Cannot scale Array down\")\n",
+    "\n",
+    "        repeat = round(n/len(array))+1\n",
+    "        scaled_length_array= []\n",
+    "\n",
+    "        for i in array:\n",
+    "            for j in range(repeat) :\n",
+    "                if(len(scaled_length_array) == n): break\n",
+    "                scaled_length_array.append(i)\n",
+    "\n",
+    "        scaled_batch.append(scaled_length_array)\n",
+    "\n",
+    "    return torch.tensor(scaled_batch)\n",
+    "\n",
+    "\n",
+    "def load_paths(wavs_path):\n",
+    "    waveforms = []\n",
+    "    for path in wavs_path :\n",
+    "        waveform, _ = torchaudio.load(path)\n",
+    "        waveforms.append(waveform.squeeze(0))\n",
+    "    # normalize array length to the bigger arrays by pading with 0's\n",
+    "    padded_arrays = pad_sequence(waveforms, batch_first=True)\n",
+    "    return torch.tensor(padded_arrays)\n",
+    "\n",
+    "\n",
+    "\n",
+    "device = 'cuda'\n",
+    "verbose = 0\n",
+    "#FLOW LEVEL FUNCTIONS\n",
+    "def merge_strategy(embeddings1, embeddings2, embeddings3,post1, post2,post3):\n",
+    "\n",
+    "\n",
+    "    post1 = post1.to(device)\n",
+    "    post2 = post2.to(device)\n",
+    "    post3 = post3.to(device)\n",
+    "    embeddings1 = embeddings1.to(device)\n",
+    "    embeddings2 = embeddings2.to(device)\n",
+    "    embeddings3 = embeddings3.to(device)\n",
+    "\n",
+    "    posteriograms_merged = torch.cat((post1,post2,post3),dim=2)\n",
+    "    embeddings_merged = torch.cat((embeddings1,embeddings2,embeddings3),dim=2)\n",
+    "\n",
+    "    if(verbose !=0):\n",
+    "      print('MERGED POST ',posteriograms_merged.shape)\n",
+    "      print('MERGED emb ',embeddings_merged.shape)\n",
+    "\n",
+    "    return torch.cat((posteriograms_merged,embeddings_merged),dim=2).to(device)\n",
+    "\n",
+    "def decode(model,wavs,wav_lens):\n",
+    "\n",
+    "    with torch.no_grad():\n",
+    "        wav_lens = wav_lens.to(model.device)\n",
+    "        encoder_out = model.encode_batch(wavs, wav_lens)\n",
+    "        predictions = model.decoding_function(encoder_out, wav_lens)\n",
+    "        return predictions\n",
+    "\n",
+    "def middle_layer(batch, lens):\n",
+    "\n",
+    "    tn_embeddings, tn_posteriogram = asr_brain.custom_encode(batch,None)\n",
+    "\n",
+    "    fr_embeddings = french_asr_model.mods.encoder.wav2vec2(batch)\n",
+    "    fr_posteriogram =french_asr_model.encode_batch(batch,lens)\n",
+    "    en_embeddings = english_asr_model.modules.wav2vec2(batch, lens)\n",
+    "    x = english_asr_model.modules.enc(en_embeddings)\n",
+    "    en_posteriogram = english_asr_model.modules.ctc_lin(x)\n",
+    "    #scores, en_posteriogram = english_asr_model.mods.decoder(en_embeddings ,lens)\n",
+    "    if(verbose !=0):\n",
+    "      print('[EMBEDDINGS] FR:',fr_embeddings.shape, \"EN:\",en_embeddings.shape, \"TN:\", tn_embeddings.shape)\n",
+    "      print('[POSTERIOGRAM] FR:',fr_posteriogram.shape, \"EN:\",en_posteriogram.shape,\"TN:\",tn_posteriogram.shape)\n",
+    "\n",
+    "\n",
+    "    bilangual_sample = merge_strategy(fr_embeddings,en_embeddings,tn_embeddings,fr_posteriogram,en_posteriogram,tn_posteriogram)\n",
+    "    return bilangual_sample\n",
+    "\n",
+    "class Mixer(sb.core.Brain):\n",
+    "\n",
+    "    def compute_forward(self, batch, stage):\n",
+    "        \"\"\"Forward computations from the waveform batches to the output probabilities.\"\"\"\n",
+    "        wavs, wav_lens = batch.sig\n",
+    "        wavs, wav_lens = wavs.to(self.device), wav_lens.to(self.device)\n",
+    "\n",
+    "        if stage == sb.Stage.TRAIN:\n",
+    "            if hasattr(self.hparams, \"augmentation\"):\n",
+    "                wavs = self.hparams.augmentation(wavs, wav_lens)\n",
+    "\n",
+    "        multi_langual_feats = middle_layer(wavs, wav_lens)\n",
+    "        multi_langual_feats= multi_langual_feats.to(device)\n",
+    "        feats, _ = self.modules.enc(multi_langual_feats)\n",
+    "        logits = self.modules.ctc_lin(feats)\n",
+    "        p_ctc = self.hparams.log_softmax(logits)\n",
+    "        \n",
+    "        if stage!= sb.Stage.TRAIN:\n",
+    "            p_tokens = sb.decoders.ctc_greedy_decode(\n",
+    "                p_ctc, wav_lens, blank_id=self.hparams.blank_index\n",
+    "            )\n",
+    "        else : \n",
+    "            p_tokens = None\n",
+    "        return p_ctc, wav_lens, p_tokens\n",
+    "    \n",
+    "    \n",
+    "    def treat_wav(self,sig):\n",
+    "        multi_langual_feats = middle_layer(sig.to(\"cpu\"), torch.tensor([1]).to(\"cpu\"))\n",
+    "        multi_langual_feats= multi_langual_feats.to(device)\n",
+    "        feats, _ = self.modules.enc(multi_langual_feats)\n",
+    "        logits = self.modules.ctc_lin(feats)\n",
+    "        p_ctc = self.hparams.log_softmax(logits)\n",
+    "        predicted_words =[]\n",
+    "        for logs in p_ctc:\n",
+    "            text = decoder.decode(logs.detach().cpu().numpy())\n",
+    "            predicted_words.append(text.split(\" \"))\n",
+    "        return \" \".join(predicted_words[0])\n",
+    "        \n",
+    "\n",
+    "    def compute_objectives(self, predictions, batch, stage):\n",
+    "        \"\"\"Computes the loss (CTC) given predictions and targets.\"\"\"\n",
+    "\n",
+    "        p_ctc, wav_lens , predicted_tokens= predictions\n",
+    "\n",
+    "        ids = batch.id\n",
+    "        tokens, tokens_lens = batch.tokens\n",
+    "\n",
+    "        loss = self.hparams.ctc_cost(p_ctc, tokens, wav_lens, tokens_lens)\n",
+    "\n",
+    "\n",
+    "        if stage == sb.Stage.VALID:\n",
+    "            predicted_words = [\n",
+    "                \"\".join(self.tokenizer.decode_ndim(utt_seq)).split(\" \")\n",
+    "                for utt_seq in predicted_tokens\n",
+    "            ]\n",
+    "            target_words = [wrd.split(\" \") for wrd in batch.wrd]\n",
+    "            self.wer_metric.append(ids, predicted_words, target_words)\n",
+    "            self.cer_metric.append(ids, predicted_words, target_words)\n",
+    "        if stage ==sb.Stage.TEST : \n",
+    "            if self.hparams.language_modelling:\n",
+    "                predicted_words = []\n",
+    "                for logs in p_ctc:\n",
+    "                    text = decoder.decode(logs.detach().cpu().numpy())\n",
+    "                    predicted_words.append(text.split(\" \"))\n",
+    "            else : \n",
+    "                predicted_words = [\n",
+    "                    \"\".join(self.tokenizer.decode_ndim(utt_seq)).split(\" \")\n",
+    "                    for utt_seq in predicted_tokens\n",
+    "                ]\n",
+    "\n",
+    "            target_words = [wrd.split(\" \") for wrd in batch.wrd]\n",
+    "            self.wer_metric.append(ids, predicted_words, target_words)\n",
+    "            self.cer_metric.append(ids, predicted_words, target_words)\n",
+    "\n",
+    "        return loss\n",
+    "\n",
+    "    def fit_batch(self, batch):\n",
+    "        \"\"\"Train the parameters given a single batch in input\"\"\"\n",
+    "        should_step = self.step % self.grad_accumulation_factor == 0\n",
+    "        # Managing automatic mixed precision\n",
+    "        # TOFIX: CTC fine-tuning currently is unstable\n",
+    "        # This is certainly due to CTC being done in fp16 instead of fp32\n",
+    "        if self.auto_mix_prec:\n",
+    "            with torch.cuda.amp.autocast():\n",
+    "                with self.no_sync():\n",
+    "                    outputs = self.compute_forward(batch, sb.Stage.TRAIN)\n",
+    "                loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)\n",
+    "            with self.no_sync(not should_step):\n",
+    "                self.scaler.scale(\n",
+    "                    loss / self.grad_accumulation_factor\n",
+    "                ).backward()\n",
+    "            if should_step:\n",
+    "\n",
+    "\n",
+    "                self.scaler.unscale_(self.model_optimizer)\n",
+    "                if self.check_gradients(loss):\n",
+    "                    self.scaler.step(self.model_optimizer)\n",
+    "                self.scaler.update()\n",
+    "                self.zero_grad()\n",
+    "                self.optimizer_step += 1\n",
+    "        else:\n",
+    "            # This is mandatory because HF models have a weird behavior with DDP\n",
+    "            # on the forward pass\n",
+    "            with self.no_sync():\n",
+    "                outputs = self.compute_forward(batch, sb.Stage.TRAIN)\n",
+    "\n",
+    "            loss = self.compute_objectives(outputs, batch, sb.Stage.TRAIN)\n",
+    "\n",
+    "            with self.no_sync(not should_step):\n",
+    "                (loss / self.grad_accumulation_factor).backward()\n",
+    "            if should_step:\n",
+    "                if self.check_gradients(loss):\n",
+    "                    self.model_optimizer.step()\n",
+    "                self.zero_grad()\n",
+    "                self.optimizer_step += 1\n",
+    "\n",
+    "        self.on_fit_batch_end(batch, outputs, loss, should_step)\n",
+    "        return loss.detach().cpu()\n",
+    "\n",
+    "    def evaluate_batch(self, batch, stage):\n",
+    "        \"\"\"Computations needed for validation/test batches\"\"\"\n",
+    "        predictions = self.compute_forward(batch, stage=stage)\n",
+    "        with torch.no_grad():\n",
+    "            loss = self.compute_objectives(predictions, batch, stage=stage)\n",
+    "        return loss.detach()\n",
+    "\n",
+    "    def on_stage_start(self, stage, epoch):\n",
+    "        \"\"\"Gets called at the beginning of each epoch\"\"\"\n",
+    "        if stage != sb.Stage.TRAIN:\n",
+    "            self.cer_metric = self.hparams.cer_computer()\n",
+    "            self.wer_metric = self.hparams.error_rate_computer()\n",
+    "\n",
+    "    def on_stage_end(self, stage, stage_loss, epoch):\n",
+    "        \"\"\"Gets called at the end of an epoch.\"\"\"\n",
+    "        # Compute/store important stats\n",
+    "        stage_stats = {\"loss\": stage_loss}\n",
+    "        if stage == sb.Stage.TRAIN:\n",
+    "            self.train_stats = stage_stats\n",
+    "        else:\n",
+    "            stage_stats[\"CER\"] = self.cer_metric.summarize(\"error_rate\")\n",
+    "            stage_stats[\"WER\"] = self.wer_metric.summarize(\"error_rate\")\n",
+    "\n",
+    "        # Perform end-of-iteration things, like annealing, logging, etc.\n",
+    "        if stage == sb.Stage.VALID:\n",
+    "            old_lr_model, new_lr_model = self.hparams.lr_annealing_model(\n",
+    "                stage_stats[\"loss\"]\n",
+    "            )\n",
+    "            sb.nnet.schedulers.update_learning_rate(\n",
+    "                self.model_optimizer, new_lr_model\n",
+    "            )\n",
+    "            self.hparams.train_logger.log_stats(\n",
+    "                stats_meta={\n",
+    "                    \"epoch\": epoch,\n",
+    "                    \"lr_model\": old_lr_model,\n",
+    "                },\n",
+    "                train_stats=self.train_stats,\n",
+    "                valid_stats=stage_stats,\n",
+    "            )\n",
+    "            self.checkpointer.save_and_keep_only(\n",
+    "                meta={\"WER\": stage_stats[\"WER\"]}, min_keys=[\"WER\"],\n",
+    "            )\n",
+    "        elif stage == sb.Stage.TEST:\n",
+    "            self.hparams.train_logger.log_stats(\n",
+    "                stats_meta={\"Epoch loaded\": self.hparams.epoch_counter.current},\n",
+    "                test_stats=stage_stats,\n",
+    "            )\n",
+    "            with open(self.hparams.wer_file, \"w\") as w:\n",
+    "                self.wer_metric.write_stats(w)\n",
+    "\n",
+    "    def init_optimizers(self):\n",
+    "\n",
+    "        self.model_optimizer = self.hparams.model_opt_class(\n",
+    "            self.hparams.model.parameters()\n",
+    "        )\n",
+    "\n",
+    "        if self.checkpointer is not None:\n",
+    "            self.checkpointer.add_recoverable(\"modelopt\", self.model_optimizer)\n",
+    "\n",
+    "    def zero_grad(self, set_to_none=False):\n",
+    "\n",
+    "        self.model_optimizer.zero_grad(set_to_none)\n",
+    "\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "speechbrain.utils.distributed - distributed_launch flag is disabled, this experiment will be executed without DDP.\n",
+      "speechbrain.core - Beginning experiment!\n",
+      "speechbrain.core - Experiment folder: results/non_semi_final_stac\n",
+      "speechbrain.dataio.encoder - Load called, but CTCTextEncoder is not empty. Loaded data will overwrite everything. This is normal if there is e.g. an unk label defined at init.\n",
+      "pyctcdecode.decoder - Using arpa instead of binary LM file, decoder instantiation might be slow.\n",
+      "pyctcdecode.alphabet - Alphabet determined to be of regular style.\n",
+      "pyctcdecode.alphabet - Unigrams and labels don't seem to agree.\n",
+      "speechbrain.core - Info: auto_mix_prec arg from hparam file is used\n",
+      "speechbrain.core - 60.1M trainable parameters in Mixer\n",
+      "speechbrain.utils.checkpoints - Loading a checkpoint from results/non_semi_final_stac/save/CKPT+2023-09-08+01-40-18+00\n",
+      "pyctcdecode.decoder - Using arpa instead of binary LM file, decoder instantiation might be slow.\n",
+      "pyctcdecode.alphabet - Alphabet determined to be of regular style.\n",
+      "pyctcdecode.alphabet - Unigrams and labels don't seem to agree.\n",
+      "speechbrain.utils.checkpoints - Loading a checkpoint from TunisianASR/results/14epoch_tunisian/1234/save/CKPT+2023-08-03+01-38-38+00\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "<ipython-input-32-948c3f4b1130>:120: GradioDeprecationWarning: `optional` parameter is deprecated, and it has no effect\n",
+      "  inputs=[gr.Audio(source=\"microphone\", type='filepath', label = \"record\", optional = True),\n",
+      "<ipython-input-32-948c3f4b1130>:121: GradioDeprecationWarning: `optional` parameter is deprecated, and it has no effect\n",
+      "  gr.Audio(source=\"upload\", type='filepath', label=\"filein\", optional=True)]\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Running on local URL:  http://127.0.0.1:7860\n",
+      "\n",
+      "To create a public link, set `share=True` in `launch()`.\n"
+     ]
+    },
+    {
+     "data": {
+      "text/html": [
+       "<div><iframe src=\"http://127.0.0.1:7860/\" width=\"100%\" height=\"500\" allow=\"autoplay; camera; microphone; clipboard-read; clipboard-write;\" frameborder=\"0\" allowfullscreen></iframe></div>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "/home/salah/anaconda3/envs/salah/lib/python3.8/site-packages/gradio/processing_utils.py:188: UserWarning: Trying to convert audio automatically from int32 to 16-bit int format.\n",
+      "  warnings.warn(warning.format(data.dtype))\n",
+      "/home/salah/anaconda3/envs/salah/lib/python3.8/site-packages/gradio/processing_utils.py:188: UserWarning: Trying to convert audio automatically from int32 to 16-bit int format.\n",
+      "  warnings.warn(warning.format(data.dtype))\n",
+      "/home/salah/anaconda3/envs/salah/lib/python3.8/site-packages/gradio/processing_utils.py:188: UserWarning: Trying to convert audio automatically from int32 to 16-bit int format.\n",
+      "  warnings.warn(warning.format(data.dtype))\n",
+      "/home/salah/anaconda3/envs/salah/lib/python3.8/site-packages/gradio/processing_utils.py:188: UserWarning: Trying to convert audio automatically from int32 to 16-bit int format.\n",
+      "  warnings.warn(warning.format(data.dtype))\n"
+     ]
+    }
+   ],
+   "source": [
+    "hparams_file, run_opts, overrides = sb.parse_arguments([\"cs.yaml\"])\n",
+    "\n",
+    "# If distributed_launch=True then\n",
+    "# create ddp_group with the right communication protocol\n",
+    "sb.utils.distributed.ddp_init_group(run_opts)\n",
+    "\n",
+    "with open(hparams_file) as fin:\n",
+    "    hparams = load_hyperpyyaml(fin, overrides)\n",
+    "\n",
+    "# Create experiment directory\n",
+    "sb.create_experiment_directory(\n",
+    "    experiment_directory=hparams[\"output_folder\"],\n",
+    "    hyperparams_to_save=hparams_file,\n",
+    "    overrides=overrides,\n",
+    ")\n",
+    "def read_labels_file(labels_file):\n",
+    "    with open(labels_file, \"r\",encoding=\"utf-8\") as lf:\n",
+    "        lines = lf.read().splitlines()\n",
+    "        division = \"===\"\n",
+    "        numbers = {}\n",
+    "        for line in lines :\n",
+    "            if division in line :\n",
+    "                break\n",
+    "            string, number = line.split(\"=>\")\n",
+    "            number = int(number)\n",
+    "            string = string[1:-2]\n",
+    "            numbers[number] = string\n",
+    "        return [numbers[x] for x in range(len(numbers))]\n",
+    "\n",
+    "label_encoder = sb.dataio.encoder.CTCTextEncoder()\n",
+    "\n",
+    "lab_enc_file = os.path.join(hparams[\"save_folder\"], \"label_encoder.txt\")\n",
+    "special_labels = {\n",
+    "    \"blank_label\": hparams[\"blank_index\"],\n",
+    "    \"unk_label\": hparams[\"unk_index\"]\n",
+    "}\n",
+    "label_encoder.load_or_create(\n",
+    "    path=lab_enc_file,\n",
+    "    from_didatasets=[[]],\n",
+    "    output_key=\"char_list\",\n",
+    "    special_labels=special_labels,\n",
+    "    sequence_input=True,\n",
+    ")\n",
+    "\n",
+    "\n",
+    "labels = read_labels_file(os.path.join(hparams[\"save_folder\"], \"label_encoder.txt\"))\n",
+    "labels = [\"\"] + labels[1:-1] + [\"1\"] \n",
+    "if hparams[\"language_modelling\"]:\n",
+    "    decoder = build_ctcdecoder(\n",
+    "        labels,\n",
+    "        kenlm_model_path=hparams[\"ngram_lm_path\"],  # either .arpa or .bin file\n",
+    "        alpha=0.5,  # tuned on a val set\n",
+    "        beta=1,  # tuned on a val set\n",
+    "    )\n",
+    "\n",
+    "\n",
+    "\n",
+    "\n",
+    "mixer = Mixer(\n",
+    "    modules=hparams[\"modules\"],\n",
+    "    hparams=hparams,\n",
+    "    run_opts=run_opts,\n",
+    "    checkpointer=hparams[\"checkpointer\"],\n",
+    ")\n",
+    "mixer.tokenizer = label_encoder\n",
+    "mixer.checkpointer.recover_if_possible()\n",
+    "mixer.modules.eval()\n",
+    "\n",
+    "\n",
+    "label_encoder = sb.dataio.encoder.CTCTextEncoder()\n",
+    "\n",
+    "\n",
+    "# We dynamicaly add the tokenizer to our brain class.\n",
+    "# NB: This tokenizer corresponds to the one used for the LM!!\n",
+    "\n",
+    "decoder = build_ctcdecoder(\n",
+    "    labels,\n",
+    "    kenlm_model_path= \"arpas/everything.arpa\",  # either .arpa or .bin file\n",
+    "    alpha=0.5,  # tuned on a val set\n",
+    "    beta=1,  # tuned on a val set\n",
+    ")\n",
+    "\n",
+    "run_opts[\"device\"]=\"cpu\"\n",
+    "\n",
+    "\n",
+    "device = \"cpu\"\n",
+    "mixer.device= \"cpu\"\n",
+    "mixer.modules.to(\"cpu\")\n",
+    "\n",
+    "from enum import Enum, auto\n",
+    "class Stage(Enum):\n",
+    "    TRAIN = auto()\n",
+    "    VALID = auto()\n",
+    "    TEST = auto()\n",
+    "\n",
+    "asr_brain.on_evaluate_start()\n",
+    "asr_brain.modules.eval()\n",
+    "\n",
+    "\n",
+    "import gradio as gr\n",
+    "\n",
+    "def treat_wav_file(file_mic,file_upload ,asr=mixer, device=\"cpu\") :\n",
+    "    if (file_mic is not None) and (file_upload is not None):\n",
+    "        warn_output = \"WARNING: You've uploaded an audio file and used the microphone. The recorded file from the microphone will be used and the uploaded audio will be discarded.\\n\"\n",
+    "        wav = file_mic\n",
+    "    elif (file_mic is None) and (file_upload is None):\n",
+    "        return \"ERROR: You have to either use the microphone or upload an audio file\"\n",
+    "    elif file_mic is not None:\n",
+    "        wav = file_mic\n",
+    "    else:\n",
+    "        wav = file_upload\n",
+    "    sig, sr = torchaudio.load(wav)\n",
+    "    tensor_wav = sig.to(device)\n",
+    "    resampled = torchaudio.functional.resample( tensor_wav, sr, 16000)\n",
+    "    sentence = asr.treat_wav(resampled)\n",
+    "    return sentence\n",
+    "\n",
+    "gr.Interface(\n",
+    "    fn=treat_wav_file, \n",
+    "    inputs=[gr.Audio(source=\"microphone\", type='filepath', label = \"record\", optional = True), \n",
+    "            gr.Audio(source=\"upload\", type='filepath', label=\"filein\", optional=True)]\n",
+    "    ,outputs=\"text\").launch(share= False, debug = True)\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

wav2vec2-FR-7K-large

@@ -0,0 +1 @@
+Subproject commit 7fa1111246fca6eb198d1caab50fd2e4469bf659

wav2vec2-large-lv60

@@ -0,0 +1 @@
+Subproject commit 0cde644b64dac88d8416bec1c92a4099b850ba0b

wavlm-large

@@ -0,0 +1 @@
+Subproject commit c1423ed94bb01d80a3f5ce5bc39f6026a0f4828c