Update README.md

update evaluation using bert/T-TA and evaluation time

README.md

@@ -21,7 +21,7 @@ model-index:
     metrics:
        - name: Test CER
          type: cer
-         value: 16.41
+         value: 25.57
 ---
 
 # Wav2Vec2-Large-XLSR-53-tw-gpt
@@ -48,7 +48,8 @@ model_name = "voidful/wav2vec2-large-xlsr-53-tw-gpt"
 device = "cuda"
 processor_name = "voidful/wav2vec2-large-xlsr-53-tw-gpt"
 
-chars_to_ignore_regex = r"[¥•＂＃＄％＆＇（）＊＋，－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､　、〃〈〉《》「」『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘’‛“”„‟…‧﹏﹑﹔·'℃°•·．﹑︰〈〉─《﹖﹣﹂﹁﹔！？｡。＂＃＄％＆＇（）＊＋，﹐－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､、〃》「」『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘’‛“”„‟…‧﹏.．!\\\\\\\\"#$%&()*+,\\\\\\\\-.\\\\\\\\:;<=>?@\\\\\\\\[\\\\\\\\]\\\\\\\\\\\\\\\\\\\\\\\\/^_`{|}~]"
+chars_to_ignore_regex = r"[¥•＂＃＄％＆＇（）＊＋，－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､　、〃〈〉《》「」『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘’‛“”„‟…‧﹏﹑﹔·'℃°•·．﹑︰〈〉─《﹖﹣﹂﹁﹔！？｡。＂＃＄％＆＇（）＊＋，﹐－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､、〃》「」『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘’‛“”„‟…‧﹏.．!\"#$%&()*+,\-.\:;<=>?@\[\]\\\/^_`{|}~]"
+
 
 model = Wav2Vec2ForCTC.from_pretrained(model_name).to(device)
 processor = Wav2Vec2Processor.from_pretrained(processor_name)
@@ -95,10 +96,17 @@ predict(load_file_to_data('voice file path'))
 The model can be evaluated as follows on the zh-tw test data of Common Voice.   
 CER calculation refer to https://huggingface.co/ctl/wav2vec2-large-xlsr-cantonese 
 
-```python
+env setup:
+```
 !mkdir cer
+!wget -O cer/cer.py https://huggingface.co/ctl/wav2vec2-large-xlsr-cantonese/raw/main/cer.py
 !pip install jiwer
+!pip install torchaudio
+!pip install datasets transformers
+```
 
+## Evaluation without LM:
+```python
 import torchaudio
 from datasets import load_dataset, load_metric
 from transformers import (
@@ -113,7 +121,8 @@ model_name = "voidful/wav2vec2-large-xlsr-53-tw-gpt"
 device = "cuda"
 processor_name = "voidful/wav2vec2-large-xlsr-53-tw-gpt"
 
-chars_to_ignore_regex = r"[¥•＂＃＄％＆＇（）＊＋，－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､　、〃〈〉《》「」『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘’‛“”„‟…‧﹏﹑﹔·'℃°•·．﹑︰〈〉─《﹖﹣﹂﹁﹔！？｡。＂＃＄％＆＇（）＊＋，﹐－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､、〃》「」『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘’‛“”„‟…‧﹏.．!\\\\\\\\"#$%&()*+,\\\\\\\\-.\\\\\\\\:;<=>?@\\\\\\\\[\\\\\\\\]\\\\\\\\\\\\\\\\\\\\\\\\/^_`{|}~]"
+chars_to_ignore_regex = r"[¥•＂＃＄％＆＇（）＊＋，－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､　、〃〈〉《》「」『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘’‛“”„‟…‧﹏﹑﹔·'℃°•·．﹑︰〈〉─《﹖﹣﹂﹁﹔！？｡。＂＃＄％＆＇（）＊＋，﹐－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､、〃》「」『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘’‛“”„‟…‧﹏.．!\"#$%&()*+,\-.\:;<=>?@\[\]\\\/^_`{|}~]"
+
 
 model = Wav2Vec2ForCTC.from_pretrained(model_name).to(device)
 processor = Wav2Vec2Processor.from_pretrained(processor_name)
@@ -148,14 +157,11 @@ cer = load_metric("./cer")
 print("CER: {:2f}".format(100 * cer.compute(predictions=result["predicted"], references=result["target"])))
 ```
 
-`CER: 28.734822`
+`CER: 28.79`.  
+`TIME: 05:23 min`
 
 ## Evaluation with GPT:
 ```python
-!mkdir cer
-!wget -O cer/cer.py https://huggingface.co/ctl/wav2vec2-large-xlsr-cantonese/raw/main/cer.py
-!pip install jiwer
-
 import torchaudio
 from datasets import load_dataset, load_metric
 from transformers import (
@@ -170,10 +176,10 @@ from transformers import AutoTokenizer, AutoModelWithLMHead
 model_name = "voidful/wav2vec2-large-xlsr-53-tw-gpt"
 device = "cuda"
 processor_name = "voidful/wav2vec2-large-xlsr-53-tw-gpt"
-chars_to_ignore_regex = r"""[¥•＂＃＄％＆＇（）＊＋，－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､　、〃〈〉《》「」『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘’‛“”„‟…‧﹏﹑﹔·'℃°•·．﹑︰〈〉─《﹖﹣﹂﹁﹔！？｡。＂＃＄％＆＇（）＊＋，﹐－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､、〃》「」『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘’‛“”„‟…‧﹏.．!\\\\\\\\"#$%&()*+,\\\\\\\\-.\\\\\\\\:;<=>?@\\\\\\\\[\\\\\\\\]\\\\\\\\\\\\\\\\\\\\\\\\/^_`{|}~]"""
+chars_to_ignore_regex = r"[¥•���＃＄％＆＇（）＊＋，－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､　、〃〈〉《》「」『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘’‛“”„‟…‧﹏﹑﹔·'℃°•·．﹑︰〈〉─《﹖﹣﹂﹁﹔！？｡。＂＃＄％＆＇（）＊＋，﹐－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､、〃》「」『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘’‛“”„‟…‧﹏.．!\"#$%&()*+,\-.\:;<=>?@\[\]\\\/^_`{|}~]"
 
 tokenizer = AutoTokenizer.from_pretrained("ckiplab/gpt2-base-chinese")  
-gpt_model = AutoModelWithLMHead.from_pretrained("ckiplab/gpt2-base-chinese").to(device)
+lm_model = AutoModelWithLMHead.from_pretrained("ckiplab/gpt2-base-chinese").to(device)
 model = Wav2Vec2ForCTC.from_pretrained(model_name).to(device)
 processor = Wav2Vec2Processor.from_pretrained(processor_name)
 
@@ -196,18 +202,173 @@ def map_to_pred(batch):
     attention_mask = features.attention_mask.to(device)
     with torch.no_grad():
         logits = model(input_values, attention_mask=attention_mask).logits
-    
+
     decoded_results = []
     for logit in logits:
         pred_ids = torch.argmax(logit, dim=-1)
         mask = pred_ids.ge(1).unsqueeze(-1).expand(logit.size())
         vocab_size = logit.size()[-1]
         voice_prob = torch.nn.functional.softmax((torch.masked_select(logit, mask).view(-1,vocab_size)),dim=-1)
-        gpt_input = torch.cat((torch.tensor([tokenizer.cls_token_id]).to(device),pred_ids[pred_ids>0]), 0)
-        gpt_prob = torch.nn.functional.softmax(gpt_model(gpt_input).logits, dim=-1)[:voice_prob.size()[0],:]
-        comb_pred_ids = torch.argmax(gpt_prob*voice_prob, dim=-1)
+        lm_input = torch.cat((torch.tensor([tokenizer.cls_token_id]).to(device),pred_ids[pred_ids>0]), 0)
+        lm_prob = torch.nn.functional.softmax(lm_model(lm_input).logits, dim=-1)[:voice_prob.size()[0],:]
+        comb_pred_ids = torch.argmax(lm_prob*voice_prob, dim=-1)
         decoded_results.append(processor.decode(comb_pred_ids))
-        
+
+    batch["predicted"] = decoded_results
+    batch["target"] = batch["sentence"]
+    return batch
+
+
+result = ds.map(map_to_pred, batched=True, batch_size=16, remove_columns=list(ds.features.keys()))
+
+cer = load_metric("./cer")
+print("CER: {:2f}".format(100 * cer.compute(predictions=result["predicted"], references=result["target"])))
+```
+
+`CER 25.75`.  
+`TIME: 06:04 min`
+
+## Evaluation with BERT:
+```python
+import torchaudio
+from datasets import load_dataset, load_metric
+from transformers import (
+    Wav2Vec2ForCTC,
+    Wav2Vec2Processor,
+)
+import torch
+import re
+import sys
+from transformers import AutoTokenizer, AutoModelForMaskedLM 
+
+model_name = "voidful/wav2vec2-large-xlsr-53-tw-gpt"
+device = "cuda"
+processor_name = "voidful/wav2vec2-large-xlsr-53-tw-gpt"
+chars_to_ignore_regex = r"[¥•＂＃＄％＆＇（）＊＋，－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､　、〃〈〉《》「」『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘’‛“”„‟…‧﹏﹑﹔·'℃°•·．﹑︰〈〉─《﹖﹣﹂﹁﹔！？｡。＂＃＄％＆＇（）＊＋，﹐－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､、〃》「」『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘’‛“”„‟…‧﹏.．!\"#$%&()*+,\-.\:;<=>?@\[\]\\\/^_`{|}~]"
+
+tokenizer = AutoTokenizer.from_pretrained("bert-base-chinese")  
+lm_model = AutoModelForMaskedLM.from_pretrained("bert-base-chinese").to(device)
+model = Wav2Vec2ForCTC.from_pretrained(model_name).to(device)
+processor = Wav2Vec2Processor.from_pretrained(processor_name)
+
+ds = load_dataset("common_voice", 'zh-TW', data_dir="./cv-corpus-6.1-2020-12-11", split="test")
+
+resampler = torchaudio.transforms.Resample(orig_freq=48_000, new_freq=16_000)
+
+def map_to_array(batch):
+    speech, _ = torchaudio.load(batch["path"])
+    batch["speech"] = resampler.forward(speech.squeeze(0)).numpy()
+    batch["sampling_rate"] = resampler.new_freq
+    batch["sentence"] = re.sub(chars_to_ignore_regex, '', batch["sentence"]).lower().replace("’", "'")
+    return batch
+
+ds = ds.map(map_to_array)
+
+def map_to_pred(batch):
+    features = processor(batch["speech"], sampling_rate=batch["sampling_rate"][0], padding=True, return_tensors="pt")
+    input_values = features.input_values.to(device)
+    attention_mask = features.attention_mask.to(device)
+    with torch.no_grad():
+        logits = model(input_values, attention_mask=attention_mask).logits
+
+    decoded_results = []
+    for logit in logits:
+        pred_ids = torch.argmax(logit, dim=-1)
+        mask = ~pred_ids.eq(tokenizer.pad_token_id).unsqueeze(-1).expand(logit.size())
+        vocab_size = logit.size()[-1]
+        voice_prob = torch.nn.functional.softmax((torch.masked_select(logit, mask).view(-1,vocab_size)),dim=-1)
+        lm_input = torch.masked_select(pred_ids, ~pred_ids.eq(tokenizer.pad_token_id)).unsqueeze(0)
+        mask_lm_prob = voice_prob.clone()
+        for i in range(lm_input.shape[-1]):
+            masked_lm_input = lm_input.clone()
+            masked_lm_input[0][i] = torch.tensor(tokenizer.mask_token_id).to('cuda')
+            lm_prob = torch.nn.functional.softmax(lm_model(masked_lm_input).logits, dim=-1).squeeze(0)
+            mask_lm_prob[i] = lm_prob[i]
+        comb_pred_ids = torch.argmax(mask_lm_prob*voice_prob, dim=-1)
+        decoded_results.append(processor.decode(comb_pred_ids))
+
+    batch["predicted"] = decoded_results
+    batch["target"] = batch["sentence"]
+    return batch
+
+
+result = ds.map(map_to_pred, batched=True, batch_size=1, remove_columns=list(ds.features.keys()))
+
+cer = load_metric("./cer")
+print("CER: {:2f}".format(100 * cer.compute(predictions=result["predicted"], references=result["target"])))
+```
+`CER 25.57`.  
+`TIME: 09:49 min`
+
+## Evaluation with T-TA:
+setup
+```
+!git clone https://github.com/voidful/pytorch-tta.git
+!mv ./pytorch-tta/tta ./tta
+!wget https://github.com/voidful/pytorch-tta/releases/download/wiki_zh/wiki_zh.pt
+```
+
+```python
+import torchaudio
+from datasets import load_dataset, load_metric
+from transformers import (
+    Wav2Vec2ForCTC,
+    Wav2Vec2Processor,
+)
+import torch
+import re
+import sys
+from tta.modeling_tta import TTALMModel
+from transformers import AutoTokenizer
+import torch
+
+
+
+model_name = "voidful/wav2vec2-large-xlsr-53-tw-gpt"
+device = "cuda"
+processor_name = "voidful/wav2vec2-large-xlsr-53-tw-gpt"
+chars_to_ignore_regex = r"[¥•＂＃＄％＆＇（）＊＋，－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､　、〃〈〉《》「」『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘’‛“”„‟…‧﹏﹑﹔·'℃°•·．﹑︰〈〉─《﹖﹣﹂﹁﹔！？｡。＂＃＄％＆＇（）＊＋，﹐－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､、〃》「」『』【】〔〕〖〗〘〙〚〛〜〝〞〟〰〾〿–—‘’‛“”„‟…‧﹏.．!\"#$%&()*+,\-.\:;<=>?@\[\]\\\/^_`{|}~]"
+
+tokenizer = AutoTokenizer.from_pretrained("bert-base-chinese")  
+lm_model = TTALMModel("bert-base-chinese")
+tokenizer = AutoTokenizer.from_pretrained("bert-base-chinese")
+lm_model.load_state_dict(torch.load("./wiki_zh.pt",map_location=torch.device('cuda')))
+lm_model.to('cuda')
+lm_model.eval()
+model = Wav2Vec2ForCTC.from_pretrained(model_name).to(device)
+processor = Wav2Vec2Processor.from_pretrained(processor_name)
+
+ds = load_dataset("common_voice", 'zh-TW', data_dir="./cv-corpus-6.1-2020-12-11", split="test")
+
+resampler = torchaudio.transforms.Resample(orig_freq=48_000, new_freq=16_000)
+
+def map_to_array(batch):
+    speech, _ = torchaudio.load(batch["path"])
+    batch["speech"] = resampler.forward(speech.squeeze(0)).numpy()
+    batch["sampling_rate"] = resampler.new_freq
+    batch["sentence"] = re.sub(chars_to_ignore_regex, '', batch["sentence"]).lower().replace("’", "'")
+    return batch
+
+ds = ds.map(map_to_array)
+
+def map_to_pred(batch):
+    features = processor(batch["speech"], sampling_rate=batch["sampling_rate"][0], padding=True, return_tensors="pt")
+    input_values = features.input_values.to(device)
+    attention_mask = features.attention_mask.to(device)
+    with torch.no_grad():
+        logits = model(input_values, attention_mask=attention_mask).logits
+
+    decoded_results = []
+    for logit in logits:
+        pred_ids = torch.argmax(logit, dim=-1)
+        mask = ~pred_ids.eq(tokenizer.pad_token_id).unsqueeze(-1).expand(logit.size())
+        vocab_size = logit.size()[-1]
+        voice_prob = torch.nn.functional.softmax((torch.masked_select(logit, mask).view(-1,vocab_size)),dim=-1)
+        lm_input = torch.masked_select(pred_ids, ~pred_ids.eq(tokenizer.pad_token_id)).unsqueeze(0)
+        lm_prob = torch.nn.functional.softmax(lm_model.forward(lm_input)[0], dim=-1).squeeze(0)
+        comb_pred_ids = torch.argmax(lm_prob*voice_prob, dim=-1)
+        decoded_results.append(processor.decode(comb_pred_ids))
+
     batch["predicted"] = decoded_results
     batch["target"] = batch["sentence"]
     return batch
@@ -219,4 +380,5 @@ cer = load_metric("./cer")
 print("CER: {:2f}".format(100 * cer.compute(predictions=result["predicted"], references=result["target"])))
 ```
 
-`CER 25.69`
+`CER: 25.77`.   
+`TIME: 06:01 min`