Update README.md

README.md

@@ -74,8 +74,6 @@ The model can be evaluated as follows on the zh-CN test data of Common Voice.
 Original CER calculation refer to https://huggingface.co/ctl/wav2vec2-large-xlsr-cantonese
 
 ```python
-!mkdir cer
-!wget -O cer/cer.py https://huggingface.co/ctl/wav2vec2-large-xlsr-cantonese/raw/main/cer.py
 !pip install jiwer
 
 import torch
@@ -83,15 +81,41 @@ import torchaudio
 from datasets import load_dataset, load_metric
 from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor
 import re
-
+import jiwer
+
+def chunked_cer(targets, predictions, chunk_size=None):
+
+    _predictions = [char for seq in predictions for char in list(seq)]
+    _targets = [char for seq in targets for char in list(seq)]
+    
+    if chunk_size is None: return jiwer.wer(_targets, _predictions)
+    
+    start = 0
+    end = chunk_size
+    H, S, D, I = 0, 0, 0, 0
+    
+    while start < len(targets):
+    
+        _predictions = [char for seq in predictions[start:end] for char in list(seq)]
+        _targets = [char for seq in targets[start:end] for char in list(seq)]
+        chunk_metrics = jiwer.compute_measures(_targets, _predictions)
+        H = H + chunk_metrics["hits"]
+        S = S + chunk_metrics["substitutions"]
+        D = D + chunk_metrics["deletions"]
+        I = I + chunk_metrics["insertions"]
+        start += chunk_size
+        end += chunk_size
+        
+    return float(S + D + I) / float(H + S + D)
+    
 test_dataset = load_dataset("common_voice", "zh-CN", split="test")
-wer = load_metric("wer")
 
 processor = Wav2Vec2Processor.from_pretrained("ydshieh/wav2vec2-large-xlsr-53-chinese-zh-cn-gpt")
 model = Wav2Vec2ForCTC.from_pretrained("ydshieh/wav2vec2-large-xlsr-53-chinese-zh-cn-gpt")
 model.to("cuda")
 
-chars_to_ignore_regex = '[\,\?\.\!\-\;\:"\“\%\‘\”\�\．\⋯\！\－\：\–\。\》\,\）\,\？\；\～\~\…\︰\，\（\」\‧\《\﹔\、\—\／\,\「\﹖\·\'\×\̃\̌\ε\λ\μ\и\т\─\□\〈\〉\『\』\ア\オ\カ\チ\ド\ベ\ャ\ヤ\ン\・\丶\ａ\ｂ\ｆ\ｇ\ｉ\ｎ\ｐ\ｔ]'
+chars_to_ignore_regex = '[\,\?\.\!\-\;\:"\“\%\‘\”\�\．\⋯\！\－\：\–\。\》\,\）\,\？\；\～\~\…\︰\，\（\」\‧\《\﹔\、\—\／\,\「\﹖\·\×\̃\̌\ε\λ\μ\и\т\─\□\〈\〉\『\』\ア\オ\カ\チ\ド\ベ\ャ\ヤ\ン\・\丶\ａ\ｂ\ｆ\ｇ\ｉ\ｎ\ｐ\ｔ' + "\']"
+
 resampler = torchaudio.transforms.Resample(48_000, 16_000)
 
 # Preprocessing the datasets.
@@ -118,7 +142,7 @@ def evaluate(batch):
 
 result = test_dataset.map(evaluate, batched=True, batch_size=8)
 
-print("WER: {:2f}".format(100 * wer.compute(predictions=result["pred_strings"], references=result["sentence"])))
+print("CER: {:2f}".format(100 * chunked_cer(predictions=result["pred_strings"], targets=result["sentence"], chunk_size=1000)))
 ```
 
 **Test Result**: 43.00 %