add evaluation

README.md

@@ -22,16 +22,36 @@ model-index:
     metrics:
        - name: Test WER
          type: wer
-         value: 9.88
+         value: 16.52
        - name: Test CER
          type: cer
-         value: 2.32
+         value: 4.62
        - name: Test WER (+LM)
          type: wer
-         value: 7.07
+         value: 12.46
        - name: Test CER (+LM)
          type: cer
-         value: 1.88
+         value: 3.98
+  - task: 
+      name: Automatic Speech Recognition
+      type: automatic-speech-recognition
+    dataset:
+      name: Robust Speech Event - Dev Data
+      type: speech-recognition-community-v2/dev_data
+      args: sv
+    metrics:
+       - name: Test WER
+         type: wer
+         value: 23.96
+       - name: Test CER
+         type: cer
+         value: 8.88
+       - name: Test WER (+LM)
+         type: wer
+         value: 15.88
+       - name: Test CER (+LM)
+         type: cer
+         value: 7.42
 ---
 
 # XLS-R-1B-RUSSIAN

eval.py

@@ -1,12 +1,11 @@
 #!/usr/bin/env python3
 from datasets import load_dataset, load_metric, Audio, Dataset
-from transformers import pipeline, AutoFeatureExtractor, AutoTokenizer
+from transformers import pipeline, AutoFeatureExtractor, AutoTokenizer, AutoConfig, AutoModelForCTC, Wav2Vec2Processor, Wav2Vec2ProcessorWithLM
 import re
 import torch
 import argparse
 from typing import Dict
 
-
 def log_results(result: Dataset, args: Dict[str, str]):
     """ DO NOT CHANGE. This function computes and logs the result metrics. """
 
@@ -68,17 +67,30 @@ def main(args):
     # dataset = dataset.select(range(10))
 
     # load processor
-    feature_extractor = AutoFeatureExtractor.from_pretrained(args.model_id)
-    sampling_rate = feature_extractor.sampling_rate
+    if args.greedy:
+        processor = Wav2Vec2Processor.from_pretrained(args.model_id)
+        decoder = None
+    else:
+        processor = Wav2Vec2ProcessorWithLM.from_pretrained(args.model_id)
+        decoder = processor.decoder
+
+    feature_extractor = processor.feature_extractor
+    tokenizer = processor.tokenizer
 
     # resample audio
-    dataset = dataset.cast_column("audio", Audio(sampling_rate=sampling_rate))
+    dataset = dataset.cast_column("audio", Audio(sampling_rate=feature_extractor.sampling_rate))
 
     # load eval pipeline
     if args.device is None:
         args.device = 0 if torch.cuda.is_available() else -1
-    asr = pipeline("automatic-speech-recognition", model=args.model_id, device=args.device)
-
+    
+    config = AutoConfig.from_pretrained(args.model_id)
+    model = AutoModelForCTC.from_pretrained(args.model_id)
+    
+    #asr = pipeline("automatic-speech-recognition", model=args.model_id, device=args.device)
+    asr = pipeline("automatic-speech-recognition", config=config, model=model, tokenizer=tokenizer, 
+                   feature_extractor=feature_extractor, decoder=decoder, device=args.device)
+    
     # build normalizer config
     tokenizer = AutoTokenizer.from_pretrained(args.model_id)
     tokens = [x for x in tokenizer.convert_ids_to_tokens(range(0, tokenizer.vocab_size))]
@@ -106,6 +118,9 @@ def main(args):
     # run inference on all examples
     result = dataset.map(map_to_pred, remove_columns=dataset.column_names)
 
+    # filtering out empty targets
+    result = result.filter(lambda example: example["target"] != "")
+
     # compute and log_results
     # do not change function below
     log_results(result, args)
@@ -135,6 +150,9 @@ if __name__ == "__main__":
     parser.add_argument(
         "--log_outputs", action='store_true', help="If defined, write outputs to log file for analysis."
     )
+    parser.add_argument(
+        "--greedy", action='store_true', help="If defined, the LM will be ignored during inference."
+    )
     parser.add_argument(
         "--device",
         type=int,

full_eval.sh

@@ -0,0 +1,15 @@
+# CV 8 - TEST
+
+python eval.py --model_id jonatasgrosman/wav2vec2-xls-r-1b-russian --dataset mozilla-foundation/common_voice_8_0 --config ru --split test --log_outputs --greedy
+mv log_mozilla-foundation_common_voice_8_0_ru_test_predictions.txt log_mozilla-foundation_common_voice_8_0_ru_test_predictions_greedy.txt
+mv mozilla-foundation_common_voice_8_0_ru_test_eval_results.txt mozilla-foundation_common_voice_8_0_ru_test_eval_results_greedy.txt
+
+python eval.py --model_id jonatasgrosman/wav2vec2-xls-r-1b-russian --dataset mozilla-foundation/common_voice_8_0 --config ru --split test --log_outputs
+
+# HF EVENT - DEV
+
+python eval.py --model_id jonatasgrosman/wav2vec2-xls-r-1b-russian --dataset speech-recognition-community-v2/dev_data --config ru --split validation --chunk_length_s 5.0 --stride_length_s 1.0 --log_outputs --greedy
+mv log_speech-recognition-community-v2_dev_data_ru_validation_predictions.txt log_speech-recognition-community-v2_dev_data_ru_validation_predictions_greedy.txt
+mv speech-recognition-community-v2_dev_data_ru_validation_eval_results.txt speech-recognition-community-v2_dev_data_ru_validation_eval_results_greedy.txt
+
+python eval.py --model_id jonatasgrosman/wav2vec2-xls-r-1b-russian --dataset speech-recognition-community-v2/dev_data --config ru --split validation --chunk_length_s 5.0 --stride_length_s 1.0 --log_outputs

mozilla-foundation_common_voice_8_0_ru_test_eval_results.txt

@@ -1,2 +1,2 @@
-WER: 0.07074739140985198
-CER: 0.01881101267576011
+WER: 0.12460567823343849
+CER: 0.039853492631028165

mozilla-foundation_common_voice_8_0_ru_test_eval_results_greedy.txt

@@ -1,2 +1,2 @@
-WER: 0.09881096821159913
-CER: 0.02324829818558626
+WER: 0.16526328561028877
+CER: 0.04621616676669796

speech-recognition-community-v2_dev_data_ru_validation_eval_results.txt

@@ -0,0 +1,2 @@
+WER: 0.15887177213786988
+CER: 0.07426400420951243

speech-recognition-community-v2_dev_data_ru_validation_eval_results_greedy.txt

@@ -0,0 +1,2 @@
+WER: 0.23960988584727919
+CER: 0.08884568391199266