up

eval.py

@@ -3,6 +3,7 @@ from datasets import load_dataset, load_metric, Audio, Dataset
 from transformers import pipeline, AutoFeatureExtractor
 import re
 import argparse
+import unicodedata
 from typing import Dict
 
 
@@ -52,14 +53,14 @@ def normalize_text(text: str) -> str:
 
     chars_to_ignore_regex = '[,?.!\-\;\:\"“%‘”�—’…–]'  # noqa: W605 IMPORTANT: this should correspond to the chars that were ignored during training
 
-    text = re.sub(chars_to_ignore_regex, "", text.lower())
+    text = text.lower()
+    # normalize non-standard (stylized) unicode characters
+    text = unicodedata.normalize('NFKC', text)
+    # remove punctuation
+    text = re.sub(chars_to_ignore_regex, "", text)
 
-    # In addition, we can normalize the target text, e.g. removing new lines characters etc...
-    # note that order is important here!
-    token_sequences_to_ignore = ["\n\n", "\n", "   ", "  "]
-
-    for t in token_sequences_to_ignore:
-        text = " ".join(text.split(t))
+    # Let's also make sure we split on all kinds of newlines, spaces, etc...
+    text = " ".join(text.split())
 
     return text
 
@@ -113,10 +114,10 @@ if __name__ == "__main__":
         "--split", type=str, required=True, help="Split of the dataset. *E.g.* `'test'`"
     )
     parser.add_argument(
-        "--chunk_length_s", type=float, default=None, help="Chunk length in seconds. Defaults to 5 seconds."
+        "--chunk_length_s", type=float, default=None, help="Chunk length in seconds. Defaults to None. For long audio files a good value would be 5.0 seconds."
     )
     parser.add_argument(
-        "--stride_length_s", type=float, default=None, help="Stride of the audio chunks. Defaults to 1 second."
+        "--stride_length_s", type=float, default=None, help="Stride of the audio chunks. Defaults to None. For long audio files a good value would be 1.0 seconds."
     )
     parser.add_argument(
         "--log_outputs", action='store_true', help="If defined, write outputs to log file for analysis."