fix oom vocab building. adjust run params

run.sh

@@ -1,21 +1,22 @@
 python run_speech_recognition_ctc.py \
-	--dataset_name="mozilla-foundation/common_voice_7_0" \
+	--dataset_name="mozilla-foundation/common_voice_7_0,marinone94/nst_sv" \
 	--model_name_or_path="KBLab/wav2vec2-large-voxrex" \
-	--dataset_config_name="sv-SE" \
+	--dataset_config_name="sv-SE,distant_channel" \
+	--train_split_name="train+validation,train"
+	--eval_split_name="test,None"
 	--output_dir="./" \
 	--overwrite_output_dir \
-	--num_train_epochs="50" \
-	--per_device_train_batch_size="8" \
-	--per_device_eval_batch_size="8" \
+	--num_train_epochs="5" \
+	--per_device_train_batch_size="16" \
+	--per_device_eval_batch_size="16" \
 	--gradient_accumulation_steps="4" \
 	--learning_rate="7.5e-5" \
-	--warmup_steps="2000" \
+	--warmup_steps="1000" \
 	--length_column_name="input_length" \
-	--evaluation_strategy="steps" \
+	--evaluation_strategy="epoch" \
+	--save_strategy="epoch" \
 	--text_column_name="sentence" \
-	--chars_to_ignore , ? . ! \- \; \: \" “ % ‘ ” � — ’ … – \
-	--save_steps="500" \
-	--eval_steps="500" \
+	--chars_to_ignore , ? . ! \- \; \: \" “ % ‘ ” � — ’ … – / \\ \
 	--logging_steps="100" \
 	--layerdrop="0.0" \
 	--activation_dropout="0.1" \

run_speech_recognition_ctc.py

@@ -329,8 +329,8 @@ def create_vocabulary_from_data(
     vocabs = datasets.map(
         extract_all_chars,
         batched=True,
-        batch_size=-1,
-        keep_in_memory=True,
+        batch_size=10000,
+        keep_in_memory=False,
         remove_columns=datasets["train"].column_names,
     )
 
@@ -449,6 +449,8 @@ def main():
                             )
                         ]
                     )
+            else:
+                logging.warning(f"{dataset_name} {dataset_config_name} as split is {train_split_name}")
 
         if data_args.audio_column_name not in raw_datasets["train"].column_names:
             raise ValueError(
@@ -491,11 +493,13 @@ def main():
                             load_dataset(
                                 dataset_name,
                                 dataset_config_name,
-                                split=train_split_name,
+                                split=eval_split_name,
                                 use_auth_token=data_args.use_auth_token,
                             )
                         ]
                     )
+            else:
+                logging.warning(f"{dataset_name} {dataset_config_name} as split is {eval_split_name}")
 
         if data_args.max_eval_samples is not None:
             raw_datasets["eval"] = raw_datasets["eval"].select(range(data_args.max_eval_samples))
@@ -509,6 +513,12 @@ def main():
     )
     text_column_name = data_args.text_column_name
 
+    def is_text_valid(text):
+        for token in text.split():
+            if len(token) > 1:
+                return True
+        return False
+
     def remove_special_characters(batch):
         if chars_to_ignore_regex is not None:
             batch["target_text"] = re.sub(chars_to_ignore_regex, "", batch[text_column_name]).lower() + " "
@@ -516,6 +526,7 @@ def main():
             batch["target_text"] = batch[text_column_name].lower() + " "
         return batch
 
+    num_workers = data_args.preprocessing_num_workers
     with training_args.main_process_first(desc="dataset map special characters removal"):
         raw_datasets = raw_datasets.map(
             remove_special_characters,
@@ -523,6 +534,13 @@ def main():
             desc="remove special characters from datasets",
         )
 
+        raw_datasets = raw_datasets.filter(
+            is_text_valid,
+            num_proc=num_workers,
+            input_columns=["input_length"],
+            desc="remove single words, single chars and 'W O R D S'",
+        )
+
     # save special tokens for tokenizer
     word_delimiter_token = data_args.word_delimiter_token
     unk_token = data_args.unk_token
@@ -638,7 +656,6 @@ def main():
     max_input_length = data_args.max_duration_in_seconds * feature_extractor.sampling_rate
     min_input_length = data_args.min_duration_in_seconds * feature_extractor.sampling_rate
     audio_column_name = data_args.audio_column_name
-    num_workers = data_args.preprocessing_num_workers
 
     # `phoneme_language` is only relevant if the model is fine-tuned on phoneme classification
     phoneme_language = data_args.phoneme_language