tests/test_pronunciationtrainer.py

import unittest

import numpy as np
import torch
from torchaudio.transforms import Resample

from aip_trainer import pronunciationTrainer, sample_rate_start
from aip_trainer.lambdas.lambdaSpeechToScore import soundfile_load
from aip_trainer.utils import utilities
from tests import EVENTS_FOLDER
from tests.lambdas.test_lambdaSpeechToScore import set_seed


phrases = {
    "de": {
        "real": "Hallo, wie geht es dir?",
        "transcribed": 'hallo wie geht es dir',
        "partial": 'hallo wie geht ',
        "incorrect": 'hail wi git es dir'
    },
    "en": {
        "real": "Hi there, how are you?",
        "transcribed": 'i there how are you',
        "partial": 'i there how',
        "incorrect": "I here how re youth"
    }
}
trainer_SST_lambda_de = pronunciationTrainer.getTrainer("de")
trainer_SST_lambda_en = pronunciationTrainer.getTrainer("en")
signal_de, samplerate = soundfile_load(str(EVENTS_FOLDER / "test_de_easy.wav"))
signal_en, samplerate = soundfile_load(str(EVENTS_FOLDER / "test_en_easy.wav"))
transform = Resample(orig_freq=sample_rate_start, new_freq=16000)


class TestScore(unittest.TestCase):
    def test_getTrainer(self):
        self.assertIsInstance(trainer_SST_lambda_de, pronunciationTrainer.PronunciationTrainer)
        self.assertIsInstance(trainer_SST_lambda_en, pronunciationTrainer.PronunciationTrainer)

    def test_exact_transcription_de(self):
        set_seed()
        phrase_real = phrases["de"]["real"]
        real_and_transcribed_words, real_and_transcribed_words_ipa, mapped_words_indices = trainer_SST_lambda_de.matchSampleAndRecordedWords(phrase_real, phrase_real)
        self.assertEqual(real_and_transcribed_words_ipa, [('haloː,', 'haloː,'), ('viː', 'viː'), ('ɡeːt', 'ɡeːt'), ('ɛːs', 'ɛːs'), ('diːr?', 'diːr?')])
        self.assertEqual(mapped_words_indices, [0, 1, 2, 3, 4])
        pronunciation_accuracy, current_words_pronunciation_accuracy = trainer_SST_lambda_de.getPronunciationAccuracy(real_and_transcribed_words)
        self.assertEqual(int(pronunciation_accuracy), 100)
        self.assertEqual(current_words_pronunciation_accuracy, [100, 100, 100, 100, 100])

    def test_transcription_de(self):
        set_seed()
        phrase_real = phrases["de"]["real"]
        phrase_transcribed = phrases["de"]["transcribed"]
        real_and_transcribed_words, real_and_transcribed_words_ipa, mapped_words_indices = trainer_SST_lambda_de.matchSampleAndRecordedWords(phrase_real, phrase_transcribed)
        self.assertEqual(real_and_transcribed_words_ipa, [('haloː,', 'haloː'), ('viː', 'viː'), ('ɡeːt', 'ɡeːt'), ('ɛːs', 'ɛːs'), ('diːr?', 'diːɐ̯')])
        self.assertEqual(mapped_words_indices, [0, 1, 2, 3, 4])
        pronunciation_accuracy, current_words_pronunciation_accuracy= trainer_SST_lambda_de.getPronunciationAccuracy(real_and_transcribed_words)
        self.assertEqual(int(pronunciation_accuracy), 100)
        self.assertEqual(current_words_pronunciation_accuracy, [100, 100, 100, 100, 100])

    def test_partial_transcription_de(self):
        set_seed()
        phrase_real = phrases["de"]["real"]
        phrase_partial = phrases["de"]["partial"]
        real_and_transcribed_words, real_and_transcribed_words_ipa, mapped_words_indices = trainer_SST_lambda_de.matchSampleAndRecordedWords(phrase_real, phrase_partial)
        pronunciation_accuracy, current_words_pronunciation_accuracy= trainer_SST_lambda_de.getPronunciationAccuracy(real_and_transcribed_words)
        self.assertEqual(real_and_transcribed_words_ipa, [('haloː,', 'haloː'), ('viː', 'viː'), ('ɡeːt', 'ɡeːt'), ('ɛːs', '-'), ('diːr?', '-')])
        self.assertEqual(mapped_words_indices, [0, 1, 2, -1, -1])
        self.assertEqual(int(pronunciation_accuracy), 71)
        self.assertEqual(current_words_pronunciation_accuracy, [100, 100, 100, 0, 0])

    def test_incorrect_transcription_with_correct_words_de(self):
        set_seed()
        phrase_real = phrases["de"]["real"]
        phrase_transcribed_incorrect = phrases["de"]["incorrect"]
        real_and_transcribed_words, real_and_transcribed_words_ipa, mapped_words_indices = trainer_SST_lambda_de.matchSampleAndRecordedWords(phrase_real, phrase_transcribed_incorrect)
        self.assertEqual(real_and_transcribed_words_ipa, [('haloː,', 'haɪ̯l'), ('viː', 'viː'), ('ɡeːt', 'ɡiːt'), ('ɛːs', 'ɛːs'), ('diːr?', 'diːɐ̯')])
        self.assertEqual(mapped_words_indices, [0, 1, 2, 3, 4])
        pronunciation_accuracy, current_words_pronunciation_accuracy= trainer_SST_lambda_de.getPronunciationAccuracy(real_and_transcribed_words)
        self.assertEqual(int(pronunciation_accuracy), 71)
        for accuracy, expected_accuracy in zip(current_words_pronunciation_accuracy, [60.0, 66.666666, 50.0, 100.0, 100.0]):
            self.assertAlmostEqual(accuracy, expected_accuracy, places=2)

    def test_exact_transcription_en(self):
        set_seed()
        phrase_real = phrases["en"]["real"]
        real_and_transcribed_words, real_and_transcribed_words_ipa, mapped_words_indices = trainer_SST_lambda_en.matchSampleAndRecordedWords(phrase_real, phrase_real)
        self.assertEqual(real_and_transcribed_words_ipa, [('haɪ', 'haɪ'), ('ðɛr,', 'ðɛr,'), ('haʊ', 'haʊ'), ('ər', 'ər'), ('ju?', 'ju?')])
        self.assertEqual(mapped_words_indices, [0, 1, 2, 3, 4])
        pronunciation_accuracy, current_words_pronunciation_accuracy= trainer_SST_lambda_en.getPronunciationAccuracy(real_and_transcribed_words)
        self.assertEqual(int(pronunciation_accuracy), 100)
        self.assertEqual(current_words_pronunciation_accuracy, [100, 100, 100, 100, 100])

    def test_transcription_en(self):
        set_seed()
        phrase_real = phrases["en"]["real"]
        phrase_transcribed = phrases["en"]["transcribed"]
        real_and_transcribed_words, real_and_transcribed_words_ipa, mapped_words_indices = trainer_SST_lambda_en.matchSampleAndRecordedWords(phrase_real, phrase_transcribed)
        self.assertEqual(real_and_transcribed_words_ipa, [('haɪ', 'aɪ'), ('ðɛr,', 'ðɛr'), ('haʊ', 'haʊ'), ('ər', 'ər'), ('ju?', 'ju')])
        self.assertEqual(mapped_words_indices, [0, 1, 2, 3, 4])
        pronunciation_accuracy, current_words_pronunciation_accuracy= trainer_SST_lambda_en.getPronunciationAccuracy(real_and_transcribed_words)
        self.assertEqual(int(pronunciation_accuracy), 94)
        self.assertEqual(current_words_pronunciation_accuracy, [50.0, 100.0, 100.0, 100.0, 100.0])

    def test_partial_transcription_en(self):
        set_seed()
        phrase_real = phrases["en"]["real"]
        phrase_partial = phrases["en"]["partial"]
        real_and_transcribed_words, real_and_transcribed_words_ipa, mapped_words_indices = trainer_SST_lambda_en.matchSampleAndRecordedWords(phrase_real, phrase_partial)
        self.assertEqual(real_and_transcribed_words_ipa, [('haɪ', 'aɪ'), ('ðɛr,', 'ðɛr'), ('haʊ', 'haʊ'), ('ər', ''), ('ju?', '')])
        self.assertEqual(mapped_words_indices, [0, 1, 2, -1, -1])
        pronunciation_accuracy, current_words_pronunciation_accuracy= trainer_SST_lambda_en.getPronunciationAccuracy(real_and_transcribed_words)
        self.assertEqual(int(pronunciation_accuracy), 56)
        self.assertEqual(current_words_pronunciation_accuracy, [50.0, 100.0, 100.0, 0.0, 0.0])

    def test_incorrect_transcription_with_correct_words_en(self):
        set_seed()
        phrase_real = phrases["en"]["real"]
        phrase_transcribed_incorrect = phrases["en"]["incorrect"]
        real_and_transcribed_words, real_and_transcribed_words_ipa, mapped_words_indices = trainer_SST_lambda_en.matchSampleAndRecordedWords(phrase_real, phrase_transcribed_incorrect)
        self.assertEqual(real_and_transcribed_words_ipa, [('haɪ', 'aɪ'), ('ðɛr,', 'hir'), ('haʊ', 'haʊ'), ('ər', 'ri'), ('ju?', 'juθ')])
        self.assertEqual(mapped_words_indices, [0, 1, 2, 3, 4])
        pronunciation_accuracy, current_words_pronunciation_accuracy= trainer_SST_lambda_en.getPronunciationAccuracy(real_and_transcribed_words)
        self.assertEqual(int(pronunciation_accuracy), 69)
        for accuracy, expected_accuracy in zip(current_words_pronunciation_accuracy, [50.0, 80.0, 100.0, 66.666666, 33.333333]):
            self.assertAlmostEqual(accuracy, expected_accuracy, places=2)

    def test_processAudioForGivenText_getTranscriptAndWordsLocations_de(self):
        set_seed()
        phrase_real = phrases["de"]["real"]
        signal_de_shape = signal_de.shape[0]
        signal_transformed = transform(torch.Tensor(signal_de)).unsqueeze(0)
        result = trainer_SST_lambda_de.processAudioForGivenText(signal_transformed, phrase_real)
        expected_result = {
            'recording_transcript': 'hallo wie geht es dir',
            'real_and_transcribed_words': [('Hallo,', 'hallo'), ('wie', 'wie'), ('geht', 'geht'), ('es', 'es'), ('dir?', 'dir')],
            'recording_ipa': 'haloː viː ɡeːt ɛːs diːɐ̯', 'start_time': '0.0 0.3733125 0.60425 0.7966875 0.989125', 'end_time': '0.4733125 0.70425 0.8966875 1.089125 1.3200625',
            'real_and_transcribed_words_ipa': [('haloː,', 'haloː'), ('viː', 'viː'), ('ɡeːt', 'ɡeːt'), ('ɛːs', 'ɛːs'), ('diːr?', 'diːɐ̯')],
            'pronunciation_accuracy': 100.0,
            'pronunciation_categories': [0, 0, 0, 0, 0]
        }
        self.assertDictEqual(result, expected_result)
        transcript, word_locations = trainer_SST_lambda_de.getTranscriptAndWordsLocations(signal_de_shape)
        assert transcript == phrases["de"]["transcribed"]
        assert word_locations == [(0, 7573), (5973, 11268), (9668, 14347), (12747, 17426), (15826, 21121)]

    def test_processAudioForGivenText_de(self):
        set_seed()
        phrase_real = phrases["de"]["real"]
        signal_transformed = transform(torch.Tensor(signal_de)).unsqueeze(0)
        expected_result = {
            'recording_transcript': 'hallo wie geht es dir',
            'real_and_transcribed_words': [('Hallo,', 'hallo'), ('wie', 'wie'), ('geht', 'geht'), ('es', 'es'), ('dir?', 'dir')],
            'recording_ipa': 'haloː viː ɡeːt ɛːs diːɐ̯', 'start_time': '0.0 0.3733125 0.60425 0.7966875 0.989125', 'end_time': '0.4733125 0.70425 0.8966875 1.089125 1.3200625',
            'real_and_transcribed_words_ipa': [('haloː,', 'haloː'), ('viː', 'viː'), ('ɡeːt', 'ɡeːt'), ('ɛːs', 'ɛːs'), ('diːr?', 'diːɐ̯')],
            'pronunciation_accuracy': 100.0,
            'pronunciation_categories': [0, 0, 0, 0, 0],
            "start_time": "0.0 0.3733125 0.60425 0.7966875 0.989125",
            "end_time": "0.4733125 0.70425 0.8966875 1.089125 1.3200625",
        }
        result = trainer_SST_lambda_de.processAudioForGivenText(signal_transformed, phrase_real)
        self.assertDictEqual(result, expected_result)

    def test_removePunctuation_de(self):
        word = "glück,"
        cleaned_word = trainer_SST_lambda_de.removePunctuation(word)
        self.assertEqual(cleaned_word, "glück")
        word = "glück,\n\rhallo..."
        cleaned_word = trainer_SST_lambda_de.removePunctuation(word)
        self.assertEqual(cleaned_word, "glück\n\rhallo")

    def test_getWordsPronunciationCategory_de(self):
        accuracies = [x for x in range(-121, 121, 10)] + [np.inf, -np.inf, np.nan, 1.5, -1.5]
        expected_categories = [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2]
        categories = trainer_SST_lambda_de.getWordsPronunciationCategory(accuracies)
        self.assertEqual(categories, expected_categories)

    def test_preprocessAudio_de(self):
        output_hash = utilities.hash_calculate(signal_de, is_file=False)
        assert output_hash == b'D9pMFzYL1BSPPg89ZCQE61xzb7QICXolYtC9EJRpvS0='
        signal_transformed = transform(torch.Tensor(signal_de)).unsqueeze(0)
        preprocessed_audio = trainer_SST_lambda_de.preprocessAudio(signal_transformed)
        self.assertIsInstance(preprocessed_audio, torch.Tensor)
        self.assertEqual(preprocessed_audio.shape, (1, 23400))
        output_hash = utilities.hash_calculate(preprocessed_audio.numpy(), is_file=False)
        assert output_hash == b'Ri/1rmgYmRSWaAw/Y3PoLEu1woiczhSUdUCbaMf++EM='

    def test_preprocessAudioStandalone_de(self):
        output_hash = utilities.hash_calculate(signal_de, is_file=False)
        assert output_hash == b'D9pMFzYL1BSPPg89ZCQE61xzb7QICXolYtC9EJRpvS0='
        signal_transformed = transform(torch.Tensor(signal_de)).unsqueeze(0)
        preprocessed_audio = pronunciationTrainer.preprocessAudioStandalone(signal_transformed)
        self.assertIsInstance(preprocessed_audio, torch.Tensor)
        self.assertEqual(preprocessed_audio.shape, (1, 23400))
        output_hash = utilities.hash_calculate(preprocessed_audio.numpy(), is_file=False)
        assert output_hash == b'Ri/1rmgYmRSWaAw/Y3PoLEu1woiczhSUdUCbaMf++EM='

    def test_processAudioForGivenText_getTranscriptAndWordsLocations_en(self):
        set_seed()
        phrase_real = phrases["en"]["real"]
        signal_en_shape = signal_en.shape[0]
        signal_transformed = transform(torch.Tensor(signal_en)).unsqueeze(0)
        result = trainer_SST_lambda_en.processAudioForGivenText(signal_transformed, phrase_real)
        expected_result = {
            'recording_transcript': 'i there how are you',
            'real_and_transcribed_words': [('Hi', 'i'), ('there,', 'there'), ('how', 'how'), ('are', 'are'), ('you?', 'you')],
            'recording_ipa': 'aɪ ðɛr haʊ ər ju', 'start_time': '0.0 0.0625 0.2875 0.475 0.7', 'end_time': '0.1625 0.3875 0.575 0.8 0.9875',
            'real_and_transcribed_words_ipa': [('haɪ', 'aɪ'), ('ðɛr,', 'ðɛr'), ('haʊ', 'haʊ'), ('ər', 'ər'), ('ju?', 'ju')],
            'pronunciation_accuracy': 94.0, 'pronunciation_categories': [2, 0, 0, 0, 0]
        }
        self.assertDictEqual(result, expected_result)
        transcript, word_locations = trainer_SST_lambda_en.getTranscriptAndWordsLocations(signal_en_shape)
        assert transcript == phrases["en"]["transcribed"]
        assert word_locations == [(0, 2600), (1000, 6200), (4600, 9200), (7600, 12800), (11200, 15800)]

    def test_processAudioForGivenText_en(self):
        set_seed()
        phrase_real = phrases["en"]["real"]
        signal_transformed = transform(torch.Tensor(signal_en)).unsqueeze(0)
        expected_result = {
            'recording_transcript': 'i there how are you',
            'real_and_transcribed_words': [('Hi', 'i'), ('there,', 'there'), ('how', 'how'), ('are', 'are'), ('you?', 'you')],
            'recording_ipa': 'aɪ ðɛr haʊ ər ju', 'start_time': '0.0 0.0625 0.2875 0.475 0.7', 'end_time': '0.1625 0.3875 0.575 0.8 0.9875',
            'real_and_transcribed_words_ipa': [('haɪ', 'aɪ'), ('ðɛr,', 'ðɛr'), ('haʊ', 'haʊ'), ('ər', 'ər'), ('ju?', 'ju')],
            'pronunciation_accuracy': 94.0, 'pronunciation_categories': [2, 0, 0, 0, 0],
            'start_time': '0.0 0.0625 0.2875 0.475 0.7',
            'end_time': '0.1625 0.3875 0.575 0.8 0.9875'
        }
        result = trainer_SST_lambda_en.processAudioForGivenText(signal_transformed, phrase_real)
        self.assertDictEqual(result, expected_result)

    def test_getPronunciationCategoryFromAccuracy_en(self):
        accuracies = [x for x in range(-121, 121, 10)] + [np.inf, -np.inf, np.nan, 1.5, -1.5]
        expected_categories = [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2]
        all_categories = []
        for accuracy in accuracies:
            category = trainer_SST_lambda_en.getPronunciationCategoryFromAccuracy(accuracy)
            all_categories.append(category)
        self.assertEqual(all_categories, expected_categories)

    def test_matchSampleAndRecordedWords(self):
        set_seed()
        phrase_real = phrases["de"]["real"]
        phrase_transcribed = phrases["de"]["transcribed"]
        real_and_transcribed_words, real_words, transcribed_words = trainer_SST_lambda_de.matchSampleAndRecordedWords(phrase_real, phrase_transcribed)
        self.assertIsInstance(real_and_transcribed_words, list)
        self.assertIsInstance(real_words, list)
        self.assertIsInstance(transcribed_words, list)
        self.assertEqual(len(real_and_transcribed_words), len(real_words))
        self.assertEqual(len(real_and_transcribed_words), len(transcribed_words))

    def test_removePunctuation_en(self):
        word = "hello,"
        cleaned_word = trainer_SST_lambda_en.removePunctuation(word)
        self.assertEqual(cleaned_word, "hello")
        word = "hello,\n\rworld..."
        cleaned_word = trainer_SST_lambda_en.removePunctuation(word)
        self.assertEqual(cleaned_word, "hello\n\rworld")

    def test_getWordsPronunciationCategory_en(self):
        accuracies = [x for x in range(-121, 121, 10)] + [np.inf, -np.inf, np.nan, 1.5, -1.5]
        expected_categories = [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2]
        categories = trainer_SST_lambda_en.getWordsPronunciationCategory(accuracies)
        self.assertEqual(categories, expected_categories)

    def test_preprocessAudio_en(self):
        output_hash = utilities.hash_calculate(signal_en, is_file=False)
        assert output_hash == b'zBAV/y7mecyPHLGiitHRP9vK7oU9hnYvyuatU0PQfts='
        signal_transformed = transform(torch.Tensor(signal_en)).unsqueeze(0)
        preprocessed_audio = trainer_SST_lambda_en.preprocessAudio(signal_transformed)
        self.assertIsInstance(preprocessed_audio, torch.Tensor)
        self.assertEqual(preprocessed_audio.shape, (1, 16800))
        output_hash = utilities.hash_calculate(preprocessed_audio.numpy(), is_file=False)
        assert output_hash == b'KsyH1MXIc+5e5B6CcijhitsGPUDRJjrJU2qg8bQi600='

    def test_preprocessAudioStandalone_en(self):
        output_hash = utilities.hash_calculate(signal_en, is_file=False)
        assert output_hash == b'zBAV/y7mecyPHLGiitHRP9vK7oU9hnYvyuatU0PQfts='
        signal_transformed = transform(torch.Tensor(signal_en)).unsqueeze(0)
        preprocessed_audio = pronunciationTrainer.preprocessAudioStandalone(signal_transformed)
        self.assertIsInstance(preprocessed_audio, torch.Tensor)
        self.assertEqual(preprocessed_audio.shape, (1, 16800))
        output_hash = utilities.hash_calculate(preprocessed_audio.numpy(), is_file=False)
        assert output_hash == b'KsyH1MXIc+5e5B6CcijhitsGPUDRJjrJU2qg8bQi600='


if __name__ == '__main__':
    unittest.main()