First repository code commit

AIModels.py

@@ -0,0 +1,65 @@
+import ModelInterfaces
+import torch
+import numpy as np
+
+
+class NeuralASR(ModelInterfaces.IASRModel):
+    word_locations_in_samples = None
+    audio_transcript = None
+
+    def __init__(self, model: torch.nn.Module, decoder) -> None:
+        super().__init__()
+        self.model = model
+        self.decoder = decoder  # Decoder from CTC-outputs to transcripts
+
+    def getTranscript(self) -> str:
+        """Get the transcripts of the process audio"""
+        assert(self.audio_transcript != None,
+               'Can get audio transcripts without having processed the audio')
+        return self.audio_transcript
+
+    def getWordLocations(self) -> list:
+        """Get the pair of words location from audio"""
+        assert(self.word_locations_in_samples != None,
+               'Can get word locations without having processed the audio')
+
+        return self.word_locations_in_samples
+
+    def processAudio(self, audio: torch.Tensor):
+        """Process the audio"""
+        audio_length_in_samples = audio.shape[1]
+        with torch.inference_mode():
+            nn_output = self.model(audio)
+
+            self.audio_transcript, self.word_locations_in_samples = self.decoder(
+                nn_output[0, :, :].detach(), audio_length_in_samples, word_align=True)
+
+
+class NeuralTTS(ModelInterfaces.ITextToSpeechModel):
+    def __init__(self, model: torch.nn.Module, sampling_rate: int) -> None:
+        super().__init__()
+        self.model = model
+        self.sampling_rate = sampling_rate
+
+    def getAudioFromSentence(self, sentence: str) -> np.array:
+        with torch.inference_mode():
+            audio_transcript = self.model.apply_tts(texts=[sentence],
+                                                    sample_rate=self.sampling_rate)[0]
+
+        return audio_transcript
+
+
+class NeuralTranslator(ModelInterfaces.ITranslationModel):
+    def __init__(self, model: torch.nn.Module, tokenizer) -> None:
+        super().__init__()
+        self.model = model
+        self.tokenizer = tokenizer
+
+    def translateSentence(self, sentence: str) -> str:
+        """Get the transcripts of the process audio"""
+        tokenized_text = self.tokenizer(sentence, return_tensors='pt')
+        translation = self.model.generate(**tokenized_text)
+        translated_text = self.tokenizer.batch_decode(
+            translation, skip_special_tokens=True)[0]
+
+        return translated_text

ModelInterfaces.py

@@ -0,0 +1,65 @@
+
+import abc
+import numpy as np
+
+
+class IASRModel(metaclass=abc.ABCMeta):
+    @classmethod
+    def __subclasshook__(cls, subclass):
+        return (hasattr(subclass, 'getTranscript') and
+                callable(subclass.getTranscript) and
+                hasattr(subclass, 'getWordLocations') and
+                callable(subclass.getWordLocations) and
+                hasattr(subclass, 'processAudio') and
+                callable(subclass.processAudio))
+
+    @abc.abstractmethod
+    def getTranscript(self) -> str:
+        """Get the transcripts of the process audio"""
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def getWordLocations(self) -> list:
+        """Get the pair of words location from audio"""
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def processAudio(self, audio):
+        """Process the audio"""
+        raise NotImplementedError
+
+
+class ITranslationModel(metaclass=abc.ABCMeta):
+    @classmethod
+    def __subclasshook__(cls, subclass):
+        return (hasattr(subclass, 'translateSentence') and
+                callable(subclass.translateSentence))
+
+    @abc.abstractmethod
+    def translateSentence(self, str) -> str:
+        """Get the translation of the sentence"""
+        raise NotImplementedError
+
+
+class ITextToSpeechModel(metaclass=abc.ABCMeta):
+    @classmethod
+    def __subclasshook__(cls, subclass):
+        return (hasattr(subclass, 'getAudioFromSentence') and
+                callable(subclass.getAudioFromSentence))
+
+    @abc.abstractmethod
+    def getAudioFromSentence(self, str) -> np.array:
+        """Get audio from sentence"""
+        raise NotImplementedError
+
+
+class ITextToPhonemModel(metaclass=abc.ABCMeta):
+    @classmethod
+    def __subclasshook__(cls, subclass):
+        return (hasattr(subclass, 'convertToPhonem') and
+                callable(subclass.convertToPhonem))
+
+    @abc.abstractmethod
+    def convertToPhonem(self, str) -> str:
+        """Convert sentence to phonemes"""
+        raise NotImplementedError

README.md

@@ -1 +1,24 @@
-# ai-pronunciation-trainer
+## Motivation
+This tool uses AI to evaluate your pronunciation so you can be understood more clearly. To directly test the tool, go to https://aipronunciationtr.com (please use the chrome browser for desktop). 
+
+Often, when we want to improve our pronunciation, it is very difficult to self-assess how good we’re speaking. Asking a native, or language instructor, to constantly correct us is either impractical, due to monetary constrains, or annoying due to simply being too boring for this other person. Additionally, they may often say “it sounds okay” after your 10th try to not discourage you, even though you may still have some mistakes in your pronunciation. 
+
+The AI pronunciation trainer is a way to provide objective feedback on how well your pronunciation is in an automatic and scalable fashion, so the only limit to your improvement is your own dedication. 
+
+This project originated from a small program that I did to improve my own pronunciation.  When I finished it, I believed it could be a useful tool also to other people trying to be better understood, so I decided to make a simple, more user friendly version of it. 
+
+## Installation 
+To run the program, all you need to do is to install the requirements and run the main python file
+```
+pip install -r requirements.txt
+python webApp.py
+```
+The code is pure python, thus you should be able to run it without any major issue as long as you’re a using a recent python 3.X version.  
+
+## Online version
+For the people who don’t feel comfortable running code or just want to have a quick way to use the tool, I hosted an online version of it at https://aipronunciationtr.com . It should work without any major issues in desktop-chrome, any other browser is not officially supported, although most of the functionality should work fine. 
+ 
+Please be aware that the usage is limited by day (I’m still not rich ;) ). If, for some reason, you would like to skip the daily usage limit, just enter in contact and we can negotiate an additional API key only for you. 
+
+## Disclaimer 
+Even though the tool can be useful, my intention here was not to make an industry-grade program with 100% test coverage, no errors and full support, but rather some easy to use tool that may be useful for some people even though it may not be perfect.  Thus, be aware that some small bugs may be presented. In case you find something not working, all feedback is welcome and issues may be adressed depending on their severity.

RuleBasedModels.py

@@ -0,0 +1,29 @@
+import ModelInterfaces
+import torch
+import numpy as np
+import epitran
+import eng_to_ipa
+
+
+class EpitranPhonemConverter(ModelInterfaces.ITextToPhonemModel):
+    word_locations_in_samples = None
+    audio_transcript = None
+
+    def __init__(self, epitran_model) -> None:
+        super().__init__()
+        self.epitran_model = epitran_model
+
+    def convertToPhonem(self, sentence: str) -> str:
+        phonem_representation = self.epitran_model.transliterate(sentence)
+        return phonem_representation
+
+
+class EngPhonemConverter(ModelInterfaces.ITextToPhonemModel):
+
+    def __init__(self,) -> None:
+        super().__init__()
+
+    def convertToPhonem(self, sentence: str) -> str:
+        phonem_representation = eng_to_ipa.convert(sentence)
+        phonem_representation = phonem_representation.replace('*','')
+        return phonem_representation

WordMatching.py

@@ -0,0 +1,181 @@
+import WordMetrics
+from ortools.sat.python import cp_model
+import numpy as np
+from string import punctuation
+from dtwalign import dtw_from_distance_matrix
+import time
+
+offset_blank = 1
+TIME_THRESHOLD_MAPPING = 5.0
+
+
+def get_word_distance_matrix(words_estimated: list, words_real: list) -> np.array:
+    number_of_real_words = len(words_real)
+    number_of_estimated_words = len(words_estimated)
+
+    word_distance_matrix = np.zeros(
+        (number_of_estimated_words+offset_blank, number_of_real_words))
+    for idx_estimated in range(number_of_estimated_words):
+        for idx_real in range(number_of_real_words):
+            word_distance_matrix[idx_estimated, idx_real] = WordMetrics.edit_distance_python(
+                words_estimated[idx_estimated], words_real[idx_real])
+
+    if offset_blank == 1:
+        for idx_real in range(number_of_real_words):
+            word_distance_matrix[number_of_estimated_words,
+                                 idx_real] = len(words_real[idx_real])
+    return word_distance_matrix
+
+
+def get_best_path_from_distance_matrix(word_distance_matrix):
+    modelCpp = cp_model.CpModel()
+
+    number_of_real_words = word_distance_matrix.shape[1]
+    number_of_estimated_words = word_distance_matrix.shape[0]-1
+
+    number_words = np.maximum(number_of_real_words, number_of_estimated_words)
+
+    estimated_words_order = [modelCpp.NewIntVar(0, int(
+        number_words - 1 + offset_blank), 'w%i' % i) for i in range(number_words+offset_blank)]
+
+    # They are in ascending order
+    for word_idx in range(number_words-1):
+        modelCpp.Add(
+            estimated_words_order[word_idx+1] >= estimated_words_order[word_idx])
+
+    total_phoneme_distance = 0
+    real_word_at_time = {}
+    for idx_estimated in range(number_of_estimated_words):
+        for idx_real in range(number_of_real_words):
+            real_word_at_time[idx_estimated, idx_real] = modelCpp.NewBoolVar(
+                'real_word_at_time'+str(idx_real)+'-'+str(idx_estimated))
+            modelCpp.Add(estimated_words_order[idx_estimated] == idx_real).OnlyEnforceIf(
+                real_word_at_time[idx_estimated, idx_real])
+            total_phoneme_distance += word_distance_matrix[idx_estimated,
+                                                           idx_real]*real_word_at_time[idx_estimated, idx_real]
+
+    # If no word in time, difference is calculated from empty string
+    for idx_real in range(number_of_real_words):
+        word_has_a_match = modelCpp.NewBoolVar(
+            'word_has_a_match'+str(idx_real))
+        modelCpp.Add(sum([real_word_at_time[idx_estimated, idx_real] for idx_estimated in range(
+            number_of_estimated_words)]) == 1).OnlyEnforceIf(word_has_a_match)
+        total_phoneme_distance += word_distance_matrix[number_of_estimated_words,
+                                                       idx_real]*word_has_a_match.Not()
+
+    # Loss should be minimized
+    modelCpp.Minimize(total_phoneme_distance)
+
+    solver = cp_model.CpSolver()
+    solver.parameters.max_time_in_seconds = TIME_THRESHOLD_MAPPING
+    status = solver.Solve(modelCpp)
+
+    mapped_indices = []
+    try:
+        for word_idx in range(number_words):
+            mapped_indices.append(
+                (solver.Value(estimated_words_order[word_idx])))
+
+        return np.array(mapped_indices, dtype=np.int)
+    except:
+        return []
+
+
+def get_resulting_string(mapped_indices: np.array, words_estimated: list, words_real: list) -> list:
+    mapped_words = []
+    mapped_words_indices = []
+    WORD_NOT_FOUND_TOKEN = '-'
+    number_of_real_words = len(words_real)
+    for word_idx in range(number_of_real_words):
+        position_of_real_word_indices = np.where(
+            mapped_indices == word_idx)[0].astype(np.int)
+
+        if len(position_of_real_word_indices) == 0:
+            mapped_words.append(WORD_NOT_FOUND_TOKEN)
+            mapped_words_indices.append(-1)
+            continue
+
+        if len(position_of_real_word_indices) == 1:
+            mapped_words.append(
+                words_estimated[position_of_real_word_indices[0]])
+            mapped_words_indices.append(position_of_real_word_indices[0])
+            continue
+        # Check which index gives the lowest error
+        if len(position_of_real_word_indices) > 1:
+            error = 99999
+            best_possible_combination = ''
+            best_possible_idx = -1
+            for single_word_idx in position_of_real_word_indices:
+                idx_above_word = single_word_idx >= len(words_estimated)
+                if idx_above_word:
+                    continue
+                error_word = WordMetrics.edit_distance_python(
+                    words_estimated[single_word_idx], words_real[word_idx])
+                if error_word < error:
+                    error = error_word*1
+                    best_possible_combination = words_estimated[single_word_idx]
+                    best_possible_idx = single_word_idx
+
+            mapped_words.append(best_possible_combination)
+            mapped_words_indices.append(best_possible_idx)
+            continue
+
+    return mapped_words, mapped_words_indices
+
+
+def get_best_mapped_words(words_estimated: list, words_real: list) -> list:
+
+    word_distance_matrix = get_word_distance_matrix(
+        words_estimated, words_real)
+
+    start = time.time()
+    mapped_indices = get_best_path_from_distance_matrix(word_distance_matrix)
+
+    duration_of_mapping = time.time()-start
+    # In case or-tools doesn't converge, go to a faster, low-quality solution
+    if len(mapped_indices) == 0 or duration_of_mapping > TIME_THRESHOLD_MAPPING+0.5:
+        mapped_indices = (dtw_from_distance_matrix(
+            word_distance_matrix)).path[:len(words_estimated), 1]
+
+    mapped_words, mapped_words_indices = get_resulting_string(
+        mapped_indices, words_estimated, words_real)
+
+    return mapped_words, mapped_words_indices
+
+
+# Faster, but not optimal
+def get_best_mapped_words_dtw(words_estimated: list, words_real: list) -> list:
+
+    from dtwalign import dtw_from_distance_matrix
+    word_distance_matrix = get_word_distance_matrix(
+        words_estimated, words_real)
+    mapped_indices = dtw_from_distance_matrix(
+        word_distance_matrix).path[:-1, 0]
+
+    mapped_words, mapped_words_indices = get_resulting_string(
+        mapped_indices, words_estimated, words_real)
+    return mapped_words, mapped_words_indices
+
+
+def getWhichLettersWereTranscribedCorrectly(real_word, transcribed_word):
+    is_leter_correct = [None]*len(real_word)
+    for idx, letter in enumerate(real_word):
+        if letter == transcribed_word[idx] or letter in punctuation:
+            is_leter_correct[idx] = 1
+        else:
+            is_leter_correct[idx] = 0
+    return is_leter_correct
+
+
+def parseLetterErrorsToHTML(word_real, is_leter_correct):
+    word_colored = ''
+    correct_color_start = '*'
+    correct_color_end = '*'
+    wrong_color_start = '-'
+    wrong_color_end = '-'
+    for idx, letter in enumerate(word_real):
+        if is_leter_correct[idx] == 1:
+            word_colored += correct_color_start + letter+correct_color_end
+        else:
+            word_colored += wrong_color_start + letter+wrong_color_end
+    return word_colored

WordMetrics.py

@@ -0,0 +1,55 @@
+import numpy as np
+
+# ref from https://gitlab.com/-/snippets/1948157
+# For some variants, look here https://en.wikibooks.org/wiki/Algorithm_Implementation/Strings/Levenshtein_distance#Python
+
+# Pure python
+def edit_distance_python2(a, b):
+    # This version is commutative, so as an optimization we force |a|>=|b|
+    if len(a) < len(b):
+        return edit_distance_python(b, a)
+    if len(b) == 0:  # Can deal with empty sequences faster
+        return len(a)
+    # Only two rows are really needed: the one currently filled in, and the previous
+    distances = []
+    distances.append([i for i in range(len(b)+1)])
+    distances.append([0 for _ in range(len(b)+1)])
+    # We can prefill the first row:
+    costs = [0 for _ in range(3)]
+    for i, a_token in enumerate(a, start=1):
+        distances[1][0] += 1  # Deals with the first column.
+        for j, b_token in enumerate(b, start=1):
+            costs[0] = distances[1][j-1] + 1
+            costs[1] = distances[0][j] + 1
+            costs[2] = distances[0][j-1] + (0 if a_token == b_token else 1)
+            distances[1][j] = min(costs)
+        # Move to the next row:
+        distances[0][:] = distances[1][:]
+    return distances[1][len(b)]
+
+#https://stackabuse.com/levenshtein-distance-and-text-similarity-in-python/
+def edit_distance_python(seq1, seq2):
+    size_x = len(seq1) + 1
+    size_y = len(seq2) + 1
+    matrix = np.zeros ((size_x, size_y))
+    for x in range(size_x):
+        matrix [x, 0] = x
+    for y in range(size_y):
+        matrix [0, y] = y
+
+    for x in range(1, size_x):
+        for y in range(1, size_y):
+            if seq1[x-1] == seq2[y-1]:
+                matrix [x,y] = min(
+                    matrix[x-1, y] + 1,
+                    matrix[x-1, y-1],
+                    matrix[x, y-1] + 1
+                )
+            else:
+                matrix [x,y] = min(
+                    matrix[x-1,y] + 1,
+                    matrix[x-1,y-1] + 1,
+                    matrix[x,y-1] + 1
+                )
+    #print (matrix)
+    return (matrix[size_x - 1, size_y - 1])

data_de_en_2.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30ec872918777a5c766ab26d78f33383f6b36fccb00af7fc5c543bd43c98ffa4
+size 1056086

lambdaGetSample.py

@@ -0,0 +1,89 @@
+
+import pandas as pd
+import json
+import RuleBasedModels
+import epitran
+import random
+import pickle
+
+
+class TextDataset():
+    def __init__(self, table, language='-'):
+        self.table_dataframe = table
+        self.number_of_samples = len(table)
+        self.language = language
+
+    def __getitem__(self, idx):
+
+        if self.language == 'de':
+            line = [self.table_dataframe['de_sentence'].iloc[idx]]
+        elif self.language == 'en':
+            line = [self.table_dataframe['en_sentence'].iloc[idx]]
+        else:
+            line = [self.table_dataframe['sentence'].iloc[idx]]
+        return line
+
+    def __len__(self):
+        return self.number_of_samples
+
+
+sample_folder = "./"
+lambda_database = {}
+lambda_ipa_converter = {}
+
+with open(sample_folder+'data_de_en_2.pickle', 'rb') as handle:
+    df = pickle.load(handle)
+
+lambda_database['de'] = TextDataset(df, 'de')
+lambda_database['en'] = TextDataset(df, 'en')
+lambda_translate_new_sample = False
+lambda_ipa_converter['de'] = RuleBasedModels.EpitranPhonemConverter(
+    epitran.Epitran('deu-Latn'))
+lambda_ipa_converter['en'] = RuleBasedModels.EngPhonemConverter()
+
+
+def lambda_handler(event, context):
+
+    body = json.loads(event['body'])
+
+    category = int(body['category'])
+
+    language = body['language']
+
+    sample_in_category = False
+
+    while(not sample_in_category):
+        valid_sequence = False
+        while not valid_sequence:
+            try:
+                sample_idx = random.randint(0, len(lambda_database[language]))
+                current_transcript = lambda_database[language][
+                    sample_idx]
+                valid_sequence = True
+            except:
+                pass
+
+        sentence_category = getSentenceCategory(
+            current_transcript[0])
+
+        sample_in_category = (sentence_category ==
+                              category) or category == 0
+
+    translated_trascript = ""
+
+    current_ipa = lambda_ipa_converter[language].convertToPhonem(
+        current_transcript[0])
+
+    result = {'real_transcript': current_transcript,
+              'ipa_transcript': current_ipa,
+              'transcript_translation': translated_trascript}
+
+    return json.dumps(result)
+
+
+def getSentenceCategory(sentence) -> int:
+    number_of_words = len(sentence.split())
+    categories_word_limits = [0, 8, 20, 100000]
+    for category in range(len(categories_word_limits)-1):
+        if number_of_words > categories_word_limits[category] and number_of_words <= categories_word_limits[category+1]:
+            return category+1

lambdaSpeechToScore.py

@@ -0,0 +1,196 @@
+
+import torch
+import json
+import os
+import WordMatching as wm
+import utilsFileIO
+import pronunciationTrainer
+import base64
+import time
+import audioread
+import numpy as np
+from torchaudio.transforms import Resample
+
+
+trainer_SST_lambda = {}
+trainer_SST_lambda['de'] = pronunciationTrainer.getTrainer("de")
+trainer_SST_lambda['en'] = pronunciationTrainer.getTrainer("en")
+
+transform = Resample(orig_freq=48000, new_freq=16000)
+
+
+def lambda_handler(event, context):
+
+    data = json.loads(event['body'])
+
+    real_text = data['title']
+    file_bytes = base64.b64decode(
+        data['base64Audio'][22:].encode('utf-8'))
+    language = data['language']
+
+    if len(real_text) == 0:
+        return {
+            'statusCode': 200,
+            'headers': {
+                'Access-Control-Allow-Headers': '*',
+                'Access-Control-Allow-Credentials': "true",
+                'Access-Control-Allow-Origin': '*',
+                'Access-Control-Allow-Methods': 'OPTIONS,POST,GET'
+            },
+            'body': ''
+        }
+
+    start = time.time()
+    random_file_name = './'+utilsFileIO.generateRandomString()+'.ogg'
+    f = open(random_file_name, 'wb')
+    f.write(file_bytes)
+    f.close()
+    print('Time for saving binary in file: ', str(time.time()-start))
+
+    start = time.time()
+    signal, fs = audioread_load(random_file_name)
+
+    signal = transform(torch.Tensor(signal)).unsqueeze(0)
+
+    print('Time for loading .ogg file file: ', str(time.time()-start))
+
+    result = trainer_SST_lambda[language].processAudioForGivenText(
+        signal, real_text)
+
+    start = time.time()
+    os.remove(random_file_name)
+    print('Time for deleting file: ', str(time.time()-start))
+
+    start = time.time()
+    real_transcripts_ipa = ' '.join(
+        [word[0] for word in result['real_and_transcribed_words_ipa']])
+    matched_transcripts_ipa = ' '.join(
+        [word[1] for word in result['real_and_transcribed_words_ipa']])
+
+    real_transcripts = ' '.join(
+        [word[0] for word in result['real_and_transcribed_words']])
+    matched_transcripts = ' '.join(
+        [word[1] for word in result['real_and_transcribed_words']])
+
+    words_real = real_transcripts.lower().split()
+    mapped_words = matched_transcripts.split()
+
+    is_letter_correct_all_words = ''
+    for idx, word_real in enumerate(words_real):
+
+        mapped_letters, mapped_letters_indices = wm.get_best_mapped_words(
+            mapped_words[idx], word_real)
+
+        is_letter_correct = wm.getWhichLettersWereTranscribedCorrectly(
+            word_real, mapped_letters)  # , mapped_letters_indices)
+
+        is_letter_correct_all_words += ''.join([str(is_correct)
+                                                for is_correct in is_letter_correct]) + ' '
+
+    pair_accuracy_category = ' '.join(
+        [str(category) for category in result['pronunciation_categories']])
+    print('Time to post-process results: ', str(time.time()-start))
+
+    res = {'real_transcript': result['recording_transcript'],
+           'ipa_transcript': result['recording_ipa'],
+           'pronunciation_accuracy': str(int(result['pronunciation_accuracy'])),
+           'real_transcripts': real_transcripts, 'matched_transcripts': matched_transcripts,
+           'real_transcripts_ipa': real_transcripts_ipa, 'matched_transcripts_ipa': matched_transcripts_ipa,
+           'pair_accuracy_category': pair_accuracy_category,
+           'start_time': result['start_time'],
+           'end_time': result['end_time'],
+           'is_letter_correct_all_words': is_letter_correct_all_words}
+
+    return json.dumps(res)
+
+# From Librosa
+
+
+def audioread_load(path, offset=0.0, duration=None, dtype=np.float32):
+    """Load an audio buffer using audioread.
+
+    This loads one block at a time, and then concatenates the results.
+    """
+
+    y = []
+    with audioread.audio_open(path) as input_file:
+        sr_native = input_file.samplerate
+        n_channels = input_file.channels
+
+        s_start = int(np.round(sr_native * offset)) * n_channels
+
+        if duration is None:
+            s_end = np.inf
+        else:
+            s_end = s_start + \
+                (int(np.round(sr_native * duration)) * n_channels)
+
+        n = 0
+
+        for frame in input_file:
+            frame = buf_to_float(frame, dtype=dtype)
+            n_prev = n
+            n = n + len(frame)
+
+            if n < s_start:
+                # offset is after the current frame
+                # keep reading
+                continue
+
+            if s_end < n_prev:
+                # we're off the end.  stop reading
+                break
+
+            if s_end < n:
+                # the end is in this frame.  crop.
+                frame = frame[: s_end - n_prev]
+
+            if n_prev <= s_start <= n:
+                # beginning is in this frame
+                frame = frame[(s_start - n_prev):]
+
+            # tack on the current frame
+            y.append(frame)
+
+    if y:
+        y = np.concatenate(y)
+        if n_channels > 1:
+            y = y.reshape((-1, n_channels)).T
+    else:
+        y = np.empty(0, dtype=dtype)
+
+    return y, sr_native
+
+# From Librosa
+
+
+def buf_to_float(x, n_bytes=2, dtype=np.float32):
+    """Convert an integer buffer to floating point values.
+    This is primarily useful when loading integer-valued wav data
+    into numpy arrays.
+
+    Parameters
+    ----------
+    x : np.ndarray [dtype=int]
+        The integer-valued data buffer
+
+    n_bytes : int [1, 2, 4]
+        The number of bytes per sample in ``x``
+
+    dtype : numeric type
+        The target output type (default: 32-bit float)
+
+    Returns
+    -------
+    x_float : np.ndarray [dtype=float]
+        The input data buffer cast to floating point
+    """
+
+    # Invert the scale of the data
+    scale = 1.0 / float(1 << ((8 * n_bytes) - 1))
+
+    # Construct the format string
+    fmt = "<i{:d}".format(n_bytes)
+
+    # Rescale and format the data buffer
+    return scale * np.frombuffer(x, fmt).astype(dtype)

lambdaTTS.py

@@ -0,0 +1,46 @@
+
+import models
+import soundfile as sf
+import json
+import AIModels
+#from flask import Response
+import utilsFileIO
+import os
+import base64
+
+sampling_rate = 16000
+model_TTS_lambda = AIModels.NeuralTTS(models.getTTSModel('de'), sampling_rate)
+
+
+def lambda_handler(event, context):
+
+    body = json.loads(event['body'])
+
+    text_string = body['value']
+
+    linear_factor = 0.2
+    audio = model_TTS_lambda.getAudioFromSentence(
+        text_string).detach().numpy()*linear_factor
+    random_file_name = utilsFileIO.generateRandomString(20)+'.wav'
+
+    sf.write('./'+random_file_name, audio, 16000)
+
+    with open(random_file_name, "rb") as f:
+        audio_byte_array = f.read()
+
+    os.remove(random_file_name)
+
+
+    return {
+        'statusCode': 200,
+        'headers': {
+            'Access-Control-Allow-Headers': '*',
+            'Access-Control-Allow-Origin': '*',
+            'Access-Control-Allow-Methods': 'OPTIONS,POST,GET'
+        },
+        'body': json.dumps(
+            {
+                "wavBase64": str(base64.b64encode(audio_byte_array))[2:-1],
+            },
+        )
+    }

models.py

@@ -0,0 +1,71 @@
+import torch
+import torch.nn as nn
+
+import pickle
+
+
+import pickle
+
+
+def getASRModel(language: str) -> nn.Module:
+
+    if language == 'de':
+
+        model, decoder, utils = torch.hub.load(repo_or_dir='snakers4/silero-models',
+                                               model='silero_stt',
+                                               language='de',
+                                               device=torch.device('cpu'))
+
+    elif language == 'en':
+        model, decoder, utils = torch.hub.load(repo_or_dir='snakers4/silero-models',
+                                               model='silero_stt',
+                                               language='en',
+                                               device=torch.device('cpu'))
+    elif language == 'fr':
+        model, decoder, utils = torch.hub.load(repo_or_dir='snakers4/silero-models',
+                                               model='silero_stt',
+                                               language='fr',
+                                               device=torch.device('cpu'))
+
+    return (model, decoder)
+
+
+def getTTSModel(language: str) -> nn.Module:
+
+    if language == 'de':
+
+        speaker = 'thorsten_v2'  # 16 kHz
+        model, _ = torch.hub.load(repo_or_dir='snakers4/silero-models',
+                                  model='silero_tts',
+                                  language=language,
+                                  speaker=speaker)
+
+    elif language == 'en':
+        speaker = 'lj_16khz'  # 16 kHz
+        model = torch.hub.load(repo_or_dir='snakers4/silero-models',
+                               model='silero_tts',
+                               language=language,
+                               speaker=speaker)
+    else:
+        raise ValueError('Language not implemented')
+
+    return model
+
+
+def getTranslationModel(language: str) -> nn.Module:
+    from transformers import AutoTokenizer
+    from transformers import AutoModelForSeq2SeqLM
+    if language == 'de':
+        model = AutoModelForSeq2SeqLM.from_pretrained(
+            "Helsinki-NLP/opus-mt-de-en")
+        tokenizer = AutoTokenizer.from_pretrained(
+            "Helsinki-NLP/opus-mt-de-en")
+        # Cache models to avoid Hugging face processing
+        with open('translation_model_de.pickle', 'wb') as handle:
+            pickle.dump(model, handle)
+        with open('translation_tokenizer_de.pickle', 'wb') as handle:
+            pickle.dump(tokenizer, handle)
+    else:
+        raise ValueError('Language not implemented')
+
+    return model, tokenizer

pronunciationTrainer.py

@@ -0,0 +1,201 @@
+
+import torch
+import numpy as np
+import models as mo
+import WordMetrics
+import WordMatching as wm
+import epitran
+import ModelInterfaces as mi
+import AIModels
+import RuleBasedModels
+from string import punctuation
+import time
+
+
+def getTrainer(language: str):
+
+    device = torch.device('cpu')
+
+    model, decoder = mo.getASRModel(language)
+    model = model.to(device)
+    model.eval()
+    asr_model = AIModels.NeuralASR(model, decoder)
+
+    if language == 'de':
+        phonem_converter = RuleBasedModels.EpitranPhonemConverter(
+            epitran.Epitran('deu-Latn'))
+    elif language == 'en':
+        phonem_converter = RuleBasedModels.EngPhonemConverter()
+    else:
+        raise ValueError('Language not implemented')
+
+    trainer = PronunciationTrainer(
+        asr_model, phonem_converter)
+
+    return trainer
+
+
+class PronunciationTrainer:
+    current_transcript: str
+    current_ipa: str
+
+    current_recorded_audio: torch.Tensor
+    current_recorded_transcript: str
+    current_recorded_word_locations: list
+    current_recorded_intonations: torch.tensor
+    current_words_pronunciation_accuracy = []
+    categories_thresholds = np.array([80, 60, 59])
+
+    sampling_rate = 16000
+
+    def __init__(self, asr_model: mi.IASRModel, word_to_ipa_coverter: mi.ITextToPhonemModel) -> None:
+        self.asr_model = asr_model
+        self.ipa_converter = word_to_ipa_coverter
+
+    def getTranscriptAndWordsLocations(self, audio_length_in_samples: int):
+
+        audio_transcript = self.asr_model.getTranscript()
+        word_locations_in_samples = self.asr_model.getWordLocations()
+
+        fade_duration_in_samples = 0.05*self.sampling_rate
+        word_locations_in_samples = [(int(np.maximum(0, word['start_ts']-fade_duration_in_samples)), int(np.minimum(
+            audio_length_in_samples-1, word['end_ts']+fade_duration_in_samples))) for word in word_locations_in_samples]
+
+        return audio_transcript, word_locations_in_samples
+
+    def getWordsRelativeIntonation(self, Audio: torch.tensor, word_locations: list):
+        intonations = torch.zeros((len(word_locations), 1))
+        intonation_fade_samples = 0.3*self.sampling_rate
+        print(intonations.shape)
+        for word in range(len(word_locations)):
+            intonation_start = int(np.maximum(
+                0, word_locations[word][0]-intonation_fade_samples))
+            intonation_end = int(np.minimum(
+                Audio.shape[1]-1, word_locations[word][1]+intonation_fade_samples))
+            intonations[word] = torch.sqrt(torch.mean(
+                Audio[0][intonation_start:intonation_end]**2))
+
+        intonations = intonations/torch.mean(intonations)
+        return intonations
+
+    ##################### ASR Functions ###########################
+
+    def processAudioForGivenText(self, recordedAudio: torch.Tensor = None, real_text=None):
+
+        start = time.time()
+        recording_transcript, recording_ipa, word_locations = self.getAudioTranscript(
+            recordedAudio)
+        print('Time for NN to transcript audio: ', str(time.time()-start))
+
+        start = time.time()
+        real_and_transcribed_words, real_and_transcribed_words_ipa, mapped_words_indices = self.matchSampleAndRecordedWords(
+            real_text, recording_transcript)
+        print('Time for matching transcripts: ', str(time.time()-start))
+
+        start_time, end_time = self.getWordLocationsFromRecordInSeconds(
+            word_locations, mapped_words_indices)
+
+        pronunciation_accuracy, current_words_pronunciation_accuracy = self.getPronunciationAccuracy(
+            real_and_transcribed_words)  # _ipa
+
+        pronunciation_categories = self.getWordsPronunciationCategory(
+            current_words_pronunciation_accuracy)
+
+        result = {'recording_transcript': recording_transcript,
+                  'real_and_transcribed_words': real_and_transcribed_words,
+                  'recording_ipa': recording_ipa, 'start_time': start_time, 'end_time': end_time,
+                  'real_and_transcribed_words_ipa': real_and_transcribed_words_ipa, 'pronunciation_accuracy': pronunciation_accuracy,
+                  'pronunciation_categories': pronunciation_categories}
+
+        return result
+
+    def getAudioTranscript(self, recordedAudio: torch.Tensor = None):
+        current_recorded_audio = recordedAudio
+
+        current_recorded_audio = self.preprocessAudio(
+            current_recorded_audio)
+
+        self.asr_model.processAudio(current_recorded_audio)
+
+        current_recorded_transcript, current_recorded_word_locations = self.getTranscriptAndWordsLocations(
+            current_recorded_audio.shape[1])
+        current_recorded_ipa = self.ipa_converter.convertToPhonem(
+            current_recorded_transcript)
+
+        return current_recorded_transcript, current_recorded_ipa, current_recorded_word_locations
+
+    def getWordLocationsFromRecordInSeconds(self, word_locations, mapped_words_indices) -> list:
+        start_time = []
+        end_time = []
+        for word_idx in range(len(mapped_words_indices)):
+            start_time.append(float(word_locations[mapped_words_indices[word_idx]]
+                                    [0])/self.sampling_rate)
+            end_time.append(float(word_locations[mapped_words_indices[word_idx]]
+                                  [1])/self.sampling_rate)
+        return ' '.join([str(time) for time in start_time]), ' '.join([str(time) for time in end_time])
+
+    ##################### END ASR Functions ###########################
+
+    ##################### Evaluation Functions ###########################
+    def matchSampleAndRecordedWords(self, real_text, recorded_transcript):
+        words_estimated = recorded_transcript.split()
+
+        if real_text is None:
+            words_real = self.current_transcript[0].split()
+        else:
+            words_real = real_text.split()
+
+        mapped_words, mapped_words_indices = wm.get_best_mapped_words(
+            words_estimated, words_real)
+
+        real_and_transcribed_words = []
+        real_and_transcribed_words_ipa = []
+        for word_idx in range(len(words_real)):
+            if word_idx >= len(mapped_words)-1:
+                mapped_words.append('-')
+            real_and_transcribed_words.append(
+                (words_real[word_idx],    mapped_words[word_idx]))
+            real_and_transcribed_words_ipa.append((self.ipa_converter.convertToPhonem(words_real[word_idx]),
+                                                   self.ipa_converter.convertToPhonem(mapped_words[word_idx])))
+        return real_and_transcribed_words, real_and_transcribed_words_ipa, mapped_words_indices
+
+    def getPronunciationAccuracy(self, real_and_transcribed_words_ipa) -> float:
+        total_mismatches = 0.
+        number_of_phonemes = 0.
+        current_words_pronunciation_accuracy = []
+        for pair in real_and_transcribed_words_ipa:
+
+            real_without_punctuation = self.removePunctuation(pair[0]).lower()
+            number_of_word_mismatches = WordMetrics.edit_distance_python(
+                real_without_punctuation, self.removePunctuation(pair[1]).lower())
+            total_mismatches += number_of_word_mismatches
+            number_of_phonemes_in_word = len(real_without_punctuation)
+            number_of_phonemes += number_of_phonemes_in_word
+
+            current_words_pronunciation_accuracy.append(float(
+                number_of_phonemes_in_word-number_of_word_mismatches)/number_of_phonemes_in_word*100)
+
+        percentage_of_correct_pronunciations = (
+            number_of_phonemes-total_mismatches)/number_of_phonemes*100
+
+        return np.round(percentage_of_correct_pronunciations), current_words_pronunciation_accuracy
+
+    def removePunctuation(self, word: str) -> str:
+        return ''.join([char for char in word if char not in punctuation])
+
+    def getWordsPronunciationCategory(self, accuracies) -> list:
+        categories = []
+
+        for accuracy in accuracies:
+            categories.append(
+                self.getPronunciationCategoryFromAccuracy(accuracy))
+
+        return categories
+
+    def getPronunciationCategoryFromAccuracy(self, accuracy) -> int:
+        return np.argmin(abs(self.categories_thresholds-accuracy))
+
+    def preprocessAudio(self, audio: torch.tensor) -> torch.tensor:
+        audio = audio-torch.mean(audio)
+        audio = audio/torch.max(torch.abs(audio))
+        return audio

requirements.txt

@@ -0,0 +1,18 @@
+-f https://download.pytorch.org/whl/torch_stable.html
+torch==1.10.1
+torchaudio==0.10.1
+soundfile==0.10.3.post1
+omegaconf
+epitran==1.15
+audioread
+requests
+dtwalign
+eng_to_ipa
+pandas
+flask
+flask_cors
+pickle-mixin
+sqlalchemy
+transformers
+sentencepiece
+ortools==9.2.9972

static/css/style-new.css

@@ -0,0 +1,471 @@
+body {
+	background: #f2f2f2;
+}
+
+
+.expanded {
+	margin: auto;
+	align-content: center;
+}
+
+p {
+	overflow: auto;
+}
+
+h1 {
+	margin-left: 2%;
+}
+
+a.disabled {
+	pointer-events: none;
+	color: #ccc;
+	background-color: #ccc;
+}
+
+
+.horizontal-flexbox {
+	height: 100%;
+	width: 100%;
+	display: flex;
+}
+
+/* ############## Next button ##### */
+.button-next {
+	border-radius: 4px;
+	display: block;
+	border: none;
+	color: #FFFFFF;
+	text-align: left;
+	font-size: 3em;
+	box-sizing: border-box;
+	position: absolute;
+	top: 0;
+	left: 0%;
+	right: 2%;
+	bottom: 2%;
+	background-color: #58636d;
+	width: 10em;
+
+	transition: all 0.5s;
+	cursor: pointer;
+}
+
+.button-next:hover {
+	background-color: #6383a1 !important;
+}
+
+.button-next span {
+	cursor: pointer;
+	display: inline-block;
+	position: relative;
+	transition: 0.5s;
+}
+
+/*
+.button-next span:after {
+	content: '\00bb';
+	position: absolute;
+	opacity: 0;
+	top: 0;
+	right: -20px;
+	transition: 0.5s;
+
+}*/
+
+.button-next:hover span {
+	padding-right: 25px;
+}
+
+.button-next:hover span:after {
+	opacity: 1;
+	right: 0;
+}
+
+
+
+/* ############# Texts ##############  */
+
+.main-text {
+	font-size: 2.5em;
+	max-width: 87%;
+}
+
+.ipa-text {
+	font-size: 1.8em;
+	max-width: 87%;
+}
+
+.ipa-text-small {
+	font-size: 1.5em;
+}
+
+.accuracy-text {
+	/*font-family: "Dank Mono", ui-monospace, monospace;*/
+	background: linear-gradient(to right,
+			rgb(54, 56, 80),
+			rgb(21, 60, 87));
+	background-clip: text;
+	-webkit-background-clip: text;
+	-webkit-text-fill-color: transparent;
+	text-align: center;
+	font-size: 2em;
+	margin-left: 2%;
+	left: 0%;
+}
+
+.main-text-div {
+	overflow-y: auto;
+	position: absolute;
+	left: 10%;
+	right: 10%;
+	top: 2%;
+	bottom: 2%;
+}
+
+/* ############# Card Container ##############  */
+.container {
+	display: block;
+	position: absolute;
+	left: 2%;
+	top: 18%;
+	transform: translate(-0%, -0%);
+	height: 59%;
+	width: 96%;
+	max-width: 96%;
+	background: #ffff;
+	overflow: hidden;
+	border-radius: 20px;
+	box-shadow: 0 0 20px 8px #d0d0d0;
+}
+
+.container-small {
+	position: fixed;
+	left: 68%;
+	top: 79%;
+	transform: translate(-0%, -0%);
+	height: 7%;
+	width: 30%;
+	background: #ffff;
+	overflow: hidden;
+	border-radius: 20px;
+	box-shadow: 0 0 20px 8px #d0d0d0;
+}
+
+/* ############# Icon Button ##############  */
+
+.round-button {
+	box-sizing: border-box;
+	display: block;
+	width: 3em;
+	/* 80px */
+	height: 3em;
+	left: 0%;
+	padding-top: 14px;
+	padding-left: 0px;
+	line-height: 0px;
+	border: 6px solid #fff;
+	border-radius: 50%;
+	color: #f5f5f5;
+	text-align: center;
+	text-decoration: none;
+	background-color: #467387;
+	font-size: 20px;
+	font-weight: bold;
+	transition: all 0.3s ease;
+}
+
+.round-button:hover {
+	background-color: rgba(0, 0, 0, 0.8);
+	box-shadow: 0px 0px 10px #61a4d4;
+	text-shadow: 0px 0px 10px #61a4d4;
+}
+
+.icon-text {
+	font-size: 1em !important;
+	text-align: center;
+}
+
+.round-button-mic {
+	box-sizing: border-box;
+	display: block;
+	width: 4.5em;
+	/* 80px */
+	height: 4.5em;
+	padding-top: 14px;
+	padding-left: -2.25em;
+	line-height: 0px;
+	border: 6px solid #fff;
+	border-radius: 50%;
+	color: #f5f5f5;
+	text-align: center;
+	text-decoration: none;
+	background-color: #49d67d;
+	/*#467387;*/
+	font-size: 20px;
+	font-weight: bold;
+	transition: all 0.3s ease;
+}
+
+.round-button-mic:hover {
+	background-color: #477c5b;
+	/*rgba(0,0,0,0.8);*/
+	box-shadow: 0px 0px 10px #61a4d4;
+	text-shadow: 0px 0px 10px #61a4d4;
+}
+
+.icon-text-mic {
+	font-size: 2.5em !important;
+}
+
+.icon-text-home {
+	font-size: 3.5em !important;
+}
+
+.mic-button-div {
+	position: fixed;
+	left: 50%;
+	top: 80%
+}
+
+/*############### Drop-down ############# */
+.dropbtn {
+	background-color: #ffffff;
+	color: rgb(50, 71, 165);
+	padding: 0px;
+	font-size: 16px;
+	border: none;
+}
+
+.dropdown {
+	position: relative;
+	display: inline-block;
+}
+
+.dropdown-content {
+	display: none;
+	position: absolute;
+	background-color: #ffffff;
+	min-width: 160px;
+	box-shadow: 0px 8px 16px 0px rgba(0, 0, 0, 0.2);
+	z-index: 1;
+}
+
+.dropdown-content a {
+	color: black;
+	padding: 12px 16px;
+	text-decoration: none;
+	display: block;
+}
+
+.dropdown-content a:hover {
+	background-color: #ddd;
+}
+
+.dropdown:hover .dropdown-content {
+	display: block;
+}
+
+.dropdown:hover .dropbtn {
+	background-color: #3e8e41;
+}
+
+/* ############# Arrow ############## position: relative;  position: absolute;*/
+.load-more {
+	position: fixed;
+	cursor: pointer;
+	width: 100px;
+	height: 100px;
+	margin: -0px 0 0 -0px;
+	min-width: 10px;
+	min-height: 10px;
+
+	left: 90%;
+	top: 45%;
+	border-width: 2px;
+	border-style: solid;
+	border-color: transparent;
+	border-bottom-color: #000;
+	border-right-color: #000;
+	border-radius: 0 0 5px 0;
+
+	transform: translate(-0%, -0%) rotate(-45deg);
+}
+
+/* ######## Radio Buttons ############## */
+.radio {
+	background: #f6f7fd;
+	padding: 4px;
+	border-radius: 3px;
+	box-shadow: inset 0 0 0 3px rgba(35, 33, 45, 0.3),
+		0 0 0 3px rgba(185, 185, 185, 0.3);
+	position: relative;
+}
+
+.radio input {
+	width: max-content;
+	height: 100%;
+	appearance: none;
+	outline: none;
+	cursor: pointer;
+	border-radius: 2px;
+	padding: 4px 8px;
+	background: #454857;
+	color: #bdbdbdbd;
+	font-size: 0.8em;
+	font-family: system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto,
+		"Helvetica Neue", Arial, "Noto Sans", sans-serif, "Apple Color Emoji",
+		"Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji";
+	transition: all 100ms linear;
+}
+
+.radio input:checked {
+	background-image: linear-gradient(180deg, #4e70ce, #5197d8);
+	color: #fff;
+	box-shadow: 0 1px 1px #0000002e;
+	text-shadow: 0 1px 0px #79485f7a;
+}
+
+.radio input:before {
+	content: attr(label);
+	display: inline-block;
+	text-align: center;
+	width: 100%;
+}
+
+/* ############ Links and credits ####*/
+
+.link-icon-div {
+	position: fixed;
+	left: 90.0%;
+	top: 0.0%;
+	vertical-align: middle;
+	align-content: flex-start;
+}
+
+.credits-icon-div {
+	position: fixed;
+	left: 90.5%;
+	top: 95%;
+	font-size: x-small;
+}
+
+.svg-icon {
+	padding-top: 1em;
+	width: 50px;
+	height: 50px;
+
+}
+
+
+/* ######## Switch ############## */
+@media only screen and (max-width: 1200px) {
+	.round-button {
+		box-sizing: border-box;
+		display: block;
+		width: 2em;
+		/* 80px */
+		height: 2em;
+		left: -2.5%;
+		padding-top: 0.3em;
+		padding-left: 0px;
+		line-height: 0px;
+		border: 6px solid #fff;
+		border-radius: 50%;
+		color: #f5f5f5;
+		text-align: center;
+		text-decoration: none;
+		background-color: #467387;
+		font-size: 1em;
+		font-weight: bold;
+		transition: all 0.3s ease;
+	}
+
+	.container {
+		display: block;
+		position: absolute;
+		left: 2%;
+		top: 22%;
+		transform: translate(-0%, -0%);
+		height: 55%;
+		width: 96%;
+		max-width: 96%;
+		background: #ffff;
+		overflow: hidden;
+		border-radius: 20px;
+		box-shadow: 0 0 20px 8px #d0d0d0;
+	}
+
+	.icon-text {
+		font-size: 0.8em !important;
+		text-align: center;
+	}
+
+	.ipa-text-small {
+		font-size: small;
+	}
+
+	.round-button-mic {
+		box-sizing: border-box;
+		display: block;
+		width: 3.5em;
+		/* 80px */
+		height: 3.5em;
+		padding-top: 0.4em;
+		left: 40%;
+		line-height: 0px;
+		border: 6px solid #fff;
+		border-radius: 50%;
+		color: #f5f5f5;
+		text-align: center;
+		text-decoration: none;
+		background-color: #49d67d;
+		font-size: 20px;
+		font-weight: bold;
+		transition: all 0.3s ease;
+	}
+
+	.mic-button-div {
+		position: fixed;
+		left: 40%;
+		top: 80%
+	}
+
+	.link-icon-div {
+		position: fixed;
+		left: 89.0%;
+		top: 0.0%;
+		vertical-align: middle;
+	}
+
+	.credits-icon-div {
+		position: fixed;
+		left: 78.5%;
+		top: 95%;
+		font-size: x-small;
+	}
+
+	.svg-icon {
+		padding-top: 1em;
+		width: 40px;
+		height: 40px;
+	}
+
+
+	.icon-text-home {
+		font-size: 2.5em !important;
+	}
+
+	.accuracy-text {
+		font-family: "Dank Mono", ui-monospace, monospace;
+		background: linear-gradient(to right,
+				rgb(54, 56, 80),
+				rgb(21, 60, 87));
+		left: -5.0%;
+		background-clip: text;
+		-webkit-background-clip: text;
+		-webkit-text-fill-color: transparent;
+		text-align: center;
+		font-size: 0.8em;
+	}
+
+}

static/javascript/callbacks.js

@@ -0,0 +1,584 @@
+
+
+// Audio context initialization
+let mediaRecorder, audioChunks, audioBlob, stream, audioRecorded;
+const ctx = new AudioContext();
+let currentAudioForPlaying;
+let lettersOfWordAreCorrect = [];
+
+// UI-related variables
+const page_title = "AI Pronunciation Trainer";
+const accuracy_colors = ["green", "orange", "red"];
+let badScoreThreshold = 30;
+let mediumScoreThreshold = 70;
+let currentSample = 0;
+let currentScore = 0.;
+let sample_difficult = 0;
+let scoreMultiplier = 1;
+let playAnswerSounds = true;
+let isNativeSelectedForPlayback = true;
+let isRecording = false;
+let serverIsInitialized = false;
+let serverWorking = true;
+let languageFound = true;
+let currentSoundRecorded = false;
+let currentText, currentIpa, real_transcripts_ipa, matched_transcripts_ipa;
+let wordCategories;
+let startTime, endTime;
+
+// API related variables 
+let AILanguage = "de"; // Standard is German
+
+
+let STScoreAPIKey = 'rll5QsTiv83nti99BW6uCmvs9BDVxSB39SVFceYb'; // Public Key. If, for some reason, you would like a private one, send-me a message and we can discuss some possibilities
+let apiMainPathSample = '';// 'http://127.0.0.1:3001';// 'https://a3hj0l2j2m.execute-api.eu-central-1.amazonaws.com/Prod';
+let apiMainPathSTS = '';// 'https://wrg7ayuv7i.execute-api.eu-central-1.amazonaws.com/Prod';
+
+
+// Variables to playback accuracy sounds
+let soundsPath = '../static';//'https://stscore-sounds-bucket.s3.eu-central-1.amazonaws.com';
+let soundFileGood = null;
+let soundFileOkay = null;
+let soundFileBad = null;
+
+// Speech generation
+var synth = window.speechSynthesis;
+let voice_idx = 0;
+let voice_synth = null;
+
+//############################ UI general control functions ###################
+const unblockUI = () => {
+    document.getElementById("recordAudio").classList.remove('disabled');
+    document.getElementById("playSampleAudio").classList.remove('disabled');
+    document.getElementById("buttonNext").onclick = () => getNextSample();
+    document.getElementById("nextButtonDiv").classList.remove('disabled');
+    document.getElementById("original_script").classList.remove('disabled');
+    document.getElementById("buttonNext").style["background-color"] = '#58636d';
+
+    if (currentSoundRecorded)
+        document.getElementById("playRecordedAudio").classList.remove('disabled');
+
+
+};
+
+const blockUI = () => {
+
+    document.getElementById("recordAudio").classList.add('disabled');
+    document.getElementById("playSampleAudio").classList.add('disabled');
+    document.getElementById("buttonNext").onclick = null;
+    document.getElementById("original_script").classList.add('disabled');
+    document.getElementById("playRecordedAudio").classList.add('disabled');
+
+    document.getElementById("buttonNext").style["background-color"] = '#adadad';
+
+
+};
+
+const UIError = () => {
+    blockUI();
+    document.getElementById("buttonNext").onclick = () => getNextSample(); //If error, user can only try to get a new sample
+    document.getElementById("buttonNext").style["background-color"] = '#58636d';
+
+    document.getElementById("recorded_ipa_script").innerHTML = "";
+    document.getElementById("single_word_ipa_pair").innerHTML = "Error";
+    document.getElementById("ipa_script").innerHTML = "Error"
+
+    document.getElementById("main_title").innerHTML = 'Server Error';
+    document.getElementById("original_script").innerHTML = 'Server error. Either the daily quota of the server is over or there was some internal error. You can try to generate a new sample in a few seconds. If the error persist, try comming back tomorrow or download the local version from Github :)';
+};
+
+const UINotSupported = () => {
+    unblockUI();
+
+    document.getElementById("main_title").innerHTML = "Browser unsupported";
+
+}
+
+const UIRecordingError = () => {
+    unblockUI();
+    document.getElementById("main_title").innerHTML = "Recording error, please try again or restart page.";
+    startMediaDevice();
+}
+
+
+
+//################### Application state functions #######################
+function updateScore(currentPronunciationScore) {
+
+    if (isNaN(currentPronunciationScore))
+        return;
+    currentScore += currentPronunciationScore * scoreMultiplier;
+    currentScore = Math.round(currentScore);
+}
+
+const cacheSoundFiles = async () => {
+    await fetch(soundsPath + '/ASR_good.wav').then(data => data.arrayBuffer()).
+        then(arrayBuffer => ctx.decodeAudioData(arrayBuffer)).
+        then(decodeAudioData => {
+            soundFileGood = decodeAudioData;
+        });
+
+    await fetch(soundsPath + '/ASR_okay.wav').then(data => data.arrayBuffer()).
+        then(arrayBuffer => ctx.decodeAudioData(arrayBuffer)).
+        then(decodeAudioData => {
+            soundFileOkay = decodeAudioData;
+        });
+
+    await fetch(soundsPath + '/ASR_bad.wav').then(data => data.arrayBuffer()).
+        then(arrayBuffer => ctx.decodeAudioData(arrayBuffer)).
+        then(decodeAudioData => {
+            soundFileBad = decodeAudioData;
+        });
+}
+
+const getNextSample = async () => {
+
+
+
+    blockUI();
+
+    if (!serverIsInitialized)
+        await initializeServer();
+
+    if (!serverWorking) {
+        UIError();
+        return;
+    }
+
+    if (soundFileBad == null)
+        cacheSoundFiles();
+
+
+
+    updateScore(parseFloat(document.getElementById("pronunciation_accuracy").innerHTML));
+
+    document.getElementById("main_title").innerHTML = "Processing new sample...";
+
+
+    if (document.getElementById('lengthCat1').checked) {
+        sample_difficult = 0;
+        scoreMultiplier = 1.3;
+    }
+    else if (document.getElementById('lengthCat2').checked) {
+        sample_difficult = 1;
+        scoreMultiplier = 1;
+    }
+    else if (document.getElementById('lengthCat3').checked) {
+        sample_difficult = 2;
+        scoreMultiplier = 1.3;
+    }
+    else if (document.getElementById('lengthCat4').checked) {
+        sample_difficult = 3;
+        scoreMultiplier = 1.6;
+    }
+
+    try {
+        await fetch(apiMainPathSample + '/getSample', {
+            method: "post",
+            body: JSON.stringify({
+                "category": sample_difficult.toString(), "language": AILanguage
+            }),
+            headers: { "X-Api-Key": STScoreAPIKey }
+        }).then(res => res.json()).
+            then(data => {
+
+
+
+                let doc = document.getElementById("original_script");
+                currentText = data.real_transcript;
+                doc.innerHTML = currentText;
+
+                currentIpa = data.ipa_transcript
+
+                let doc_ipa = document.getElementById("ipa_script");
+                doc_ipa.innerHTML = "/ " + currentIpa + " /";
+
+                document.getElementById("recorded_ipa_script").innerHTML = ""
+                document.getElementById("pronunciation_accuracy").innerHTML = "";
+                document.getElementById("single_word_ipa_pair").innerHTML = "Reference | Spoken"
+                document.getElementById("section_accuracy").innerHTML = "| Score: " + currentScore.toString() + " - (" + currentSample.toString() + ")";
+                currentSample += 1;
+
+                document.getElementById("main_title").innerHTML = page_title;
+
+                document.getElementById("translated_script").innerHTML = data.transcript_translation;
+
+                currentSoundRecorded = false;
+                unblockUI();
+                document.getElementById("playRecordedAudio").classList.add('disabled');
+
+            })
+    }
+    catch
+    {
+        UIError();
+    }
+
+
+};
+
+const updateRecordingState = async () => {
+    if (isRecording) {
+        stopRecording();
+        return
+    }
+    else {
+        recordSample()
+        return;
+    }
+}
+
+const generateWordModal = (word_idx) => {
+
+    document.getElementById("single_word_ipa_pair").innerHTML = wrapWordForPlayingLink(real_transcripts_ipa[word_idx], word_idx, false, "black")
+        + ' | ' + wrapWordForPlayingLink(matched_transcripts_ipa[word_idx], word_idx, true, accuracy_colors[parseInt(wordCategories[word_idx])])
+}
+
+const recordSample = async () => {
+
+    document.getElementById("main_title").innerHTML = "Recording... click again when done speaking";
+    document.getElementById("recordIcon").innerHTML = 'pause_presentation';
+    blockUI();
+    document.getElementById("recordAudio").classList.remove('disabled');
+    audioChunks = [];
+    isRecording = true;
+    mediaRecorder.start();
+
+}
+
+const changeLanguage = (language, generateNewSample = false) => {
+    voices = synth.getVoices();
+    AILanguage = language;
+    languageFound = false;
+    let languageIdentifier, languageName;
+    switch (language) {
+        case 'de':
+
+            document.getElementById("languageBox").innerHTML = "German";
+            languageIdentifier = 'de';
+            languageName = 'Anna';
+            break;
+
+        case 'en':
+
+            document.getElementById("languageBox").innerHTML = "English";
+            languageIdentifier = 'en';
+            languageName = 'Daniel';
+            break;
+    };
+
+    for (idx = 0; idx < voices.length; idx++) {
+        if (voices[idx].lang.slice(0, 2) == languageIdentifier && voices[idx].name == languageName) {
+            voice_synth = voices[idx];
+            languageFound = true;
+            break;
+        }
+
+    }
+    // If specific voice not found, search anything with the same language 
+    if (!languageFound) {
+        for (idx = 0; idx < voices.length; idx++) {
+            if (voices[idx].lang.slice(0, 2) == languageIdentifier) {
+                voice_synth = voices[idx];
+                languageFound = true;
+                break;
+            }
+        }
+    }
+    if (generateNewSample)
+        getNextSample();
+}
+
+//################### Speech-To-Score function ########################
+const mediaStreamConstraints = {
+    audio: {
+        channelCount: 1,
+        sampleRate: 48000
+    }
+}
+
+
+const startMediaDevice = () => {
+    navigator.mediaDevices.getUserMedia(mediaStreamConstraints).then(_stream => {
+        stream = _stream
+        mediaRecorder = new MediaRecorder(stream);
+
+        let currentSamples = 0
+        mediaRecorder.ondataavailable = event => {
+
+            currentSamples += event.data.length
+            audioChunks.push(event.data);
+        };
+
+        mediaRecorder.onstop = async () => {
+
+
+            document.getElementById("recordIcon").innerHTML = 'mic';
+            blockUI();
+
+
+            audioBlob = new Blob(audioChunks, { type: 'audio/ogg;' });
+
+            let audioUrl = URL.createObjectURL(audioBlob);
+            audioRecorded = new Audio(audioUrl);
+
+            let audioBase64 = await convertBlobToBase64(audioBlob);
+
+            let minimumAllowedLength = 6;
+            if (audioBase64.length < minimumAllowedLength) {
+                setTimeout(UIRecordingError, 50); // Make sure this function finished after get called again
+                return;
+            }
+
+            try {
+                await fetch(apiMainPathSTS + '/GetAccuracyFromRecordedAudio', {
+                    method: "post",
+                    body: JSON.stringify({ "title": currentText[0], "base64Audio": audioBase64, "language": AILanguage }),
+                    headers: { "X-Api-Key": STScoreAPIKey }
+
+                }).then(res => res.json()).
+                    then(data => {
+
+                        if (playAnswerSounds)
+                            playSoundForAnswerAccuracy(parseFloat(data.pronunciation_accuracy))
+
+                        document.getElementById("recorded_ipa_script").innerHTML = "/ " + data.ipa_transcript + " /";
+                        document.getElementById("recordAudio").classList.add('disabled');
+                        document.getElementById("main_title").innerHTML = page_title;
+                        document.getElementById("pronunciation_accuracy").innerHTML = data.pronunciation_accuracy + "%";
+
+                        lettersOfWordAreCorrect = data.is_letter_correct_all_words.split(" ")
+
+
+                        startTime = data.start_time;
+                        endTime = data.end_time;
+
+
+                        real_transcripts_ipa = data.real_transcripts_ipa.split(" ")
+                        matched_transcripts_ipa = data.matched_transcripts_ipa.split(" ")
+                        wordCategories = data.pair_accuracy_category.split(" ")
+                        let currentTextWords = currentText[0].split(" ")
+
+                        coloredWords = "";
+                        for (let word_idx = 0; word_idx < currentTextWords.length; word_idx++) {
+
+                            wordTemp = '';
+                            for (let letter_idx = 0; letter_idx < currentTextWords[word_idx].length; letter_idx++) {
+                                letter_is_correct = lettersOfWordAreCorrect[word_idx][letter_idx] == '1'
+                                if (letter_is_correct)
+                                    color_letter = 'green'
+                                else
+                                    color_letter = 'red'
+
+                                wordTemp += '<font color=' + color_letter + '>' + currentTextWords[word_idx][letter_idx] + "</font>"
+                            }
+                            currentTextWords[word_idx]
+                            coloredWords += " " + wrapWordForIndividualPlayback(wordTemp, word_idx)
+                        }
+
+
+
+                        document.getElementById("original_script").innerHTML = coloredWords
+
+                        currentSoundRecorded = true;
+                        unblockUI();
+                        document.getElementById("playRecordedAudio").classList.remove('disabled');
+
+                    });
+            }
+            catch {
+                UIError();
+            }
+        };
+
+    });
+};
+startMediaDevice();
+
+// ################### Audio playback ##################
+const playSoundForAnswerAccuracy = async (accuracy) => {
+
+    currentAudioForPlaying = soundFileGood;
+    if (accuracy < mediumScoreThreshold) {
+        if (accuracy < badScoreThreshold) {
+            currentAudioForPlaying = soundFileBad;
+        }
+        else {
+            currentAudioForPlaying = soundFileOkay;
+        }
+    }
+    playback();
+
+}
+
+const playAudio = async () => {
+
+    document.getElementById("main_title").innerHTML = "Generating sound...";
+    playWithMozillaApi(currentText[0]);
+    document.getElementById("main_title").innerHTML = "Current Sound was played";
+
+};
+
+function playback() {
+    const playSound = ctx.createBufferSource();
+    playSound.buffer = currentAudioForPlaying;
+    playSound.connect(ctx.destination);
+    playSound.start(ctx.currentTime)
+}
+
+
+const playRecording = async (start = null, end = null) => {
+    blockUI();
+
+    try {
+        if (start == null || end == null) {
+            endTimeInMs = Math.round(audioRecorded.duration * 1000)
+            audioRecorded.addEventListener("ended", function () {
+                audioRecorded.currentTime = 0;
+                unblockUI();
+                document.getElementById("main_title").innerHTML = "Recorded Sound was played";
+            });
+            await audioRecorded.play();
+
+        }
+        else {
+            audioRecorded.currentTime = start;
+            audioRecorded.play();
+            durationInSeconds = end - start;
+            endTimeInMs = Math.round(durationInSeconds * 1000);
+            setTimeout(function () {
+                unblockUI();
+                audioRecorded.pause();
+                audioRecorded.currentTime = 0;
+                document.getElementById("main_title").innerHTML = "Recorded Sound was played";
+            }, endTimeInMs);
+
+        }
+    }
+    catch {
+        UINotSupported();
+    }
+};
+
+const playNativeAndRecordedWord = async (word_idx) => {
+
+    if (isNativeSelectedForPlayback)
+        playCurrentWord(word_idx)
+    else
+        playRecordedWord(word_idx);
+
+    isNativeSelectedForPlayback = !isNativeSelectedForPlayback;
+}
+
+const stopRecording = () => {
+    isRecording = false
+    mediaRecorder.stop()
+    document.getElementById("main_title").innerHTML = "Processing audio...";
+}
+
+
+const playCurrentWord = async (word_idx) => {
+
+    document.getElementById("main_title").innerHTML = "Generating word...";
+    playWithMozillaApi(currentText[0].split(' ')[word_idx]);
+    document.getElementById("main_title").innerHTML = "Word was played";
+}
+
+// TODO: Check if fallback is correct
+const playWithMozillaApi = (text) => {
+
+    if (languageFound) {
+        blockUI();
+        if (voice_synth == null)
+            changeLanguage(AILanguage);
+
+        var utterThis = new SpeechSynthesisUtterance(text);
+        utterThis.voice = voice_synth;
+        utterThis.rate = 0.7;
+        utterThis.onend = function (event) {
+            unblockUI();
+        }
+        synth.speak(utterThis);
+    }
+    else {
+        UINotSupported();
+    }
+}
+
+const playRecordedWord = (word_idx) => {
+
+    wordStartTime = parseFloat(startTime.split(' ')[word_idx]);
+    wordEndTime = parseFloat(endTime.split(' ')[word_idx]);
+
+    playRecording(wordStartTime, wordEndTime);
+
+}
+
+// ############# Utils #####################
+const convertBlobToBase64 = async (blob) => {
+    return await blobToBase64(blob);
+}
+
+const blobToBase64 = blob => new Promise((resolve, reject) => {
+    const reader = new FileReader();
+    reader.readAsDataURL(blob);
+    reader.onload = () => resolve(reader.result);
+    reader.onerror = error => reject(error);
+});
+
+const wrapWordForPlayingLink = (word, word_idx, isFromRecording, word_accuracy_color) => {
+    if (isFromRecording)
+        return '<a style = " white-space:nowrap; color:' + word_accuracy_color + '; " href="javascript:playRecordedWord(' + word_idx.toString() + ')"  >' + word + '</a> '
+    else
+        return '<a style = " white-space:nowrap; color:' + word_accuracy_color + '; " href="javascript:playCurrentWord(' + word_idx.toString() + ')" >' + word + '</a> '
+}
+
+const wrapWordForIndividualPlayback = (word, word_idx) => {
+
+
+    return '<a onmouseover="generateWordModal(' + word_idx.toString() + ')" style = " white-space:nowrap; " href="javascript:playNativeAndRecordedWord(' + word_idx.toString() + ')"  >' + word + '</a> '
+
+}
+
+// ########## Function to initialize server ###############
+// This is to try to avoid aws lambda cold start 
+try {
+    fetch(apiMainPathSTS + '/GetAccuracyFromRecordedAudio', {
+        method: "post",
+        body: JSON.stringify({ "title": '', "base64Audio": '', "language": AILanguage }),
+        headers: { "X-Api-Key": STScoreAPIKey }
+
+    });
+}
+catch { }
+
+const initializeServer = async () => {
+
+    valid_response = false;
+    document.getElementById("main_title").innerHTML = 'Initializing server, this may take up to 2 minutes...';
+    let number_of_tries = 0;
+    let maximum_number_of_tries = 4;
+
+    while (!valid_response) {
+        if (number_of_tries > maximum_number_of_tries) {
+            serverWorking = false;
+            break;
+        }
+
+        try {
+            await fetch(apiMainPathSTS + '/GetAccuracyFromRecordedAudio', {
+                method: "post",
+                body: JSON.stringify({ "title": '', "base64Audio": '', "language": AILanguage }),
+                headers: { "X-Api-Key": STScoreAPIKey }
+
+            }).then(
+                valid_response = true);
+            serverIsInitialized = true;
+        }
+        catch
+        {
+            number_of_tries += 1;
+        }
+    }
+}
+

templates/main.html

@@ -0,0 +1,181 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+
+    <link href="https://cdn.jsdelivr.net/npm/bootstrap@5.1.3/dist/css/bootstrap.min.css" rel="stylesheet"
+        integrity="sha384-1BmE4kWBq78iYhFldvKuhfTAU6auU8tT94WrHftjDbrCEXSU1oBoqyl2QvZ6jIW3" crossorigin="anonymous">
+
+    </script>
+    <link rel="stylesheet" href="../static/css/style-new.css">
+    <script src="../static/javascript/callbacks.js"></script>
+
+
+    <title>AI pronunciation trainer</title>
+
+    <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.7.1/jquery.min.js" type="text/javascript"></script>
+
+    <link href="https://fonts.googleapis.com/icon?family=Material+Icons" rel="stylesheet">
+
+
+
+</head>
+
+<body style="height: 100%; width: 100%; background-color: white; max-width: 90%;">
+
+    <div>
+
+        <div style="display:flex; flex-direction: row;">
+
+
+            <div style="display:inline-block; margin-left: 1.5em;">
+                <i class="material-icons icon-text-home" style="text-align: right;" onclick="history.go(0)">home</i>
+            </div>
+
+            <h1 id='main_title'> AI Pronunciation Trainer
+            </h1>
+        </div>
+
+
+        <div class="expanded">
+            <div class="horizontal-flexbox" style="display:flex; flex-direction: row;">
+
+
+                <p class="accuracy-text" style="font-size:1em; text-align: left; padding-top: 3px; padding-left: 5px;">
+                    Language: </p>
+                <!--dropbtn accuracy-text-->
+                <div class="dropdown">
+                    <button id="languageBox" class="dropbtn accuracy-text"
+                        style="font-size:1em; text-align: left; padding-top: 3px; padding-left: 0px;">German</button>
+                    <div class="dropdown-content">
+                        <a href="javascript:changeLanguage('de',true)" class="accuracy-text"
+                            style="padding-top: 3px; ">German</a>
+                        <a href="javascript:changeLanguage('en',true)" class="accuracy-text ;"
+                            style="padding-top: 3px; ">English</a>
+                    </div>
+                </div>
+
+                <p id="section_accuracy" class="accuracy-text"
+                    style="text-align: left; color: black; font-size: larger;">
+                    | Score: 0
+                </p>
+
+            </div>
+        </div>
+
+        <div style="margin-bottom: 200px;">
+
+        </div>
+
+
+        <div class="container">
+
+            <div class="horizontal-flexbox" style="position: absolute; top: 2%; ">
+
+                <a id="playSampleAudio" href="javascript:playAudio()" class="round-button disabled" style="color:white; text-align:center;
+                    position: absolute; top: 2%; "><i class="material-icons icon-text">play_arrow</i>
+                </a>
+
+                <a id="playRecordedAudio" href="javascript:playRecording()" class="round-button disabled"
+                    style="color:white; text-align:center; position: absolute; top: 15%; "><i
+                        class="material-icons icon-text">record_voice_over</i>
+                </a>
+                <p id="pronunciation_accuracy" class="expanded accuracy-text"
+                    style="text-align: center; color: black; position: absolute;  top: 27%; ">
+                    -
+                </p>
+
+            </div>
+
+            <div id="text-area" class="main-text-div">
+
+                <p id="original_script" class=" bigger-text text-primary main-text">Click on the bar on the
+                    right
+                    to
+                    generate a
+                    new sentence (please use chrome web browser).
+                </p>
+                <p id="ipa_script" class="text-muted bigger-text ipa-text"> Before speaking, click on the mic button
+                    below to start recording and then click again when you're done.
+                </p>
+                <p id="recorded_ipa_script" class="text-primary ipa-text">On the left bottom you can choose the
+                    difficult. On the upper left you can choose the language.
+                </p>
+                <p id="translated_script" class="text-muted medium-text ipa-text"> The corresponding IPA reading of each
+                    sentence will also be displayed. If you never heard from IPA, you can check out this
+                    <a href="https://www.youtube.com/watch?v=mzrLZi6fipA&list=RDCMUCQAUWk_yGz7bk1181DrijNw&start_radio=1&rv=mzrLZi6fipA&t=22&ab_channel=FluentForever"
+                        target=”_blank”>playlist</a>. Try to get at least 690 points a day. Don't be shy! You can do it
+                    :)
+                </p>
+
+            </div>
+
+            <div id="nextButtonDiv" style="position: absolute; left: 90%; top:0%; height: 100%;" class="flex-container">
+                <button id="buttonNext" class="expanded button-next" onclick="javascript:getNextSample()">
+                    <span></span></a>
+            </div>
+        </div>
+
+
+
+
+        <div class="container-small flex expand"
+            style="align-items: center; text-align: center; vertical-align:middle; ">
+            <p id="single_word_ipa_pair" class="expand ipa-text-small"
+                style="text-align: center; vertical-align: middle;">Reference | Spoken
+            </p>
+        </div>
+
+
+        <div id="btn-record" class="expanded mic-button-div">
+            <a id="recordAudio" href="javascript:updateRecordingState()" class="round-button-mic disabled"
+                style="color:white; text-align:center; "><i id="recordIcon" class="material-icons icon-text-mic">mic</i>
+            </a>
+        </div>
+
+
+        <div id="radio-difficulty" class="radio" style="position: fixed; top: 95%; left: 2%;">
+            <input label="Random" type="radio" id="lengthCat1" name='length' onclick="javascript:getNextSample()">
+            <input label="Easy" type="radio" id="lengthCat2" name='length' checked onclick="javascript:getNextSample()">
+            <input label="Medium" type="radio" id="lengthCat3" name='length' onclick="javascript:getNextSample()">
+            <input label="Hard" type="radio" id="lengthCat4" name='length' onclick="javascript:getNextSample()">
+        </div>
+
+    </div>
+
+
+    <p class="credits-icon-div">By Thiago
+        Lobato.</p>
+
+    <div class="link-icon-div">
+        <a href="https://github.com/Thiagohgl/ai-pronunciation-trainer" target=”_blank”
+            style="text-decoration:none; vertical-align: middle; ">
+            <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" class="svg-icon">
+                <path
+                    d="M12 0c-6.626 0-12 5.373-12 12 0 5.302 3.438 9.8 8.207 11.387.599.111.793-.261.793-.577v-2.234c-3.338.726-4.033-1.416-4.033-1.416-.546-1.387-1.333-1.756-1.333-1.756-1.089-.745.083-.729.083-.729 1.205.084 1.839 1.237 1.839 1.237 1.07 1.834 2.807 1.304 3.492.997.107-.775.418-1.305.762-1.604-2.665-.305-5.467-1.334-5.467-5.931 0-1.311.469-2.381 1.236-3.221-.124-.303-.535-1.524.117-3.176 0 0 1.008-.322 3.301 1.23.957-.266 1.983-.399 3.003-.404 1.02.005 2.047.138 3.006.404 2.291-1.552 3.297-1.23 3.297-1.23.653 1.653.242 2.874.118 3.176.77.84 1.235 1.911 1.235 3.221 0 4.609-2.807 5.624-5.479 5.921.43.372.823 1.102.823 2.222v3.293c0 .319.192.694.801.576 4.765-1.589 8.199-6.086 8.199-11.386 0-6.627-5.373-12-12-12z" />
+            </svg>
+        </a>
+
+        <a href="https://www.linkedin.com/in/thiagohgl/" target=”_blank”
+            style="text-decoration:none; vertical-align: middle; padding-top: 2.3em; ">
+            <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" class="svg-icon">
+                <path
+                    d="M19 0h-14c-2.761 0-5 2.239-5 5v14c0 2.761 2.239 5 5 5h14c2.762 0 5-2.239 5-5v-14c0-2.761-2.238-5-5-5zm-11 19h-3v-11h3v11zm-1.5-12.268c-.966 0-1.75-.79-1.75-1.764s.784-1.764 1.75-1.764 1.75.79 1.75 1.764-.783 1.764-1.75 1.764zm13.5 12.268h-3v-5.604c0-3.368-4-3.113-4 0v5.604h-3v-11h3v1.765c1.396-2.586 7-2.777 7 2.476v6.759z" />
+            </svg>
+
+        </a>
+    </div>
+
+
+
+
+
+
+
+</body>
+
+
+
+</html>

unitTests.py

@@ -0,0 +1,99 @@
+import unittest
+
+import ModelInterfaces
+import lambdaGetSample
+import RuleBasedModels
+import epitran
+import json
+import pronunciationTrainer
+
+
+def test_category(category: int, threshold_min: int, threshold_max: int):
+    event = {'body': json.dumps({'category': category, 'language': 'de'})}
+    for _ in range(1000):
+        response = lambdaGetSample.lambda_handler(event, [])
+        response_dict = json.loads(response)
+        number_of_words = len(
+            response_dict['real_transcript'][0].split())
+        length_valid = number_of_words > threshold_min and number_of_words <= threshold_max
+        if not length_valid:
+            print('Category ', category,
+                  ' had a sentence with length ', number_of_words)
+            return False
+    return True
+
+
+class TestDataset(unittest.TestCase):
+
+    def test_random_sentences(self):
+
+        self.assertFalse(test_category(0, 0, 8))
+
+    def test_easy_sentences(self):
+
+        self.assertTrue(test_category(1, 0, 8))
+
+    def test_normal_sentences(self):
+        self.assertTrue(test_category(2, 8, 20))
+
+    def test_hard_sentences(self):
+        self.assertTrue(test_category(3, 20, 10000))
+
+
+def check_phonem_converter(converter: ModelInterfaces.ITextToPhonemModel, input: str, expected_output: str):
+    output = converter.convertToPhonem(input)
+
+    is_correct = output == expected_output
+    if not is_correct:
+        print('Conversion from "', input, '" should be "',
+              expected_output, '", but was "', output, '"')
+    return is_correct
+
+
+class TestPhonemConverter(unittest.TestCase):
+
+    def test_english(self):
+        phonem_converter = RuleBasedModels.EngPhonemConverter()
+        self.assertTrue(check_phonem_converter(
+            phonem_converter, 'Hello, this is a test', 'hɛˈloʊ, ðɪs ɪz ə tɛst'))
+
+    def test_german(self):
+        phonem_converter = RuleBasedModels.EpitranPhonemConverter(
+            epitran.Epitran('deu-Latn'))
+
+        self.assertTrue(check_phonem_converter(
+            phonem_converter, 'Hallo, das ist ein Test', 'haloː, dɑːs ɪst ain tɛst'))
+
+
+trainer_SST_lambda = {}
+trainer_SST_lambda['de'] = pronunciationTrainer.getTrainer("de")
+
+
+class TestScore(unittest.TestCase):
+
+    def test_exact_transcription(self):
+        words_real = 'Ich habe sehr viel glück, am leben und gesund zu sein'
+
+        real_and_transcribed_words, _, _ = trainer_SST_lambda['de'].matchSampleAndRecordedWords(
+            words_real, words_real)
+
+        pronunciation_accuracy, _ = trainer_SST_lambda['de'].getPronunciationAccuracy(
+            real_and_transcribed_words)
+
+        self.assertTrue(int(pronunciation_accuracy) == 100)
+
+    def test_incorrect_transcription(self):
+        words_real = 'Ich habe sehr viel glück, am leben und gesund zu sein'
+        words_transcribed = 'Ic hab zeh viel guck am und gesund tu sein'
+
+        real_and_transcribed_words, _, _ = trainer_SST_lambda['de'].matchSampleAndRecordedWords(
+            words_real, words_transcribed)
+
+        pronunciation_accuracy, _ = trainer_SST_lambda['de'].getPronunciationAccuracy(
+            real_and_transcribed_words)
+
+        self.assertTrue(int(pronunciation_accuracy) == 71)
+
+
+if __name__ == '__main__':
+    unittest.main()

utilsFileIO.py

@@ -0,0 +1,9 @@
+import string 
+import random 
+
+
+def generateRandomString(str_length: int = 20):
+
+    # printing lowercase
+    letters = string.ascii_lowercase
+    return ''.join(random.choice(letters) for i in range(str_length))

webApp.py

@@ -0,0 +1,47 @@
+from flask import Flask, render_template, request
+import webbrowser
+import os
+from flask_cors import CORS
+import json
+
+import lambdaTTS
+import lambdaSpeechToScore
+import lambdaGetSample
+
+app = Flask(__name__)
+cors = CORS(app)
+app.config['CORS_HEADERS'] = '*'
+
+rootPath = ''
+
+
+@app.route(rootPath+'/')
+def main():
+    return render_template('main.html')
+
+
+@app.route(rootPath+'/getAudioFromText', methods=['POST'])
+def getAudioFromText():
+    event = {'body': json.dumps(request.get_json(force=True))}
+    return lambdaTTS.lambda_handler(event, [])
+
+
+@app.route(rootPath+'/getSample', methods=['POST'])
+def getNext():
+    event = {'body':  json.dumps(request.get_json(force=True))}
+    return lambdaGetSample.lambda_handler(event, [])
+
+
+@app.route(rootPath+'/GetAccuracyFromRecordedAudio', methods=['POST'])
+def GetAccuracyFromRecordedAudio():
+
+    event = {'body': json.dumps(request.get_json(force=True))}
+    lambda_correct_output = lambdaSpeechToScore.lambda_handler(event, [])
+    return lambda_correct_output
+
+
+if __name__ == "__main__":
+    language = 'de'
+    print(os.system('pwd'))
+    webbrowser.open_new('http://127.0.0.1:3000/')
+    app.run(host="0.0.0.0", port=3000)