app

app.py

@@ -1,17 +1,7 @@
 import gradio as gr
 import util
 import tts
-
-# Functions
-def check_pronunciation(input_text, script, user_audio):
-    # Placeholder logic for pronunciation checking
-    transcript_ugArab_box = "Automatic transcription of your audio (Arabic)..."
-    transcript_ugLatn_box = "Automatic transcription of your audio (Latin)..."
-    correct_pronunciation = "Correct pronunciation in IPA"
-    user_pronunciation = "User pronunciation in IPA"
-    pronunciation_match = "Matching segments in green, mismatched in red"
-    pronunciation_score = 85.7  # Replace with actual score calculation
-    return transcript_ugArab_box, transcript_ugLatn_box, correct_pronunciation, user_pronunciation, pronunciation_match, pronunciation_score
+import asr
 
 # Front-End
 with gr.Blocks() as app:
@@ -101,13 +91,13 @@ with gr.Blocks() as app:
     )
     
     tts_btn.click(
-        tts.generate_example_pronunciation,
+        tts.generate_audio,
         inputs=[input_text, script_choice],
         outputs=[example_audio]
     )
 
     check_btn.click(
-        check_pronunciation,
+        asr.check_pronunciation,
         inputs=[input_text, script_choice, user_audio],
         outputs=[transcript_ugArab_box, transcript_ugLatn_box, correct_pronunciation_box, user_pronunciation_box, match_box, score_box]
     )

asr.py

@@ -0,0 +1,44 @@
+from transformers import Wav2Vec2Processor, Wav2Vec2ForCTC
+import torch
+from umsc import UgMultiScriptConverter
+import util 
+
+# Model ID and setup
+model_id = 'ixxan/wav2vec2-large-mms-1b-uyghur-latin'
+asr_model = Wav2Vec2ForCTC.from_pretrained(model_id, target_lang="uig-script_latin")
+asr_processor = Wav2Vec2Processor.from_pretrained(model_id)
+asr_processor.tokenizer.set_target_lang("uig-script_latin")
+
+# Automatically allocate the device
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+asr_model = asr_model.to(device) 
+
+def asr(user_audio):
+    # Load and resample user audio
+    audio_input, sampling_rate = util.load_and_resample_audio(user_audio, target_rate=16000)
+    
+    # Process audio through ASR model
+    inputs = asr_processor(audio_input.squeeze(), sampling_rate=sampling_rate, return_tensors="pt", padding=True)
+    inputs = {key: val.to(device) for key, val in inputs.items()}
+    with torch.no_grad():
+        logits = asr_model(**inputs).logits
+        predicted_ids = torch.argmax(logits, dim=-1)
+        transcript = asr_processor.batch_decode(predicted_ids)[0]
+    return transcript
+
+    
+def check_pronunciation(input_text, script, user_audio):
+    # Transcripts from user input audio
+    transcript_ugLatn_box = asr(user_audio)
+    ug_latn_to_arab = UgMultiScriptConverter('ULS', 'UAS')
+    transcript_ugArab_box = ug_latn_to_arab(transcript_ugLatn_box)
+
+    # Get IPA and Pronunciation Feedback
+    if script == 'Uyghur Latin':
+        input_text = ug_latn_to_arab(input_text) # make sure input text is arabic script to IPA conversion
+    correct_pronunciation, user_pronunciation, pronunciation_match, pronunciation_score = util.calculate_pronunciation_accuracy(
+        reference_text = input_text, 
+        output_text = transcript_ugArab_box, 
+        language_code='uig-Arab')
+    
+    return transcript_ugArab_box, transcript_ugLatn_box, correct_pronunciation, user_pronunciation, pronunciation_match, pronunciation_score

tts.py

@@ -2,20 +2,31 @@ from transformers import VitsModel, AutoTokenizer
 import torch
 from umsc import UgMultiScriptConverter
 import scipy.io.wavfile
-import os
 
-tts_tokenizer = AutoTokenizer.from_pretrained("facebook/mms-tts-uig-script_arabic")
-tts_model = VitsModel.from_pretrained("facebook/mms-tts-uig-script_arabic")
+# Model ID and setup
+model_id = "facebook/mms-tts-uig-script_arabic"
+tts_tokenizer = AutoTokenizer.from_pretrained(model_id)
+tts_model = VitsModel.from_pretrained(model_id)
 
-def generate_example_pronunciation(input_text, script):
-    # Convert text to uyghur_arabic
+# Automatically allocate the device
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+tts_model = tts_model.to(device) 
+
+def generate_audio(input_text, script):
+    """
+    Generate audio for the given input text and script
+    """
+    # Convert text to Uyghur Arabic if needed
     ug_latn_to_arab = UgMultiScriptConverter('ULS', 'UAS')
-    if not script == "Uyghur Arabic":
+    if script != "Uyghur Arabic":
         input_text = ug_latn_to_arab(input_text)
 
-    tts_inputs = tts_tokenizer(input_text, return_tensors="pt")
+    # Tokenize and move inputs to the same device as the model
+    tts_inputs = tts_tokenizer(input_text, return_tensors="pt").to(device)
+    
+    # Perform inference
     with torch.no_grad():
-        tts_output = tts_model(**tts_inputs).waveform
+        tts_output = tts_model(**tts_inputs).waveform.cpu()  # Move output back to CPU for saving
 
     # Save to a temporary file
     output_path = "tts_output.wav"

util.py

@@ -1,16 +1,21 @@
 import random
 from umsc import UgMultiScriptConverter
+import torchaudio
+import string
+import epitran
+from difflib import SequenceMatcher
 
 # Lists of Uyghur short and long texts
 short_texts = [
     "سالام", "رەھمەت", "ياخشىمۇسىز"
 ]
 long_texts = [
-    "مەكتەپكە بارغاندا تېخىمۇ بىلىملىك بولۇپ قېلىمەن.",
+    "مەكتەپكە بارغاندا تېخىمۇ بىلىملىك بولۇمەن.",
     "يېزا مەنزىرىسى ھەقىقەتەن گۈزەل.",
-    "پېقىرلارغا ياردەم قىلىش مەنەم پەرزەندە."
+    "بىزنىڭ ئۆيدەپ تۆت تەكچە تۆتىلىسى تەكتەكچە"
 ]
 
+# Front-End Utils
 def generate_short_text(script_choice):
     """Generate a random Uyghur short text based on the type."""
     ug_arab_to_latn = UgMultiScriptConverter('UAS', 'ULS')
@@ -27,4 +32,66 @@ def generate_long_text(script_choice):
     text = random.choice(long_texts)
     if script_choice == "Uyghur Latin":
         return ug_arab_to_latn(text)
-    return text
+    return text
+
+# ASR Utils
+def load_and_resample_audio(file_path, target_rate):
+    """Load audio and resample based on target sample rate"""
+    audio_input, sampling_rate = torchaudio.load(file_path)
+    if sampling_rate != target_rate:
+        resampler = torchaudio.transforms.Resample(sampling_rate, target_rate)
+        audio_input = resampler(audio_input)
+    return audio_input, target_rate
+
+def calculate_pronunciation_accuracy(reference_text, output_text, language_code='uig-Arab'):
+    """
+    Calculate pronunciation accuracy between reference and ASR output text using Epitran.
+
+    Args:
+        reference_text (str): The ground truth text in Uyghur (Arabic script).
+        output_text (str): The ASR output text in Uyghur (Arabic script).
+        language_code (str): Epitran language code (default is 'uig-Arab' for Uyghur).
+
+    Returns:
+        float: Pronunciation accuracy as a percentage.
+        str: IPA transliteration of the reference text.
+        str: IPA transliteration of the output text.
+    """
+    # Initialize Epitran for Uyghur (Arabic script)
+    ipa_converter = epitran.Epitran(language_code)
+
+    # Remove punctuation from both texts
+    reference_text_clean = remove_punctuation(reference_text)
+    output_text_clean = remove_punctuation(output_text)
+
+    # Transliterate both texts to IPA
+    reference_ipa = ipa_converter.transliterate(reference_text_clean)
+    output_ipa = ipa_converter.transliterate(output_text_clean)
+
+    # Calculate pronunciation accuracy using SequenceMatcher
+    matcher = SequenceMatcher(None, reference_ipa, output_ipa)
+    match_ratio = matcher.ratio()  # This is the fraction of matching characters
+
+    # Convert to percentage
+    pronunciation_accuracy = match_ratio * 100
+
+    # Generate HTML for comparison
+    comparison_html = ""
+    for opcode, i1, i2, j1, j2 in matcher.get_opcodes():
+        ref_segment = reference_ipa[i1:i2]
+        out_segment = output_ipa[j1:j2]
+
+        if opcode == 'equal':  # Matching characters
+            comparison_html += f'<span style="color: green">{ref_segment}</span>'
+        elif opcode == 'replace':  # Mismatched characters
+            comparison_html += f'<span style="color: red">{ref_segment}</span>'
+        elif opcode == 'delete':  # Characters in reference but not in output
+            comparison_html += f'<span style="color: red">{ref_segment}</span>'
+        elif opcode == 'insert':  # Characters in output but not in reference
+            comparison_html += f'<span style="color: red">{out_segment}</span>'
+
+    return reference_ipa, output_ipa, comparison_html, pronunciation_accuracy
+
+def remove_punctuation(text):
+  """Helper function to remove punctuation from text."""
+  return text.translate(str.maketrans('', '', string.punctuation))