feat: handle example selection and input change by resetting audio and text outputs

aip_trainer/lambdas/lambdaSpeechToScore.py

@@ -1,4 +1,3 @@
-
 import base64
 import json
 import os
@@ -176,8 +175,13 @@ def soundfile_write(audiofile: str | Path, data: np.ndarray, samplerate: int):
 
 
 def get_selected_word(idx_recorded_word: int, raw_json_output: str) -> str:
-    json_output = json.loads(raw_json_output)
-    list_audio_files = json_output["audio_files"]
+    recognition_output = json.loads(raw_json_output)
+    list_audio_files = recognition_output["audio_files"]
+    real_transcripts = recognition_output["real_transcripts"]
+    real_transcripts_list = real_transcripts.split()
+    app_logger.info(f"idx_recorded_word:{idx_recorded_word} ...")
+    current_word = real_transcripts_list[idx_recorded_word]
+    app_logger.info(f"real_transcripts, current word:{current_word} ...")
     return list_audio_files[idx_recorded_word]
 
 

app.py

@@ -12,6 +12,11 @@ css = """
 .speech-output-container {min-height: 60px;}
 .speech-output-html {text-align: left; }
 """
+word_idx_text = "Recognized word index"
+
+
+def get_textbox_hidden():
+    return gr.Textbox(visible=False)
 
 
 def clear():
@@ -118,31 +123,18 @@ with gr.Blocks(css=css, head=js.head_driver_tour) as gradio_app:
                 gr.Markdown("### Speech accuracy score (%)", elem_classes="speech-accuracy-score-container row1", elem_id="speech-accuracy-score-container-id-element")
             with gr.Row():
                 with gr.Column(min_width=100, elem_classes="speech-accuracy-score-container row2 col1"):
-                    number_pronunciation_accuracy = gr.Number(label="Current score", elem_id="number-pronunciation-accuracy-id-element")
+                    num_pronunciation_accuracy = gr.Number(label="Current score", elem_id="number-pronunciation-accuracy-id-element")
                 with gr.Column(min_width=100, elem_classes="speech-accuracy-score-container row2 col2"):
-                    number_score_de = gr.Number(label="Global score DE", value=0, interactive=False, elem_id="number-score-de-id-element")
+                    num_score_de = gr.Number(label="Global score DE", value=0, interactive=False, elem_id="number-score-de-id-element")
                 with gr.Column(min_width=100, elem_classes="speech-accuracy-score-container row2 col3"):
-                    number_score_en = gr.Number(label="Global score EN", value=0, interactive=False, elem_id="number-score-en-id-element")
+                    num_score_en = gr.Number(label="Global score EN", value=0, interactive=False, elem_id="number-score-en-id-element")
             with gr.Row():
                 btn_recognize_speech_accuracy = gr.Button(value="Recognize speech accuracy", elem_id="btn-recognize-speech-accuracy-id-element")
             with gr.Row():
                 with gr.Column(scale=1, min_width=50):
                     num_tot_recognized_words = gr.Number(label="Total recognized words", visible=True, minimum=0, interactive=False)
                 with gr.Column(scale=1, min_width=50):
-                    num_selected_recognized_words = gr.Number(label="Recognized word index", visible=True, minimum=0, value=0)
-
-                    def change_max_selected_words(n):
-                        app_logger.info(f"change_max_selected_words: {n} ...")
-                        app_logger.info(f"num_selected_recognized_words.maximum, pre: {num_selected_recognized_words.maximum} ...")
-                        new_num_selected_recognized_words = gr.Number(label=f"Recognized word index, max {n}!", visible=True, value=0, minimum=0, maximum=n)
-                        app_logger.info(f"num_selected_recognized_words.maximum, post: {num_selected_recognized_words.maximum} ...")
-                        return new_num_selected_recognized_words
-
-                    num_tot_recognized_words.change(
-                        change_max_selected_words,
-                        inputs=[num_tot_recognized_words],
-                         outputs=[num_selected_recognized_words],
-                    )
+                    num_selected_recognized_words = gr.Number(label=word_idx_text, visible=True, minimum=0, value=0)
                 with gr.Column(scale=2, min_width=100):
                     # todo: use https://www.gradio.app/docs/gradio/multimodaltextbox
                     audio_splitted_student_recording_stt = gr.Audio(
@@ -170,11 +162,11 @@ with gr.Blocks(css=css, head=js.head_driver_tour) as gradio_app:
 
     def get_updated_score_by_language(text: str, audio_rec: str | Path, lang: str, score_de: float, score_en: float):
         _transcribed_text, _letter_correctness, _pronunciation_accuracy, _recording_ipa, _ideal_ipa, _num_tot_recognized_words, _res = lambdaSpeechToScore.get_speech_to_score_tuple(text, audio_rec, lang, remove_random_file=False)
-        new_num_selected_recognized_words = gr.Number(label="Recognized word index", visible=True, value=0)
+        new_num_selected_recognized_words = gr.Number(label=word_idx_text, visible=True, value=0)
         output = {
             text_transcribed_hidden: _transcribed_text,
             text_letter_correctness: _letter_correctness,
-            number_pronunciation_accuracy: _pronunciation_accuracy,
+            num_pronunciation_accuracy: _pronunciation_accuracy,
             text_recording_ipa: _recording_ipa,
             text_ideal_ipa: _ideal_ipa,
             text_raw_json_output_hidden: _res,
@@ -184,14 +176,14 @@ with gr.Blocks(css=css, head=js.head_driver_tour) as gradio_app:
         match lang:
             case "de":
                 return {
-                    number_score_de: float(score_de) + float(_pronunciation_accuracy),
-                    number_score_en: float(score_en),
+                    num_score_de: float(score_de) + float(_pronunciation_accuracy),
+                    num_score_en: float(score_en),
                     **output
                 }
             case "en":
                 return {
-                    number_score_en: float(score_en) + float(_pronunciation_accuracy),
-                    number_score_de: float(score_de),
+                    num_score_en: float(score_en) + float(_pronunciation_accuracy),
+                    num_score_de: float(score_de),
                     **output
                 }
             case _:
@@ -199,20 +191,68 @@ with gr.Blocks(css=css, head=js.head_driver_tour) as gradio_app:
 
     btn_recognize_speech_accuracy.click(
         get_updated_score_by_language,
-        inputs=[text_student_transcription, audio_student_recording_stt, radio_language, number_score_de, number_score_en],
+        inputs=[text_student_transcription, audio_student_recording_stt, radio_language, num_score_de, num_score_en],
         outputs=[
             text_transcribed_hidden,
             text_letter_correctness,
-            number_pronunciation_accuracy,
+            num_pronunciation_accuracy,
             text_recording_ipa,
             text_ideal_ipa,
             text_raw_json_output_hidden,
-            number_score_de,
-            number_score_en,
+            num_score_de,
+            num_score_en,
             num_tot_recognized_words,
             num_selected_recognized_words
         ],
     )
+
+    def change_max_selected_words(n):
+        app_logger.info(f"change_max_selected_words: {n} ...")
+        app_logger.info(f"num_selected_recognized_words.maximum, pre: {num_selected_recognized_words.maximum} ...")
+        label = word_idx_text if n == 0 else f"{word_idx_text}, max {n}!"
+        new_num_selected_recognized_words = gr.Number(label=label, visible=True, value=0, minimum=0, maximum=n)
+        app_logger.info(f"num_selected_recognized_words.maximum, post: {num_selected_recognized_words.maximum} ...")
+        return new_num_selected_recognized_words
+
+    num_tot_recognized_words.change(
+        change_max_selected_words,
+        inputs=[num_tot_recognized_words],
+        outputs=[num_selected_recognized_words],
+    )
+
+    def clear3():
+        return None, None, None, None, None, None, 0, 0, 0
+
+    text_student_transcription.change(
+        clear3,
+        inputs=[],
+        outputs=[
+            audio_student_recording_stt, audio_tts, audio_splitted_student_recording_stt, text_recording_ipa, text_ideal_ipa, text_transcribed_hidden,
+            num_pronunciation_accuracy, num_selected_recognized_words, num_pronunciation_accuracy
+        ],
+    )
+
+    def reset_max_total_recognized_words(content_text_recording_ipa, content_num_tot_recognized_words):
+        if content_text_recording_ipa is None or content_text_recording_ipa == "":
+            app_logger.info("reset_max_total_recognized_words...")
+            new_num_tot_recognized_words = gr.Number(label="Total recognized words", visible=True, value=0, minimum=0, interactive=False)
+            return new_num_tot_recognized_words
+        return content_num_tot_recognized_words
+
+    text_recording_ipa.change(
+        reset_max_total_recognized_words,
+        inputs=[text_recording_ipa, num_tot_recognized_words],
+        outputs=[
+            num_tot_recognized_words
+        ],
+    )
+    text_recording_ipa.change(
+        None,
+        inputs=[get_textbox_hidden(), get_textbox_hidden()],
+        outputs=[html_output],
+        js=js.js_update_ipa_output,
+    )
+
     btn_run_tts.click(fn=None, inputs=[text_student_transcription, radio_language], outputs=audio_tts, js=js.js_play_audio)
     btn_run_tts_backend.click(
         fn=lambdaTTS.get_tts,
@@ -236,12 +276,12 @@ with gr.Blocks(css=css, head=js.head_driver_tour) as gradio_app:
         js=js.js_update_ipa_output,
     )
     
-    @gradio_app.load(inputs=[local_storage], outputs=[number_score_de, number_score_en])
+    @gradio_app.load(inputs=[local_storage], outputs=[num_score_de, num_score_en])
     def load_from_local_storage(saved_values):
         print("loading from local storage", saved_values)
         return saved_values[0], saved_values[1]
 
-    @gr.on([number_score_de.change, number_score_en.change], inputs=[number_score_de, number_score_en], outputs=[local_storage])
+    @gr.on([num_score_de.change, num_score_en.change], inputs=[num_score_de, num_score_en], outputs=[local_storage])
     def save_to_local_storage(score_de, score_en):
         return [score_de, score_en]