app.py

from pathlib import Path
import gradio as gr

from aip_trainer import PROJECT_ROOT_FOLDER, app_logger, sample_rate_start
from aip_trainer.lambdas import js, lambdaGetSample, lambdaSpeechToScore, lambdaTTS


css = """
.speech-output-label p {color: grey;}
.speech-output-container {align-items: center; min-height: 60px; padding-left: 8px; padding-right: 8px; margin-top: -12px; border-width: 1px; border-style: solid; border-color: lightgrey;}
"""

js_play_audio = """
function playAudio(text, language) {
    let voice_idx = 0;
    let voice_synth = null;
    let synth = window.speechSynthesis;

    function setSpeech() {
        return new Promise(
            function (resolve, reject) {
                let id;

                id = setInterval(() => {
                    if (synth.getVoices().length !== 0) {
                        resolve(synth.getVoices());
                        clearInterval(id);
                    }
                }, 10);
            }
        )
    }

    let s = setSpeech();
    s.then((voices) => {
        for (idx = 0; idx < voices.length; idx++) {
            if (voices[idx].lang.slice(0, 2) == language) {
                voice_synth = voices[idx];
                break;
            }
        }

        var utterThis = new SpeechSynthesisUtterance(text);
        utterThis.voice = voice_synth;
        utterThis.rate = 0.7;

        synth.speak(utterThis);
        return utterThis;
    });
}
"""


def clear():
    return None


def clear2():
    return None, None


with gr.Blocks(css=css) as gradio_app:
    local_storage = gr.BrowserState([0.0, 0.0])
    app_logger.info("start gradio app building...")

    project_root_folder = Path(PROJECT_ROOT_FOLDER)
    with open(project_root_folder / "aip_trainer" / "lambdas" / "app_description.md", "r", encoding="utf-8") as app_description_src:
        md_app_description = app_description_src.read()
        gr.Markdown(md_app_description.format(sample_rate_start=sample_rate_start))
    with gr.Row():
        with gr.Column(scale=4, min_width=300):
            with gr.Row():
                with gr.Column(scale=2, min_width=80):
                    radio_language = gr.Radio(["de", "en"], label="Language", value="en")
                with gr.Column(scale=5, min_width=160):
                    radio_difficulty = gr.Radio(
                        label="Difficulty",
                        value=0,
                        choices=[
                            ("random", 0),
                            ("easy", 1),
                            ("medium", 2),
                            ("hard", 3),
                        ],
                    )
                with gr.Column(scale=1, min_width=100):
                    btn_random_phrase = gr.Button(value="Choose a random phrase")
            with gr.Row():
                with gr.Column(scale=7, min_width=300):
                    text_learner_transcription = gr.Textbox(
                        lines=3,
                        label="Learner Transcription",
                        value="Hi there, how are you?",
                    )
            with gr.Row():
                audio_tts = gr.Audio(label="Audio TTS")
            with gr.Row():
                btn_run_tts = gr.Button(value="TTS in browser")
                btn_run_tts_backend = gr.Button(value="TTS backend")
                btn_clear_tts = gr.Button(value="Clear TTS backend")
                btn_clear_tts.click(clear, inputs=[], outputs=[audio_tts])
            with gr.Row():
                audio_learner_recording_stt = gr.Audio(
                    label="Learner Recording",
                    sources=["microphone", "upload"],
                    type="filepath",
                    show_download_button=True,
                )
        with gr.Column(scale=4, min_width=320):
            text_transcribed_hidden = gr.Textbox(
                placeholder=None, label="Transcribed text", visible=False
            )
            text_letter_correctness = gr.Textbox(
                placeholder=None,
                label="Letters correctness",
                visible=False,
            )
            text_recording_ipa = gr.Textbox(
                placeholder=None, label="Learner phonetic transcription"
            )
            text_ideal_ipa = gr.Textbox(
                placeholder=None, label="Ideal phonetic transcription"
            )
            text_raw_json_output_hidden = gr.Textbox(placeholder=None, label="text_raw_json_output_hidden", visible=False)
            gr.Markdown("Speech accuracy output", elem_classes="speech-output-label")
            with gr.Row(elem_classes="speech-output-container"):
                html_output = gr.HTML(
                    label="Speech accuracy output",
                    elem_id="speech-output",
                    show_label=False,
                    visible=True,
                    render=True,
                    value=" - ",
                    elem_classes="speech-output",
                )
            with gr.Row():
                gr.Markdown("### Speech accuracy score (%)", elem_classes="speech-accuracy-score-container row1")
            with gr.Row():
                with gr.Column(min_width=100, elem_classes="speech-accuracy-score-container row2 col1"):
                    number_pronunciation_accuracy = gr.Number(label="Current score")
                with gr.Column(min_width=100, elem_classes="speech-accuracy-score-container row2 col2"):
                    number_score_de = gr.Number(label="Global score DE", value=0, interactive=False)
                with gr.Column(min_width=100, elem_classes="speech-accuracy-score-container row2 col3"):
                    number_score_en = gr.Number(label="Global score EN", value=0, interactive=False)
            with gr.Row():
                btn = gr.Button(value="Recognize speech accuracy")
            with gr.Accordion("Click here to expand the table examples", open=False):
                examples_text = gr.Examples(
                    examples=[
                        ["Hallo, wie geht es dir?", "de", 1],
                        ["Hi there, how are you?", "en", 1],
                        ["Die König-Ludwig-Eiche ist ein Naturdenkmal im Staatsbad Brückenau.", "de", 2,],
                        ["Rome is home to some of the most beautiful monuments in the world.", "en", 2],
                        ["Die König-Ludwig-Eiche ist ein Naturdenkmal im Staatsbad Brückenau, einem Ortsteil des drei Kilometer nordöstlich gelegenen Bad Brückenau im Landkreis Bad Kissingen in Bayern.", "de", 3],
                        ["Some machine learning models are designed to understand and generate human-like text based on the input they receive.", "en", 3],
                    ],
                    inputs=[text_learner_transcription, radio_language, radio_difficulty],
                )

    def get_updated_score_by_language(text: str, audio_rec: str | Path, lang: str, score_de: float, score_en: float):
        _transcribed_text, _letter_correctness, _pronunciation_accuracy, _recording_ipa, _ideal_ipa, _res = lambdaSpeechToScore.get_speech_to_score_tuple(text, audio_rec, lang)
        output = {
            text_transcribed_hidden: _transcribed_text,
            text_letter_correctness: _letter_correctness,
            number_pronunciation_accuracy: _pronunciation_accuracy,
            text_recording_ipa: _recording_ipa,
            text_ideal_ipa: _ideal_ipa,
            text_raw_json_output_hidden: _res,
        }
        match lang:
            case "de":
                return {
                    number_score_de: float(score_de) + float(_pronunciation_accuracy),
                    number_score_en: float(score_en),
                    **output
                }
            case "en":
                return {
                    number_score_en: float(score_en) + float(_pronunciation_accuracy),
                    number_score_de: float(score_de),
                    **output
                }
            case _:
                raise NotImplementedError(f"Language {lang} not supported")

    btn.click(
        get_updated_score_by_language,
        inputs=[text_learner_transcription, audio_learner_recording_stt, radio_language, number_score_de, number_score_en],
        outputs=[
            text_transcribed_hidden,
            text_letter_correctness,
            number_pronunciation_accuracy,
            text_recording_ipa,
            text_ideal_ipa,
            text_raw_json_output_hidden,
            number_score_de, number_score_en
        ],
    )
    btn_run_tts.click(fn=None, inputs=[text_learner_transcription, radio_language], outputs=audio_tts, js=js_play_audio)
    btn_run_tts_backend.click(
        fn=lambdaTTS.get_tts,
        inputs=[text_learner_transcription, radio_language],
        outputs=audio_tts,
    )
    btn_random_phrase.click(
        lambdaGetSample.get_random_selection,
        inputs=[radio_language, radio_difficulty],
        outputs=[text_learner_transcription],
    )
    btn_random_phrase.click(
        clear2,
        inputs=[],
        outputs=[audio_learner_recording_stt, audio_tts]
    )
    html_output.change(
        None,
        inputs=[text_transcribed_hidden, text_letter_correctness],
        outputs=[html_output],
        js=js.js_update_ipa_output,
    )
    
    @gradio_app.load(inputs=[local_storage], outputs=[number_score_de, number_score_en])
    def load_from_local_storage(saved_values):
        print("loading from local storage", saved_values)
        return saved_values[0], saved_values[1]

    @gr.on([number_score_de.change, number_score_en.change], inputs=[number_score_de, number_score_en], outputs=[local_storage])
    def save_to_local_storage(score_de, score_en):
        return [score_de, score_en]


if __name__ == "__main__":
    gradio_app.launch()