app.py

app.py

@@ -0,0 +1,82 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Copyright 2023 Imperial College London (Pingchuan Ma)
+# Apache 2.0  (http://www.apache.org/licenses/LICENSE-2.0)
+import os
+import datetime
+import subprocess
+import gradio as gr
+from pipelines.pipeline import InferencePipeline
+
+FFMPEG_COMMAND = "-loglevel error -y -r 25 -pix_fmt yuv420p -f mp4"
+pipelines = {
+    "VSR": InferencePipeline("./configs/LRS3_V_WER19.1.ini", device="cuda:0", face_track=True, detector="retinaface"),
+    "VSR(fast)": InferencePipeline("./configs/LRS3_V_WER19.1.ini", device="cuda:0", face_track=True, detector="mediapipe"),
+    "ASR": InferencePipeline("./configs/LRS3_A_WER1.0.ini", device="cuda:0", face_track=True, detector="retinaface"),
+    "AVSR": InferencePipeline("./configs/LRS3_AV_WER0.9.ini", device="cuda:0", face_track=True, detector="retinaface"),
+    "AVSR(fast)": InferencePipeline("./configs/LRS3_AV_WER0.9.ini", device="cuda:0", face_track=True, detector="mediapipe")
+}
+print("Step 0. Model has been loaded.")
+
+def fn(pipeline_type, filename):
+    directory = "./tmp_video"
+    if not os.path.exists(directory):
+        os.makedirs(directory)
+    now = datetime.datetime.now()
+    timestamp = now.strftime("%Y-%m-%d_%H-%M-%S")
+    dst_filename = f"{directory}/file_{timestamp}.mp4"
+    command_string = f"ffmpeg -i {filename} {FFMPEG_COMMAND} {dst_filename}"
+    print("Step 0. Video has been uploaded.")
+    os.system(command_string)
+    selected_pipeline_instance = pipelines[pipeline_type]
+    print("Step 1. Video has been converted.")
+    landmarks = selected_pipeline_instance.process_landmarks(dst_filename, landmarks_filename=None)
+    print("Step 2. Landmarks have been detected.")
+    data = selected_pipeline_instance.dataloader.load_data(dst_filename, landmarks)
+    print("Step 3. Data has been preprocessed.")
+    transcript = selected_pipeline_instance.model.infer(data)
+    print("Step 4. Inference has been done.")
+    print(f"transcript: {transcript}")
+    return transcript
+
+demo = gr.Blocks()
+
+with demo:
+    gr.HTML(
+        """
+            <div style="text-align: center; max-width: 700px; margin: 0 auto;">
+              <div
+                style="
+                  display: inline-flex;
+                  align-items: center;
+                  gap: 0.8rem;
+                  font-size: 1.75rem;
+                "
+              >
+                <h1 style="font-weight: 900; margin-bottom: 7px; line-height: normal;">
+                  Auto-AVSR: Audio-Visual Speech Recognition with Automatic Labels
+                </h1>
+              </div>
+              <p style="margin-bottom: 10px; font-size: 94%">
+              </p>
+            </div>
+        """
+    )
+    dropdown_list = gr.inputs.Dropdown(["VSR", "ASR", "AVSR", "VSR(fast)", "AVSR(fast)"], label="model")
+    video_file = gr.Video(label="INPUT VIDEO", include_audio=True)
+    text = gr.Textbox(label="PREDICTION")
+    btn = gr.Button("Submit").style(full_width=True)
+
+    btn.click(fn, inputs=[dropdown_list, video_file], outputs=text)
+
+    with gr.Accordion("Additional information", open=False):
+        gr.HTML(
+            """
+            <div class="acknowledgments">
+                <p> We share this demo only for non-commercial purposes. </p>
+                        </div>
+            """
+        )
+
+demo.launch(share=True)