Create app.py

app.py

@@ -0,0 +1,72 @@
+import gradio as gr
+from PIL import Image
+from transformers import pipeline
+import scipy.io.wavfile as wavfile
+import numpy as np
+
+caption_image = pipeline("image-to-text", model="Salesforce/blip-image-captioning-large")
+
+Narrator = pipeline("text-to-speech", model="kakao-enterprise/vits-ljs")
+
+
+def generate_audio(text):
+    # Generate speech from the input text using the Narrator (VITS model)
+    Narrated_Text = Narrator(text)
+    
+    # Extract the audio data and sampling rate
+    audio_data = np.array(Narrated_Text["audio"][0])
+    sampling_rate = Narrated_Text["sampling_rate"]
+    
+    # Save the generated speech as a WAV file
+    wavfile.write("generated_audio.wav", rate=sampling_rate, data=audio_data)
+    
+    # Return the filename of the saved audio file
+    return "generated_audio.wav"
+
+
+def caption_my_image(pil_image):
+    # Use BLIP to generate a text description of the input image
+    semantics = caption_image(images=pil_image)[0]["generated_text"]
+    
+    # Generate audio from the text description
+    return generate_audio(semantics)
+
+
+# Main functionality tab
+main_tab = gr.Interface(
+    fn=caption_my_image,
+    inputs=[gr.Image(label="Select Image", type="pil")],
+    outputs=[gr.Audio(label="Generated Audio")],
+    title="Image Audio Captioning App",
+    description="Upload an image, and the app will generate a textual description and convert it into speech."
+)
+
+# Information tab
+info_tab = gr.Markdown("""
+# Image Audio Captioning App
+
+### Purpose
+This application is designed to assist visually impaired users by providing audio descriptions of images. It can also be used in various scenarios such as creating audio captions for educational materials, enhancing accessibility for digital content, and more.
+
+### How to Use
+- **Step 1:** Click on the 'Select Image' button to upload an image.
+- **Step 2:** Wait for the application to generate the audio description.
+- **Step 3:** Listen to the generated audio file.
+
+### Limits
+- The quality of the description depends on the image clarity and content.
+- The application might not work well with images that have complex scenes or unclear subjects.
+- Audio generation time may vary depending on the input image size and content.
+
+### Note
+- Ensure the uploaded image is clear and well-defined for the best results.
+- This app is a prototype and may have limitations in real-world applications.
+""")
+
+# Combine both tabs into a single app
+demo = gr.TabbedInterface(
+    [main_tab, info_tab],
+    tab_names=["Main", "Information"]
+)
+
+demo.launch()