Update app.py

app.py

@@ -11,6 +11,7 @@ import scipy.io.wavfile
 import re
 import glob
 import os
+from io import BytesIO
 
 # 定义图像到文本函数
 def img2text(image):
@@ -62,7 +63,6 @@ def text2text(user_input):
     completion = response.json()
     return completion['choices'][0]['message']['content']
 
-# 定义文本到视频函数
 def text2vid(input_text):
     sentences = re.findall(r'\[\d+\] (.+?)(?:\n|\Z)', input_text)
     adapter = MotionAdapter.from_pretrained("wangfuyun/AnimateLCM", config_file="wangfuyun/AnimateLCM/config.json", torch_dtype=torch.float16)
@@ -75,7 +75,8 @@ def text2vid(input_text):
         print("Ignoring the error:", str(e))
     pipe.enable_vae_slicing()
     pipe.enable_model_cpu_offload()
-    for index, sentence in enumerate(sentences):
+    video_clips = []
+    for sentence in sentences:
         output = pipe(
             prompt=sentence + ", 4k, high resolution",
             negative_prompt="bad quality, worse quality, low resolution",
@@ -85,7 +86,10 @@ def text2vid(input_text):
             generator=torch.Generator("cpu").manual_seed(0)
         )
         frames = output.frames[0]
-        export_to_gif(frames, f"./{index+1}.gif")
+        video_clip = frames_to_video_clip(frames)
+        video_clips.append(video_clip)
+    final_clip = concatenate_videoclips(video_clips, method="compose")
+    return final_clip
 
 # 定义生成最终视频的函数
 def video_generate():
@@ -95,49 +99,35 @@ def video_generate():
     final_clip = concatenate_videoclips(clips, method="compose")
     final_clip.write_videofile('output_video.mp4', codec='libx264')
 
-# 定义文本到音频函数
+# 修改音频生成函数
 def text2audio(text_input, duration_seconds):
     processor = AutoProcessor.from_pretrained("facebook/musicgen-small")
     model = MusicgenForConditionalGeneration.from_pretrained("facebook/musicgen-small")
     inputs = processor(text=[text_input], padding=True, return_tensors="pt")
     max_new_tokens = int((duration_seconds / 5) * 256)
     audio_values = model.generate(**inputs, max_new_tokens=max_new_tokens)
-    scipy.io.wavfile.write("bgm.wav", rate=model.config.audio_encoder.sampling_rate, data=audio_values[0, 0].numpy())
+    audio_array = audio_values[0, 0].numpy()
+    audio_clip = numpy_array_to_audio_clip(audio_array, rate=model.config.audio_encoder.sampling_rate)
+    return audio_clip
 
-# 定义生成结果视频的函数
-def result_generate():
-    video = VideoFileClip("output_video.mp4")
-    audio = AudioFileClip("bgm.wav")
-    video = video.set_audio(audio)
-    video.write_videofile("result.mp4", codec="libx264", audio_codec="aac")
+# 修改最终视频生成函数
+def result_generate(video_clip, audio_clip):
+    video = video_clip.set_audio(audio_clip)
+    video_bytes = video_clip_to_bytes(video)
+    return video_bytes
 
-# 定义删除所有文件的函数
-def delete_all_files(directory):
-    for filename in os.listdir(directory):
-        file_path = os.path.join(directory, filename)
-        try:
-            if os.path.isfile(file_path):
-                os.remove(file_path)
-                print(f"Deleted {filename}")
-            elif os.path.isdir(file_path):
-                os.rmdir(file_path)
-                print(f"Deleted empty directory {filename}")
-        except Exception as e:
-            print(f"Failed to delete {filename}. Reason: {e}")
 
-# 整合所有步骤到主函数
+# 主函数，结合上述修改
 def generate_video(image):
-    #delete_all_files("data")
     text = img2text(image)
     sentences = text2text(text)
-    text2vid(sentences)
-    video_generate()
-    video = VideoFileClip("output_video.mp4")
+    final_video_clip = text2vid(sentences)
+    video = VideoFileClip(final_video_clip)
     duration = video.duration
     audio_text = text2text(text)
-    text2audio(audio_text, duration)
-    result_generate()
-    return "result.mp4"
+    audio_clip = text2audio(audio_text, duration)
+    result_video = result_generate(final_video_clip, audio_clip)
+    return result_video
 
 # 定义 Gradio 接口
 # interface = gr.Interface(