Spaces:
Running
on
Zero
Running
on
Zero
File size: 5,277 Bytes
6a87547 dae6484 6a87547 702754c 6a87547 b832af5 6a87547 702754c 6a87547 702754c 6a87547 dae6484 b832af5 6a87547 b832af5 6a87547 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 |
import os
import torch
import gradio as gr
from PIL import Image, ImageOps
from huggingface_hub import snapshot_download
from pyramid_dit import PyramidDiTForVideoGeneration
from diffusers.utils import export_to_video
import spaces
# Constants
MODEL_PATH = "pyramid-flow-model"
MODEL_REPO = "rain1011/pyramid-flow-sd3"
MODEL_VARIANT = "diffusion_transformer_768p"
MODEL_DTYPE = "bf16"
def center_crop(image, target_width, target_height):
width, height = image.size
aspect_ratio_target = target_width / target_height
aspect_ratio_image = width / height
if aspect_ratio_image > aspect_ratio_target:
# Crop the width (left and right)
new_width = int(height * aspect_ratio_target)
left = (width - new_width) // 2
right = left + new_width
top, bottom = 0, height
else:
# Crop the height (top and bottom)
new_height = int(width / aspect_ratio_target)
top = (height - new_height) // 2
bottom = top + new_height
left, right = 0, width
image = image.crop((left, top, right, bottom))
return image
# Download and load the model
def load_model():
if not os.path.exists(MODEL_PATH):
snapshot_download(MODEL_REPO, local_dir=MODEL_PATH, local_dir_use_symlinks=False, repo_type='model')
model = PyramidDiTForVideoGeneration(
MODEL_PATH,
MODEL_DTYPE,
model_variant=MODEL_VARIANT,
)
model.vae.to("cuda")
model.dit.to("cuda")
model.text_encoder.to("cuda")
model.vae.enable_tiling()
return model
# Global model variable
model = load_model()
# Text-to-video generation function
@spaces.GPU(duration=240)
def generate_video(prompt, duration, guidance_scale, video_guidance_scale):
temp = int(duration * 2.4) # Convert seconds to temp value (assuming 24 FPS)
torch_dtype = torch.bfloat16 if MODEL_DTYPE == "bf16" else torch.float32
with torch.no_grad(), torch.cuda.amp.autocast(enabled=True, dtype=torch_dtype):
frames = model.generate(
prompt=prompt,
num_inference_steps=[20, 20, 20],
video_num_inference_steps=[10, 10, 10],
height=768,
width=1280,
temp=temp,
guidance_scale=guidance_scale,
video_guidance_scale=video_guidance_scale,
output_type="pil",
save_memory=True,
)
output_path = "output_video.mp4"
export_to_video(frames, output_path, fps=24)
return output_path
# Image-to-video generation function
@spaces.GPU(duration=240)
def generate_video_from_image(image, prompt, duration, video_guidance_scale):
temp = int(duration * 2.4) # Convert seconds to temp value (assuming 24 FPS)
torch_dtype = torch.bfloat16 if MODEL_DTYPE == "bf16" else torch.float32
target_size = (1280, 720)
cropped_image = center_crop(image, 1280, 720)
resized_image = cropped_image.resize((1280, 720))
with torch.no_grad(), torch.cuda.amp.autocast(enabled=True, dtype=torch_dtype):
frames = model.generate_i2v(
prompt=prompt,
input_image=resized_image,
num_inference_steps=[10, 10, 10],
temp=temp,
guidance_scale=7.0,
video_guidance_scale=video_guidance_scale,
output_type="pil",
save_memory=True,
)
output_path = "output_video_i2v.mp4"
export_to_video(frames, output_path, fps=24)
return output_path
# Gradio interface
with gr.Blocks() as demo:
gr.Markdown("# Pyramid Flow Video Generation Demo")
with gr.Tab("Text-to-Video"):
with gr.Row():
with gr.Column():
t2v_prompt = gr.Textbox(label="Prompt")
t2v_duration = gr.Slider(minimum=1, maximum=10, value=5, step=1, label="Duration (seconds)")
t2v_guidance_scale = gr.Slider(minimum=1, maximum=15, value=9, step=0.1, label="Guidance Scale")
t2v_video_guidance_scale = gr.Slider(minimum=1, maximum=15, value=5, step=0.1, label="Video Guidance Scale")
t2v_generate_btn = gr.Button("Generate Video")
with gr.Column():
t2v_output = gr.Video(label="Generated Video")
t2v_generate_btn.click(
generate_video,
inputs=[t2v_prompt, t2v_duration, t2v_guidance_scale, t2v_video_guidance_scale],
outputs=t2v_output
)
with gr.Tab("Image-to-Video"):
with gr.Row():
with gr.Column():
i2v_image = gr.Image(type="pil", label="Input Image")
i2v_prompt = gr.Textbox(label="Prompt")
i2v_duration = gr.Slider(minimum=1, maximum=10, value=5, step=1, label="Duration (seconds)")
i2v_video_guidance_scale = gr.Slider(minimum=1, maximum=15, value=4, step=0.1, label="Video Guidance Scale")
i2v_generate_btn = gr.Button("Generate Video")
with gr.Column():
i2v_output = gr.Video(label="Generated Video")
i2v_generate_btn.click(
generate_video_from_image,
inputs=[i2v_image, i2v_prompt, i2v_duration, i2v_video_guidance_scale],
outputs=i2v_output
)
demo.launch() |