import random import gradio as gr import numpy as np import spaces import torch from diffusers import DiffusionPipeline from PIL import Image device = "cuda" if torch.cuda.is_available() else "cpu" repo_id = "black-forest-labs/FLUX.1-dev" adapter_id = "alvarobartt/ghibli-characters-flux-lora" pipeline = DiffusionPipeline.from_pretrained(repo_id, torch_dtype=torch.bfloat16) pipeline.load_lora_weights(adapter_id) pipeline = pipeline.to(device) MAX_SEED = np.iinfo(np.int32).max MAX_IMAGE_SIZE = 1024 @spaces.GPU(duration=120) def inference( prompt: str, seed: int, randomize_seed: bool, width: int, height: int, guidance_scale: float, num_inference_steps: int, lora_scale: float, progress: gr.Progress = gr.Progress(track_tqdm=True), ): if randomize_seed: seed = random.randint(0, MAX_SEED) generator = torch.Generator(device=device).manual_seed(seed) image = pipeline( prompt=prompt, guidance_scale=guidance_scale, num_inference_steps=num_inference_steps, width=width, height=height, generator=generator, joint_attention_kwargs={"scale": lora_scale}, ).images[0] return image, seed examples = [ ( "Ghibli style futuristic stormtrooper with glossy white armor and a sleek helmet," " standing heroically on a lush alien planet, vibrant flowers blooming around, soft" " sunlight illuminating the scene, a gentle breeze rustling the leaves" ), ] css = """ #col-container { margin: 0 auto; max-width: 640px; } """ with gr.Blocks(css=css) as demo: with gr.Column(elem_id="col-container"): gr.Markdown("# FLUX.1 Studio Ghibli LoRA") gr.Markdown( "LoRA fine-tune of [FLUX.1-dev](https://huggingface.co/black-forest-labs/FLUX.1-dev)" " with [alvarobartt/ghibli-characters](https://huggingface.co/datasets/alvarobartt/ghibli-characters)." ) with gr.Accordion("How to generate nice prompts?", open=False): gr.Markdown( "What worked best for me to generate high-quality prompts, was to prompt" " [Claude 3 Haiku](https://claude.ai) with the following:\n\nYou are an" " expert prompt writer for diffusion text to image models, and you've been provided" " the following prompt template:\n\n\"Ghibli style [character description] with" " [distinctive features], [action or pose], [environment or background]," " [lighting or atmosphere], [additional details].\"\n\nCould you create a prompt" " to generate [CHARACTER NAME] as a Studio Ghibli character following that template?" " [MORE DETAILS IF NEEDED]\n" ) with gr.Row(): prompt = gr.Text( label="Prompt", show_label=False, max_lines=1, placeholder="Enter your prompt", container=False, ) run_button = gr.Button("Run", scale=0) result = gr.Image(label="Result", show_label=False) with gr.Accordion("Advanced Settings", open=False): seed = gr.Slider( label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=42, ) randomize_seed = gr.Checkbox(label="Randomize seed", value=True) with gr.Row(): width = gr.Slider( label="Width", minimum=256, maximum=MAX_IMAGE_SIZE, step=32, value=1024, ) height = gr.Slider( label="Height", minimum=256, maximum=MAX_IMAGE_SIZE, step=32, value=768, ) with gr.Row(): guidance_scale = gr.Slider( label="Guidance scale", minimum=0.0, maximum=10.0, step=0.1, value=3.5, ) num_inference_steps = gr.Slider( label="Number of inference steps", minimum=1, maximum=50, step=1, value=30, ) lora_scale = gr.Slider( label="LoRA scale", minimum=0.0, maximum=1.0, step=0.1, value=1.0, ) gr.Examples( examples=examples, fn=lambda x: (Image.open("./example.jpg"), 42), inputs=[prompt], outputs=[result, seed], run_on_click=True, ) gr.Markdown( "### Disclaimer\n\n" "License is non-commercial for both FLUX.1-dev and the Studio Ghibli dataset;" " but free to use for personal and non-commercial purposes." ) gr.on( triggers=[run_button.click, prompt.submit], fn=inference, inputs=[ prompt, seed, randomize_seed, width, height, guidance_scale, num_inference_steps, lora_scale, ], outputs=[result, seed], ) demo.queue() demo.launch()