|
import gradio as gr |
|
import numpy as np |
|
import random |
|
from diffusers import DiffusionPipeline |
|
import torch |
|
from huggingface_hub import hf_hub_download |
|
from safetensors.torch import load_file |
|
from cog_sdxl.dataset_and_utils import TokenEmbeddingsHandler |
|
from cog_sdxl.no_init import no_init_or_tensor |
|
from diffusers.models.attention_processor import LoRAAttnProcessor2_0 |
|
|
|
device = "cuda" if torch.cuda.is_available() else "cpu" |
|
|
|
pipe = DiffusionPipeline.from_pretrained( |
|
"stabilityai/stable-diffusion-xl-base-1.0", |
|
torch_dtype=torch.float16, |
|
use_safetensors=True, |
|
variant="fp16", |
|
).to(device) |
|
|
|
unet = pipe.unet |
|
|
|
lora_path = hf_hub_download(repo_id="SvenN/sdxl-emoji", filename="lora.safetensors", repo_type="model") |
|
embeddings_path = hf_hub_download(repo_id="SvenN/sdxl-emoji", filename="embeddings.pti", repo_type="model") |
|
|
|
|
|
tensors = load_file(lora_path) |
|
unet_lora_attn_procs = {} |
|
name_rank_map = {} |
|
for tk, tv in tensors.items(): |
|
|
|
tensors[tk] = tv.half() |
|
if tk.endswith("up.weight"): |
|
proc_name = ".".join(tk.split(".")[:-3]) |
|
r = tv.shape[1] |
|
name_rank_map[proc_name] = r |
|
|
|
for name, attn_processor in unet.attn_processors.items(): |
|
cross_attention_dim = ( |
|
None |
|
if name.endswith("attn1.processor") |
|
else unet.config.cross_attention_dim |
|
) |
|
if name.startswith("mid_block"): |
|
hidden_size = unet.config.block_out_channels[-1] |
|
elif name.startswith("up_blocks"): |
|
block_id = int(name[len("up_blocks.")]) |
|
hidden_size = list(reversed(unet.config.block_out_channels))[ |
|
block_id |
|
] |
|
elif name.startswith("down_blocks"): |
|
block_id = int(name[len("down_blocks.")]) |
|
hidden_size = unet.config.block_out_channels[block_id] |
|
with no_init_or_tensor(): |
|
module = LoRAAttnProcessor2_0( |
|
hidden_size=hidden_size, |
|
cross_attention_dim=cross_attention_dim, |
|
rank=name_rank_map[name], |
|
).half() |
|
unet_lora_attn_procs[name] = module.to("cuda", non_blocking=True) |
|
|
|
unet.set_attn_processor(unet_lora_attn_procs) |
|
unet.load_state_dict(tensors, strict=False) |
|
|
|
|
|
|
|
handler = TokenEmbeddingsHandler( |
|
[pipe.text_encoder, pipe.text_encoder_2], [pipe.tokenizer, pipe.tokenizer_2] |
|
) |
|
handler.load_embeddings(embeddings_path) |
|
|
|
|
|
MAX_SEED = np.iinfo(np.int32).max |
|
MAX_IMAGE_SIZE = 1024 |
|
|
|
def infer(prompt, negative_prompt, seed, randomize_seed, width, height, guidance_scale, num_inference_steps): |
|
|
|
if randomize_seed: |
|
seed = random.randint(0, MAX_SEED) |
|
|
|
generator = torch.Generator().manual_seed(seed) |
|
|
|
image = pipe( |
|
prompt = prompt, |
|
negative_prompt = negative_prompt, |
|
guidance_scale = guidance_scale, |
|
num_inference_steps = num_inference_steps, |
|
width = width, |
|
height = height, |
|
generator = generator, |
|
cross_attention_kwargs={"scale": 0.6}, |
|
).images[0] |
|
|
|
return image |
|
|
|
examples = [ |
|
"Astronaut in a jungle, cold color palette, muted colors, detailed, 8k", |
|
"An astronaut riding a green horse", |
|
"A delicious ceviche cheesecake slice", |
|
] |
|
|
|
css=""" |
|
#col-container { |
|
margin: 0 auto; |
|
max-width: 520px; |
|
} |
|
""" |
|
|
|
with gr.Blocks(css=css) as demo: |
|
|
|
with gr.Column(elem_id="col-container"): |
|
gr.Markdown(f""" |
|
# SDXL Emoji running on diffusers 0.25.0 with cog-sdxl custom classes |
|
""") |
|
|
|
with gr.Row(): |
|
|
|
prompt = gr.Text( |
|
label="Prompt", |
|
show_label=False, |
|
max_lines=1, |
|
placeholder="Enter your prompt", |
|
container=False, |
|
) |
|
|
|
run_button = gr.Button("Run", scale=0) |
|
|
|
result = gr.Image(label="Result", show_label=False) |
|
|
|
with gr.Accordion("Advanced Settings", open=False): |
|
|
|
negative_prompt = gr.Text( |
|
label="Negative prompt", |
|
max_lines=1, |
|
placeholder="Enter a negative prompt", |
|
visible=False, |
|
) |
|
|
|
seed = gr.Slider( |
|
label="Seed", |
|
minimum=0, |
|
maximum=MAX_SEED, |
|
step=1, |
|
value=0, |
|
) |
|
|
|
randomize_seed = gr.Checkbox(label="Randomize seed", value=True) |
|
|
|
with gr.Row(): |
|
|
|
width = gr.Slider( |
|
label="Width", |
|
minimum=256, |
|
maximum=MAX_IMAGE_SIZE, |
|
step=32, |
|
value=1024, |
|
) |
|
|
|
height = gr.Slider( |
|
label="Height", |
|
minimum=256, |
|
maximum=MAX_IMAGE_SIZE, |
|
step=32, |
|
value=1024, |
|
) |
|
|
|
with gr.Row(): |
|
|
|
guidance_scale = gr.Slider( |
|
label="Guidance scale", |
|
minimum=0.0, |
|
maximum=10.0, |
|
step=0.1, |
|
value=7.5, |
|
) |
|
|
|
num_inference_steps = gr.Slider( |
|
label="Number of inference steps", |
|
minimum=1, |
|
maximum=50, |
|
step=1, |
|
value=50, |
|
) |
|
|
|
gr.Examples( |
|
examples = examples, |
|
inputs = [prompt] |
|
) |
|
|
|
run_button.click( |
|
fn = infer, |
|
inputs = [prompt, negative_prompt, seed, randomize_seed, width, height, guidance_scale, num_inference_steps], |
|
outputs = [result] |
|
) |
|
|
|
demo.queue().launch() |