Upload 4 files

app.py

@@ -2,10 +2,12 @@ import spaces
 import gradio as gr
 import json
 import torch
-from diffusers import DiffusionPipeline, AutoencoderTiny, AutoencoderKL, AutoPipelineForImage2Image, AutoPipelineForInpainting
+from diffusers import DiffusionPipeline, AutoencoderTiny, AutoencoderKL, AutoPipelineForImage2Image, AutoPipelineForInpainting, GGUFQuantizationConfig
 from live_preview_helpers import calculate_shift, retrieve_timesteps, flux_pipe_call_that_returns_an_iterable_of_images
 from diffusers.utils import load_image
-from diffusers import FluxControlNetPipeline, FluxControlNetModel, FluxMultiControlNetModel, FluxControlNetImg2ImgPipeline, FluxTransformer2DModel, FluxControlNetInpaintPipeline, FluxInpaintPipeline
+from diffusers import (FluxControlNetPipeline, FluxControlNetModel, FluxMultiControlNetModel, FluxControlNetImg2ImgPipeline,
+                       FluxTransformer2DModel, FluxControlNetInpaintPipeline, FluxImg2ImgPipeline, FluxInpaintPipeline, FluxFillPipeline, FluxControlPipeline)
+from transformers import T5EncoderModel
 from huggingface_hub import hf_hub_download, HfFileSystem, ModelCard, snapshot_download, HfApi
 import os
 import copy
@@ -16,10 +18,10 @@ import pandas as pd
 import numpy as np
 from pathlib import Path
 
-from env import models, num_loras, num_cns, HF_TOKEN, single_file_base_models
+from env import models, models_dev, models_schnell, models_fill, models_canny, models_depth, num_loras, num_cns, HF_TOKEN, single_file_base_models
 from mod import (clear_cache, get_repo_safetensors, is_repo_name, is_repo_exists, get_model_trigger,
-                 description_ui, compose_lora_json, is_valid_lora, fuse_loras, save_image, preprocess_i2i_image,
-                 get_trigger_word, enhance_prompt, set_control_union_image,
+                 description_ui, compose_lora_json, is_valid_lora, fuse_loras, turbo_loras, save_image, preprocess_i2i_image,
+                 get_trigger_word, enhance_prompt, set_control_union_image, get_canny_image, get_depth_image,
                  get_control_union_mode, set_control_union_mode, get_control_params, translate_to_en)
 from modutils import (search_civitai_lora, select_civitai_lora, search_civitai_lora_json,
                       download_my_lora_flux, get_all_lora_tupled_list, apply_lora_prompt_flux,
@@ -42,39 +44,33 @@ controlnet_model_union_repo = 'InstantX/FLUX.1-dev-Controlnet-Union'
 #controlnet_model_union_repo = 'Shakker-Labs/FLUX.1-dev-ControlNet-Union-Pro'
 dtype = torch.bfloat16
 #dtype = torch.float8_e4m3fn
-CACHE_MODEL = False
 device = "cuda" if torch.cuda.is_available() else "cpu"
 taef1 = AutoencoderTiny.from_pretrained("madebyollin/taef1", torch_dtype=dtype, token=HF_TOKEN)
 good_vae = AutoencoderKL.from_pretrained(base_model, subfolder="vae", torch_dtype=dtype, token=HF_TOKEN)
 pipe = DiffusionPipeline.from_pretrained(base_model, torch_dtype=dtype, vae=taef1, token=HF_TOKEN)
-if CACHE_MODEL:
-    taef1.to(device)
-    good_vae.to(device)
-    pipe.to(device)
-    pipe.transformer.to("cpu")
 pipe_i2i = AutoPipelineForImage2Image.from_pretrained(base_model, vae=good_vae, transformer=pipe.transformer, text_encoder=pipe.text_encoder,
  tokenizer=pipe.tokenizer, text_encoder_2=pipe.text_encoder_2, tokenizer_2=pipe.tokenizer_2, torch_dtype=dtype, token=HF_TOKEN)
-pipe_ip = AutoPipelineForInpainting.from_pretrained(base_model, vae=good_vae, transformer=pipe.transformer, text_encoder=pipe.text_encoder,
- tokenizer=pipe.tokenizer, text_encoder_2=pipe.text_encoder_2, tokenizer_2=pipe.tokenizer_2, torch_dtype=dtype, token=HF_TOKEN)
 controlnet_union = None
 controlnet = None
 last_model = models[0]
 last_cn_on = False
+last_task = "Text-to-Image"
+last_dtype_str = "BF16"
 #controlnet_union = FluxControlNetModel.from_pretrained(controlnet_model_union_repo, torch_dtype=dtype)
 #controlnet = FluxMultiControlNetModel([controlnet_union])
 #controlnet.config = controlnet_union.config
 
 MAX_SEED = 2**32-1
+TASK_TYPE_T2I = ["Text-to-Image"]
+TASK_TYPE_I2I = ["Image-to-Image", "Inpainting", "Flux Fill"] # , "Canny", "Depth"
 
 def unload_lora():
-    global pipe, pipe_i2i, pipe_ip
+    global pipe, pipe_i2i
     try:
         #pipe.unfuse_lora()
         pipe.unload_lora_weights()
         #pipe_i2i.unfuse_lora()
         pipe_i2i.unload_lora_weights()
-        #pipe_ip.unfuse_lora()
-        pipe_ip.unload_lora_weights()
     except Exception as e:
         print(e)
 
@@ -83,90 +79,104 @@ def download_file_mod(url, directory=os.getcwd()):
     if not path: raise Exception(f"Download error: {url}")
     return path
 
+def print_progress(desc: str, proceed: float=0.0, progress=gr.Progress(track_tqdm=True)):
+    progress(proceed, desc=desc)
+    print(desc)
+
+#@spaces.GPU(duration=30)
+def load_quantized_control(control_repo: str, dtype, hf_token):
+    transformer = FluxTransformer2DModel.from_pretrained(control_repo, subfolder="transformer", torch_dtype=dtype, token=hf_token).to("cpu")
+    text_encoder_2 = T5EncoderModel.from_pretrained(control_repo, subfolder="text_encoder_2", torch_dtype=dtype, token=hf_token).to("cpu")
+    return transformer, text_encoder_2
+
+def load_pipeline(repo_id: str, cn_on: bool, model_type: str, task: str, dtype_str: str, hf_token: str, progress=gr.Progress(track_tqdm=True)):
+    pipe = None
+    pipe_i2i = None
+    try:
+        controlnet_model_union_repo = 'InstantX/FLUX.1-dev-Controlnet-Union'
+        if task == "Flux Fill" or repo_id in models_fill:
+            model_type = "fill"
+            if repo_id in set(models_dev + models_schnell): repo_id = models_fill[0]
+        if dtype_str == "BF16": dtype = torch.bfloat16
+        else: dtype = torch.bfloat16
+        single_file_base_model = single_file_base_models.get(model_type, models[0])
+        kwargs = {}
+        transformer_model = FluxTransformer2DModel
+        t5_model = T5EncoderModel
+        if task == "Flux Fill":
+            pipeline = FluxFillPipeline
+            pipeline_i2i = FluxFillPipeline
+        elif task == "Canny" or task == "Depth":
+            pipeline = DiffusionPipeline
+            pipeline_i2i = FluxControlPipeline
+        elif cn_on: # with ControlNet
+            print_progress(f"Loading model: {repo_id} / Loading ControlNet: {controlnet_model_union_repo}", 0, progress)
+            controlnet_union = FluxControlNetModel.from_pretrained(controlnet_model_union_repo, torch_dtype=dtype, token=hf_token)
+            controlnet = FluxMultiControlNetModel([controlnet_union])
+            controlnet.config = controlnet_union.config
+            pipeline = FluxControlNetPipeline
+            pipeline_i2i = FluxControlNetInpaintPipeline if task == "Inpainting" else FluxControlNetImg2ImgPipeline
+            kwargs["controlnet"] = controlnet
+        else: # without ControlNet
+            print_progress(f"Loading model: {repo_id}", 0, progress)
+            pipeline = DiffusionPipeline
+            pipeline_i2i = AutoPipelineForInpainting if task == "Inpainting" else AutoPipelineForImage2Image
+        if task == "Canny" or task == "Depth": # FluxControlPipeline
+            if task == "Canny": control_repo = models_canny[0]
+            elif task == "Depth": control_repo = models_depth[0]
+            transformer = transformer_model.from_pretrained(control_repo, subfolder="transformer", torch_dtype=dtype, token=hf_token)
+            text_encoder_2 = t5_model.from_pretrained(control_repo, subfolder="text_encoder_2", torch_dtype=dtype, token=hf_token)
+            #transformer, text_encoder_2 = load_quantized_control(control_repo, dtype, hf_token)
+            pipe = pipeline.from_pretrained(models_dev[0], transformer=transformer, text_encoder_2=text_encoder_2, torch_dtype=dtype, token=hf_token)
+            pipe_i2i = pipeline_i2i.from_pipe(pipe, transformer=transformer, text_encoder_2=text_encoder_2, torch_dtype=dtype)
+        elif ".safetensors" in repo_id or ".gguf" in repo_id: # from single file
+            file_url = repo_id.replace("/resolve/main/", "/blob/main/").replace("?download=true", "")
+            if ".gguf" in file_url: transformer_model.from_single_file(file_url, subfolder="transformer",
+                quantization_config=GGUFQuantizationConfig(compute_dtype=dtype), torch_dtype=dtype, config=single_file_base_model)
+            else: transformer = transformer_model.from_single_file(file_url, subfolder="transformer", torch_dtype=dtype, config=single_file_base_model)
+            pipe = pipeline.from_pretrained(single_file_base_model, transformer=transformer, torch_dtype=dtype, token=hf_token, **kwargs)
+            pipe_i2i = pipeline_i2i.from_pretrained(single_file_base_model, vae=pipe.vae, transformer=pipe.transformer,
+                text_encoder=pipe.text_encoder, tokenizer=pipe.tokenizer, text_encoder_2=pipe.text_encoder_2, tokenizer_2=pipe.tokenizer_2,
+                torch_dtype=dtype, token=hf_token, **kwargs)
+        else: # from diffusers repo
+            pipe = pipeline.from_pretrained(repo_id, torch_dtype=dtype, token=hf_token, **kwargs)
+            pipe_i2i = pipeline_i2i.from_pretrained(repo_id, vae=pipe.vae, transformer=pipe.transformer, text_encoder=pipe.text_encoder,
+                tokenizer=pipe.tokenizer, text_encoder_2=pipe.text_encoder_2, tokenizer_2=pipe.tokenizer_2, torch_dtype=dtype, token=hf_token, **kwargs)
+        if cn_on: print_progress(f"Model loaded: {repo_id} / ControlNet Loaded: {controlnet_model_union_repo}", 1, progress)
+        else: print_progress(f"Model loaded: {repo_id}", 1, progress)
+    except Exception as e:
+        print(e)
+        gr.Warning(f"Failed to load pipeline: {e}")
+    finally:
+        return pipe, pipe_i2i
+
+#load_pipeline.zerogpu = True
+
 # https://huggingface.co/InstantX/FLUX.1-dev-Controlnet-Union
 # https://huggingface.co/spaces/jiuface/FLUX.1-dev-Controlnet-Union
 # https://huggingface.co/docs/diffusers/main/en/api/pipelines/flux
 #@spaces.GPU()
-def change_base_model(repo_id: str, cn_on: bool, disable_model_cache: bool, model_type: str, progress=gr.Progress(track_tqdm=True)):
-    global pipe, pipe_i2i, pipe_ip, taef1, good_vae, controlnet_union, controlnet, last_model, last_cn_on, dtype
-    safetensors_file = None
-    single_file_base_model = single_file_base_models.get(model_type, models[0])
+def change_base_model(repo_id: str, cn_on: bool, disable_model_cache: bool, model_type: str, task: str, dtype_str: str, progress=gr.Progress(track_tqdm=True)):
+    global pipe, pipe_i2i, taef1, good_vae, controlnet_union, controlnet, last_model, last_cn_on, last_task, last_dtype_str, dtype
     try:
-        #if not disable_model_cache and (repo_id == last_model and cn_on is last_cn_on) or not is_repo_name(repo_id) or not is_repo_exists(repo_id): return gr.update(visible=True)
-        if not disable_model_cache and (repo_id == last_model and cn_on is last_cn_on) or ((not is_repo_name(repo_id) or not is_repo_exists(repo_id)) and not ".safetensors" in repo_id): return gr.update()
+        if not disable_model_cache and (repo_id == last_model and cn_on is last_cn_on and task == last_task and dtype_str == last_dtype_str)\
+            or ((not is_repo_name(repo_id) or not is_repo_exists(repo_id)) and not ".safetensors" in repo_id): return gr.update()
         unload_lora()
         pipe.to("cpu")
         pipe_i2i.to("cpu")
-        pipe_ip.to("cpu")
         good_vae.to("cpu")
         taef1.to("cpu")
         if controlnet is not None: controlnet.to("cpu")
         if controlnet_union is not None: controlnet_union.to("cpu")
         clear_cache()
-        if cn_on:
-            progress(0, desc=f"Loading model: {repo_id} / Loading ControlNet: {controlnet_model_union_repo}")
-            print(f"Loading model: {repo_id} / Loading ControlNet: {controlnet_model_union_repo}")
-            controlnet_union = FluxControlNetModel.from_pretrained(controlnet_model_union_repo, torch_dtype=dtype, token=HF_TOKEN)
-            controlnet = FluxMultiControlNetModel([controlnet_union])
-            controlnet.config = controlnet_union.config
-            if ".safetensors" in repo_id:
-                safetensors_file = download_file_mod(repo_id)
-                transformer = FluxTransformer2DModel.from_single_file(safetensors_file, subfolder="transformer", torch_dtype=dtype, config=single_file_base_model)
-                if CACHE_MODEL:
-                    pipe = FluxControlNetPipeline.from_pretrained(single_file_base_model, controlnet=controlnet, vae=good_vae, transformer=transformer, text_encoder=pipe.text_encoder,
-                     tokenizer=pipe.tokenizer, text_encoder_2=pipe.text_encoder_2, tokenizer_2=pipe.tokenizer_2, torch_dtype=dtype, token=HF_TOKEN)
-                else: pipe = FluxControlNetPipeline.from_pretrained(single_file_base_model, transformer=transformer, controlnet=controlnet, torch_dtype=dtype, token=HF_TOKEN)
-                pipe_i2i = FluxControlNetImg2ImgPipeline.from_pretrained(single_file_base_model, controlnet=controlnet, vae=good_vae, transformer=pipe.transformer, text_encoder=pipe.text_encoder,
-                 tokenizer=pipe.tokenizer, text_encoder_2=pipe.text_encoder_2, tokenizer_2=pipe.tokenizer_2, torch_dtype=dtype, token=HF_TOKEN)
-                pipe_ip = FluxControlNetInpaintPipeline.from_pretrained(single_file_base_model, controlnet=controlnet, vae=good_vae, transformer=pipe.transformer, text_encoder=pipe.text_encoder,
-                 tokenizer=pipe.tokenizer, text_encoder_2=pipe.text_encoder_2, tokenizer_2=pipe.tokenizer_2, torch_dtype=dtype, token=HF_TOKEN)
-            else:
-                if CACHE_MODEL:
-                    transformer = FluxTransformer2DModel.from_pretrained(repo_id, subfolder="transformer", torch_dtype=dtype, token=HF_TOKEN)
-                    pipe = FluxControlNetPipeline.from_pretrained(repo_id, controlnet=controlnet, vae=good_vae, transformer=transformer, text_encoder=pipe.text_encoder,
-                     tokenizer=pipe.tokenizer, text_encoder_2=pipe.text_encoder_2, tokenizer_2=pipe.tokenizer_2, torch_dtype=dtype, token=HF_TOKEN)
-                pipe = FluxControlNetPipeline.from_pretrained(repo_id, controlnet=controlnet, torch_dtype=dtype, token=HF_TOKEN)
-                pipe_i2i = FluxControlNetImg2ImgPipeline.from_pretrained(repo_id, controlnet=controlnet, vae=good_vae, transformer=pipe.transformer, text_encoder=pipe.text_encoder,
-                 tokenizer=pipe.tokenizer, text_encoder_2=pipe.text_encoder_2, tokenizer_2=pipe.tokenizer_2, torch_dtype=dtype, token=HF_TOKEN)
-                pipe_ip = FluxControlNetInpaintPipeline.from_pretrained(repo_id, controlnet=controlnet, vae=good_vae, transformer=pipe.transformer, text_encoder=pipe.text_encoder,
-                 tokenizer=pipe.tokenizer, text_encoder_2=pipe.text_encoder_2, tokenizer_2=pipe.tokenizer_2, torch_dtype=dtype, token=HF_TOKEN)
-            last_model = repo_id
-            last_cn_on = cn_on
-            progress(1, desc=f"Model loaded: {repo_id} / ControlNet Loaded: {controlnet_model_union_repo}")
-            print(f"Model loaded: {repo_id} / ControlNet Loaded: {controlnet_model_union_repo}")
-        else:
-            progress(0, desc=f"Loading model: {repo_id}")
-            print(f"Loading model: {repo_id}")
-            if ".safetensors" in repo_id:
-                safetensors_file = download_file_mod(repo_id)
-                transformer = FluxTransformer2DModel.from_single_file(safetensors_file, subfolder="transformer", torch_dtype=dtype, config=single_file_base_model, token=HF_TOKEN)
-                if CACHE_MODEL:
-                    pipe = DiffusionPipeline.from_pretrained(single_file_base_model, vae=taef1, transformer=transformer, text_encoder=pipe.text_encoder,
-                     tokenizer=pipe.tokenizer, text_encoder_2=pipe.text_encoder_2, tokenizer_2=pipe.tokenizer_2, torch_dtype=dtype, token=HF_TOKEN)
-                else: pipe = DiffusionPipeline.from_pretrained(single_file_base_model, transformer=transformer, torch_dtype=dtype, token=HF_TOKEN)
-                pipe_i2i = AutoPipelineForImage2Image.from_pretrained(single_file_base_model, vae=good_vae, transformer=pipe.transformer, text_encoder=pipe.text_encoder,
-                 tokenizer=pipe.tokenizer, text_encoder_2=pipe.text_encoder_2, tokenizer_2=pipe.tokenizer_2, torch_dtype=dtype, token=HF_TOKEN)
-                pipe_ip = AutoPipelineForInpainting.from_pretrained(single_file_base_model, vae=good_vae, transformer=pipe.transformer, text_encoder=pipe.text_encoder,
-                 tokenizer=pipe.tokenizer, text_encoder_2=pipe.text_encoder_2, tokenizer_2=pipe.tokenizer_2, torch_dtype=dtype, token=HF_TOKEN)
-            else:
-                if CACHE_MODEL:
-                    transformer = FluxTransformer2DModel.from_pretrained(repo_id, subfolder="transformer", torch_dtype=dtype, token=HF_TOKEN)
-                    pipe = DiffusionPipeline.from_pretrained(repo_id, vae=taef1, transformer=transformer, text_encoder=pipe.text_encoder,
-                     tokenizer=pipe.tokenizer, text_encoder_2=pipe.text_encoder_2, tokenizer_2=pipe.tokenizer_2, torch_dtype=dtype, token=HF_TOKEN)
-                else: pipe = DiffusionPipeline.from_pretrained(repo_id, torch_dtype=dtype, token=HF_TOKEN)
-                pipe_i2i = AutoPipelineForImage2Image.from_pretrained(repo_id, vae=good_vae, transformer=pipe.transformer, text_encoder=pipe.text_encoder,
-                 tokenizer=pipe.tokenizer, text_encoder_2=pipe.text_encoder_2, tokenizer_2=pipe.tokenizer_2, torch_dtype=dtype, token=HF_TOKEN)
-                pipe_ip = AutoPipelineForInpainting.from_pretrained(repo_id, vae=good_vae, transformer=pipe.transformer, text_encoder=pipe.text_encoder,
-                 tokenizer=pipe.tokenizer, text_encoder_2=pipe.text_encoder_2, tokenizer_2=pipe.tokenizer_2, torch_dtype=dtype, token=HF_TOKEN)
-            last_model = repo_id
-            last_cn_on = cn_on
-            progress(1, desc=f"Model loaded: {repo_id}")
-            print(f"Model loaded: {repo_id}")
+        pipe, pipe_i2i = load_pipeline(repo_id, cn_on, model_type, task, dtype_str, HF_TOKEN, progress)
+        last_model = repo_id
+        last_cn_on = cn_on
+        last_task = task
+        last_dtype_str = dtype_str
     except Exception as e:
         print(f"Model load Error: {repo_id} {e}")
         raise gr.Error(f"Model load Error: {repo_id} {e}") from e
-    finally:
-        if safetensors_file and Path(safetensors_file).exists(): Path(safetensors_file).unlink()
     return gr.update()
 
 change_base_model.zerogpu = True
@@ -441,7 +451,7 @@ def remove_custom_lora(selected_indices, current_loras, gallery):
 
 @spaces.GPU(duration=70)
 @torch.inference_mode()
-def generate_image(prompt_mash, steps, seed, cfg_scale, width, height, sigmas_factor, cn_on, progress=gr.Progress(track_tqdm=True)):
+def generate_image(prompt_mash: str, steps: int, seed: int, cfg_scale: float, width: int, height: int, sigmas_factor: float, cn_on: bool, progress=gr.Progress(track_tqdm=True)):
     global pipe, taef1, good_vae, controlnet, controlnet_union
     try:
         good_vae.to(device)
@@ -452,11 +462,11 @@ def generate_image(prompt_mash, steps, seed, cfg_scale, width, height, sigmas_fa
         with calculateDuration("Generating image"):
             # Generate image
             modes, images, scales = get_control_params()
-            if not cn_on or len(modes) == 0:
+            if not cn_on or len(modes) == 0: # without ControlNet
                 pipe.to(device)
                 pipe.vae = taef1
                 pipe.flux_pipe_call_that_returns_an_iterable_of_images = flux_pipe_call_that_returns_an_iterable_of_images.__get__(pipe)
-                progress(0, desc="Start Inference.")
+                print_progress("Start Inference.")
                 for img in pipe.flux_pipe_call_that_returns_an_iterable_of_images(
                     prompt=prompt_mash,
                     num_inference_steps=steps,
@@ -470,13 +480,13 @@ def generate_image(prompt_mash, steps, seed, cfg_scale, width, height, sigmas_fa
                     sigmas=sigmas,
                 ):
                     yield img
-            else:
+            else: # with ControlNet
                 pipe.to(device)
                 pipe.vae = good_vae
                 if controlnet_union is not None: controlnet_union.to(device)
                 if controlnet is not None: controlnet.to(device)
                 pipe.enable_model_cpu_offload()
-                progress(0, desc="Start Inference with ControlNet.")
+                print_progress("Start Inference with ControlNet.")
                 for img in pipe(
                     prompt=prompt_mash,
                     control_image=images,
@@ -497,122 +507,94 @@ def generate_image(prompt_mash, steps, seed, cfg_scale, width, height, sigmas_fa
 
 @spaces.GPU(duration=70)
 @torch.inference_mode()
-def generate_image_to_image(prompt_mash, image_input_path_dict, image_strength, is_inpaint, blur_mask, blur_factor, steps, cfg_scale, width, height,
-                            sigmas_factor, seed, cn_on, progress=gr.Progress(track_tqdm=True)):
-    global pipe_i2i, pipe_ip, good_vae, controlnet, controlnet_union
+def generate_image_to_image(prompt_mash: str, image_input_path_dict: dict, image_strength: float, task_type: str, blur_mask: bool, blur_factor: float,
+                            steps: int, cfg_scale: float, width: int, height: int, sigmas_factor: float, seed: int, cn_on: bool, progress=gr.Progress(track_tqdm=True)):
+    global pipe_i2i, good_vae, controlnet, controlnet_union
     try:
         good_vae.to(device)
         generator = torch.Generator(device=device).manual_seed(int(float(seed)))
         image_input_path = image_input_path_dict['background']
         mask_path = image_input_path_dict['layers'][0]
-        sigmas = calc_sigmas(steps, sigmas_factor)
+        is_mask = True if task_type == "Inpainting" or task_type == "Flux Fill" else False
+        is_fill = True if task_type == "Flux Fill" else False
+        is_depth = True if task_type == "Depth" else False
+        is_canny = True if task_type == "Canny" else False
+        kwargs = {}
+        if not is_fill: kwargs["strength"] = image_strength
+        if sigmas_factor < 1.0 and task_type != "Image-to-Image": kwargs["sigmas"] = calc_sigmas(steps, sigmas_factor)
 
         with calculateDuration("Generating image"):
             # Generate image
             modes, images, scales = get_control_params()
-            if not cn_on or len(modes) == 0:
-                if is_inpaint: # Inpainting
-                    pipe_ip.to(device)
-                    pipe_ip.vae = good_vae
-                    image_input = load_image(image_input_path)
+            if not cn_on or len(modes) == 0: # without ControlNet
+                pipe_i2i.to(device)
+                pipe_i2i.vae = good_vae
+                image_input = load_image(image_input_path)
+                if is_mask:
                     mask_input = load_image(mask_path)
-                    if blur_mask: mask_input = pipe_ip.mask_processor.blur(mask_input, blur_factor=blur_factor)
-                    progress(0, desc="Start Inpainting Inference.")
-                    final_image = pipe_ip(
-                        prompt=prompt_mash,
-                        image=image_input,
-                        mask_image=mask_input,
-                        strength=image_strength,
-                        num_inference_steps=steps,
-                        guidance_scale=cfg_scale,
-                        width=width,
-                        height=height,
-                        generator=generator,
-                        joint_attention_kwargs={"scale": 1.0},
-                        output_type="pil",
-                        #sigmas=sigmas,
-                    ).images[0]
-                    return final_image 
-                else:
-                    pipe_i2i.to(device)
-                    pipe_i2i.vae = good_vae
-                    image_input = load_image(image_input_path)
-                    progress(0, desc="Start I2I Inference.")
-                    final_image = pipe_i2i(
-                        prompt=prompt_mash,
-                        image=image_input,
-                        strength=image_strength,
-                        num_inference_steps=steps,
-                        guidance_scale=cfg_scale,
-                        width=width,
-                        height=height,
-                        generator=generator,
-                        joint_attention_kwargs={"scale": 1.0},
-                        output_type="pil",
-                        #sigmas=sigmas,
-                    ).images[0]
-                    return final_image 
-            else:
-                if is_inpaint: # Inpainting
-                    pipe_ip.to(device)
-                    pipe_ip.vae = good_vae
-                    image_input = load_image(image_input_path)
+                    if blur_mask: mask_input = pipe_i2i.mask_processor.blur(mask_input, blur_factor=blur_factor)
+                    kwargs["mask_image"] = mask_input
+                    if is_fill: print_progress("Start Flux Fill Inference.")
+                    else: print_progress("Start Inpainting Inference.")
+                elif is_canny:
+                    image_input = get_canny_image(image_input, height, width)
+                    print_progress("Start Canny Inference.")
+                elif is_depth:
+                    image_input = get_depth_image(image_input, height, width)
+                    print_progress("Start Depth Inference.")
+                else: print_progress("Start I2I Inference.")
+                final_image = pipe_i2i(
+                    prompt=prompt_mash,
+                    image=image_input,
+                    num_inference_steps=steps,
+                    guidance_scale=cfg_scale,
+                    #width=width,
+                    #height=height,
+                    generator=generator,
+                    joint_attention_kwargs={"scale": 1.0},
+                    output_type="pil",
+                    **kwargs,
+                ).images[0]
+                return final_image 
+            else: # with ControlNet
+                pipe_i2i.to(device)
+                pipe_i2i.vae = good_vae
+                image_input = load_image(image_input_path)
+                if controlnet_union is not None: controlnet_union.to(device)
+                if controlnet is not None: controlnet.to(device)
+                if is_mask:
                     mask_input = load_image(mask_path)
-                    if blur_mask: mask_input = pipe_ip.mask_processor.blur(mask_input, blur_factor=blur_factor)
-                    if controlnet_union is not None: controlnet_union.to(device)
-                    if controlnet is not None: controlnet.to(device)
-                    pipe_ip.enable_model_cpu_offload()
-                    progress(0, desc="Start Inpainting Inference with ControlNet.")
-                    final_image = pipe_ip(
-                        prompt=prompt_mash,
-                        control_image=images,
-                        control_mode=modes,
-                        image=image_input,
-                        mask_image=mask_input,
-                        strength=image_strength,
-                        num_inference_steps=steps,
-                        guidance_scale=cfg_scale,
-                        width=width,
-                        height=height,
-                        controlnet_conditioning_scale=scales,
-                        generator=generator,
-                        joint_attention_kwargs={"scale": 1.0},
-                        output_type="pil",
-                        #sigmas=sigmas,
-                    ).images[0]
-                    return final_image
-                else:
-                    pipe_i2i.to(device)
-                    pipe_i2i.vae = good_vae
-                    image_input = load_image(image_input_path['background'])
-                    if controlnet_union is not None: controlnet_union.to(device)
-                    if controlnet is not None: controlnet.to(device)
-                    pipe_i2i.enable_model_cpu_offload()
-                    progress(0, desc="Start I2I Inference with ControlNet.")
-                    final_image = pipe_i2i(
-                        prompt=prompt_mash,
-                        control_image=images,
-                        control_mode=modes,
-                        image=image_input,
-                        strength=image_strength,
-                        num_inference_steps=steps,
-                        guidance_scale=cfg_scale,
-                        width=width,
-                        height=height,
-                        controlnet_conditioning_scale=scales,
-                        generator=generator,
-                        joint_attention_kwargs={"scale": 1.0},
-                        output_type="pil",
-                        #sigmas=sigmas,
-                    ).images[0]
-                    return final_image
+                    if blur_mask: mask_input = pipe_i2i.mask_processor.blur(mask_input, blur_factor=blur_factor)
+                    kwargs["mask_image"] = mask_input
+                    if is_fill: print_progress("Start Flux Fill Inference with ControlNet.")
+                    else: print_progress("Start Inpainting Inference with ControlNet.")
+                else: print_progress("Start I2I Inference with ControlNet.")
+                pipe_i2i.enable_model_cpu_offload()
+                final_image = pipe_i2i(
+                    prompt=prompt_mash,
+                    control_image=images,
+                    control_mode=modes,
+                    image=image_input,
+                    num_inference_steps=steps,
+                    guidance_scale=cfg_scale,
+                    #width=width,
+                    #height=height,
+                    controlnet_conditioning_scale=scales,
+                    generator=generator,
+                    joint_attention_kwargs={"scale": 1.0},
+                    output_type="pil",
+                    **kwargs,
+                ).images[0]
+                return final_image
     except Exception as e:
         print(e)
         raise gr.Error(f"I2I Inference Error: {e}") from e
 
-def run_lora(prompt, image_input, image_strength, task_type, blur_mask, blur_factor, cfg_scale, steps, selected_indices, lora_scale_1, lora_scale_2,
-             randomize_seed, seed, width, height, sigmas_factor, loras_state, lora_json, cn_on, translate_on, progress=gr.Progress(track_tqdm=True)):
-    global pipe, pipe_i2i, pipe_ip
+def run_lora(prompt: str, image_input: dict, image_strength: float, task_type: str, turbo_mode: str, blur_mask: bool, blur_factor: float,
+             cfg_scale: float, steps: int, selected_indices, lora_scale_1: float, lora_scale_2: float,
+             randomize_seed: bool, seed: int, width: int, height: int, sigmas_factor: float, loras_state,
+             lora_json, cn_on: bool, translate_on: bool, progress=gr.Progress(track_tqdm=True)):
+    global pipe, pipe_i2i
     if not selected_indices and not is_valid_lora(lora_json):
         gr.Info("LoRA isn't selected.")
     #    raise gr.Error("You must select a LoRA before proceeding.")
@@ -620,16 +602,8 @@ def run_lora(prompt, image_input, image_strength, task_type, blur_mask, blur_fac
 
     selected_loras = [loras_state[idx] for idx in selected_indices]
 
-    if task_type == "Inpainting":
-        is_inpaint = True
-        is_i2i = True
-    elif task_type == "Image-to-Image":
-        is_inpaint = False
-        is_i2i = True
-    else: # "Text-to-Image"
-        is_inpaint = False
-        is_i2i = False
-
+    if task_type in set(TASK_TYPE_I2I): is_i2i = True
+    else: is_i2i = False
     if translate_on: prompt = translate_to_en(prompt)
 
     # Build the prompt with trigger words
@@ -651,7 +625,6 @@ def run_lora(prompt, image_input, image_strength, task_type, blur_mask, blur_fac
 
     print(pipe.get_active_adapters()) #
     print(pipe_i2i.get_active_adapters()) #
-    print(pipe_ip.get_active_adapters()) #
 
     clear_cache() #
 
@@ -659,13 +632,16 @@ def run_lora(prompt, image_input, image_strength, task_type, blur_mask, blur_fac
     prompt_mash = prompt_mash + get_model_trigger(last_model)
     lora_names = []
     lora_weights = []
-    if is_valid_lora(lora_json): # Load External LoRA weights
+    # Load Turbo LoRA weights
+    if turbo_mode != "None":
+        if is_i2i: pipe_i2i, lora_names, lora_weights, steps = turbo_loras(pipe_i2i, turbo_mode, lora_names, lora_weights)
+        else: pipe, lora_names, lora_weights, steps = turbo_loras(pipe, turbo_mode, lora_names, lora_weights)
+
+    # Load External LoRA weights
+    if is_valid_lora(lora_json):
         with calculateDuration("Loading External LoRA weights"):
-            if is_inpaint:
-                pipe_ip, lora_names, lora_weights = fuse_loras(pipe_ip, lora_json)
-            elif is_i2i:
-                pipe_i2i, lora_names, lora_weights = fuse_loras(pipe_i2i, lora_json)
-            else: pipe, lora_names, lora_weights = fuse_loras(pipe, lora_json)
+            if is_i2i: pipe_i2i, lora_names, lora_weights = fuse_loras(pipe_i2i, lora_json, lora_names, lora_weights)
+            else: pipe, lora_names, lora_weights = fuse_loras(pipe, lora_json, lora_names, lora_weights)
             trigger_word = get_trigger_word(lora_json)
             prompt_mash = f"{prompt_mash} {trigger_word}"
     print("Prompt Mash: ", prompt_mash) #
@@ -681,15 +657,7 @@ def run_lora(prompt, image_input, image_strength, task_type, blur_mask, blur_fac
                 lora_path = lora['repo']
                 weight_name = lora.get("weights")
                 print(f"Lora Path: {lora_path}")
-                if is_inpaint:
-                    pipe_ip.load_lora_weights(
-                        lora_path, 
-                        weight_name=weight_name if weight_name else None,
-                        low_cpu_mem_usage=False,
-                        adapter_name=lora_name,
-                        token=HF_TOKEN
-                    )
-                elif is_i2i:
+                if is_i2i:
                     pipe_i2i.load_lora_weights(
                         lora_path, 
                         weight_name=weight_name if weight_name else None,
@@ -707,16 +675,11 @@ def run_lora(prompt, image_input, image_strength, task_type, blur_mask, blur_fac
                     )
             print("Loaded LoRAs:", lora_names)
     if selected_indices or is_valid_lora(lora_json):
-        if is_inpaint:
-            pipe_ip.set_adapters(lora_names, adapter_weights=lora_weights)
-        elif is_i2i:
-            pipe_i2i.set_adapters(lora_names, adapter_weights=lora_weights)
-        else:
-            pipe.set_adapters(lora_names, adapter_weights=lora_weights)
+        if is_i2i: pipe_i2i.set_adapters(lora_names, adapter_weights=lora_weights)
+        else: pipe.set_adapters(lora_names, adapter_weights=lora_weights)
 
     print(pipe.get_active_adapters()) #
     print(pipe_i2i.get_active_adapters()) #
-    print(pipe_ip.get_active_adapters()) #
 
     # Set random seed for reproducibility
     with calculateDuration("Randomizing seed"):
@@ -726,7 +689,7 @@ def run_lora(prompt, image_input, image_strength, task_type, blur_mask, blur_fac
     # Generate image
     progress(0, desc="Running Inference.")
     if is_i2i:
-        final_image = generate_image_to_image(prompt_mash, image_input, image_strength, is_inpaint, blur_mask, blur_factor,
+        final_image = generate_image_to_image(prompt_mash, image_input, image_strength, task_type, blur_mask, blur_factor,
                                               steps, cfg_scale, width, height, sigmas_factor, seed, cn_on)
         yield save_image(final_image, None, last_model, prompt_mash, height, width, steps, cfg_scale, seed), seed, gr.update(visible=False)
     else:
@@ -898,7 +861,6 @@ with gr.Blocks(theme='NoCrypt/miku@>=1.2.2', fill_width=True, css=css, delete_ca
                                                  show_share_button=False, show_download_button=True)
                     history_clear_button = gr.Button(value="Clear History", variant="secondary")
                     history_clear_button.click(lambda: ([], []), None, [history_gallery, history_files], queue=False, show_api=False)
-
         with gr.Row():
             with gr.Accordion("Advanced Settings", open=True):
                 with gr.Tab("Generation Settings"):
@@ -906,9 +868,9 @@ with gr.Blocks(theme='NoCrypt/miku@>=1.2.2', fill_width=True, css=css, delete_ca
                         with gr.Group():
                             with gr.Row():
                                 model_name = gr.Dropdown(label="Base Model", info="You can enter a huggingface model repo_id or path of single safetensors file to want to use.",
-                                                        choices=models, value=models[0], allow_custom_value=True, min_width=320, scale=5)
+                                                         choices=models, value=models[0], allow_custom_value=True, min_width=320, scale=5)
                                 model_type = gr.Radio(label="Model type", info="Model type of single safetensors file",
-                                                    choices=list(single_file_base_models.keys()), value=list(single_file_base_models.keys())[0], scale=1)
+                                                      choices=list(single_file_base_models.keys()), value=list(single_file_base_models.keys())[0], scale=1)
                             model_info = gr.Markdown(elem_classes="info")
                         with gr.Row():
                             width = gr.Slider(label="Width", minimum=256, maximum=1536, step=64, value=1024)
@@ -918,17 +880,21 @@ with gr.Blocks(theme='NoCrypt/miku@>=1.2.2', fill_width=True, css=css, delete_ca
                         with gr.Row():
                             randomize_seed = gr.Checkbox(True, label="Randomize seed")
                             seed = gr.Slider(label="Seed", minimum=0, maximum=MAX_SEED, step=1, value=0, randomize=True)
+                        with gr.Row():
+                            turbo_mode = gr.Radio(label="Turbo mode", choices=["None", "Hyper-FLUX.1-dev-8steps", "Hyper-FLUX.1-dev-16steps", "FLUX.1-Turbo-Alpha 8-steps"], value="None")
                             sigmas_factor = gr.Slider(label="Sigmas factor", minimum=0.01, maximum=1.00, step=0.01, value=0.95)
+                            data_type = gr.Radio(label="Data type", choices=["BF16"], value="BF16", visible=False)
                             disable_model_cache = gr.Checkbox(False, label="Disable model caching")
                 with gr.Tab("Image-to-Image"):
                     with gr.Row():
                         with gr.Column():
                             #input_image = gr.Image(label="Input image", type="filepath", height=256, sources=["upload", "clipboard"], show_share_button=False)
                             input_image = gr.ImageEditor(label='Input image', type='filepath', sources=["upload", "clipboard"], image_mode='RGB', show_share_button=False, show_fullscreen_button=False,
-                                                        layers=False, brush=gr.Brush(colors=["white"], color_mode="fixed", default_size=32), eraser=gr.Eraser(default_size="32"), value=None,
-                                                        canvas_size=(384, 384), width=384, height=512)
+                                                         layers=False, brush=gr.Brush(colors=["white"], color_mode="fixed", default_size=32), eraser=gr.Eraser(default_size="32"), value=None,
+                                                         canvas_size=(384, 384), width=384, height=512)
                         with gr.Column():
-                            task_type = gr.Radio(label="Task", choices=["Text-to-Image", "Image-to-Image", "Inpainting"], value="Text-to-Image")
+
+                            task_type = gr.Radio(label="Task", choices=TASK_TYPE_T2I+TASK_TYPE_I2I, value=TASK_TYPE_T2I[0])
                             image_strength = gr.Slider(label="Strength", info="Lower means more image influence in I2I, opposite in Inpaint", minimum=0.01, maximum=1.0, step=0.01, value=0.75)
                             blur_mask = gr.Checkbox(label="Blur mask", value=False)
                             blur_factor = gr.Slider(label="Blur factor", minimum=0, maximum=50, step=1, value=33)
@@ -1033,27 +999,23 @@ with gr.Blocks(theme='NoCrypt/miku@>=1.2.2', fill_width=True, css=css, delete_ca
     gr.on(
         triggers=[generate_button.click, prompt.submit],
         fn=change_base_model,
-        inputs=[model_name, cn_on, disable_model_cache, model_type],
+        inputs=[model_name, cn_on, disable_model_cache, model_type, task_type, data_type],
         outputs=[result],
         queue=True,
         show_api=False,
         trigger_mode="once",
     ).success(
         fn=run_lora,
-        inputs=[prompt, input_image, image_strength, task_type, blur_mask, blur_factor, cfg_scale, steps, selected_indices, lora_scale_1, lora_scale_2,
+        inputs=[prompt, input_image, image_strength, task_type, turbo_mode, blur_mask, blur_factor, cfg_scale, steps, selected_indices, lora_scale_1, lora_scale_2,
                 randomize_seed, seed, width, height, sigmas_factor, loras_state, lora_repo_json, cn_on, auto_trans], 
         outputs=[result, seed, progress_bar],
         queue=True,
         show_api=True,
-    #).then(  # Update the history gallery
-    #    fn=lambda x, history: update_history(x, history),
-    #    inputs=[result, history_gallery],
-    #    outputs=history_gallery,
     ).success(save_image_history, [result, history_gallery, history_files, model_name], [history_gallery, history_files], queue=False, show_api=False)
 
     input_image.clear(lambda: gr.update(value="Text-to-Image"), None, [task_type], queue=False, show_api=False)
-    input_image.upload(preprocess_i2i_image, [input_image, input_image_preprocess, height, width], [input_image], queue=False, show_api=False)\
-    .success(lambda: gr.update(value="Image-to-Image"), None, [task_type], queue=False, show_api=False)
+    input_image.upload(preprocess_i2i_image, [input_image, input_image_preprocess, height, width], [input_image], queue=False, show_api=False)#\
+    #.success(lambda: gr.update(value="Image-to-Image"), None, [task_type], queue=False, show_api=False)
     gr.on(
         triggers=[model_name.change, cn_on.change],
         fn=get_t2i_model_info,
@@ -1062,7 +1024,7 @@ with gr.Blocks(theme='NoCrypt/miku@>=1.2.2', fill_width=True, css=css, delete_ca
         queue=False,
         show_api=False,
         trigger_mode="once",
-    )#.then(change_base_model, [model_name, cn_on, disable_model_cache, model_type], [result], queue=True, show_api=False)
+    )
     prompt_enhance.click(enhance_prompt, [prompt], [prompt], queue=False, show_api=False)
 
     gr.on(

env.py

@@ -10,9 +10,8 @@ num_loras = 3
 num_cns = 2
 
 
-models = [
+models_dev = [
     "camenduru/FLUX.1-dev-diffusers",
-    "black-forest-labs/FLUX.1-schnell",
     "sayakpaul/FLUX.1-merged",
     "ostris/OpenFLUX.1",
     "multimodalart/FLUX.1-dev2pro-full",
@@ -63,11 +62,27 @@ models = [
     "John6666/xe-asian-flux-01-fp8-flux",
     "John6666/fluxescore-dev-v10fp16-fp8-flux",
     "John6666/2758-flux-asian-utopian-v30fp8noclip-fp8-flux",
+    "trongg/FLUX_dev2pro_nsfw_context_lora",
     "https://huggingface.co/StableDiffusionVN/SDVN11-Ghibli-Flux/blob/main/SDVN11-Ghibli-Flux_fp8-hyper.safetensors",
     "https://huggingface.co/datasets/John6666/flux1-backup-202410/blob/main/iniverseMixXLSFWNSFW_f1dFP16V10.safetensors",
+    "https://huggingface.co/city96/FLUX.1-dev-gguf/blob/main/flux1-dev-Q2_K.gguf",
     # "",
 ]
 
+models_schnell = [
+    "black-forest-labs/FLUX.1-schnell",
+]
+
+models_fill = ["fuliucansheng/FLUX.1-Fill-dev-diffusers"]
+
+models_dedistill = []
+
+models_canny = ["sayakpaul/FLUX.1-Canny-dev-nf4"]
+
+models_depth = ["sayakpaul/FLUX.1-Depth-dev-nf4"]
+
+models = models_dev + models_schnell + models_fill
+
 model_trigger = {
     "Raelina/Raemu-Flux": "anime",
     "John6666/raemu-flux-v10-fp8-flux": "anime",
@@ -77,6 +92,7 @@ model_trigger = {
 single_file_base_models = {
     "dev": "camenduru/FLUX.1-dev-diffusers",
     "schnell": "black-forest-labs/FLUX.1-schnell",
+    "fill": "fuliucansheng/FLUX.1-Fill-dev-diffusers",
 }
 
 # List all Models for specified user

mod.py

@@ -8,7 +8,7 @@ import subprocess
 import os
 import re
 from translatepy import Translator
-from huggingface_hub import HfApi
+from huggingface_hub import HfApi, hf_hub_download
 from env import num_cns, model_trigger, HF_TOKEN, CIVITAI_API_KEY, DOWNLOAD_LORA_LIST, DIRECTORY_LORAS
 from modutils import download_things
 
@@ -232,6 +232,14 @@ def set_control_union_image(i: int, mode: str, image: Image.Image | None, height
     return control_images[i]
 
 
+def get_canny_image(image: Image.Image, height: int, width: int):
+    return preprocess_image(image, "canny", height, width, 384)
+
+
+def get_depth_image(image: Image.Image, height: int, width: int):
+    return preprocess_image(image, "depth_midas", height, width, 384)
+
+
 def preprocess_i2i_image(image_path_dict: dict, is_preprocess: bool, height: int, width: int):
     try:
         if not is_preprocess: return gr.update()
@@ -275,24 +283,22 @@ def get_model_trigger(model_name: str):
 
 # https://huggingface.co/docs/diffusers/v0.23.1/en/api/loaders#diffusers.loaders.LoraLoaderMixin.fuse_lora
 # https://github.com/huggingface/diffusers/issues/4919
-def fuse_loras(pipe, lorajson: list[dict]):
+def fuse_loras(pipe, lorajson: list[dict], a_list: list, w_list: list):
     try:
-        if not lorajson or not isinstance(lorajson, list): return pipe, [], []
-        a_list = []
-        w_list = []
+        if not lorajson or not isinstance(lorajson, list): return pipe, a_list, w_list
         for d in lorajson:
             if not d or not isinstance(d, dict) or not d["name"] or d["name"] == "None": continue
             k = d["name"]
             if is_repo_name(k) and is_repo_exists(k):
                 a_name = Path(k).stem
-                pipe.load_lora_weights(k, weight_name=d["filename"], adapter_name = a_name, low_cpu_mem_usage=True)
+                pipe.load_lora_weights(k, weight_name=d["filename"], adapter_name = a_name, low_cpu_mem_usage=False)
             elif not Path(k).exists():
                 print(f"LoRA not found: {k}")
                 continue
             else:
                 w_name = Path(k).name
                 a_name = Path(k).stem
-                pipe.load_lora_weights(k, weight_name = w_name, adapter_name = a_name, low_cpu_mem_usage=True)
+                pipe.load_lora_weights(k, weight_name = w_name, adapter_name = a_name, low_cpu_mem_usage=False)
             a_list.append(a_name)
             w_list.append(d["scale"])
         if not a_list: return pipe, [], []
@@ -305,6 +311,25 @@ def fuse_loras(pipe, lorajson: list[dict]):
         raise Exception(f"External LoRA Error: {e}") from e
 
 
+def turbo_loras(pipe, turbo_mode: str, lora_names: list, lora_weights: list):
+    if turbo_mode == "Hyper-FLUX.1-dev-8steps":
+        lora_names.append("Hyper-FLUX1-dev-8steps")
+        lora_weights.append(0.125)
+        pipe.load_lora_weights(hf_hub_download("ByteDance/Hyper-SD", "Hyper-FLUX.1-dev-8steps-lora.safetensors"), adapter_name=lora_names[-1], low_cpu_mem_usage=False)
+        steps = 8
+    elif turbo_mode == "Hyper-FLUX.1-dev-16steps":
+        lora_names.append("Hyper-FLUX1-dev-16steps")
+        lora_weights.append(0.125)
+        pipe.load_lora_weights(hf_hub_download("ByteDance/Hyper-SD", "Hyper-FLUX.1-dev-16steps-lora.safetensors"), adapter_name=lora_names[-1], low_cpu_mem_usage=False)
+        steps = 16
+    elif turbo_mode == "FLUX.1-Turbo-Alpha 8-steps":
+        lora_names.append("FLUX1-Turbo-Alpha 8-steps")
+        lora_weights.append(1.0)
+        pipe.load_lora_weights("alimama-creative/FLUX.1-Turbo-Alpha", adapter_name=lora_names[-1], low_cpu_mem_usage=False)
+        steps = 8
+    return pipe, lora_names, lora_weights, steps
+
+
 def description_ui():
     gr.Markdown(
         """

requirements.txt

@@ -16,4 +16,7 @@ deepspeed
 mediapipe
 openai>=1.37.0
 translatepy
-unidecode
+unidecode
+optimum-quanto
+bitsandbytes
+gguf