utils.py

import gradio as gr
from huggingface_hub import HfApi, HfFolder, hf_hub_download, snapshot_download
import os
from pathlib import Path
import shutil
import gc
import re
import urllib.parse
import subprocess
import time


def get_token():
    try:
        token = HfFolder.get_token()
    except Exception:
        token = ""
    return token


def set_token(token):
    try:
        HfFolder.save_token(token)
    except Exception:
        print(f"Error: Failed to save token.")


def get_user_agent():
    return 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:127.0) Gecko/20100101 Firefox/127.0'


def is_repo_exists(repo_id: str, repo_type: str="model"):
    hf_token = get_token()
    api = HfApi(token=hf_token)
    try:
        if api.repo_exists(repo_id=repo_id, repo_type=repo_type, token=hf_token): return True
        else: return False
    except Exception as e:
        print(f"Error: Failed to connect {repo_id} ({repo_type}). {e}")
        return True # for safe


MODEL_TYPE_CLASS = {
    "diffusers:StableDiffusionPipeline": "SD 1.5",
    "diffusers:StableDiffusionXLPipeline": "SDXL",
    "diffusers:FluxPipeline": "FLUX",
}


def get_model_type(repo_id: str):
    hf_token = get_token()
    api = HfApi(token=hf_token)
    lora_filename = "pytorch_lora_weights.safetensors"
    diffusers_filename = "model_index.json"
    default = "SDXL"
    try:
        if api.file_exists(repo_id=repo_id, filename=lora_filename, token=hf_token): return "LoRA"
        if not api.file_exists(repo_id=repo_id, filename=diffusers_filename, token=hf_token): return "None"
        model = api.model_info(repo_id=repo_id, token=hf_token)
        tags = model.tags
        for tag in tags:
            if tag in MODEL_TYPE_CLASS.keys(): return MODEL_TYPE_CLASS.get(tag, default)
    except Exception:
        return default
    return default


def list_uniq(l):
    return sorted(set(l), key=l.index)


def list_sub(a, b):
    return [e for e in a if e not in b]


def is_repo_name(s):
    return re.fullmatch(r'^[\w_\-\.]+/[\w_\-\.]+$', s)


def get_hf_url(repo_id: str, repo_type: str="model"):
    if repo_type == "dataset": url = f"https://huggingface.co/datasets/{repo_id}"
    elif repo_type == "space": url = f"https://huggingface.co/spaces/{repo_id}"
    else: url = f"https://huggingface.co/{repo_id}"
    return url


def split_hf_url(url: str):
    try:
        s = list(re.findall(r'^(?:https?://huggingface.co/)(?:(datasets|spaces)/)?(.+?/.+?)/\w+?/.+?/(?:(.+)/)?(.+?.\w+)(?:\?download=true)?$', url)[0])
        if len(s) < 4: return "", "", "", ""
        repo_id = s[1]
        if s[0] == "datasets": repo_type = "dataset"
        elif s[0] == "spaces": repo_type = "space"
        else: repo_type = "model"
        subfolder = urllib.parse.unquote(s[2]) if s[2] else None
        filename = urllib.parse.unquote(s[3])
        return repo_id, filename, subfolder, repo_type
    except Exception as e:
        print(e)


def download_hf_file(directory, url, progress=gr.Progress(track_tqdm=True)):
    hf_token = get_token()
    repo_id, filename, subfolder, repo_type = split_hf_url(url)
    try:
        print(f"Downloading {url} to {directory}")
        if subfolder is not None: path = hf_hub_download(repo_id=repo_id, filename=filename, subfolder=subfolder, repo_type=repo_type, local_dir=directory, token=hf_token)
        else: path = hf_hub_download(repo_id=repo_id, filename=filename, repo_type=repo_type, local_dir=directory, token=hf_token)
        return path
    except Exception as e:
        print(f"Failed to download: {e}")
        return None


def download_thing(directory, url, civitai_api_key="", progress=gr.Progress(track_tqdm=True)): # requires aria2, gdown
    url = url.strip()
    if "drive.google.com" in url:
        original_dir = os.getcwd()
        os.chdir(directory)
        os.system(f"gdown --fuzzy {url}")
        os.chdir(original_dir)
    elif "huggingface.co" in url:
        url = url.replace("?download=true", "")
        if "/blob/" in url: url = url.replace("/blob/", "/resolve/")
        download_hf_file(directory, url)
    elif "civitai.com" in url:
        if "?" in url:
            url = url.split("?")[0]
        if civitai_api_key:
            url = url + f"?token={civitai_api_key}"
            os.system(f"aria2c --console-log-level=error --summary-interval=10 -c -x 16 -k 1M -s 16 -d {directory} {url}")
        else:
            print("You need an API key to download Civitai models.")
    else:
        os.system(f"aria2c --console-log-level=error --summary-interval=10 -c -x 16 -k 1M -s 16 -d {directory} {url}")


def get_local_file_list(dir_path):
    file_list = []
    for file in Path(dir_path).glob("**/*.*"):
        if file.is_file():
            file_path = str(file)
            file_list.append(file_path)
    return file_list


def get_download_file(temp_dir, url, civitai_key, progress=gr.Progress(track_tqdm=True)):
    if not "http" in url and is_repo_name(url) and not Path(url).exists():
        print(f"Use HF Repo: {url}")
        new_file = url
    elif not "http" in url and Path(url).exists():
        print(f"Use local file: {url}")
        new_file = url
    elif Path(f"{temp_dir}/{url.split('/')[-1]}").exists():
        print(f"File to download alreday exists: {url}")
        new_file = f"{temp_dir}/{url.split('/')[-1]}"
    else:
        print(f"Start downloading: {url}")
        before = get_local_file_list(temp_dir)
        try:
            download_thing(temp_dir, url.strip(), civitai_key)
        except Exception:
            print(f"Download failed: {url}")
            return ""
        after = get_local_file_list(temp_dir)
        new_file = list_sub(after, before)[0] if list_sub(after, before) else ""
    if not new_file:
        print(f"Download failed: {url}")
        return ""
    print(f"Download completed: {url}")
    return new_file


def download_repo(repo_id: str, dir_path: str, progress=gr.Progress(track_tqdm=True)): # for diffusers repo
    hf_token = get_token()
    try:
        snapshot_download(repo_id=repo_id, local_dir=dir_path, token=hf_token, allow_patterns=["*.safetensors", "*.bin"],
                          ignore_patterns=["*.fp16.*", "/*.safetensors", "/*.bin"], force_download=True)
        return True
    except Exception as e:
        print(f"Error: Failed to download {repo_id}. {e}")
        gr.Warning(f"Error: Failed to download {repo_id}. {e}")
        return False


def upload_repo(repo_id: str, dir_path: str, is_private: bool, progress=gr.Progress(track_tqdm=True)): # for diffusers repo
    hf_token = get_token()
    api = HfApi(token=hf_token)
    try:
        progress(0, desc="Start uploading...")
        api.create_repo(repo_id=repo_id, token=hf_token, private=is_private, exist_ok=True)
        for path in Path(dir_path).glob("*"):
            if path.is_dir():
                api.upload_folder(repo_id=repo_id, folder_path=str(path), path_in_repo=path.name, token=hf_token)
            elif path.is_file():
                api.upload_file(repo_id=repo_id, path_or_fileobj=str(path), path_in_repo=path.name, token=hf_token)
        progress(1, desc="Uploaded.")
        return get_hf_url(repo_id, "model")
    except Exception as e:
        print(f"Error: Failed to upload to {repo_id}. {e}")
        return ""


HF_SUBFOLDER_NAME = ["None", "user_repo"]


def duplicate_hf_repo(src_repo: str, dst_repo: str, src_repo_type: str, dst_repo_type: str,

                      is_private: bool, subfolder_type: str=HF_SUBFOLDER_NAME[1], progress=gr.Progress(track_tqdm=True)):
    hf_token = get_token()
    api = HfApi(token=hf_token)
    try:
        if subfolder_type == "user_repo": subfolder = src_repo.replace("/", "_")
        else: subfolder = ""
        progress(0, desc="Start duplicating...")
        api.create_repo(repo_id=dst_repo, repo_type=dst_repo_type, private=is_private, exist_ok=True, token=hf_token)
        for path in api.list_repo_files(repo_id=src_repo, repo_type=src_repo_type, token=hf_token):
            file = hf_hub_download(repo_id=src_repo, filename=path, repo_type=src_repo_type, token=hf_token)
            if not Path(file).exists(): continue
            if Path(file).is_dir(): # unused for now
                api.upload_folder(repo_id=dst_repo, folder_path=file, path_in_repo=f"{subfolder}/{path}" if subfolder else path,
                                  repo_type=dst_repo_type, token=hf_token)
            elif Path(file).is_file():
                api.upload_file(repo_id=dst_repo, path_or_fileobj=file, path_in_repo=f"{subfolder}/{path}" if subfolder else path,
                                repo_type=dst_repo_type, token=hf_token)
            if Path(file).exists(): Path(file).unlink()
        progress(1, desc="Duplicated.")
        return f"{get_hf_url(dst_repo, dst_repo_type)}/tree/main/{subfolder}" if subfolder else get_hf_url(dst_repo, dst_repo_type)
    except Exception as e:
        print(f"Error: Failed to duplicate repo {src_repo} to {dst_repo}. {e}")
        return ""


BASE_DIR = str(Path(__file__).resolve().parent.resolve())
CIVITAI_API_KEY = os.environ.get("CIVITAI_API_KEY")


def get_file(url: str, path: str): # requires aria2, gdown
    print(f"Downloading {url} to {path}...")
    get_download_file(path, url, CIVITAI_API_KEY)


def git_clone(url: str, path: str, pip: bool=False, addcmd: str=""): # requires git
    os.makedirs(str(Path(BASE_DIR, path)), exist_ok=True)
    os.chdir(Path(BASE_DIR, path))
    print(f"Cloning {url} to {path}...")
    cmd = f'git clone {url}'
    print(f'Running {cmd} at {Path.cwd()}')
    i = subprocess.run(cmd, shell=True).returncode
    if i != 0: print(f'Error occured at running {cmd}')
    p = url.split("/")[-1]
    if not Path(p).exists: return
    if pip:
        os.chdir(Path(BASE_DIR, path, p))
        cmd = f'pip install -r requirements.txt'
        print(f'Running {cmd} at {Path.cwd()}')
        i = subprocess.run(cmd, shell=True).returncode
        if i != 0: print(f'Error occured at running {cmd}')
    if addcmd:
        os.chdir(Path(BASE_DIR, path, p))
        cmd = addcmd
        print(f'Running {cmd} at {Path.cwd()}')
        i = subprocess.run(cmd, shell=True).returncode
        if i != 0: print(f'Error occured at running {cmd}')


def run(cmd: str, timeout: float=0):
    print(f'Running {cmd} at {Path.cwd()}')
    if timeout == 0:
        i = subprocess.run(cmd, shell=True).returncode
        if i != 0: print(f'Error occured at running {cmd}')
    else:
        p = subprocess.Popen(cmd, shell=True)
        time.sleep(timeout)
        p.terminate()
        print(f'Terminated in {timeout} seconds')