Upload 8 files

README.md

@@ -4,7 +4,7 @@ emoji: 🐶
 colorFrom: yellow
 colorTo: red
 sdk: gradio
-sdk_version: 4.38.1
+sdk_version: 4.44.0
 app_file: app.py
 pinned: false
 ---

app.py

@@ -1,30 +1,40 @@
 import gradio as gr
 import os
-from convert_repo_to_safetensors_sd_gr import convert_repo_to_safetensors_multi_sd
+from convert_repo_to_safetensors_sd_gr import convert_repo_to_safetensors_multi_sd, clear_safetensors
 os.environ['HF_OUTPUT_REPO'] = 'John6666/safetensors_converting_test'
 
 css = """"""
 
-with gr.Blocks(theme="NoCrypt/miku@>=1.2.2", css=css) as demo:
+with gr.Blocks(theme="NoCrypt/miku@>=1.2.2", fill_width=True, css=css, delete_cache=(60, 3600)) as demo:
     gr.Markdown(
             f"""
     - [A CLI version of this tool is available here](https://huggingface.co/spaces/John6666/convert_repo_to_safetensors_sd/tree/main/local).
     """)
     with gr.Column(): 
         repo_id = gr.Textbox(label="Repo ID", placeholder="author/model", value="", lines=1)
-        is_half = gr.Checkbox(label="Half precision", value=True)
         is_upload = gr.Checkbox(label="Upload safetensors to HF Repo", info="Fast download, but files will be public.", value=False)
-        uploaded_urls = gr.CheckboxGroup(visible=False, choices=[], value=None)
+        with gr.Accordion("Advanced", open=False):
+            dtype = gr.Radio(label="Output data type", choices=["fp16", "fp32", "bf16", "default"], value="fp16")
+            with gr.Row():
+                hf_token = gr.Textbox(label="Your HF write token", placeholder="hf_...", value="", max_lines=1)
+                gr.Markdown("Your token is available at [hf.co/settings/tokens](https://huggingface.co/settings/tokens).")
+            with gr.Row():
+                newrepo_id = gr.Textbox(label="Upload repo ID", placeholder="yourid/newrepo", value="", max_lines=1)
+                newrepo_type = gr.Radio(label="Upload repo type", choices=["model", "dataset"], value="model")
+                is_private = gr.Checkbox(label="Create / Use private repo", value=True)
+        uploaded_urls = gr.CheckboxGroup(visible=False, choices=[], value=None) # hidden
         run_button = gr.Button(value="Convert")
         st_file = gr.Files(label="Output", interactive=False)
         st_md = gr.Markdown()
+        delete_button = gr.Button(value="Delete Safetensors")
 
     gr.on(
         triggers=[repo_id.submit, run_button.click],
         fn=convert_repo_to_safetensors_multi_sd,
-        inputs=[repo_id, st_file, is_upload, uploaded_urls, is_half],
+        inputs=[repo_id, hf_token, st_file, uploaded_urls, dtype, is_upload, newrepo_id, newrepo_type, is_private],
         outputs=[st_file, uploaded_urls, st_md],
     )
+    delete_button.click(clear_safetensors, None, [st_file], queue=False, show_api=False)
 
 demo.queue()
 demo.launch()

convert_repo_to_safetensors_sd.py

@@ -281,7 +281,7 @@ def convert_text_enc_state_dict(text_enc_dict):
     return text_enc_dict
 
 
-def convert_diffusers_to_safetensors(model_path, checkpoint_path, half = True):
+def convert_diffusers_to_safetensors(model_path, checkpoint_path, dtype="fp16"):
     # Path for safetensors
     unet_path = osp.join(model_path, "unet", "diffusion_pytorch_model.safetensors")
     vae_path = osp.join(model_path, "vae", "diffusion_pytorch_model.safetensors")
@@ -328,8 +328,10 @@ def convert_diffusers_to_safetensors(model_path, checkpoint_path, half = True):
 
     # Put together new checkpoint
     state_dict = {**unet_state_dict, **vae_state_dict, **text_enc_dict}
-    if half:
-        state_dict = {k: v.half() for k, v in state_dict.items()}
+    
+    if dtype == "fp16": state_dict = {k: v.half() for k, v in state_dict.items()}
+    elif dtype == "fp32": state_dict = {k: v.to(torch.float32) for k, v in state_dict.items()}
+    elif dtype == "bf16": state_dict = {k: v.to(torch.bfloat16) for k, v in state_dict.items()}
 
     save_file(state_dict, checkpoint_path)
 
@@ -343,11 +345,11 @@ def download_repo(repo_id, dir_path):
         return
 
 
-def convert_repo_to_safetensors(repo_id, half = True):
+def convert_repo_to_safetensors(repo_id, dtype="fp16"):
     download_dir = f"{repo_id.split('/')[0]}_{repo_id.split('/')[-1]}"
     output_filename = f"{repo_id.split('/')[0]}_{repo_id.split('/')[-1]}.safetensors"
     download_repo(repo_id, download_dir)
-    convert_diffusers_to_safetensors(download_dir, output_filename, half)
+    convert_diffusers_to_safetensors(download_dir, output_filename, dtype)
     return output_filename
 
 
@@ -355,9 +357,9 @@ if __name__ == "__main__":
     parser = argparse.ArgumentParser()
 
     parser.add_argument("--repo_id", default=None, type=str, required=True, help="HF Repo ID of the model to convert.")
-    parser.add_argument("--half", default=True, help="Save weights in half precision.")
+    parser.add_argument("--dtype", default="fp16", type=str, choices=["fp16", "fp32", "bf16", "default"], help='Output data type. (Default: "fp16")')
 
     args = parser.parse_args()
     assert args.repo_id is not None, "Must provide a Repo ID!"
 
-    convert_repo_to_safetensors(args.repo_id, args.half)
+    convert_repo_to_safetensors(args.repo_id, args.dtype)

convert_repo_to_safetensors_sd_gr.py

@@ -10,6 +10,12 @@ import torch
 from safetensors.torch import load_file, save_file
 import gradio as gr
 
+from huggingface_hub import HfApi, HfFolder, hf_hub_url, snapshot_download
+import os
+from pathlib import Path
+import shutil
+import gc
+from utils import get_token, set_token, is_repo_exists
 
 # =================#
 # UNet Conversion #
@@ -282,8 +288,7 @@ def convert_text_enc_state_dict(text_enc_dict):
     return text_enc_dict
 
 
-def convert_diffusers_to_safetensors(model_path, checkpoint_path, half = True, progress=gr.Progress(track_tqdm=True)):
-    progress(0, desc="Start converting...")
+def convert_diffusers_to_safetensors(model_path, checkpoint_path, dtype="fp16", progress=gr.Progress(track_tqdm=True)):
     # Path for safetensors
     unet_path = osp.join(model_path, "unet", "diffusion_pytorch_model.safetensors")
     vae_path = osp.join(model_path, "vae", "diffusion_pytorch_model.safetensors")
@@ -330,57 +335,68 @@ def convert_diffusers_to_safetensors(model_path, checkpoint_path, half = True, p
 
     # Put together new checkpoint
     state_dict = {**unet_state_dict, **vae_state_dict, **text_enc_dict}
-    if half:
-        state_dict = {k: v.half() for k, v in state_dict.items()}
+    
+    if dtype == "fp16": state_dict = {k: v.half() for k, v in state_dict.items()}
+    elif dtype == "fp32": state_dict = {k: v.to(torch.float32) for k, v in state_dict.items()}
+    elif dtype == "bf16": state_dict = {k: v.to(torch.bfloat16) for k, v in state_dict.items()}
 
     save_file(state_dict, checkpoint_path)
 
-    progress(1, desc="Converted.")
-
 
-def download_repo(repo_id, dir_path, progress=gr.Progress(track_tqdm=True)):
-    from huggingface_hub import snapshot_download
+# https://huggingface.co/docs/huggingface_hub/v0.25.1/en/package_reference/file_download#huggingface_hub.snapshot_download
+def download_repo(repo_id, dir_path):
+    hf_token = get_token()
     try:
-        snapshot_download(repo_id=repo_id, local_dir=dir_path)
+        snapshot_download(repo_id=repo_id, local_dir=dir_path, token=hf_token, allow_patterns=["*.safetensors", "*.bin"],
+                          ignore_patterns=["*.fp16.*", "/*.safetensors", "/*.bin"])
     except Exception as e:
-        print(f"Error: Failed to download {repo_id}. ")
+        print(f"Error: Failed to download {repo_id}. {e}")
+        gr.Warning(f"Error: Failed to download {repo_id}. {e}")
         return
 
 
-def upload_safetensors_to_repo(filename, progress=gr.Progress(track_tqdm=True)):
-    from huggingface_hub import HfApi, hf_hub_url
-    import os
-    from pathlib import Path
+def upload_safetensors_to_repo(filename, repo_id, repo_type, is_private, progress=gr.Progress(track_tqdm=True)):
     output_filename = Path(filename).name
-    hf_token = os.environ.get("HF_TOKEN")
-    repo_id = os.environ.get("HF_OUTPUT_REPO")
-    api = HfApi()
+    hf_token = get_token()
+    api = HfApi(token=hf_token)
     try:
+        if not is_repo_exists(repo_id, repo_type): api.create_repo(repo_id=repo_id, repo_type=repo_type, token=hf_token, private=is_private)
         progress(0, desc="Start uploading...")
-        api.upload_file(path_or_fileobj=filename, path_in_repo=output_filename, repo_id=repo_id, token=hf_token)
+        api.upload_file(path_or_fileobj=filename, path_in_repo=output_filename, repo_type=repo_type, revision="main", token=hf_token, repo_id=repo_id)
         progress(1, desc="Uploaded.")
-        url  = hf_hub_url(repo_id=repo_id, filename=output_filename)
+        url = hf_hub_url(repo_id=repo_id, repo_type=repo_type, filename=output_filename)
     except Exception as e:
-        print(f"Error: Failed to upload to {repo_id}. ")
+        print(f"Error: Failed to upload to {repo_id}. {e}")
+        gr.Warning(f"Error: Failed to upload to {repo_id}. {e}")
         return None
     return url
 
 
-def convert_repo_to_safetensors(repo_id, half = True, progress=gr.Progress(track_tqdm=True)):
+def convert_repo_to_safetensors(repo_id, dtype="fp16", progress=gr.Progress(track_tqdm=True)):
     download_dir = f"{repo_id.split('/')[0]}_{repo_id.split('/')[-1]}"
     output_filename = f"{repo_id.split('/')[0]}_{repo_id.split('/')[-1]}.safetensors"
+    progress(0, desc="Start downloading...")
     download_repo(repo_id, download_dir)
-    convert_diffusers_to_safetensors(download_dir, output_filename, half)
+    progress(0, desc="Start converting...")
+    convert_diffusers_to_safetensors(download_dir, output_filename, dtype)
+    progress(1, desc="Converted.")
+    shutil.rmtree(download_dir)
     return output_filename
 
 
-def convert_repo_to_safetensors_multi_sd(repo_id, files, is_upload, urls, half=True, progress=gr.Progress(track_tqdm=True)):
-    file = convert_repo_to_safetensors(repo_id, half)
+def convert_repo_to_safetensors_multi_sd(repo_id, hf_token, files, urls, dtype="fp16", is_upload=False,
+                                         newrepo_id="", repo_type="model", is_private=True, progress=gr.Progress(track_tqdm=True)):
+    if hf_token: set_token(hf_token)
+    else: set_token(os.environ.get("HF_TOKEN"))
+    if is_upload and newrepo_id and not hf_token: raise gr.Error("HF write token is required for this process.")
+    if not newrepo_id: newrepo_id = os.environ.get("HF_OUTPUT_REPO")
+    file = convert_repo_to_safetensors(repo_id, dtype)
     if not urls: urls = []
     url = ""
     if is_upload:
-        url = upload_safetensors_to_repo(file)
+        url = upload_safetensors_to_repo(file, newrepo_id, repo_type, is_private)
         if url: urls.append(url)
+    progress(1, desc="Processing...")
     md = ""
     for u in urls:
         md += f"[Download {str(u).split('/')[-1]}]({str(u)})<br>"
@@ -389,13 +405,21 @@ def convert_repo_to_safetensors_multi_sd(repo_id, files, is_upload, urls, half=T
     return gr.update(value=files), gr.update(value=urls, choices=urls), gr.update(value=md)
 
 
+def clear_safetensors():
+    for p in Path('.').glob('*.safetensors'):
+        p.unlink()
+        print("Deleted.")
+    gc.collect()
+    return gr.update(value=[])
+
+
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
 
     parser.add_argument("--repo_id", default=None, type=str, required=True, help="HF Repo ID of the model to convert.")
-    parser.add_argument("--half", default=True, help="Save weights in half precision.")
+    parser.add_argument("--dtype", default="fp16", type=str, choices=["fp16", "fp32", "bf16", "default"], help='Output data type. (Default: "fp16")')
 
     args = parser.parse_args()
     assert args.repo_id is not None, "Must provide a Repo ID!"
 
-    convert_repo_to_safetensors(args.repo_id, args.half)
+    convert_repo_to_safetensors(args.repo_id, args.dtype)

utils.py

@@ -0,0 +1,161 @@
+import gradio as gr
+from huggingface_hub import HfApi, HfFolder, hf_hub_download
+import os
+from pathlib import Path
+import shutil
+import gc
+import re
+import urllib.parse
+
+
+def get_token():
+    try:
+        token = HfFolder.get_token()
+    except Exception:
+        token = ""
+    return token
+
+
+def set_token(token):
+    try:
+        HfFolder.save_token(token)
+    except Exception:
+        print(f"Error: Failed to save token.")
+
+
+def get_user_agent():
+    return 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:127.0) Gecko/20100101 Firefox/127.0'
+
+
+def is_repo_exists(repo_id: str, repo_type: str="model"):
+    hf_token = get_token()
+    api = HfApi(token=hf_token)
+    try:
+        if api.repo_exists(repo_id=repo_id, repo_type=repo_type, token=hf_token): return True
+        else: return False
+    except Exception as e:
+        print(f"Error: Failed to connect {repo_id} ({repo_type}). {e}")
+        return True # for safe
+
+
+MODEL_TYPE_CLASS = {
+    "diffusers:StableDiffusionPipeline": "SD 1.5",
+    "diffusers:StableDiffusionXLPipeline": "SDXL",
+    "diffusers:FluxPipeline": "FLUX",
+}
+
+
+def get_model_type(repo_id: str):
+    hf_token = get_token()
+    api = HfApi(token=hf_token)
+    lora_filename = "pytorch_lora_weights.safetensors"
+    diffusers_filename = "model_index.json"
+    default = "SDXL"
+    try:
+        if api.file_exists(repo_id=repo_id, filename=lora_filename, token=hf_token): return "LoRA"
+        if not api.file_exists(repo_id=repo_id, filename=diffusers_filename, token=hf_token): return "None"
+        model = api.model_info(repo_id=repo_id, token=hf_token)
+        tags = model.tags
+        for tag in tags:
+            if tag in MODEL_TYPE_CLASS.keys(): return MODEL_TYPE_CLASS.get(tag, default)
+    except Exception:
+        return default
+    return default
+
+
+def list_sub(a, b):
+    return [e for e in a if e not in b]
+
+
+def is_repo_name(s):
+    return re.fullmatch(r'^[^/,\s\"\']+/[^/,\s\"\']+$', s)
+
+
+def split_hf_url(url: str):
+    try:
+        s = list(re.findall(r'^(?:https?://huggingface.co/)(?:(datasets)/)?(.+?/.+?)/\w+?/.+?/(?:(.+)/)?(.+?.safetensors)(?:\?download=true)?$', url)[0])
+        if len(s) < 4: return "", "", "", ""
+        repo_id = s[1]
+        repo_type = "dataset" if s[0] == "datasets" else "model"
+        subfolder = urllib.parse.unquote(s[2]) if s[2] else None
+        filename = urllib.parse.unquote(s[3])
+        return repo_id, filename, subfolder, repo_type
+    except Exception as e:
+        print(e)
+
+
+def download_hf_file(directory, url, progress=gr.Progress(track_tqdm=True)):
+    hf_token = get_token()
+    repo_id, filename, subfolder, repo_type = split_hf_url(url)
+    try:
+        if subfolder is not None: hf_hub_download(repo_id=repo_id, filename=filename, subfolder=subfolder, repo_type=repo_type, local_dir=directory, token=hf_token)
+        else: hf_hub_download(repo_id=repo_id, filename=filename, repo_type=repo_type, local_dir=directory, token=hf_token)
+    except Exception as e:
+        print(f"Failed to download: {e}")
+
+
+def download_thing(directory, url, civitai_api_key="", progress=gr.Progress(track_tqdm=True)): # requires aria2, gdown
+    hf_token = get_token()
+    url = url.strip()
+    if "drive.google.com" in url:
+        original_dir = os.getcwd()
+        os.chdir(directory)
+        os.system(f"gdown --fuzzy {url}")
+        os.chdir(original_dir)
+    elif "huggingface.co" in url:
+        url = url.replace("?download=true", "")
+        if "/blob/" in url:
+            url = url.replace("/blob/", "/resolve/")
+        #user_header = f'"Authorization: Bearer {hf_token}"'
+        if hf_token:
+            download_hf_file(directory, url)
+            #os.system(f"aria2c --console-log-level=error --summary-interval=10 --header={user_header} -c -x 16 -k 1M -s 16 {url} -d {directory}  -o {url.split('/')[-1]}")
+        else:
+            os.system(f"aria2c --optimize-concurrent-downloads --console-log-level=error --summary-interval=10 -c -x 16 -k 1M -s 16 {url} -d {directory}  -o {url.split('/')[-1]}")
+    elif "civitai.com" in url:
+        if "?" in url:
+            url = url.split("?")[0]
+        if civitai_api_key:
+            url = url + f"?token={civitai_api_key}"
+            os.system(f"aria2c --console-log-level=error --summary-interval=10 -c -x 16 -k 1M -s 16 -d {directory} {url}")
+        else:
+            print("You need an API key to download Civitai models.")
+    else:
+        os.system(f"aria2c --console-log-level=error --summary-interval=10 -c -x 16 -k 1M -s 16 -d {directory} {url}")
+
+
+def get_local_model_list(dir_path):
+    model_list = []
+    valid_extensions = ('.safetensors')
+    for file in Path(dir_path).glob("**/*.*"):
+        if file.is_file() and file.suffix in valid_extensions:
+            file_path = str(file)
+            model_list.append(file_path)
+    return model_list
+
+
+def get_download_file(temp_dir, url, civitai_key, progress=gr.Progress(track_tqdm=True)):
+    if not "http" in url and is_repo_name(url) and not Path(url).exists():
+        print(f"Use HF Repo: {url}")
+        new_file = url
+    elif not "http" in url and Path(url).exists():
+        print(f"Use local file: {url}")
+        new_file = url
+    elif Path(f"{temp_dir}/{url.split('/')[-1]}").exists():
+        print(f"File to download alreday exists: {url}")
+        new_file = f"{temp_dir}/{url.split('/')[-1]}"
+    else:
+        print(f"Start downloading: {url}")
+        before = get_local_model_list(temp_dir)
+        try:
+            download_thing(temp_dir, url.strip(), civitai_key)
+        except Exception:
+            print(f"Download failed: {url}")
+            return ""
+        after = get_local_model_list(temp_dir)
+        new_file = list_sub(after, before)[0] if list_sub(after, before) else ""
+    if not new_file:
+        print(f"Download failed: {url}")
+        return ""
+    print(f"Download completed: {url}")
+    return new_file