Efreak commited on
Commit
1be89b6
1 Parent(s): a7e4e13

update to vae merger

Browse files
Files changed (3) hide show
  1. README.md +1 -1
  2. app.py +88 -21
  3. requirements.txt +2 -1
README.md CHANGED
@@ -1,5 +1,5 @@
1
  ---
2
- title: Repo_duplicator
3
  emoji: 😻
4
  colorFrom: gray
5
  colorTo: blue
 
1
  ---
2
+ title: VAE_merger
3
  emoji: 😻
4
  colorFrom: gray
5
  colorTo: blue
app.py CHANGED
@@ -1,42 +1,106 @@
1
  import gradio as gr
2
  import requests
 
 
 
3
  from huggingface_hub import whoami
4
- from huggingface_hub.utils import build_hf_headers, hf_raise_for_status
5
-
6
- ENDPOINT = "https://huggingface.co"
7
- # ENDPOINT = "http://localhost:5564"
8
 
 
9
  REPO_TYPES = ["model", "dataset", "space"]
10
 
11
-
12
- def duplicate(source_repo, dst_repo, token, repo_type):
13
  try:
14
- if not repo_type in REPO_TYPES:
15
- raise ValueError("need to select valid repo type")
16
  _ = whoami(token)
17
  # ^ this will throw if token is invalid
18
 
19
- r = requests.post(
20
- f"{ENDPOINT}/api/{repo_type}s/{source_repo}/duplicate",
21
- headers=build_hf_headers(token=token),
22
- json={"repository": dst_repo},
23
- )
24
- hf_raise_for_status(r)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
25
 
26
- repo_url = r.json().get("url")
27
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
28
  return (
29
  f'Find your repo <a href=\'{repo_url}\' target="_blank" style="text-decoration:underline">here</a>',
30
  "sp.jpg",
31
  )
32
 
33
  except Exception as e:
 
 
 
 
 
34
  return (
35
  f"""
36
  ### Error 😢😢😢
37
-
38
  {e}
39
- """,
 
 
 
40
  None,
41
  )
42
 
@@ -44,18 +108,21 @@ def duplicate(source_repo, dst_repo, token, repo_type):
44
  interface = gr.Interface(
45
  fn=duplicate,
46
  inputs=[
47
- gr.Textbox(placeholder="Source repository (e.g. osanseviero/src)"),
 
48
  gr.Textbox(placeholder="Destination repository (e.g. osanseviero/dst)"),
49
  gr.Textbox(placeholder="Write access token", type="password"),
 
 
50
  gr.Dropdown(choices=REPO_TYPES, value="model"),
51
  ],
52
  outputs=[
53
  gr.Markdown(label="output"),
54
  gr.Image(show_label=False),
55
  ],
56
- title="Duplicate your repo!",
57
- description="Duplicate a Hugging Face repository! You need to specify a write token obtained in https://hf.co/settings/tokens. This Space is a an experimental demo.",
58
- article="<p>Find your write token at <a href='https://huggingface.co/settings/tokens' target='_blank'>token settings</a></p>",
59
  allow_flagging="never",
60
  live=False,
61
  )
 
1
  import gradio as gr
2
  import requests
3
+ import subprocess
4
+ import os
5
+ import torch
6
  from huggingface_hub import whoami
7
+ from huggingface_hub import HfApi
8
+ from huggingface_hub import login
9
+ import random
10
+ import time
11
 
12
+ api=HfApi()
13
  REPO_TYPES = ["model", "dataset", "space"]
14
 
15
+ def duplicate(source_url_model, source_url_vae, dst_repo, token, new_name, dst_repo_path, repo_type):
 
16
  try:
 
 
17
  _ = whoami(token)
18
  # ^ this will throw if token is invalid
19
 
20
+ # make sure the user fills out the other required paths.
21
+ if not dst_repo_path[len(dst_repo_path)-1] == '/':
22
+ raise Exception("Your destination path *must* end with a /")
23
+ if not source_url_model:
24
+ raise Exception("You haven't chosen a model file to download!")
25
+ if not source_url_vae:
26
+ raise Exception("You haven't chosen a VAE file to download!")
27
+ if not dst_repo:
28
+ raise Exception("You haven't chosen a repo to download to")
29
+ login(token=token)
30
+
31
+ # keep things separate, partly in case people download different files with same name (`download.zip`). Especially, it also allows saving filename to work
32
+ dir="/home/user/apps/downloads/"+str(int(time.time()))+str(random.getrandbits(8))+"/"
33
+ subprocess.check_call([r"mkdir","-p",dir])
34
+ subprocess.check_call([r"aria2c","-x16","--split=16","-o","source.ckpt",source_url_model,"--dir="+dir])
35
+ subprocess.check_call([r"aria2c","-x16","--split=16","-o","vae.ckpt",source_url_vae,"--dir="+dir])
36
+
37
+ #USE AT YOUR OWN RISK
38
+
39
+ #local path to runwayML SD 1.5 checkpoint (https://huggingface.co/runwayml/stable-diffusion-v1-5)
40
+ ckpt_15 = dir+"source.ckpt"
41
+
42
+ #local path to StabilityAI finetuned autoencoder (https://huggingface.co/stabilityai/sd-vae-ft-mse)
43
+ ckpt_vae = dir+"vae.ckpt"
44
+
45
+ #path to save merged model to
46
+ ckpt_out = dir+"source_vae.ckpt"
47
+
48
+ pl_sd = torch.load(ckpt_15, map_location="cpu")
49
+ sd = pl_sd["state_dict"]
50
+
51
+ over_sd = torch.load(ckpt_vae,map_location="cpu")["state_dict"]
52
 
53
+ sdk = sd.keys()
54
 
55
+ for key in over_sd.keys():
56
+ if "first_stage_model."+key in sdk:
57
+ sd["first_stage_model."+key] = over_sd[key]
58
+ print(key,"overwritten")
59
+
60
+ torch.save(pl_sd,ckpt_out)
61
+
62
+ if new_name:
63
+ dst_repo_path=dst_repo_path
64
+ else:
65
+ dst_repo_path=dst_repo_path+"model+vae.ckpt"
66
+
67
+ api.upload_file(
68
+ path_or_fileobj=dir+"source_vae.ckpt",
69
+ path_in_repo=dst_repo_path,
70
+ repo_id=dst_repo,
71
+ repo_type=repo_type
72
+ )
73
+
74
+ # now clean up
75
+ os.remove(dir+files[0])
76
+ os.rmdir(dir)
77
+ match repo_type:
78
+ case "space":
79
+ repo_url=f"https://hf.co/spaces/{dst_repo}"
80
+ case "dataset":
81
+ repo_url=f"https://hf.co/datasets/{dst_repo}"
82
+ case "model":
83
+ repo_url=f"https://hf.co/{dst_repo}"
84
  return (
85
  f'Find your repo <a href=\'{repo_url}\' target="_blank" style="text-decoration:underline">here</a>',
86
  "sp.jpg",
87
  )
88
 
89
  except Exception as e:
90
+ blames=["grandma","my boss","your boss","God","you","you. It's *all* your fault.","the pope"]
91
+ blameweights=(1,1,1,1,4,2,1)
92
+ excuses=["I blame it all on "+random.choices(blames,weights=blameweights)[0],"It's my fault, sorry.","I did it on purpose.","That file doesn't want to be downloaded.","You nincompoop!"]
93
+ excusesweights=(12,1,1,2,3)
94
+ excuse=random.choices(excuses,weights=excusesweights)[0]
95
  return (
96
  f"""
97
  ### Error 😢😢😢
98
+
99
  {e}
100
+
101
+
102
+
103
+ <i>""" + excuse+"</i>",
104
  None,
105
  )
106
 
 
108
  interface = gr.Interface(
109
  fn=duplicate,
110
  inputs=[
111
+ gr.Textbox(placeholder="Source URL for model (e.g. civitai.com/api/download/models/4324322534)"),
112
+ gr.Textbox(placeholder="Source URL for VAE (e.g. civitai.com/api/download/models/4324322534)"),
113
  gr.Textbox(placeholder="Destination repository (e.g. osanseviero/dst)"),
114
  gr.Textbox(placeholder="Write access token", type="password"),
115
+ gr.Textbox(placeholder="Post-download name of your file, if you want it changed (e.g. stupidmodel_stupidvae.safetensors)"),
116
+ gr.Textbox(placeholder="Destination for your file within your repo. Don't include the filename, end path with a / (e.g. /models/Stable-diffusion/)"),
117
  gr.Dropdown(choices=REPO_TYPES, value="model"),
118
  ],
119
  outputs=[
120
  gr.Markdown(label="output"),
121
  gr.Image(show_label=False),
122
  ],
123
+ title="Merge a VAE with a model!",
124
+ description="Merge a VAE with your model, and export to your Hugging Face repository! You need to specify a write token obtained in https://hf.co/settings/tokens. This Space is a an experimental demo. CKPT format only; I just ripped off someone else's script, I have no idea how this works...",
125
+ article="<p>credit to <a href='https://gist.github.com/Quasimondo/f344659f57dc15bd7892a969bd58ac67'>Quasimodo's script</a></p><p>Find your write token at <a href='https://huggingface.co/settings/tokens' target='_blank'>token settings</a></p>",
126
  allow_flagging="never",
127
  live=False,
128
  )
requirements.txt CHANGED
@@ -1 +1,2 @@
1
- huggingface_hub==0.11.0
 
 
1
+ huggingface_hub==0.11.0
2
+ torch