Upload sdwui-start.ipynb
Browse files- sdwui-start.ipynb +1 -1
sdwui-start.ipynb
CHANGED
|
@@ -1 +1 @@
|
|
| 1 |
-
{"cells":[{"cell_type":"markdown","metadata":{"id":"6zE2QeUKLCtC"},"source":["# 这只是一个完整项目的一部分 不能运行的 \n","- 可以从这个地址运行 [点击打开](https://www.kaggle.com/viyiviyi/sdwui-before)\n","---"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["from pathlib import Path\n","import os\n","import time\n","import re\n","import subprocess\n","import threading\n","import sys\n","import socket\n","\n","install_path=f\"{os.environ['HOME']}/sdwebui\" # 安装目录\n","output_path=f\"{os.environ['HOME']}/sdwebui/Output\" # 输出目录 如果使用google云盘 会在google云盘增加sdwebui/Output\n","input_path = '/kaggle/input' # 输入目录\n","\n","google_drive = '' \n","# 连接谷歌云\n","try:\n"," if useGooglrDrive:\n"," from google.colab import drive\n"," drive.mount(f'~/google_drive')\n"," google_drive = f\"{os.environ['HOME']}/google_drive/MyDrive\"\n"," output_path = f'{google_drive}/sdwebui/Output'\n"," input_path = f'{google_drive}/sdwebui/Input'\n"," !mkdir -p {input_path}\n","except:\n"," useGooglrDrive = False\n","\n","!mkdir -p {install_path}\n","!mkdir -p {output_path}\n","\n","os.environ['install_path'] = install_path\n","os.environ['output_path'] = output_path\n","os.environ['google_drive'] = google_drive\n","\n"]},{"cell_type":"code","execution_count":null,"metadata":{"_kg_hide-input":true,"id":"i3LhnwYHLCtC","trusted":true},"outputs":[],"source":["# 模型目录 文件名称:链接的方式指定文件名\n","modelDirs = []+[item.strip() for item in 模型列表.split('\\n') if item.strip() ]\n","# vae列表\n","modelVaeDirs = []+[item.strip() for item in VAE列表.split('\\n') if item.strip()]\n","# 此配置内的目录下所有文件将加载到hypernetworks文件夹\n","hypernetworksModelDirs = []+[item.strip() for item in hypernetworks列表.split('\\n') if item.strip()] \n","# 此配置内的目录下所有文件将加载到embeddings文件夹\n","embeddingsModelDirs = [] +[item.strip() for item in embeddings列表.split('\\n') if item.strip()]\n","# 此配置内的目录下所有文件将加载到Lora文件夹\n","loraModelDirs = []+[item.strip() for item in Lora列表.split('\\n') if item.strip()] \n","\n","lyCORISModelDirs = []+[item.strip() for item in LyCORIS列表.split('\\n') if item.strip()] \n","# controlNet插件模型列表\n","controlNetModels=[] +[item.strip() for item in controlNet模型列表.split('\\n') if item.strip()]\n","# 插件列表\n","extensions=[]+[item.strip() for item in 插件列表.split('\\n') if item.strip()]\n","# 配置文件链接\n","linkHypernetworksDir=False # 链接 hypernetworks 目录到输出目录\n","linkEmbeddingsDir=False # 链接 embeddings 目录到输出目录\n","linkTextual_inversionDir=False # 链接 textual_inversion 目录到输出目录 如果需要保存训练过程的产出文件时建议开启\n","\n","ngrokTokenFile = os.path.join(input_path,'configs/ngrok_token.txt') # 非必填 存放ngrokToken的文件的路径\n","frpcConfigFile = os.path.join(input_path,'configs/frpc_koishi.ini') # 非必填 frp 配置文件\n","# ss证书目录 下载nginx的版本，把pem格式改成crt格式\n","frpcSSLFFlies =[os.path.join(input_path,'configs/koishi_ssl')]+[item.strip() for item in frpSSL文件.split('\\n') if item.strip() ]\n","# frpc 文件目录 如果目录不存在，会自动下载，也可以在数据集搜索 viyiviyi/utils 添加\n","frpcExePath = os.path.join(input_path,'utils-tools/frpc')\n","# 其他需要加载的webui启动参数 写到【参数列表】这个配置去\n","otherArgs = ' '.join([item.strip() for item in 参数列表.split('\\n') if item.strip()])\n","\n","venvPath = os.path.join(input_path,'sd-webui-venv/venv.tar.bak') # 安装好的python环境 sd-webui-venv是一个公开是数据集 可以搜索添加\n","\n","# 用于使用kaggle api的token文件 参考 https://www.kaggle.com/docs/api\n","# 此文件用于自动上传koishi的相关配置 也可以用于保存重要的输出文件\n","kaggleApiTokenFile = os.path.join(input_path,'configs/kaggle.json')\n","\n","# 设置文件路径\n","if setting_file== '/kaggle/working/configs/config.json':\n"," setting_file = os.path.join(output_path,'configs/config.json')\n","if ui_config_file == '/kaggle/working/configs/ui-config.json':\n"," ui_config_file = os.path.join(output_path,'configs/ui-config.json')"]},{"cell_type":"code","execution_count":null,"metadata":{"_kg_hide-input":true,"id":"a_GtG2ayLCtD","trusted":true},"outputs":[],"source":["# 这下面的是用于初始化一些值或者环境变量的，轻易别改\n","frpcStartArg = ''\n","\n","!mkdir -p $install_path/configFiles\n","if Path(frp配置文件或配置.strip()).exists():\n"," frpcConfigFile = frp配置文件或配置.strip()\n","if not Path(frpcConfigFile).exists(): \n"," if frp配置文件或配置.strip().startswith('-f'):\n"," frpcStartArg = frp配置文件或配置.strip()\n"," else:\n"," print('没有frpcp配置')\n"," useFrpc = False\n","else:\n"," os.system(f'cp -f {frpcConfigFile} {install_path}/configFiles/frpc_webui.ini')\n"," frpcConfigFile = f'{install_path}/configFiles/frpc_webui.ini'\n"," os.system(f'sed -i \"s/local_port = .*/local_port = {webuiPort}/g\" {frpcConfigFile}')\n"," frpcStartArg = f' -c {frpcConfigFile}'\n","\n","ngrokToken=''\n","if Path(ngrok配置或文件地址.strip()).exists():\n"," ngrokTokenFile = ngrok配置或文件地址.strip()\n","if Path(ngrokTokenFile).exists():\n"," with open(ngrokTokenFile,encoding = \"utf-8\") as nkfile:\n"," ngrokToken = nkfile.readline()\n","elif not ngrok配置或文件地址.strip().startswith('/'):\n"," ngrokToken=ngrok配置或文件地址.strip()\n"," \n","if not Path(venvPath).exists():\n"," venvPath = os.path.join(input_path,'sd-webui-venv/venv.zip')"]},{"cell_type":"markdown","metadata":{"id":"p0uS-BLULCtD"},"source":["# kaggle public API\n","\n","**不能使用%cd这种会改变当前工作目录的命令，会导致和其他线程冲突**\n","\n","---"]},{"cell_type":"code","execution_count":null,"metadata":{"_kg_hide-input":true,"id":"m8FJi4j0LCtD","trusted":true},"outputs":[],"source":["# 安装kaggle的api token文件\n","def initKaggleConfig():\n"," if Path('~/.kaggle/kaggle.json').exists():\n"," return True\n"," if Path(kaggleApiTokenFile).exists():\n"," !mkdir -p ~/.kaggle/\n"," os.system('cp '+kaggleApiTokenFile+' ~/.kaggle/kaggle.json')\n"," !chmod 600 ~/.kaggle/kaggle.json\n"," return True\n"," print('缺少kaggle的apiToken文件，访问：https://www.kaggle.com/你的kaggle用户名/account 获取')\n"," return False\n","\n","def getUserName():\n"," if not initKaggleConfig(): return\n"," import kaggle\n"," return kaggle.KaggleApi().read_config_file()['username']\n","\n","def createOrUpdateDataSet(path:str,datasetName:str):\n"," if not initKaggleConfig(): return\n"," print('创建或更新数据集 '+datasetName)\n"," import kaggle\n"," os.system('mkdir -p $install_path/kaggle_cache')\n"," os.system('rm -rf $install_path/kaggle_cache/*')\n"," datasetDirPath = install_path+'/kaggle_cache/'+datasetName\n"," os.system('mkdir -p '+datasetDirPath)\n"," os.system('cp -f '+path+' '+datasetDirPath+'/')\n"," username = getUserName()\n"," print(\"kaggle username:\"+username)\n"," datasetPath = username+'/'+datasetName\n"," datasetList = kaggle.api.dataset_list(mine=True,search=datasetPath)\n"," print(datasetList)\n"," if len(datasetList) == 0 or datasetPath not in [str(d) for d in datasetList]: # 创建 create\n"," os.system('kaggle datasets init -p' + datasetDirPath)\n"," metadataFile = datasetDirPath+'/dataset-metadata.json'\n"," os.system('sed -i s/INSERT_TITLE_HERE/'+ datasetName + '/g ' + metadataFile)\n"," os.system('sed -i s/INSERT_SLUG_HERE/'+ datasetName + '/g ' + metadataFile)\n"," os.system('cat '+metadataFile)\n"," os.system('kaggle datasets create -p '+datasetDirPath)\n"," print('create database done')\n"," else:\n"," kaggle.api.dataset_metadata(datasetPath,datasetDirPath)\n"," kaggle.api.dataset_create_version(datasetDirPath, 'auto update',dir_mode='zip')\n"," print('upload database done')\n","\n","def downloadDatasetFiles(datasetName:str,outputPath:str):\n"," if not initKaggleConfig(): return\n"," print('下载数据集文件 '+datasetName)\n"," import kaggle\n"," username = getUserName()\n"," datasetPath = username+'/'+datasetName\n"," datasetList = kaggle.api.dataset_list(mine=True,search=datasetPath)\n"," if datasetPath not in [str(d) for d in datasetList]:\n"," return False\n"," os.system('mkdir -p '+outputPath)\n"," kaggle.api.dataset_download_files(datasetPath,path=outputPath,unzip=True)\n"," return True\n","\n"]},{"cell_type":"markdown","metadata":{"id":"sswa04veLCtE"},"source":["# 工具函数\n","**不能使用%cd这种会改变当前工作目录的命令，会导致和其他线程冲突**\n","\n","---"]},{"cell_type":"code","execution_count":null,"metadata":{"_kg_hide-input":true,"id":"coqQvTSLLCtE","trusted":true},"outputs":[],"source":["# 绕过 os.systen 的限制执行命令\n","def run(shell:str,shellName=''):\n"," if shellName == '': shellName = str(time.time())\n"," !mkdir -p $install_path/run_cache\n"," with open(install_path+'/run_cache/run_cache.'+shellName+'.sh','w') as sh:\n"," sh.write(shell)\n"," !bash {install_path}/run_cache/run_cache.{shellName}.sh\n","\n","# 连接多个路径字符串 让路径在shell命令中能正常的执行\n","def pathJoin(*paths:str):\n"," pathStr = ''\n"," for p in paths:\n"," pathStr += '\"'+p+'\"'\n"," pathStr = '\"*\"'.join(pathStr.split('*'))\n"," pathStr = '\"$\"'.join(pathStr.split('$'))\n"," pathStr = '\"(\"'.join(pathStr.split('('))\n"," pathStr = '\")\"'.join(pathStr.split(')'))\n"," pathStr = '\"{\"'.join(pathStr.split('{'))\n"," pathStr = '\"}\"'.join(pathStr.split('}'))\n"," pathStr = re.sub(r'\"\"','',pathStr)\n"," pathStr = re.sub(r'\\*{2,}','\"',pathStr)\n"," pathStr = re.sub(r'/{2,}','/',pathStr)\n"," pathStr = re.sub(r'/\\./','/',pathStr)\n"," return pathStr\n","\n","# 判断路径是不是一个文件或者可能指向一些文件\n","def pathIsFile(path):\n"," if Path(path).is_file():\n"," return True\n"," if re.search(r'\\.(ckpt|safetensors|png|jpg|txt|pt|pth|json|yaml|\\*)$',path):\n"," return True\n"," return False\n","\n","def echoToFile(content:str,path:str):\n"," with open(path,'w') as sh:\n"," sh.write(content)\n","\n","# ngrok\n","def startNgrok(ngrokToken:str,ngrokLocalPort:int):\n"," from pyngrok import conf, ngrok\n"," try:\n"," conf.get_default().auth_token = ngrokToken\n"," conf.get_default().monitor_thread = False\n"," ssh_tunnels = ngrok.get_tunnels(conf.get_default())\n"," if len(ssh_tunnels) == 0:\n"," ssh_tunnel = ngrok.connect(ngrokLocalPort)\n"," print('ngrok 访问地址：'+ssh_tunnel.public_url)\n"," else:\n"," print('ngrok 访问地址：'+ssh_tunnels[0].public_url)\n"," except:\n"," print('启动ngrok出错')\n"," \n","def startFrpc(name,configFile):\n"," run(f'''\n","cd $install_path/frpc/\n","$install_path/frpc/frpc {configFile}\n"," ''',name)\n"," \n","def installProxyExe():\n"," if useFrpc:\n"," print('安装frpc')\n"," !mkdir -p $install_path/frpc\n"," if Path(frpcExePath).exists():\n"," os.system(f'cp -f -n {frpcExePath} $install_path/frpc/frpc')\n"," else:\n"," !wget \"https://huggingface.co/datasets/ACCA225/Frp/resolve/main/frpc\" -O $install_path/frpc/frpc\n"," \n"," for ssl in frpcSSLFFlies:\n"," if Path(ssl).exists():\n"," os.system('cp -f -n '+pathJoin(ssl,'/*')+' $install_path/frpc/')\n"," !chmod +x $install_path/frpc/frpc\n"," !$install_path/frpc/frpc -v\n"," if useNgrok:\n"," %pip install pyngrok\n","\n","def startProxy():\n"," if useNgrok:\n"," startNgrok(ngrokToken,webuiPort)\n"," if useFrpc:\n"," startFrpc('frpc_proxy',frpcStartArg)\n","\n"," \n","def zipPath(path:str,zipName:str,format='tar'):\n"," if path.startswith('$install_path'):\n"," path = path.replace('$install_path',install_path)\n"," if path.startswith('$output_path'):\n"," path = path.replace('$install_path',output_path)\n"," if not path.startswith('/'):\n"," path = pathJoin(install_path,'/stable-diffusion-webui','/',path)\n"," if Path(path).exists():\n"," if 'tar' == format:\n"," os.system('tar -cf $output_path/'+ zipName +'.tar -C '+ path +' . ')\n"," elif 'gz' == format:\n"," os.system('tar -czf $output_path/'+ zipName +'.tar.gz -C '+ path +' . ')\n"," return\n"," print('指定的目录不存在：'+path)\n","\n"," \n","def check_service(host, port):\n"," try:\n"," socket.create_connection((host, port), timeout=5)\n"," return True\n"," except socket.error:\n"," return False"]},{"cell_type":"code","execution_count":null,"metadata":{"_kg_hide-input":true,"id":"THNUsEm_LCtE","trusted":true},"outputs":[],"source":["import os\n","import re\n","import requests\n","\n","def download_file(url, filename, path):\n"," # 获取文件的真实文件名\n"," if not filename:\n"," with requests.get(url, stream=True) as r:\n"," if 'Content-Disposition' in r.headers:\n"," filename = r.headers['Content-Disposition'].split('filename=')[1].strip('\"')\n"," r.close()\n"," filename = re.sub(r'[\\\\/:*?\"<>|;]', '', filename)\n"," # 拼接文件的完整路径\n"," filepath = os.path.join(path, filename)\n"," # 判断文件是否已存在\n"," if os.path.exists(filepath):\n"," print(f'{filename} already exists in {path}')\n"," return\n"," # 下载文件\n"," with requests.get(url, stream=True) as r:\n"," r.raise_for_status()\n"," with open(filepath, 'wb') as f:\n"," for chunk in r.iter_content(chunk_size=8192):\n"," if chunk:\n"," f.write(chunk)\n"," print(f'{filename} has been downloaded to {path}')\n"," \n","# 加入文件到下载列表\n","def putDownloadFile(url:str,distDir:str,file_name:str=None):\n"," if re.match(r'^[^:]+:(https?|ftps?)://', url, flags=0):\n"," file_name = re.findall(r'^[^:]+:',url)[0][:-1]\n"," url = url[len(file_name)+1:]\n"," if not re.match(r'^(https?|ftps?)://',url):\n"," return\n"," file_name = re.sub(r'\\s+','_',file_name or '')\n"," dir = str(hash(url)).replace('-','')\n"," down_dir = f'{install_path}/down_cache/{dir}'\n"," !mkdir -p {down_dir}\n"," return [url,file_name,distDir,down_dir]\n","\n","def get_file_size_in_gb(file_path):\n"," size_in_bytes = Path(file_path).stat().st_size\n"," size_in_gb = size_in_bytes / (1024 ** 3)\n"," return '%.2f' % size_in_gb\n"," \n","# 下载文件\n","def startDownloadFiles(download_list):\n"," print('下载列表:\\n','\\n'.join([f'{item[0]} -> {item[2]}/{item[1]}' for item in download_list]))\n"," dist_list = []\n"," for dow_f in download_list:\n"," !mkdir -p {dow_f[3]}\n"," print('下载 名称：',dow_f[1],'url：',dow_f[0])\n"," download_file(dow_f[0],dow_f[1],dow_f[2])"]},{"cell_type":"code","execution_count":null,"metadata":{"_kg_hide-input":true,"trusted":true},"outputs":[],"source":["\n","# nginx 反向代理配置文件\n","def localProxy():\n"," conf = '''\n","server\n","{\n"," listen '''+str(webuiPort)+''';\n"," listen [::]:'''+str(webuiPort)+''';\n"," server_name 127.0.0.1 localhost 0.0.0.0 \"\";\n"," \n"," if ($request_method = OPTIONS) {\n"," return 200;\n"," }\n"," fastcgi_send_timeout 10m;\n"," fastcgi_read_timeout 10m;\n"," fastcgi_connect_timeout 10m;\n"," location /1/\n"," {\n"," proxy_pass http://127.0.0.1:'''+str(webuiPort+2)+'''/;\n"," add_header Set-Cookie \"subpath=1; expires=0; Path=/\";\n"," proxy_set_header Set-Cookie \"subpath=1; expires=0; Path=/\";\n"," proxy_set_header Host $host;\n"," proxy_set_header X-Real-IP $remote_addr;\n"," proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n"," proxy_set_header REMOTE-HOST $remote_addr;\n"," proxy_set_header Upgrade $http_upgrade;\n"," proxy_set_header Connection upgrade;\n"," proxy_http_version 1.1;\n"," proxy_connect_timeout 10m;\n"," proxy_read_timeout 10m;\n"," }\n"," location /\n"," {\n"," set $proxy_url http://127.0.0.1:'''+str(webuiPort+1)+''';\n"," if ($cookie_subpath = \"1\") {\n"," set $proxy_url http://127.0.0.1:'''+str(webuiPort+2)+''';\n"," }\n"," proxy_pass $proxy_url;\n"," proxy_set_header Host $host;\n"," proxy_set_header X-Real-IP $remote_addr;\n"," proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n"," proxy_set_header REMOTE-HOST $remote_addr;\n"," proxy_set_header Upgrade $http_upgrade;\n"," proxy_set_header Connection upgrade;\n"," proxy_http_version 1.1;\n"," proxy_connect_timeout 10m;\n"," proxy_read_timeout 10m;\n"," }\n","}\n","'''\n"," echoToFile(conf,'/etc/nginx/conf.d/proxy_nginx.conf')\n"," if not check_service('localhost',webuiPort):\n"," !nginx -c /etc/nginx/nginx.conf\n"," !nginx -s reload\n"," "]},{"cell_type":"code","execution_count":null,"metadata":{"_kg_hide-input":true,"trusted":true},"outputs":[],"source":["import inspect\n","import ctypes\n","\n","def _async_raise(tid, exctype):\n"," \"\"\"raises the exception, performs cleanup if needed\"\"\"\n"," tid = ctypes.c_long(tid)\n"," if not inspect.isclass(exctype):\n"," exctype = type(exctype)\n"," res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype))\n"," if res == 0:\n"," raise ValueError(\"invalid thread id\")\n"," elif res != 1:\n"," # \"\"\"if it returns a number greater than one, you're in trouble,\n"," # and you should call it again with exc=NULL to revert the effect\"\"\"\n"," ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None)\n"," raise SystemError(\"PyThreadState_SetAsyncExc failed\")\n","\n","def stop_thread(thread):\n"," _async_raise(thread.ident, SystemExit)\n","\n","def stop_solo_threads():\n"," # 获取当前所有活动的线程\n"," threads = threading.enumerate()\n"," # 关闭之前创建的子线程\n"," for thread in threads:\n"," if thread.name.startswith('solo_'):\n"," print(thread.name)\n"," try:\n"," stop_thread(thread)\n"," except socket.error:\n"," print(f'结束进程：{thread.name} 执行失败')"]},{"cell_type":"markdown","metadata":{"id":"Ve3p8oOkLCtE"},"source":["# webui 安装和配置函数\n","---"]},{"cell_type":"code","execution_count":null,"metadata":{"_kg_hide-input":true,"id":"GTjyBJihLCtE","trusted":true},"outputs":[],"source":["envInstalled=False\n","extensionsDone=False\n","\n","#安装webui\n","def install():\n"," print('安装webui')\n"," %cd $install_path\n"," if reLoad:\n"," !rm -rf stable-diffusion-webui\n"," if Path(\"stable-diffusion-webui\").exists():\n"," %cd $install_path/stable-diffusion-webui/\n"," !git checkout .\n"," !git pull\n"," else:\n"," if mobileOptimize:\n"," !git clone https://github.com/viyiviyi/stable-diffusion-webui.git stable-diffusion-webui -b local # 修改了前端界面，手机使用更方便\n"," else:\n"," !git clone https://github.com/AUTOMATIC1111/stable-diffusion-webui stable-diffusion-webui\n"," print('安装webui 完成')\n","\n","# 链接输出目录\n","def link_dir():\n"," print('链接输出目录')\n"," # 链接输出目录 因为sd被部署在了stable-diffusion-webui目录，运行结束后为了方便下载就只有outputs目录放在output_path/目录下\n"," !mkdir -p $output_path/outputs\n"," !rm -rf $install_path/stable-diffusion-webui/outputs\n"," !ln -s -r $output_path/outputs $install_path/stable-diffusion-webui/ # 输出目录\n"," !mkdir -p $output_path/log\n"," !rm -rf $install_path/stable-diffusion-webui/log\n"," !ln -s -r $output_path/log $install_path/stable-diffusion-webui/\n"," # 链接 hypernetworks 目录\n"," if linkHypernetworksDir:\n"," !rm -rf $install_path/stable-diffusion-webui/models/hypernetworks\n"," !mkdir -p $output_path/hypernetworks\n"," !ln -s -r $output_path/hypernetworks $install_path/stable-diffusion-webui/models/\n"," # 链接 embeddings 目录\n"," if linkEmbeddingsDir:\n"," !rm -rf $install_path/stable-diffusion-webui/embeddings\n"," !mkdir -p $output_path/embeddings\n"," !ln -s -r $output_path/embeddings $install_path/stable-diffusion-webui/\n"," # 链接训练输出目录 文件夹链接会导致功能不能用\n"," if linkTextual_inversionDir:\n"," !rm -rf $install_path/stable-diffusion-webui/textual_inversion\n"," !mkdir -p $output_path/textual_inversion/\n"," !ln -s -r $output_path/textual_inversion $install_path/stable-diffusion-webui/\n"," print('链接输出目录 完成')\n"," \n","# 链接模型文件\n","def load_models(skip_url=False):\n"," print(('复制' if enableLoadByCopy else '链接') + '模型文件')\n"," if enableLoadByCopy:\n"," print('如果出现 No such file or directory 错误，可以忽略')\n"," download_list = []\n"," models_dir_path = f'{install_path}/stable-diffusion-webui/models/Stable-diffusion'\n"," hypernetworks_dir_path = f'{install_path}/stable-diffusion-webui/models/hypernetworks'\n"," embeddings_dir_path = f'{install_path}/stable-diffusion-webui/models/embeddings'\n"," lora_dir_path = f'{install_path}/stable-diffusion-webui/models/Lora'\n"," lycoris_dir_path = f'{install_path}/stable-diffusion-webui/models/LyCORIS'\n"," vae_dir_path = f'{install_path}/stable-diffusion-webui/models/VAE'\n"," control_net_dir_path = f'{install_path}/stable-diffusion-webui/extensions/sd-webui-controlnet/models'\n"," \n"," def link_file(source_paths,dist:str):\n"," !mkdir -p {dist}\n"," for path in source_paths:\n"," if 'https://' in path or 'http://' in path:\n"," if skip_url:\n"," continue\n"," download_list.append(putDownloadFile(path,dist))\n"," elif pathIsFile(path):\n"," os.system(('cp -n' if enableLoadByCopy else 'ln -s')+' -f '+ pathJoin(path) +f' {dist}')\n"," else:\n"," os.system(('cp -n' if enableLoadByCopy else 'ln -s')+' -f '+ pathJoin(path,'/*.*') +f' {dist}')\n"," !rm -f {dist}/\\*.* \n"," \n"," link_file(modelDirs,models_dir_path)\n"," link_file(hypernetworksModelDirs,hypernetworks_dir_path)\n"," link_file(embeddingsModelDirs,embeddings_dir_path)\n"," link_file(loraModelDirs,lora_dir_path)\n"," link_file(lyCORISModelDirs,lycoris_dir_path)\n"," link_file(modelVaeDirs,vae_dir_path)\n"," link_file(controlNetModels,control_net_dir_path)\n"," startDownloadFiles([item for item in download_list if item])\n"," print(('复制' if enableLoadByCopy else '链接') + '模型文件 完成')\n","\n","#安装依赖\n","def install_dependencies():\n"," print('安装webui需要的python环境')\n"," global envInstalled\n"," global venvPath\n"," !sudo apt install nginx -y\n"," sh = f'cd {install_path}/stable-diffusion-webui\\n'\n"," if str(sys.version).startswith('3.8') or str(sys.version).startswith('3.9') or str(sys.version).startswith('3.10'):\n"," sh += '''\n","python3 -m venv venv\n","'''\n"," else:\n"," sh += '''\n","add-apt-repository ppa:deadsnakes/ppa -y\n","apt update\n","apt install python3.10 -y\n","python3.10 -m venv venv\n","'''\n"," if quickStart:\n"," if not Path(venvPath).exists():\n"," !mkdir -p {install_path}/venv_cache\n"," if not Path(f'{install_path}/venv_cache/venv.tar.bak').exists():\n"," download_file('https://huggingface.co/viyi/sdwui/resolve/main/venv.zip','venv.zip',f'{install_path}/venv_cache')\n"," !unzip {install_path}/venv_cache/venv.zip -d {install_path}/venv_cache\n"," venvPath = f'{install_path}/venv_cache/venv.tar.bak'\n"," !rm -rf {install_path}/venv_cache/venv.zip\n"," elif venvPath.endswith('.zip'):\n"," !mkdir -p {install_path}/venv_cache\n"," !unzip {venvPath} -d {install_path}/venv_cache\n"," venvPath = f'{install_path}/venv_cache/venv.tar.bak'\n"," \n"," sh += 'echo \"unzip venv\"\\n'\n"," sh += f'tar -xf {venvPath} -C ./venv\\n'\n"," sh += '\\n'\n"," run(sh,'dependencies')\n"," if not Path(f'{install_path}/stable-diffusion-webui/venv/bin/pip').exists():\n"," !curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py\n"," !{install_path}/stable-diffusion-webui/venv/bin/python3 get-pip.py\n"," \n"," !{install_path}/stable-diffusion-webui/venv/bin/python3 -V\n"," !{install_path}/stable-diffusion-webui/venv/bin/python3 -m pip -V\n"," \n"," if not quickStart or not Path(venvPath).exists():\n"," !{install_path}/stable-diffusion-webui/venv/bin/python3 -m pip install -U torch==2.0.0+cu117 torchvision==0.15.1+cu117 torchaudio==2.0.1 --index-url https://download.pytorch.org/whl/cu117\n"," !{install_path}/stable-diffusion-webui/venv/bin/python3 -m pip install -U open-clip-torch xformers==0.0.19\n"," \n"," envInstalled = True\n"," print('安装webui需要的python环境 完成')\n","\n","# 安装插件\n","def install_extensions():\n"," global extensionsDone\n"," print('安装插件')\n"," sh = 'mkdir -p $install_path/stable-diffusion-webui/extensions \\n'\n"," sh = 'mkdir -p $install_path/cache/extensions \\n'\n"," sh += 'cd $install_path/cache/extensions \\n'\n"," for ex in extensions:\n"," sh += 'git clone ' + ex + ' \\n'\n"," sh += 'rsync -a $install_path/cache/extensions/* $install_path/stable-diffusion-webui/extensions \\n'\n"," sh += 'cd ../ && rm -rf $install_path/cache/extensions \\n'\n"," run(sh,'extensions')\n"," extensionsDone = True\n"," print('安装插件 完成')\n"," \n","# 个性化配置 \n","def use_config():\n"," print('使用自定义配置 包括tag翻译 \\n')\n"," %cd $install_path/stable-diffusion-webui\n"," !mkdir -p tmp\n"," %cd tmp\n"," !git clone {webui_settings} sd-configs\n"," !cp -rf sd-configs/dist/* $install_path/stable-diffusion-webui\n"," if not Path(ui_config_file).exists(): # ui配置文件\n"," !mkdir -p {ui_config_file[0:ui_config_file.rfind('/')]}\n"," os.system('cp -f -n $install_path/stable-diffusion-webui/ui-config.json '+ui_config_file)\n"," if not Path(setting_file).exists(): # 设置配置文件\n"," !mkdir -p {setting_file[0:setting_file.rfind('/')]}\n"," os.system('cp -f -n $install_path/stable-diffusion-webui/config.json '+setting_file)\n","\n","def checkDefaultModel():\n"," print('检查默认模型文件是否存在 \\n')\n"," global usedCkpt\n"," if usedCkpt is not None and usedCkpt != '': # 设置启动时默认加载的模型\n"," if '.' in usedCkpt:\n"," if '/' in usedCkpt:\n"," usedCkpt = usedCkpt\n"," else:\n"," usedCkpt = install_path + '/stable-diffusion-webui/models/Stable-diffusion/' + usedCkpt\n"," else:\n"," for x in ['.ckpt','.safetensors']:\n"," if Path(install_path+'/stable-diffusion-webui/models/Stable-diffusion/' + usedCkpt+x).exists():\n"," usedCkpt = install_path+'/stable-diffusion-webui/models/Stable-diffusion/' + usedCkpt+x\n"," break\n"," if Path(usedCkpt).exists():\n"," return True\n"," else:\n"," if Path(usedCkpt).is_symlink():\n"," print('模型文件真实地址：'+os.readlink(usedCkpt))\n"," return False\n"," \n","def start_webui(i):\n"," # 只要不爆内存，其他方式关闭后会再次重启 访问地址会发生变化\n"," print(i,'--port',str(webuiPort+1+i))\n"," if i>0:\n"," print(f'使用第{i+1}张显卡启动第{i+1}个webui，通过frpc或nrgok地址后加/{i}/进行访问（不能使用同一个浏览器）')\n"," if useFrpc:\n"," while True:\n"," !venv/bin/python3 launch.py --device-id={i} --port {str(webuiPort+1+i)} # {'' if i == 0 else '--subpath=/'+str(i)}\n"," print('5秒后重启webui')\n"," time.sleep(5)\n"," else:\n"," !venv/bin/python3 launch.py --device-id={i} --port {str(webuiPort+1+i)} # {'' if i == 0 else '--subpath=/'+str(i)}\n"," \n","# 启动\n","def start():\n"," print('启动webui')\n"," %cd $install_path/stable-diffusion-webui\n"," args = '' # ' --port=' + str(webuiPort+1)\n"," if not disableShared:\n"," args += ' --share'\n"," if onlyApi:\n"," args += ' --nowebui'\n"," if ui_config_file is not None and ui_config_file != '' and Path(ui_config_file).exists(): # ui配置文件\n"," args += ' --ui-config-file=' + pathJoin(ui_config_file)\n"," if setting_file is not None and setting_file != '' and Path(setting_file).exists(): # 设置配置文件\n"," args += ' --ui-settings-file=' + pathJoin(setting_file)\n"," if not checkDefaultModel():\n"," print('默认模型文件不存在，请检查配置：'+usedCkpt)\n"," else:\n"," args += ' --ckpt='+ pathJoin(usedCkpt)\n"," if vaeHalf is False: \n"," args += ' --no-half-vae'\n"," if modelHalf is False:\n"," args += ' --no-half'\n"," if consoleProgressbars is False:\n"," args += ' --disable-console-progressbars'\n"," if consolePrompts is True:\n"," args += ' --enable-console-prompts'\n"," args += ' ' + otherArgs\n"," os.environ['COMMANDLINE_ARGS']=args\n"," !echo COMMANDLINE_ARGS=$COMMANDLINE_ARGS\n"," %env REQS_FILE=requirements.txt\n"," import torch\n"," threads = []\n"," if torch.cuda.device_count() == 0:\n"," print(\"当前环境没有GPU\")\n"," for i in range(torch.cuda.device_count()):\n"," thread = threading.Thread(target = start_webui, name='solo_start_webui_'+str(i), daemon=True,args=([i]))\n"," thread.start()\n"," threads.append(thread)\n"," while thread.is_alive() and not check_service('localhost',str(webuiPort+1+i)): # 当当前服务启动完成才允许退出此次循环\n"," time.sleep(10)\n"," time.sleep(5)\n"," for thread in threads:\n"," thread.join()"]},{"cell_type":"markdown","metadata":{"id":"qLvsk8ByLCtF"},"source":["# 入口函数\n","---"]},{"cell_type":"code","execution_count":null,"metadata":{"_kg_hide-input":true,"id":"IOKjaMlcLCtF","trusted":true},"outputs":[],"source":["\n","# 启动非webui相关的的内容，加快启动速度\n","def main():\n"," global envInstalled\n"," global extensionsDone\n"," startTicks = time.time()\n"," isInstall = True if os.getenv('IsInstall','False') == 'True' else False\n"," if isInstall is False or reLoad: \n"," install()\n"," if enableThread:\n"," threading.Thread(target = install_extensions,daemon=True, name='solo_install_extensions').start()\n"," threading.Thread(target = load_models,daemon=True).start()\n"," threading.Thread(target = install_dependencies,daemon=True, name='solo_install_dependencies').start()\n"," else:\n"," install_extensions()\n"," load_models()\n"," install_dependencies()\n"," t = 0\n"," while not envInstalled or not extensionsDone:\n"," if t%10==0:\n"," print('等待python环境和插件安装...')\n"," t = t+1\n"," time.sleep(1)\n"," link_dir()\n"," use_config()\n"," installProxyExe()\n"," os.environ['IsInstall'] = 'True'\n"," else:\n"," envInstalled = True\n"," extensionsDone = True\n","\n"," stop_solo_threads()\n"," localProxy()\n"," threading.Thread(target = startProxy, daemon=True, name='solo_startProxy').start()\n"," load_models(True)\n"," ticks = time.time()\n"," print(\"加载耗时:\",(ticks - startTicks),\"秒\")\n"," start()\n"]},{"cell_type":"markdown","metadata":{"id":"0oaCRs2gLCtF"},"source":["# 执行区域\n","---"]},{"cell_type":"code","execution_count":null,"metadata":{"_kg_hide-output":true,"id":"O3DR0DWHLCtF","scrolled":true,"trusted":true},"outputs":[],"source":["# 启动\n","# localProxy()\n","main()"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["# 停止一些不必要的后台线程\n","stop_solo_threads()"]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.12"}},"nbformat":4,"nbformat_minor":4}
|
|
|
|
| 1 |
+
{"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.12"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 这只是一个完整项目的一部分 不能运行的 \n- 可以从这个地址运行 [点击打开](https://www.kaggle.com/viyiviyi/sdwui-before)\n---","metadata":{"id":"6zE2QeUKLCtC"}},{"cell_type":"code","source":"print('''\n路径说明\n* 可以使用 $install_path 或 {install_path} 来访问安装目录，写在字符串内也会生效\n* $output_path 或 {output_path} 可以访问输出目录\n* $input_path 或 {input_path} 可以访问输入目录 在kaggle是数据集根目录\n\n更新日志\n* 23-07-18 \n* 如果有两个GPU 将会额外启动一个只支持api的的服务\n* 通过/1/作为base url来访问这个api\n* 没有启动两个webui是因为，很大概率爆内存\n''')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from pathlib import Path\nimport os\nimport time\nimport re\nimport subprocess\nimport threading\nimport sys\nimport socket\nimport torch\n\n# 检查gpu是否存在\nif torch.cuda.device_count() == 0:\n print(\"当前环境没有GPU\")\n exit()\n\ninstall_path=f\"{os.environ['HOME']}/sdwebui\" # 安装目录\noutput_path=f\"{os.environ['HOME']}/sdwebui/Output\" # 输出目录 如果使用google云盘 会在google云盘增加sdwebui/Output\ninput_path = '/kaggle/input' # 输入目录\n\ngoogle_drive = '' \n# 连接谷歌云\ntry:\n if useGooglrDrive:\n from google.colab import drive\n drive.mount(f'~/google_drive')\n google_drive = f\"{os.environ['HOME']}/google_drive/MyDrive\"\n output_path = f'{google_drive}/sdwebui/Output'\n input_path = f'{google_drive}/sdwebui/Input'\n !mkdir -p {input_path}\n # 设置文件路径\n if setting_file== '/kaggle/working/configs/config.json':\n setting_file = os.path.join(output_path,'configs/config.json')\n if ui_config_file == '/kaggle/working/configs/ui-config.json':\n ui_config_file = os.path.join(output_path,'configs/ui-config.json')\n print('''\n已经链接到谷歌云盘\n云盘根目录/sdwebui/Output 被链接为输出目录，图片和设置文件将会保存在此文件夹\n云盘根目录/sdwebui/Input 被链接为输入目录，环境变量名是 $input_path\n ''')\nexcept:\n useGooglrDrive = False\n\n!mkdir -p {install_path}\n!mkdir -p {output_path}\n\nos.environ['install_path'] = install_path\nos.environ['output_path'] = output_path\nos.environ['google_drive'] = google_drive\nos.environ['input_path'] = input_path\n\ndef replace_path(input_str:str):\n return input_str.replace('$install_path',install_path)\\\n .replace('{install_path}',install_path)\\\n .replace('$input_path',input_path)\\\n .replace('{input_path}',input_path)\\\n .replace('$output_path',output_path)\\\n .replace('{output_path}',output_path)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 模型目录 文件名称:链接的方式指定文件名\nmodelDirs = []+[replace_path(item.strip()) for item in 模型列表.split('\\n') if item.strip() ]\n# vae列表\nmodelVaeDirs = []+[replace_path(item.strip()) for item in VAE列表.split('\\n') if item.strip()]\n# 此配置内的目录下所有文件将加载到hypernetworks文件夹\nhypernetworksModelDirs = []+[replace_path(item.strip()) for item in hypernetworks列表.split('\\n') if item.strip()] \n# 此配置内的目录下所有文件将加载到embeddings文件夹\nembeddingsModelDirs = [] +[replace_path(item.strip()) for item in embeddings列表.split('\\n') if item.strip()]\n# 此配置内的目录下所有文件将加载到Lora文件夹\nloraModelDirs = []+[replace_path(item.strip()) for item in Lora列表.split('\\n') if item.strip()] \n\nlyCORISModelDirs = []+[replace_path(item.strip()) for item in LyCORIS列表.split('\\n') if item.strip()] \n# controlNet插件模型列表\ncontrolNetModels=[] +[replace_path(item.strip()) for item in controlNet模型列表.split('\\n') if item.strip()]\n# 插件列表\nextensions=[]+[replace_path(item.strip()) for item in 插件列表.split('\\n') if item.strip()]\n# 配置文件链接\nlinkHypernetworksDir=False # 链接 hypernetworks 目录到输出目录\nlinkEmbeddingsDir=False # 链接 embeddings 目录到输出目录\nlinkTextual_inversionDir=False # 链接 textual_inversion 目录到输出目录 如果需要保存训练过程的产出文件时建议开启\n\nngrokTokenFile = os.path.join(input_path,'configs/ngrok_token.txt') # 非必填 存放ngrokToken的文件的路径\nfrpcConfigFile = os.path.join(input_path,'configs/frpc_koishi.ini') # 非必填 frp 配置文件\n# ss证书目录 下载nginx的版本，把pem格式改成crt格式\nfrpcSSLFFlies =[os.path.join(input_path,'configs/koishi_ssl')]+[replace_path(item.strip()) for item in frpSSL文件.split('\\n') if item.strip() ]\n# frpc 文件目录 如果目录不存在，会自动下载，也可以在数据集搜索 viyiviyi/utils 添加\nfrpcExePath = os.path.join(input_path,'utils-tools/frpc')\n# 其他需要加载的webui启动参数 写到【参数列表】这个配置去\notherArgs = ' '.join([replace_path(item.strip()) for item in 参数列表.split('\\n') if item.strip()])\n\nvenvPath = os.path.join(input_path,'sd-webui-venv/venv.tar.bak') # 安装好的python环境 sd-webui-venv是一个公开是数据集 可以搜索添加\n\n# 用于使用kaggle api的token文件 参考 https://www.kaggle.com/docs/api\n# 此文件用于自动上传koishi的相关配置 也可以用于保存重要的输出文件\nkaggleApiTokenFile = os.path.join(input_path,'configs/kaggle.json')\n","metadata":{"_kg_hide-input":true,"id":"i3LhnwYHLCtC","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 这下面的是用于初始化一些值或者环境变量的，轻易别改\nfrpcStartArg = ''\n\nfrp配置文件或配置 = replace_path(frp配置文件或配置)\nngrok配置或文件地址 = replace_path(ngrok配置或文件地址)\nprint(frp配置文件或配置)\n\n!mkdir -p $install_path/configFiles\nif Path(frp配置文件或配置.strip()).exists():\n frpcConfigFile = frp配置文件或配置.strip()\nif not Path(frpcConfigFile).exists(): \n if frp配置文件或配置.strip().startswith('-f'):\n frpcStartArg = frp配置文件或配置.strip()\n else:\n print('没有frpcp配置')\n useFrpc = False\nelse:\n os.system(f'cp -f {frpcConfigFile} {install_path}/configFiles/frpc_webui.ini')\n frpcConfigFile = f'{install_path}/configFiles/frpc_webui.ini'\n os.system(f'sed -i \"s/local_port = .*/local_port = {webuiPort}/g\" {frpcConfigFile}')\n frpcStartArg = f' -c {frpcConfigFile}'\n\nngrokToken=''\nif Path(ngrok配置或文件地址.strip()).exists():\n ngrokTokenFile = ngrok配置或文件地址.strip()\nif Path(ngrokTokenFile).exists():\n with open(ngrokTokenFile,encoding = \"utf-8\") as nkfile:\n ngrokToken = nkfile.readline()\nelif not ngrok配置或文件地址.strip().startswith('/'):\n ngrokToken=ngrok配置或文件地址.strip()\n \nif not Path(venvPath).exists():\n venvPath = os.path.join(input_path,'sd-webui-venv/venv.zip')","metadata":{"_kg_hide-input":true,"id":"a_GtG2ayLCtD","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# kaggle public API\n\n**不能使用%cd这种会改变当前工作目录的命令，会导致和其他线程冲突**\n\n---","metadata":{"id":"p0uS-BLULCtD"}},{"cell_type":"code","source":"# 安装kaggle的api token文件\ndef initKaggleConfig():\n if Path('~/.kaggle/kaggle.json').exists():\n return True\n if Path(kaggleApiTokenFile).exists():\n !mkdir -p ~/.kaggle/\n os.system('cp '+kaggleApiTokenFile+' ~/.kaggle/kaggle.json')\n !chmod 600 ~/.kaggle/kaggle.json\n return True\n print('缺少kaggle的apiToken文件，访问：https://www.kaggle.com/你的kaggle用户名/account 获取')\n return False\n\ndef getUserName():\n if not initKaggleConfig(): return\n import kaggle\n return kaggle.KaggleApi().read_config_file()['username']\n\ndef createOrUpdateDataSet(path:str,datasetName:str):\n if not initKaggleConfig(): return\n print('创建或更新数据集 '+datasetName)\n import kaggle\n os.system('mkdir -p $install_path/kaggle_cache')\n os.system('rm -rf $install_path/kaggle_cache/*')\n datasetDirPath = install_path+'/kaggle_cache/'+datasetName\n os.system('mkdir -p '+datasetDirPath)\n os.system('cp -f '+path+' '+datasetDirPath+'/')\n username = getUserName()\n print(\"kaggle username:\"+username)\n datasetPath = username+'/'+datasetName\n datasetList = kaggle.api.dataset_list(mine=True,search=datasetPath)\n print(datasetList)\n if len(datasetList) == 0 or datasetPath not in [str(d) for d in datasetList]: # 创建 create\n os.system('kaggle datasets init -p' + datasetDirPath)\n metadataFile = datasetDirPath+'/dataset-metadata.json'\n os.system('sed -i s/INSERT_TITLE_HERE/'+ datasetName + '/g ' + metadataFile)\n os.system('sed -i s/INSERT_SLUG_HERE/'+ datasetName + '/g ' + metadataFile)\n os.system('cat '+metadataFile)\n os.system('kaggle datasets create -p '+datasetDirPath)\n print('create database done')\n else:\n kaggle.api.dataset_metadata(datasetPath,datasetDirPath)\n kaggle.api.dataset_create_version(datasetDirPath, 'auto update',dir_mode='zip')\n print('upload database done')\n\ndef downloadDatasetFiles(datasetName:str,outputPath:str):\n if not initKaggleConfig(): return\n print('下载数据集文件 '+datasetName)\n import kaggle\n username = getUserName()\n datasetPath = username+'/'+datasetName\n datasetList = kaggle.api.dataset_list(mine=True,search=datasetPath)\n if datasetPath not in [str(d) for d in datasetList]:\n return False\n os.system('mkdir -p '+outputPath)\n kaggle.api.dataset_download_files(datasetPath,path=outputPath,unzip=True)\n return True\n\n","metadata":{"_kg_hide-input":true,"id":"m8FJi4j0LCtD","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 工具函数\n**不能使用%cd这种会改变当前工作目录的命令，会导致和其他线程冲突**\n\n---","metadata":{"id":"sswa04veLCtE"}},{"cell_type":"code","source":"# 绕过 os.systen 的限制执行命令\ndef run(shell:str,shellName=''):\n if shellName == '': shellName = str(time.time())\n !mkdir -p $install_path/run_cache\n with open(install_path+'/run_cache/run_cache.'+shellName+'.sh','w') as sh:\n sh.write(shell)\n !bash {install_path}/run_cache/run_cache.{shellName}.sh\n\n# 连接多个路径字符串 让路径在shell命令中能正常的执行\ndef pathJoin(*paths:str):\n pathStr = ''\n for p in paths:\n pathStr += '\"'+p+'\"'\n pathStr = '\"*\"'.join(pathStr.split('*'))\n pathStr = '\"$\"'.join(pathStr.split('$'))\n pathStr = '\"(\"'.join(pathStr.split('('))\n pathStr = '\")\"'.join(pathStr.split(')'))\n pathStr = '\"{\"'.join(pathStr.split('{'))\n pathStr = '\"}\"'.join(pathStr.split('}'))\n pathStr = re.sub(r'\"\"','',pathStr)\n pathStr = re.sub(r'\\*{2,}','\"',pathStr)\n pathStr = re.sub(r'/{2,}','/',pathStr)\n pathStr = re.sub(r'/\\./','/',pathStr)\n return pathStr\n\n# 判断路径是不是一个文件或者可能指向一些文件\ndef pathIsFile(path):\n if Path(path).is_file():\n return True\n if re.search(r'\\.(ckpt|safetensors|png|jpg|txt|pt|pth|json|yaml|\\*)$',path):\n return True\n return False\n\ndef echoToFile(content:str,path:str):\n with open(path,'w') as sh:\n sh.write(content)\n\n# ngrok\ndef startNgrok(ngrokToken:str,ngrokLocalPort:int):\n from pyngrok import conf, ngrok\n try:\n conf.get_default().auth_token = ngrokToken\n conf.get_default().monitor_thread = False\n ssh_tunnels = ngrok.get_tunnels(conf.get_default())\n if len(ssh_tunnels) == 0:\n ssh_tunnel = ngrok.connect(ngrokLocalPort)\n print('ngrok 访问地址：'+ssh_tunnel.public_url)\n else:\n print('ngrok 访问地址：'+ssh_tunnels[0].public_url)\n except:\n print('启动ngrok出错')\n \ndef startFrpc(name,configFile):\n run(f'''\ncd $install_path/frpc/\n$install_path/frpc/frpc {configFile}\n ''',name)\n \ndef installProxyExe():\n if useFrpc:\n print('安装frpc')\n !mkdir -p $install_path/frpc\n if Path(frpcExePath).exists():\n os.system(f'cp -f -n {frpcExePath} $install_path/frpc/frpc')\n else:\n !wget \"https://huggingface.co/datasets/ACCA225/Frp/resolve/main/frpc\" -O $install_path/frpc/frpc\n \n for ssl in frpcSSLFFlies:\n if Path(ssl).exists():\n os.system('cp -f -n '+pathJoin(ssl,'/*')+' $install_path/frpc/')\n !chmod +x $install_path/frpc/frpc\n !$install_path/frpc/frpc -v\n if useNgrok:\n %pip install pyngrok\n\ndef startProxy():\n if useNgrok:\n startNgrok(ngrokToken,webuiPort)\n if useFrpc:\n startFrpc('frpc_proxy',frpcStartArg)\n\n \ndef zipPath(path:str,zipName:str,format='tar'):\n if path.startswith('$install_path'):\n path = path.replace('$install_path',install_path)\n if path.startswith('$output_path'):\n path = path.replace('$install_path',output_path)\n if not path.startswith('/'):\n path = pathJoin(install_path,'/stable-diffusion-webui','/',path)\n if Path(path).exists():\n if 'tar' == format:\n os.system('tar -cf $output_path/'+ zipName +'.tar -C '+ path +' . ')\n elif 'gz' == format:\n os.system('tar -czf $output_path/'+ zipName +'.tar.gz -C '+ path +' . ')\n return\n print('指定的目录不存在：'+path)\n\n \ndef check_service(host, port):\n try:\n socket.create_connection((host, port), timeout=5)\n return True\n except socket.error:\n return False","metadata":{"_kg_hide-input":true,"id":"coqQvTSLLCtE","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport re\nimport requests\n\ndef download_file(url, filename, path):\n # 获取文件的真实文件名\n if not filename:\n with requests.get(url, stream=True) as r:\n if 'Content-Disposition' in r.headers:\n filename = r.headers['Content-Disposition'].split('filename=')[1].strip('\"')\n r.close()\n filename = re.sub(r'[\\\\/:*?\"<>|;]', '', filename)\n # 拼接文件的完整路径\n filepath = os.path.join(path, filename)\n # 判断文件是否已存在\n if os.path.exists(filepath):\n print(f'{filename} already exists in {path}')\n return\n # 下载文件\n with requests.get(url, stream=True) as r:\n r.raise_for_status()\n with open(filepath, 'wb') as f:\n for chunk in r.iter_content(chunk_size=8192):\n if chunk:\n f.write(chunk)\n print(f'{filename} has been downloaded to {path}')\n \n# 加入文件到下载列表\ndef putDownloadFile(url:str,distDir:str,file_name:str=None):\n if re.match(r'^[^:]+:(https?|ftps?)://', url, flags=0):\n file_name = re.findall(r'^[^:]+:',url)[0][:-1]\n url = url[len(file_name)+1:]\n if not re.match(r'^(https?|ftps?)://',url):\n return\n file_name = re.sub(r'\\s+','_',file_name or '')\n dir = str(hash(url)).replace('-','')\n down_dir = f'{install_path}/down_cache/{dir}'\n !mkdir -p {down_dir}\n return [url,file_name,distDir,down_dir]\n\ndef get_file_size_in_gb(file_path):\n size_in_bytes = Path(file_path).stat().st_size\n size_in_gb = size_in_bytes / (1024 ** 3)\n return '%.2f' % size_in_gb\n \n# 下载文件\ndef startDownloadFiles(download_list):\n print('下载列表:\\n','\\n'.join([f'{item[0]} -> {item[2]}/{item[1]}' for item in download_list]))\n dist_list = []\n for dow_f in download_list:\n !mkdir -p {dow_f[3]}\n print('下载 名称：',dow_f[1],'url：',dow_f[0])\n download_file(dow_f[0],dow_f[1],dow_f[2])","metadata":{"_kg_hide-input":true,"id":"THNUsEm_LCtE","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# nginx 反向代理配置文件\ndef localProxy():\n conf = '''\nserver\n{\n listen '''+str(webuiPort)+''';\n listen [::]:'''+str(webuiPort)+''';\n server_name 127.0.0.1 localhost 0.0.0.0 \"\";\n \n if ($request_method = OPTIONS) {\n return 200;\n }\n fastcgi_send_timeout 10m;\n fastcgi_read_timeout 10m;\n fastcgi_connect_timeout 10m;\n location /1/\n {\n proxy_pass http://127.0.0.1:'''+str(webuiPort+2)+'''/;\n # add_header Set-Cookie \"subpath=1; expires=0; Path=/\";\n # proxy_set_header Set-Cookie \"subpath=1; expires=0; Path=/\";\n proxy_set_header Host $host;\n proxy_set_header X-Real-IP $remote_addr;\n proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n proxy_set_header REMOTE-HOST $remote_addr;\n proxy_set_header Upgrade $http_upgrade;\n proxy_set_header Connection upgrade;\n proxy_http_version 1.1;\n proxy_connect_timeout 10m;\n proxy_read_timeout 10m;\n }\n location /\n {\n set $proxy_url http://127.0.0.1:'''+str(webuiPort+1)+''';\n # if ($cookie_subpath = \"1\") {\n # set $proxy_url http://127.0.0.1:'''+str(webuiPort+2)+''';\n # }\n proxy_pass $proxy_url;\n proxy_set_header Host $host;\n proxy_set_header X-Real-IP $remote_addr;\n proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n proxy_set_header REMOTE-HOST $remote_addr;\n proxy_set_header Upgrade $http_upgrade;\n proxy_set_header Connection upgrade;\n proxy_http_version 1.1;\n proxy_connect_timeout 10m;\n proxy_read_timeout 10m;\n }\n}\n'''\n echoToFile(conf,'/etc/nginx/conf.d/proxy_nginx.conf')\n if not check_service('localhost',webuiPort):\n !nginx -c /etc/nginx/nginx.conf\n !nginx -s reload\n ","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import inspect\nimport ctypes\n\ndef _async_raise(tid, exctype):\n \"\"\"raises the exception, performs cleanup if needed\"\"\"\n tid = ctypes.c_long(tid)\n if not inspect.isclass(exctype):\n exctype = type(exctype)\n res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype))\n if res == 0:\n raise ValueError(\"invalid thread id\")\n elif res != 1:\n # \"\"\"if it returns a number greater than one, you're in trouble,\n # and you should call it again with exc=NULL to revert the effect\"\"\"\n ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None)\n raise SystemError(\"PyThreadState_SetAsyncExc failed\")\n\ndef stop_thread(thread):\n _async_raise(thread.ident, SystemExit)\n\ndef stop_solo_threads():\n # 获取当前所有活动的线程\n threads = threading.enumerate()\n # 关闭之前创建的子线程\n for thread in threads:\n if thread.name.startswith('solo_'):\n print(thread.name)\n try:\n stop_thread(thread)\n except socket.error:\n print(f'结束进程：{thread.name} 执行失败')","metadata":{"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# webui 安装和配置函数\n---","metadata":{"id":"Ve3p8oOkLCtE"}},{"cell_type":"code","source":"envInstalled=False\nextensionsDone=False\n\n#安装webui\ndef install():\n print('安装webui')\n %cd $install_path\n if reLoad:\n !rm -rf stable-diffusion-webui\n if Path(\"stable-diffusion-webui\").exists():\n %cd $install_path/stable-diffusion-webui/\n !git checkout .\n !git pull\n else:\n if mobileOptimize:\n !git clone https://github.com/viyiviyi/stable-diffusion-webui.git stable-diffusion-webui -b local # 修改了前端界面，手机使用更方便\n else:\n !git clone https://github.com/AUTOMATIC1111/stable-diffusion-webui stable-diffusion-webui\n print('安装webui 完成')\n\n# 链接输出目录\ndef link_dir():\n print('链接输出目录')\n # 链接输出目录 因为sd被部署在了stable-diffusion-webui目录，运行结束后为了方便下载就只有outputs目录放在output_path/目录下\n !mkdir -p $output_path/outputs\n !rm -rf $install_path/stable-diffusion-webui/outputs\n !ln -s -r $output_path/outputs $install_path/stable-diffusion-webui/ # 输出目录\n !mkdir -p $output_path/log\n !rm -rf $install_path/stable-diffusion-webui/log\n !ln -s -r $output_path/log $install_path/stable-diffusion-webui/\n # 链接 hypernetworks 目录\n if linkHypernetworksDir:\n !rm -rf $install_path/stable-diffusion-webui/models/hypernetworks\n !mkdir -p $output_path/hypernetworks\n !ln -s -r $output_path/hypernetworks $install_path/stable-diffusion-webui/models/\n # 链接 embeddings 目录\n if linkEmbeddingsDir:\n !rm -rf $install_path/stable-diffusion-webui/embeddings\n !mkdir -p $output_path/embeddings\n !ln -s -r $output_path/embeddings $install_path/stable-diffusion-webui/\n # 链接训练输出目录 文件夹链接会导致功能不能用\n if linkTextual_inversionDir:\n !rm -rf $install_path/stable-diffusion-webui/textual_inversion\n !mkdir -p $output_path/textual_inversion/\n !ln -s -r $output_path/textual_inversion $install_path/stable-diffusion-webui/\n print('链接输出目录 完成')\n \n# 链接模型文件\ndef load_models(skip_url=False):\n print(('复制' if enableLoadByCopy else '链接') + '模型文件')\n if enableLoadByCopy:\n print('如果出现 No such file or directory 错误，可以忽略')\n download_list = []\n models_dir_path = f'{install_path}/stable-diffusion-webui/models/Stable-diffusion'\n hypernetworks_dir_path = f'{install_path}/stable-diffusion-webui/models/hypernetworks'\n embeddings_dir_path = f'{install_path}/stable-diffusion-webui/models/embeddings'\n lora_dir_path = f'{install_path}/stable-diffusion-webui/models/Lora'\n lycoris_dir_path = f'{install_path}/stable-diffusion-webui/models/LyCORIS'\n vae_dir_path = f'{install_path}/stable-diffusion-webui/models/VAE'\n control_net_dir_path = f'{install_path}/stable-diffusion-webui/extensions/sd-webui-controlnet/models'\n \n def link_file(source_paths,dist:str):\n !mkdir -p {dist}\n for path in source_paths:\n if 'https://' in path or 'http://' in path:\n if skip_url:\n continue\n download_list.append(putDownloadFile(path,dist))\n elif pathIsFile(path):\n os.system(('cp -n' if enableLoadByCopy else 'ln -s')+' -f '+ pathJoin(path) +f' {dist}')\n else:\n os.system(('cp -n' if enableLoadByCopy else 'ln -s')+' -f '+ pathJoin(path,'/*.*') +f' {dist}')\n !rm -f {dist}/\\*.* \n \n link_file(modelDirs,models_dir_path)\n link_file(hypernetworksModelDirs,hypernetworks_dir_path)\n link_file(embeddingsModelDirs,embeddings_dir_path)\n link_file(loraModelDirs,lora_dir_path)\n link_file(lyCORISModelDirs,lycoris_dir_path)\n link_file(modelVaeDirs,vae_dir_path)\n link_file(controlNetModels,control_net_dir_path)\n startDownloadFiles([item for item in download_list if item])\n print(('复制' if enableLoadByCopy else '链接') + '模型文件 完成')\n\n#安装依赖\ndef install_dependencies():\n print('安装webui需要的python环境')\n global envInstalled\n global venvPath\n !sudo apt install nginx -y\n sh = f'cd {install_path}/stable-diffusion-webui\\n'\n if str(sys.version).startswith('3.8') or str(sys.version).startswith('3.9') or str(sys.version).startswith('3.10'):\n sh += '''\npython3 -m venv venv\n'''\n else:\n sh += '''\nadd-apt-repository ppa:deadsnakes/ppa -y\napt update\napt install python3.10 -y\npython3.10 -m venv venv\n'''\n if quickStart:\n if not Path(venvPath).exists():\n !mkdir -p {install_path}/venv_cache\n if not Path(f'{install_path}/venv_cache/venv.tar.bak').exists():\n download_file('https://huggingface.co/viyi/sdwui/resolve/main/venv.zip','venv.zip',f'{install_path}/venv_cache')\n !unzip {install_path}/venv_cache/venv.zip -d {install_path}/venv_cache\n venvPath = f'{install_path}/venv_cache/venv.tar.bak'\n !rm -rf {install_path}/venv_cache/venv.zip\n elif venvPath.endswith('.zip'):\n !mkdir -p {install_path}/venv_cache\n !unzip {venvPath} -d {install_path}/venv_cache\n venvPath = f'{install_path}/venv_cache/venv.tar.bak'\n \n sh += 'echo \"unzip venv\"\\n'\n sh += f'tar -xf {venvPath} -C ./venv\\n'\n sh += '\\n'\n run(sh,'dependencies')\n if not Path(f'{install_path}/stable-diffusion-webui/venv/bin/pip').exists():\n !curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py\n !{install_path}/stable-diffusion-webui/venv/bin/python3 get-pip.py\n \n !{install_path}/stable-diffusion-webui/venv/bin/python3 -V\n !{install_path}/stable-diffusion-webui/venv/bin/python3 -m pip -V\n \n if not quickStart or not Path(venvPath).exists():\n !{install_path}/stable-diffusion-webui/venv/bin/python3 -m pip install -U torch==2.0.0+cu117 torchvision==0.15.1+cu117 torchaudio==2.0.1 --index-url https://download.pytorch.org/whl/cu117\n !{install_path}/stable-diffusion-webui/venv/bin/python3 -m pip install -U open-clip-torch xformers==0.0.19\n \n envInstalled = True\n print('安装webui需要的python环境 完成')\n\n# 安装插件\ndef install_extensions():\n global extensionsDone\n print('安装插件')\n sh = 'mkdir -p $install_path/stable-diffusion-webui/extensions \\n'\n sh = 'mkdir -p $install_path/cache/extensions \\n'\n sh += 'cd $install_path/cache/extensions \\n'\n for ex in extensions:\n sh += 'git clone ' + ex + ' \\n'\n sh += 'rsync -a $install_path/cache/extensions/* $install_path/stable-diffusion-webui/extensions \\n'\n sh += 'cd ../ && rm -rf $install_path/cache/extensions \\n'\n run(sh,'extensions')\n extensionsDone = True\n print('安装插件 完成')\n \n# 个性化配置 \ndef use_config():\n print('使用自定义配置 包括tag翻译 \\n')\n %cd $install_path/stable-diffusion-webui\n !mkdir -p tmp\n %cd tmp\n !git clone {webui_settings} sd-configs\n !cp -rf sd-configs/dist/* $install_path/stable-diffusion-webui\n if not Path(ui_config_file).exists(): # ui配置文件\n !mkdir -p {ui_config_file[0:ui_config_file.rfind('/')]}\n os.system('cp -f -n $install_path/stable-diffusion-webui/ui-config.json '+ui_config_file)\n if not Path(setting_file).exists(): # 设置配置文件\n !mkdir -p {setting_file[0:setting_file.rfind('/')]}\n os.system('cp -f -n $install_path/stable-diffusion-webui/config.json '+setting_file)\n\ndef checkDefaultModel():\n print('检查默认模型文件是否存在 \\n')\n global usedCkpt\n if usedCkpt is not None and usedCkpt != '': # 设置启动时默认加载的模型\n if '.' in usedCkpt:\n if '/' in usedCkpt:\n usedCkpt = usedCkpt\n else:\n usedCkpt = install_path + '/stable-diffusion-webui/models/Stable-diffusion/' + usedCkpt\n else:\n for x in ['.ckpt','.safetensors']:\n if Path(install_path+'/stable-diffusion-webui/models/Stable-diffusion/' + usedCkpt+x).exists():\n usedCkpt = install_path+'/stable-diffusion-webui/models/Stable-diffusion/' + usedCkpt+x\n break\n if Path(usedCkpt).exists():\n return True\n else:\n if Path(usedCkpt).is_symlink():\n print('模型文件真实地址：'+os.readlink(usedCkpt))\n return False\n \ndef start_webui(i):\n # 只要不爆内存，其他方式关闭后会再次重启 访问地址会发生变化\n print(i,'--port',str(webuiPort+1+i))\n if i>0:\n print(f'使用第{i+1}张显卡启动第{i+1}个webui，通过frpc或nrgok地址后加/{i}/进行访问（不能使用同一个浏览器）')\n if useFrpc:\n while True:\n !venv/bin/python3 launch.py --device-id={i} --port {str(webuiPort+1+i)} {'' if i ==0 else '--nowebui'} # {'' if i == 0 else '--subpath=/'+str(i)}\n print('5秒后重启webui')\n time.sleep(5)\n else:\n !venv/bin/python3 launch.py --device-id={i} --port {str(webuiPort+1+i)} {'' if i ==0 else '--nowebui'} # {'' if i == 0 else '--subpath=/'+str(i)}\n \n# 启动\ndef start():\n print('启动webui')\n %cd $install_path/stable-diffusion-webui\n args = '' # ' --port=' + str(webuiPort+1)\n if not disableShared:\n args += ' --share'\n if onlyApi:\n args += ' --nowebui'\n if ui_config_file is not None and ui_config_file != '' and Path(ui_config_file).exists(): # ui配置文件\n args += ' --ui-config-file=' + pathJoin(ui_config_file)\n if setting_file is not None and setting_file != '' and Path(setting_file).exists(): # 设置配置文件\n args += ' --ui-settings-file=' + pathJoin(setting_file)\n if not checkDefaultModel():\n print('默认模型文件不存在，请检查配置：'+usedCkpt)\n else:\n args += ' --ckpt='+ pathJoin(usedCkpt)\n if vaeHalf is False: \n args += ' --no-half-vae'\n if modelHalf is False:\n args += ' --no-half'\n if consoleProgressbars is False:\n args += ' --disable-console-progressbars'\n if consolePrompts is True:\n args += ' --enable-console-prompts'\n args += ' ' + otherArgs\n os.environ['COMMANDLINE_ARGS']=args\n !echo COMMANDLINE_ARGS=$COMMANDLINE_ARGS\n %env REQS_FILE=requirements.txt\n \n threads = []\n for i in range(torch.cuda.device_count()):\n thread = threading.Thread(target = start_webui, name='solo_start_webui_'+str(i), daemon=True,args=([i]))\n thread.start()\n threads.append(thread)\n while thread.is_alive() and not check_service('localhost',str(webuiPort+1+i)): # 当当前服务启动完成才允许退出此次循环\n time.sleep(10)\n time.sleep(5)\n for thread in threads:\n thread.join()","metadata":{"_kg_hide-input":true,"id":"GTjyBJihLCtE","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 入口函数\n---","metadata":{"id":"qLvsk8ByLCtF"}},{"cell_type":"code","source":"\n# 启动非webui相关的的内容，加快启动速度\ndef main():\n global envInstalled\n global extensionsDone\n startTicks = time.time()\n isInstall = True if os.getenv('IsInstall','False') == 'True' else False\n if isInstall is False or reLoad: \n install()\n if enableThread:\n threading.Thread(target = install_extensions,daemon=True, name='solo_install_extensions').start()\n threading.Thread(target = load_models,daemon=True).start()\n threading.Thread(target = install_dependencies,daemon=True, name='solo_install_dependencies').start()\n else:\n install_extensions()\n load_models()\n install_dependencies()\n t = 0\n while not envInstalled or not extensionsDone:\n if t%10==0:\n print('等待python环境和插件安装...')\n t = t+1\n time.sleep(1)\n link_dir()\n use_config()\n installProxyExe()\n os.environ['IsInstall'] = 'True'\n else:\n envInstalled = True\n extensionsDone = True\n\n stop_solo_threads()\n localProxy()\n threading.Thread(target = startProxy, daemon=True, name='solo_startProxy').start()\n load_models(True)\n ticks = time.time()\n print(\"加载耗时:\",(ticks - startTicks),\"秒\")\n start()\n","metadata":{"_kg_hide-input":true,"id":"IOKjaMlcLCtF","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 执行区域\n---","metadata":{"id":"0oaCRs2gLCtF"}},{"cell_type":"code","source":"# 启动\n# localProxy()\nmain()","metadata":{"_kg_hide-output":true,"id":"O3DR0DWHLCtF","scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 停止一些不必要的后台线程\nstop_solo_threads()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}
|