diff --git "a/sdwui-start-util.dev.ipynb" "b/sdwui-start-util.dev.ipynb" --- "a/sdwui-start-util.dev.ipynb" +++ "b/sdwui-start-util.dev.ipynb" @@ -1 +1 @@ -{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["\n"]},{"cell_type":"code","execution_count":null,"metadata":{},"outputs":[],"source":["update_desc = '''\n","欢迎使用 stable-diffusion-webui 便捷启动脚本\n","\n","此脚本理论上可以在任何jupyter环境运行，但仅在 google colab 和 kaggle 测试。 已经无法在 google colab 免费实例上运行。\n","此脚本的【前置脚本】发布地址 https://www.kaggle.com/code/yiyiooo/stable-diffusion-webui-novelai-sdxl。\n","此脚本需配合 【前置脚本】 的脚本附带的配置项才能正常启动。\n","此脚本的内容会自动更新，你无需更新【前置脚本】就能获取到最新的功能。\n","如果新功能需要增加新的配置项，可以在更新日志中查看到。\n","\n","路径说明\n","* 为了解决平台差异，所有的安装目录和文件输出目录都被重新指定，如果你需要在配置中访问这些目录，请查看以下说明\n","*\n","* 如果链接了谷歌云盘，将会在云盘根目录创建 sdwebui 的文件夹，文件夹内的 Input 目录将会作为 输入目录； Output 将会作为输出目录\n","* 可以使用 $install_path 或 {install_path} 来访问安装目录，写在字符串内也会生效\n","* 项目跟目录固定是： {install_path}/sd_main_dir 或 $install_path/sd_main_dir\n","* $output_path 或 {output_path} 可以访问输出目录\n","* $input_path 或 {input_path} 可以访问输入目录 在kaggle是数据集根目录\n","\n","更新日志\n","* 23-11-09\n","* 可自定义方法 on_before_start 并在方法内写上启动前需要的逻辑来实现在webui启动前执行自定义逻辑\n","*\n","* 23-10-28\n","* 可通过 {install_path} {output_path} {input_path} {ui_dir_name} 来访问对应路径\n","* \n","* 23-10-20\n","* 可通过增加配置 install_path = '文件夹路径' 来指定安装位置，所有下载的内容将保存在这个目录。默认值是 {os.environ['HOME']}/sd_webui\n","* 可通过增加配置 output_path = '文件夹路径' 来指定输出目录，输出的图片将会保存在这个目录下。默认值是 /kaggle/working\n","* 可通过增加配置 input_path = '文件夹路径' 来指定数据集（输入）目录。默认值是 /kaggle/input\n","* 可通过增加配置 ui_dir_name = '文件夹名称' sd的安装路径的名称，sd的安装根目录将会以此命名。默认值是 sd_main_dir\n","*\n","* 23-09-13\n","* 修改了项目安装目录的文件夹名称，改为sd_main_dir，访问方式是{install_path}/sd_main_dir\n","*\n","* 23-08-02\n","* 需要增加配置 multi_case = True 来使用第二张显卡\n","*\n","* 23-08-01\n","* 估计是新的gradio的问题，需要禁用 --no-gradio-queue 启动参数才能正常访问，否则会一直处于加载界面，所以这个参数会被屏蔽不再生效\n","*\n","* 23-07-30\n","* 使用新的文件加载逻辑重写完成，文件加载和加载更加简单，可以支持自定义下载或加载文件的目标目录了\n","*\n","* 23-07-26\n","* 增加可隐藏启动时的不重要信息 通过增加配置 hidden_console_info = True 开启\n","*\n","* 23-07-22\n","* 更新了可自动同步收藏目录到 https://huggingface.co/ 的功能\n","* 可通过增加配置项 _huggingface_token = \"token\" 和 _huggingface_repo = “仓库id” 后使用此功能\n","*\n","* 23-07-18 \n","* 如果有两个GPU 第二个GPU将会启动api服务，可用api的方式调用，和webui互相独立，绘图时不会导致另外一边卡顿\n","* 通过/1/作为base url来访问这个api\n","* 没有启动两个webui是因为，很大概率爆内存\n","'''"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["from pathlib import Path\n","import os\n","import time\n","import re\n","import subprocess\n","import threading\n","import sys\n","import socket\n","import torch\n","from typing import List"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["# 内置参数默认值，当上下文有参数时可覆盖默认值\n","\n","_useFrpc = locals().get('useFrpc') or globals().get('useFrpc') or True\n","\n","_useNgrok = locals().get('useNgrok') or globals().get('useNgrok') or True\n","\n","_reLoad = locals().get('reLoad') or globals().get('reLoad') or False\n"," \n","_before_downloading = locals().get('before_downloading') or globals().get('before_downloading') or ''\n","\n","_async_downloading = locals().get('async_downloading') or globals().get('async_downloading') or ''\n","\n","_before_start_sync_downloading = locals().get('before_start_sync_downloading') or globals().get('before_start_sync_downloading') or ''\n","\n","_server_port = locals().get('server_port') or globals().get('server_port') or 7860\n"," \n","_sd_git_repo = locals().get('sd_git_repo') or globals().get('sd_git_repo') or 'https://github.com/viyiviyi/stable-diffusion-webui.git -b local' \n","_sd_git_repo = _sd_git_repo\\\n"," .replace('{sdwui}','stable-diffusion-webui')\\\n"," .replace('{wui}',\"webui\")\n"," \n","_sd_config_git_repu = locals().get('sd_config_git_repu') or globals().get('sd_config_git_repu') or 'https://github.com/viyiviyi/sd-configs.git'\n","_sd_config_git_repu = _sd_config_git_repu\\\n"," .replace('{sdwui}','stable-diffusion-webui')\\\n"," .replace('{wui}',\"webui\")\n"," \n"," \n","_huggingface_token = locals().get('huggingface_token') or globals().get('huggingface_token') or '{input_path}/configs/huggingface_token.txt'\n","_huggingface_token = _huggingface_token\\\n"," .replace('{sdwui}','stable-diffusion-webui')\\\n"," .replace('{wui}',\"webui\")\n"," \n","_huggingface_repo = locals().get('huggingface_repo') or globals().get('huggingface_repo') or ''\n","_huggingface_repo = _huggingface_repo\\\n"," .replace('{sdwui}','stable-diffusion-webui')\\\n"," .replace('{wui}',\"webui\")\n","\n","_link_instead_of_copy = locals().get('link_instead_of_copy') or globals().get('link_instead_of_copy') or True\n"," \n","show_shell_info = locals().get('hidden_console_info') or globals().get('hidden_console_info')\n","if show_shell_info is None: show_shell_info = False\n","\n","_multi_case = locals().get('multi_case') or globals().get('multi_case') or False\n"," \n","_skip_start = locals().get('skip_start') or globals().get('skip_start') or True\n","\n","def before_start():\n"," ''''''\n","\n","_on_before_start = locals().get('on_before_start') or globals().get('on_before_start') or before_start \n"," \n","run_by_none_device = False"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["\n","def run(command, cwd=None, desc=None, errdesc=None, custom_env=None,try_error:bool=True) -> str:\n"," global show_shell_info\n"," if desc is not None:\n"," print(desc)\n","\n"," run_kwargs = {\n"," \"args\": command,\n"," \"shell\": True,\n"," \"cwd\": cwd,\n"," \"env\": os.environ if custom_env is None else custom_env,\n"," \"encoding\": 'utf8',\n"," \"errors\": 'ignore',\n"," }\n","\n"," if not show_shell_info:\n"," run_kwargs[\"stdout\"] = run_kwargs[\"stderr\"] = subprocess.PIPE\n","\n"," result = subprocess.run(**run_kwargs)\n","\n"," if result.returncode != 0:\n"," error_bits = [\n"," f\"{errdesc or 'Error running command'}.\",\n"," f\"Command: {command}\",\n"," f\"Error code: {result.returncode}\",\n"," ]\n"," if result.stdout:\n"," error_bits.append(f\"stdout: {result.stdout}\")\n"," if result.stderr:\n"," error_bits.append(f\"stderr: {result.stderr}\")\n"," if try_error:\n"," print((RuntimeError(\"\\n\".join(error_bits))))\n"," else:\n"," raise RuntimeError(\"\\n\".join(error_bits))\n","\n"," if show_shell_info:\n"," print((result.stdout or \"\"))\n"," return (result.stdout or \"\")\n","\n","def mkdirs(path, exist_ok=True):\n"," if path and not Path(path).exists():\n"," os.makedirs(path,exist_ok=exist_ok)"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["\n","# 检查gpu是否存在\n","def check_gpu():\n"," if not run_by_none_device and torch.cuda.device_count() == 0:\n"," raise Exception('当前环境没有GPU')\n","\n","_install_path = f\"{os.environ['HOME']}/sd_webui\" # 安装目录\n","_output_path = '/kaggle/working' if os.path.exists('/kaggle/working/') else f\"{os.environ['HOME']}/.sdwui/Output\" # 输出目录 如果使用google云盘 会在google云盘增加sdwebui/Output\n","_input_path = '/kaggle/input' # 输入目录\n","_ui_dir_name = 'sd_main_dir'\n","\n","_install_path = locals().get('install_path') or globals().get('install_path') or _install_path\n","_output_path = locals().get('output_path') or globals().get('output_path') or _output_path\n","_input_path = locals().get('input_path') or globals().get('input_path') or _input_path\n","_ui_dir_name = locals().get('ui_dir_name') or globals().get('ui_dir_name') or _ui_dir_name\n","\n","install_path = _install_path\n","output_path = _output_path\n","input_path = _input_path\n","ui_dir_name = _ui_dir_name\n"," \n","google_drive = '' \n","\n","\n","_useGooglrDrive = locals().get('useGooglrDrive') or globals().get('useGooglrDrive') or True\n","\n","# 连接谷歌云\n","try:\n"," if _useGooglrDrive:\n"," from google.colab import drive\n"," drive.mount(f'~/google_drive')\n"," google_drive = f\"{os.environ['HOME']}/google_drive/MyDrive\"\n"," _output_path = f'{google_drive}/sdwebui/Output'\n"," _input_path = f'{google_drive}/sdwebui/Input'\n"," run(f'''mkdir -p {_input_path}''')\n"," print('''\n","已经链接到谷歌云盘\n","已在云盘创建Input和Output目录\n"," ''')\n","except:\n"," _useGooglrDrive = False\n","\n","run(f'''mkdir -p {_install_path}''')\n","run(f'''mkdir -p {_output_path}''')\n","\n","\n","os.environ['install_path'] = _install_path\n","os.environ['output_path'] = _output_path\n","os.environ['google_drive'] = google_drive\n","os.environ['input_path'] = _input_path\n","\n","def replace_path(input_str:str):\n"," if not input_str: return ''\n"," return input_str.replace('$install_path',_install_path)\\\n"," .replace('{install_path}',_install_path)\\\n"," .replace('$input_path',_input_path)\\\n"," .replace('{input_path}',_input_path)\\\n"," .replace('$output_path',_output_path)\\\n"," .replace('{output_path}',_output_path)\\\n"," .replace('{sdwui}','stable-diffusion-webui')\\\n"," .replace('{wui}',\"webui\")\n","\n","space_string = ' \\n\\r\\t\\'\\\",'\n","\n","def config_reader(conf:str):\n"," args = [replace_path(item.split('#')[0].strip(space_string)) for item in conf.split('\\n') if item.strip(space_string)]\n"," return [item.strip() for item in args if item.strip()]\n"]},{"cell_type":"code","execution_count":null,"metadata":{"_kg_hide-input":true,"id":"i3LhnwYHLCtC","trusted":true},"outputs":[],"source":["ngrokTokenFile = os.path.join(_input_path,'configs/ngrok_token.txt') # 非必填 存放ngrokToken的文件的路径\n","frpcConfigFile = os.path.join(_input_path,'configs/frpc_koishi.ini') # 非必填 frp 配置文件\n","# ss证书目录 下载nginx的版本，把pem格式改成crt格式\n","frpcSSLFFlies = [os.path.join(_input_path,'configs/koishi_ssl')]\n","if 'frp_ssl_dir' in locals() or 'frp_ssl_dir' in globals():\n"," frpcSSLFFlies = frpcSSLFFlies + config_reader(locals().get('frp_ssl_dir') or globals().get('frp_ssl_dir'))\n","# frpc 文件目录 如果目录不存在，会自动下载，也可以在数据集搜索 viyiviyi/utils 添加\n","frpcExePath = os.path.join(_input_path,'utils-tools/frpc')\n","# 其他需要加载的webui启动参数 写到【参数列表】这个配置去\n","otherArgs = '--xformers'\n","if 'sd_start_args' in locals() or 'sd_start_args' in globals():\n"," otherArgs = ' '.join([item for item in config_reader(locals().get('sd_start_args') or globals().get('sd_start_args')) if item != '--no-gradio-queue'])\n","venvPath = os.path.join(_input_path,'sd-webui-venv/venv.tar.bak') # 安装好的python环境 sd-webui-venv是一个公开是数据集 可以搜索添加\n","\n","# 用于使用kaggle api的token文件 参考 https://www.kaggle.com/docs/api\n","# 此文件用于自动上传koishi的相关配置 也可以用于保存重要的输出文件\n","kaggleApiTokenFile = os.path.join(_input_path,'configs/kaggle.json')\n","\n","requirements = []\n"]},{"cell_type":"code","execution_count":null,"metadata":{"_kg_hide-input":true,"id":"a_GtG2ayLCtD","trusted":true},"outputs":[],"source":["# 这下面的是用于初始化一些值或者环境变量的，轻易别改\n","_setting_file = replace_path(locals().get('setting_file') or globals().get('setting_file') or '/kaggle/working/configs/config.json')\n","\n","_ui_config_file = replace_path(locals().get('ui_config_file') or globals().get('ui_config_file') or '/kaggle/working/configs/ui-config.json')\n","\n","# 设置文件路径\n","if Path(f\"{os.environ['HOME']}/google_drive/MyDrive\").exists():\n"," if _setting_file == '/kaggle/working/configs/config.json':\n"," _setting_file = os.path.join(_output_path,'configs/config.json')\n"," if _ui_config_file == '/kaggle/working/configs/ui-config.json':\n"," _ui_config_file = os.path.join(_output_path,'configs/ui-config.json')\n"," \n","frpcStartArg = ''\n","_frp_config_or_file = replace_path(locals().get('frp_config_or_file') or globals().get('frp_config_or_file')) or frpcConfigFile\n","run(f'''mkdir -p {_install_path}/configFiles''')\n","if _frp_config_or_file:\n"," if Path(_frp_config_or_file.strip()).exists():\n"," frpcConfigFile = _frp_config_or_file.strip()\n"," if not Path(frpcConfigFile).exists(): \n"," if _frp_config_or_file.strip().startswith('-f'):\n"," frpcStartArg = _frp_config_or_file.strip()\n"," else:\n"," print('没有frpcp配置')\n"," _useFrpc = False\n"," else:\n"," run(f'''cp -f {frpcConfigFile} {_install_path}/configFiles/frpc_webui.ini''')\n"," frpcConfigFile = f'{_install_path}/configFiles/frpc_webui.ini'\n"," run(f'''sed -i \"s/local_port = .*/local_port = {_server_port}/g\" {frpcConfigFile}''')\n"," frpcStartArg = f' -c {frpcConfigFile}'\n","\n","ngrokToken=''\n","_ngrok_config_or_file = replace_path(locals().get('ngrok_config_or_file') or globals().get('ngrok_config_or_file')) or ngrokTokenFile\n","if _ngrok_config_or_file:\n"," if Path(_ngrok_config_or_file.strip()).exists():\n"," ngrokTokenFile = _ngrok_config_or_file.strip()\n"," if Path(ngrokTokenFile).exists():\n"," with open(ngrokTokenFile,encoding = \"utf-8\") as nkfile:\n"," ngrokToken = nkfile.readline()\n"," elif not _ngrok_config_or_file.strip().startswith('/'):\n"," ngrokToken=_ngrok_config_or_file.strip()\n"," \n","if not Path(venvPath).exists():\n"," venvPath = os.path.join(_input_path,'sd-webui-venv/venv.zip')\n"]},{"cell_type":"markdown","metadata":{},"source":["## 文件下载工具\n","\n","---\n","\n","link_or_download_flie(config:str, skip_url:bool=False, _link_instead_of_copy:bool=True, base_path:str = '',sync:bool=False,thread_num:int=None)"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["import concurrent.futures\n","import importlib\n","import os\n","import pprint\n","import re\n","from pathlib import Path\n","from typing import List\n","\n","import requests\n","\n","show_shell_info = False\n","\n","def is_installed(package):\n"," try:\n"," spec = importlib.util.find_spec(package)\n"," except ModuleNotFoundError:\n"," return False\n","\n"," return spec is not None\n","\n","def download_file(url:str, filename:str, dist_path:str, cache_path = '',_link_instead_of_copy:bool=True):\n"," # 获取文件的真实文件名\n"," if not filename:\n"," with requests.get(url, stream=True) as r:\n"," if 'Content-Disposition' in r.headers:\n"," filename = r.headers['Content-Disposition'].split('filename=')[1].strip('\"')\n"," r.close()\n"," if not filename and re.search(r'/[^/]+\\.[^/]+$',url):\n"," filename = url.split('/')[-1]\n"," \n"," filename = re.sub(r'[\\\\/:*?\"<>|;]', '', filename)\n"," filename = re.sub(r'[\\s\\t]+', '_', filename)\n"," \n"," if show_shell_info:\n"," print(f'下载 {filename} url: {url} --> {dist_path}')\n"," \n"," # 创建目录\n"," if cache_path and not Path(cache_path).exists():\n"," os.makedirs(cache_path,exist_ok=True)\n"," if dist_path and not Path(dist_path).exists():\n"," os.makedirs(dist_path,exist_ok=True)\n"," \n"," # 拼接文件的完整路径\n"," filepath = os.path.join(dist_path, filename)\n","\n"," if cache_path:\n"," cache_path = os.path.join(cache_path, filename)\n"," \n"," # 判断文件是否已存在\n"," if Path(filepath).exists():\n"," print(f'文件 {filename} 已存在 {dist_path}')\n"," return\n"," \n"," if cache_path and Path(cache_path).exists():\n"," run(f'cp -n -r -f {\"-s\" if _link_instead_of_copy else \"\"} {cache_path} {dist_path}')\n"," if show_shell_info:\n"," print(f'文件缓存 {cache_path} --> {dist_path}')\n"," return\n"," # 下载文件\n"," with requests.get(url, stream=True) as r:\n"," r.raise_for_status()\n"," with open(cache_path or filepath, 'wb') as f:\n"," for chunk in r.iter_content(chunk_size=8192):\n"," if chunk:\n"," f.write(chunk)\n"," # 如果使用了缓存目录 需要复制或链接文件到目标目录\n"," if cache_path:\n"," run(f'cp -n -r -f {\"-s\" if _link_instead_of_copy else \"\"} {cache_path} {dist_path}')\n"," if show_shell_info:\n"," print(f'下载完成 {filename} --> {dist_path}')\n"," \n","def download_git(url, dist_path, cache_path = '',_link_instead_of_copy:bool=True):\n"," if not Path(dist_path).exists():\n"," os.makedirs(dist_path,exist_ok=True)\n"," if show_shell_info:\n"," print(f'git 下载 {url} --> {dist_path}')\n"," if cache_path and not Path(cache_path).exists():\n"," os.makedirs(cache_path,exist_ok=True)\n"," run(f'git clone {url}',cwd = cache_path)\n"," if cache_path:\n"," run(f'cp -n -r -f {cache_path}/* {dist_path}')\n"," else:\n"," run(f'git clone {url}',cwd = dist_path)\n"," if show_shell_info:\n"," print(f'git 下载完成 {url} --> {dist_path}')\n"," \n"," \n"," \n","# 加入文件到下载列表\n","def pause_url(url:str,dist_path:str):\n"," file_name = ''\n"," if re.match(r'^[^:]+:(https?|ftps?)://', url, flags=0):\n"," file_name = re.findall(r'^[^:]+:',url)[0][:-1]\n"," url = url[len(file_name)+1:]\n"," if not re.match(r'^(https?|ftps?)://',url):\n"," return\n"," file_name = re.sub(r'\\s+','_',file_name or '')\n"," path_hash = str(hash(url)).replace('-','')\n"," \n"," return {'file_name':file_name,'path_hash':path_hash,'url':url,'dist_path':dist_path}\n","\n","def download_urls(download_list:List[dict],sync:bool=False,thread_num:int=5, \n"," cache_path:str=os.path.join(os.environ['HOME'],'.cache','download_util'),\n"," _link_instead_of_copy:bool=True,is_await:bool=False):\n"," if sync:\n"," for conf in download_list:\n"," cache_dir = os.path.join(cache_path,conf['path_hash'])\n"," if conf['url'].startswith('https://github.com'):\n"," download_git(conf['url'],conf['dist_path'],cache_path=cache_dir,_link_instead_of_copy=_link_instead_of_copy)\n"," continue\n"," download_file(conf['url'],conf['file_name'],conf['dist_path'],cache_path=cache_dir,_link_instead_of_copy=_link_instead_of_copy)\n"," else:\n"," executor = concurrent.futures.ThreadPoolExecutor(max_workers=thread_num)\n"," futures = []\n"," for conf in download_list:\n"," cache_dir = os.path.join(cache_path,conf['path_hash'])\n"," if conf['url'].startswith('https://github.com'):\n"," futures.append(executor.submit(download_git, conf['url'],conf['dist_path'],\n"," cache_path=cache_dir,_link_instead_of_copy=_link_instead_of_copy))\n"," continue\n"," futures.append(executor.submit(download_file, conf['url'],conf['file_name'],conf['dist_path'],\n"," cache_path=cache_dir,_link_instead_of_copy=_link_instead_of_copy))\n"," if is_await:\n"," concurrent.futures.wait(futures)\n"," \n"," \n","def parse_config(config:str):\n"," space_string = ' \\n\\r\\t\\'\\\",'\n"," other_flie_list = [item.split('#')[0].strip(space_string) for item in config.split('\\n') if item.strip(space_string)]\n"," other_flie_list = [item.strip() for item in other_flie_list if item.strip()]\n"," other_flie_list_store = {}\n"," other_flie_list_store_name='default'\n"," other_flie_list_store_list_cache=[]\n"," \n"," for item in other_flie_list:\n"," if item.startswith('[') and item.endswith(']'):\n"," if not other_flie_list_store_name == 'default':\n"," other_flie_list_store[other_flie_list_store_name]=other_flie_list_store_list_cache\n"," other_flie_list_store_list_cache = []\n"," other_flie_list_store_name = item[1:-1]\n"," else:\n"," other_flie_list_store_list_cache.append(item)\n"," other_flie_list_store[other_flie_list_store_name]=other_flie_list_store_list_cache\n"," \n"," return other_flie_list_store\n","\n","\n","def link_or_download_flie(config:str, skip_url:bool=False, _link_instead_of_copy:bool=True, base_path:str = '',\n"," sync:bool=False,thread_num:int=None, is_await:bool=False):\n"," store:dict[str,List[str]] = parse_config(config)\n"," download_list = []\n"," for dist_dir in store.keys():\n"," dist_path = os.path.join(base_path,dist_dir)\n"," os.makedirs(dist_path,exist_ok=True)\n"," for path in store[dist_dir]:\n"," if 'https://' in path or 'http://' in path:\n"," if skip_url:\n"," continue\n"," if sync:\n"," download_urls([pause_url(path,dist_path)],_link_instead_of_copy = _link_instead_of_copy, sync=sync)\n"," continue\n"," download_list.append(pause_url(path,dist_path))\n"," else:\n"," run(f'cp -n -r -f {\"-s\" if _link_instead_of_copy else \"\"} {path} {dist_path}')\n"," if show_shell_info:\n"," print(f'{\"链接\" if _link_instead_of_copy else \"复制\"} {path} --> {dist_path}')\n"," run(f'rm -f {dist_path}/\\*.* ')\n"," if not skip_url:\n"," if show_shell_info:\n"," pprint.pprint(download_list)\n"," download_urls(download_list,_link_instead_of_copy = _link_instead_of_copy, sync=sync, thread_num=thread_num or 2,is_await=is_await)"]},{"cell_type":"markdown","metadata":{"id":"p0uS-BLULCtD"},"source":["## kaggle public API\n","\n","**不能使用%cd这种会改变当前工作目录的命令，会导致和其他线程冲突**\n","\n","---"]},{"cell_type":"code","execution_count":null,"metadata":{"_kg_hide-input":true,"id":"m8FJi4j0LCtD","trusted":true},"outputs":[],"source":["# 安装kaggle的api token文件\n","def initKaggleConfig():\n"," if Path('~/.kaggle/kaggle.json').exists():\n"," return True\n"," if Path(kaggleApiTokenFile).exists():\n"," run(f'''mkdir -p ~/.kaggle/''')\n"," run('cp '+kaggleApiTokenFile+' ~/.kaggle/kaggle.json')\n"," run(f'''chmod 600 ~/.kaggle/kaggle.json''')\n"," return True\n"," print('缺少kaggle的apiToken文件，访问：https://www.kaggle.com/你的kaggle用户名/account 获取')\n"," return False\n","\n","def getUserName():\n"," if not initKaggleConfig(): return\n"," import kaggle\n"," return kaggle.KaggleApi().read_config_file()['username']\n","\n","def createOrUpdateDataSet(path:str,datasetName:str):\n"," if not initKaggleConfig(): return\n"," print('创建或更新数据集 '+datasetName)\n"," import kaggle\n"," run(f'mkdir -p {_install_path}/kaggle_cache')\n"," run(f'rm -rf {_install_path}/kaggle_cache/*')\n"," datasetDirPath = _install_path+'/kaggle_cache/'+datasetName\n"," run('mkdir -p '+datasetDirPath)\n"," run('cp -f '+path+' '+datasetDirPath+'/')\n"," username = getUserName()\n"," print(\"kaggle username:\"+username)\n"," datasetPath = username+'/'+datasetName\n"," datasetList = kaggle.api.dataset_list(mine=True,search=datasetPath)\n"," print(datasetList)\n"," if len(datasetList) == 0 or datasetPath not in [str(d) for d in datasetList]: # 创建 create\n"," run('kaggle datasets init -p' + datasetDirPath)\n"," metadataFile = datasetDirPath+'/dataset-metadata.json'\n"," run('sed -i s/INSERT_TITLE_HERE/'+ datasetName + '/g ' + metadataFile)\n"," run('sed -i s/INSERT_SLUG_HERE/'+ datasetName + '/g ' + metadataFile)\n"," run('cat '+metadataFile)\n"," run('kaggle datasets create -p '+datasetDirPath)\n"," print('create database done')\n"," else:\n"," kaggle.api.dataset_metadata(datasetPath,datasetDirPath)\n"," kaggle.api.dataset_create_version(datasetDirPath, 'auto update',dir_mode='zip')\n"," print('upload database done')\n","\n","def downloadDatasetFiles(datasetName:str,outputPath:str):\n"," if not initKaggleConfig(): return\n"," print('下载数据集文件 '+datasetName)\n"," import kaggle\n"," username = getUserName()\n"," datasetPath = username+'/'+datasetName\n"," datasetList = kaggle.api.dataset_list(mine=True,search=datasetPath)\n"," if datasetPath not in [str(d) for d in datasetList]:\n"," return False\n"," run('mkdir -p '+outputPath)\n"," kaggle.api.dataset_download_files(datasetPath,path=outputPath,unzip=True)\n"," return True\n","\n"]},{"cell_type":"markdown","metadata":{},"source":["## 同步文件夹到 huggingface\n","\n","---"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["# 文件夹与 huggingface 同步\n","if _huggingface_token and _huggingface_repo:\n"," if not is_installed('watchdog'):\n"," requirements.append('watchdog')\n"," if not is_installed('huggingface_hub'):\n"," requirements.append('huggingface_hub')\n"," else:\n"," try:\n"," from huggingface_hub import HfApi,login,snapshot_download\n"," except:\n"," requirements.append('huggingface_hub')\n","\n","huggingface_is_init = False\n","\n","def init_huggingface():\n"," if not _huggingface_token or not _huggingface_repo:\n"," print('huggingface 功能不会启动，可增加配置项 huggingface_token = \"token\" 和 huggingface_repo = “仓库id” 后启用 huggingface 功能')\n"," return False\n","\n"," global huggingface_is_init\n"," \n"," from huggingface_hub import HfApi,login,snapshot_download\n"," token = replace_path(_huggingface_token)\n"," if Path(token).exists():\n"," with open(token,encoding = \"utf-8\") as nkfile:\n"," token = nkfile.readline()\n"," if not token.startswith('hf_'):\n"," print('huggingface token 不正确，请将 token 或 仅存放token 的txt文件路径填入 _huggingface_token 配置')\n"," return False\n"," login(token,add_to_git_credential=True)\n"," huggingface_is_init = True\n"," return True\n","\n","\n","def download__huggingface_repo(repo_id:str,dist_directory:str=None,repo_type='dataset',callback=None):\n"," if not huggingface_is_init:\n"," print('huggingface 相关功能未初始化 请调用 init_huggingface() 初始化')\n"," \n"," if not dist_directory:\n"," dist_directory = f'{_install_path}/{_ui_dir_name}/log'\n"," \n"," from huggingface_hub import HfApi,login,snapshot_download\n"," \n"," api = HfApi()\n"," \n"," print('下载收藏的图片')\n"," if not Path(f'{_install_path}/cache/huggingface/huggingface_repo').exists():\n"," mkdirs(f'{_install_path}/cache/huggingface')\n"," repo_path = ''\n"," if repo_type == 'dataset':\n"," repo_path = 'datasets'\n"," if repo_type == 'space':\n"," repo_path = 'spaces'\n"," if repo_path:\n"," run(f'git clone https://huggingface.co/{repo_path}/{repo_id} huggingface_repo',cwd=f'{_install_path}/cache/huggingface')\n"," else:\n"," run(f'git clone https://huggingface.co/{repo_id} huggingface_repo',cwd=f'{_install_path}/cache/huggingface')\n","\n"," run(f'cp -r -f -n -s {_install_path}/cache/huggingface/huggingface_repo/* {dist_directory}')\n","# snapshot_download(repo_id = repo_id, local_dir = dist_directory, local_dir_use_symlinks = \"auto\", token=True, repo_type=repo_type )\n"," if callback:\n"," callback()\n","\n","def start_sync_log_to_huggingface(repo_id:str,directory_to_watch:str=None,repo_type='dataset'):\n"," if not huggingface_is_init:\n"," print('huggingface 相关功能未初始化 请调用 init_huggingface() 初始化')\n"," \n"," from watchdog.observers import Observer\n"," from watchdog.events import FileSystemEventHandler\n"," from huggingface_hub import HfApi,login,snapshot_download\n"," \n"," # 配置监视的目录和 Hugging Face 仓库信息\n"," class FileChangeHandler(FileSystemEventHandler):\n"," def __init__(self, api, repo_id, repo_type,directory_to_watch):\n"," self.api = api\n"," self.repo_id = repo_id\n"," self.repo_type = repo_type\n"," self.directory_to_watch = directory_to_watch\n"," def on_created(self, event):\n"," if not event.is_directory:\n"," # 上传新文件到 Hugging Face 仓库\n"," file_path = event.src_path\n"," file_name:str = os.path.basename(file_path)\n"," print(file_name)\n"," if file_name[file_name.rindex('.'):] not in ['.png','.jpg','.txt','.webp','.jpeg']: return\n"," print(file_name,'>>','huggingface')\n"," try:\n"," self.api.upload_file(\n"," path_or_fileobj=file_path,\n"," path_in_repo=file_path.replace(self.directory_to_watch,''),\n"," repo_id=self.repo_id,\n"," repo_type=self.repo_type,\n"," )\n"," except IOError as error:\n"," print(error)\n","\n"," def on_deleted(self, event):\n"," if not event.is_directory:\n"," # 从 Hugging Face 仓库删除文件\n"," file_path = event.src_path\n"," file_name = os.path.basename(file_path)\n"," if file_name[file_name.rindex('.'):] not in ['.png','.jpg','.txt','.webp','.jpeg']: return\n"," try:\n"," self.api.delete_file(\n"," path_in_repo=file_path.replace(self.directory_to_watch,''),\n"," repo_id=self.repo_id,\n"," repo_type=self.repo_type,\n"," )\n"," except IOError as error:\n"," print(error)\n","\n"," def on_modified(self, event):\n"," if not event.is_directory:\n"," # 更新 Hugging Face 仓库中的文件\n"," file_path = event.src_path\n"," file_name = os.path.basename(file_path)\n"," if file_name[file_name.rindex('.'):] not in ['.png','.jpg','.txt','.webp','.jpeg']: return\n"," try:\n"," self.api.upload_file(\n"," path_or_fileobj=file_path,\n"," path_in_repo=file_path.replace(self.directory_to_watch,''),\n"," repo_id=self.repo_id,\n"," repo_type=self.repo_type,\n"," )\n"," except IOError as error:\n"," print(error)\n","\n"," def on_moved(self, event):\n"," if not event.is_directory:\n"," file_path = event.dest_path\n"," file_name = os.path.basename(file_path)\n"," if file_name[file_name.rindex('.'):] not in ['.png','.jpg','.txt','.webp','.jpeg']: return\n"," if event.dest_path.startswith(self.directory_to_watch):\n"," try:\n"," self.api.upload_file(\n"," path_or_fileobj=file_path,\n"," path_in_repo=file_path.replace(self.directory_to_watch,''),\n"," repo_id=self.repo_id,\n"," repo_type=self.repo_type,\n"," )\n"," except IOError as error:\n"," print(error)\n","\n"," api = HfApi()\n"," \n"," if not directory_to_watch:\n"," directory_to_watch = f'{_install_path}/{_ui_dir_name}/log'\n"," # 创建观察者对象并注册文件变化处理程序\n"," event_handler = FileChangeHandler(api,repo_id,repo_type,directory_to_watch)\n"," observer = Observer()\n"," observer.schedule(event_handler, directory_to_watch, recursive=True)\n","\n"," # 启动观察者\n"," observer.name = \"solo_directory_to_watch\"\n"," print(f'启动收藏图片文件夹监听，并自动同步到 huggingface {repo_type} : {repo_id}')\n"," observer.start()"]},{"cell_type":"markdown","metadata":{"id":"sswa04veLCtE"},"source":["## 工具函数\n","**不能使用%cd这种会改变当前工作目录的命令，会导致和其他线程冲突**\n","\n","---"]},{"cell_type":"code","execution_count":null,"metadata":{"_kg_hide-input":true,"trusted":true},"outputs":[],"source":["def echoToFile(content:str,path:str):\n"," if path.find('/') >= 0:\n"," _path = '/'.join(path.split('/')[:-1])\n"," run(f'''mkdir -p {_path}''')\n"," with open(path,'w') as sh:\n"," sh.write(content)\n","\n","def zipPath(path:str,zipName:str,format='tar'):\n"," if path.startswith('$install_path'):\n"," path = path.replace('$install_path',_install_path)\n"," if path.startswith('$output_path'):\n"," path = path.replace('$install_path',_output_path)\n"," if not path.startswith('/'):\n"," path = f'{_install_path}/{_ui_dir_name}/{path}'\n"," if Path(path).exists():\n"," if 'tar' == format:\n"," run(f'tar -cf {_output_path}/'+ zipName +'.tar -C '+ path +' . ')\n"," elif 'gz' == format:\n"," run(f'tar -czf {_output_path}/'+ zipName +'.tar.gz -C '+ path +' . ')\n"," return\n"," print('指定的目录不存在：'+path)\n","\n","# 检查网络\n","def check_service(host, port):\n"," try:\n"," socket.create_connection((host, port), timeout=5)\n"," return True\n"," except socket.error:\n"," return False"]},{"cell_type":"markdown","metadata":{},"source":["## 内网穿���\n","\n","---"]},{"cell_type":"code","execution_count":null,"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"id":"coqQvTSLLCtE","trusted":true},"outputs":[],"source":["# ngrok\n","def startNgrok(ngrokToken:str,ngrokLocalPort:int):\n"," if not is_installed('pyngrok'):\n"," run(f'pip install pyngrok')\n"," from pyngrok import conf, ngrok\n"," try:\n"," conf.get_default().auth_token = ngrokToken\n"," conf.get_default().monitor_thread = False\n"," ssh_tunnels = ngrok.get_tunnels(conf.get_default())\n"," url = ''\n"," if len(ssh_tunnels) == 0:\n"," ssh_tunnel = ngrok.connect(ngrokLocalPort)\n"," url = ssh_tunnel.public_url\n"," print('ngrok 访问地址：'+ssh_tunnel.public_url)\n"," else:\n"," print('ngrok 访问地址：'+ssh_tunnels[0].public_url)\n"," url = ssh_tunnels[0].public_url\n"," def auto_request_ngrok():\n"," if url:\n"," while(True):\n"," time.sleep(60*10)\n"," res = requests.get(url+'/sdapi/v1/samplers',headers={\"ngrok-skip-browser-warning\" : \"1\"})\n"," # print('自动调用ngrok链接以保存链接不会断开',res.status_code)\n","\n"," threading.Thread(target = auto_request_ngrok,daemon=True,name='solo_auto_request_ngrok').start()\n"," except:\n"," print('启动ngrok出错')\n"," \n","def startFrpc(name,configFile):\n"," echoToFile(f'''\n","cd {_install_path}/frpc/\n","{_install_path}/frpc/frpc {configFile}\n","''',f'{_install_path}/frpc/start.sh')\n"," get_ipython().system(f'''bash {_install_path}/frpc/start.sh''')\n"," \n","def installProxyExe():\n"," if _useFrpc:\n"," print('安装frpc')\n"," run(f'mkdir -p {_install_path}/frpc')\n"," if Path(frpcExePath).exists():\n"," run(f'cp -f -n {frpcExePath} {_install_path}/frpc/frpc')\n"," else:\n"," run(f'wget \"https://huggingface.co/datasets/ACCA225/Frp/resolve/main/frpc\" -O {_install_path}/frpc/frpc')\n"," \n"," for ssl in frpcSSLFFlies:\n"," if Path(ssl).exists():\n"," run(f'cp -f -n {ssl}/* {_install_path}/frpc/')\n"," run(f'chmod +x {_install_path}/frpc/frpc')\n"," run(f'{_install_path}/frpc/frpc -v')\n"," if _useNgrok and not is_installed('pyngrok'):\n"," run('pip install pyngrok')\n","\n","def startProxy():\n"," if _useNgrok:\n"," startNgrok(ngrokToken,_server_port)\n"," if _useFrpc:\n"," startFrpc('frpc_proxy',frpcStartArg)"]},{"cell_type":"markdown","metadata":{},"source":["## NGINX 反向代理\n","\n","---"]},{"cell_type":"code","execution_count":null,"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"outputs":[],"source":["\n","# nginx 反向代理配置文件\n","def localProxy():\n"," conf = '''\n","server\n","{\n"," listen '''+str(_server_port)+''';\n"," listen [::]:'''+str(_server_port)+''';\n"," server_name 127.0.0.1 localhost 0.0.0.0 \"\";\n"," \n"," if ($request_method = OPTIONS) {\n"," return 200;\n"," }\n"," fastcgi_send_timeout 10m;\n"," fastcgi_read_timeout 10m;\n"," fastcgi_connect_timeout 10m;\n"," location /1/\n"," {\n"," proxy_pass http://127.0.0.1:'''+str(_server_port+2)+'''/;\n"," # add_header Set-Cookie \"subpath=1; expires=0; Path=/\";\n"," # proxy_set_header Set-Cookie \"subpath=1; expires=0; Path=/\";\n"," proxy_set_header Host $host;\n"," proxy_set_header X-Real-IP $remote_addr;\n"," proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n"," proxy_set_header REMOTE-HOST $remote_addr;\n"," proxy_set_header Upgrade $http_upgrade;\n"," proxy_set_header Connection upgrade;\n"," proxy_http_version 1.1;\n"," proxy_connect_timeout 10m;\n"," proxy_read_timeout 10m;\n"," }\n"," location /\n"," {\n"," set $proxy_url http://127.0.0.1:'''+str(_server_port+1)+''';\n"," # if ($cookie_subpath = \"1\") {\n"," # set $proxy_url http://127.0.0.1:'''+str(_server_port+2)+''';\n"," # }\n"," proxy_pass $proxy_url;\n"," proxy_set_header Host $host;\n"," proxy_set_header X-Real-IP $remote_addr;\n"," proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n"," proxy_set_header REMOTE-HOST $remote_addr;\n"," proxy_set_header Upgrade $http_upgrade;\n"," proxy_set_header Connection upgrade;\n"," proxy_http_version 1.1;\n"," proxy_connect_timeout 10m;\n"," proxy_read_timeout 10m;\n"," }\n","}\n","'''\n"," echoToFile(conf,'/etc/nginx/conf.d/proxy_nginx.conf')\n"," if not check_service('localhost',_server_port):\n"," run(f'''nginx -c /etc/nginx/nginx.conf''')\n"," run(f'''nginx -s reload''')\n"," "]},{"cell_type":"markdown","metadata":{},"source":["## 线程清理工具\n","\n","---\n","\n","清理线程名以 solo_ 开头的所有线���"]},{"cell_type":"code","execution_count":null,"metadata":{"_kg_hide-input":true,"trusted":true},"outputs":[],"source":["import inspect\n","import ctypes\n","\n","def _async_raise(tid, exctype):\n"," \"\"\"raises the exception, performs cleanup if needed\"\"\"\n"," tid = ctypes.c_long(tid)\n"," if not inspect.isclass(exctype):\n"," exctype = type(exctype)\n"," res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype))\n"," if res == 0:\n"," raise ValueError(\"invalid thread id\")\n"," elif res != 1:\n"," # \"\"\"if it returns a number greater than one, you're in trouble,\n"," # and you should call it again with exc=NULL to revert the effect\"\"\"\n"," ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None)\n"," raise SystemError(\"PyThreadState_SetAsyncExc failed\")\n","\n","def stop_thread(thread):\n"," _async_raise(thread.ident, SystemExit)\n","\n","def stop_solo_threads():\n"," # 获取当前所有活动的线程\n"," threads = threading.enumerate()\n"," # 关闭之前创建的子线程\n"," for thread in threads:\n"," if thread.name.startswith('solo_'):\n"," print(f'结束线程：{thread.name}')\n"," try:\n"," stop_thread(thread)\n"," except socket.error:\n"," print(f'结束线程：{thread.name} 执行失败')"]},{"cell_type":"markdown","metadata":{"id":"Ve3p8oOkLCtE"},"source":["# webui 安装和配置函数\n","---"]},{"cell_type":"code","execution_count":null,"metadata":{"_kg_hide-input":true,"id":"GTjyBJihLCtE","trusted":true},"outputs":[],"source":["envInstalled=False\n","quickStart = True\n","#安装\n","def install():\n"," print('安装')\n"," os.chdir(f'''{_install_path}''')\n"," run(f'''git lfs install''')\n"," run(f'''git config --global credential.helper store''')\n"," for requirement in requirements:\n"," run(f'pip install {requirement}')\n"," if _reLoad:\n"," run(f'''rm -rf {_install_path}/{_ui_dir_name}''')\n"," if Path(f\"{_ui_dir_name}\").exists():\n"," os.chdir(f'''{_install_path}/{_ui_dir_name}/''')\n"," run(f'''git checkout .''')\n"," run(f'''git pull''')\n"," else:\n"," run(f'''git clone {_sd_git_repo} {_ui_dir_name}''')\n"," os.chdir(f'''{_install_path}/{_ui_dir_name}''')\n"," print('安装 完成')\n","\n","# 链接输出目录\n","def link_dir():\n"," print('链接输出目录')\n"," # 链接图片输出目录\n"," run(f'''mkdir -p {_output_path}/outputs''')\n"," run(f'''rm -rf {_install_path}/{_ui_dir_name}/outputs''')\n"," run(f'''ln -s -r {_output_path}/outputs {_install_path}/{_ui_dir_name}/''')\n"," # 输出收藏目录\n"," run(f'''mkdir -p {_output_path}/log''')\n"," run(f'''rm -rf {_install_path}/{_ui_dir_name}/log''')\n"," run(f'''ln -s -r {_output_path}/log {_install_path}/{_ui_dir_name}/''')\n"," # 链接训练输出目录 文件夹链接会导致功能不能用\n"," run(f'''rm -rf {_install_path}/{_ui_dir_name}/textual_inversion''')\n"," run(f'''mkdir -p {_output_path}/textual_inversion/''')\n"," run(f'''ln -s -r {_output_path}/textual_inversion {_install_path}/{_ui_dir_name}/''')\n"," print('链接输出目录 完成') \n","\n","def install_optimizing():\n"," run('apt install nginx -y')\n"," run('env TF_CPP_MIN_LOG_LEVEL=1')\n"," \n","#安装依赖\n","def install_dependencies():\n"," print('安装需要的python环境')\n"," import venv\n"," global envInstalled\n"," global venvPath\n"," if Path(f'{_install_path}/{_ui_dir_name}/venv').exists():\n"," print('跳过安装python环境')\n"," envInstalled = True\n"," return\n"," \n"," if quickStart:\n"," if not Path(venvPath).exists():\n"," mkdirs(f'{_install_path}/venv_cache',True)\n"," if not Path(f'{_install_path}/venv_cache/venv.tar.bak').exists():\n"," print('下载 venv.zip')\n"," download_file('https://huggingface.co/viyi/sdwui/resolve/main/venv.zip','venv.zip',f'{_install_path}/venv_cache')\n"," run(f'''unzip {_install_path}/venv_cache/venv.zip -d {_install_path}/venv_cache''')\n"," venvPath = f'{_install_path}/venv_cache/venv.tar.bak'\n"," run(f'''rm -rf {_install_path}/venv_cache/venv.zip''')\n"," elif venvPath.endswith('.zip'):\n"," mkdirs(f'{_install_path}/venv_cache',True)\n"," run(f'''unzip {venvPath} -d {_install_path}/venv_cache''')\n"," venvPath = f'{_install_path}/venv_cache/venv.tar.bak'\n"," print('解压环境')\n"," # mkdirs(f'{_install_path}/{_ui_dir_name}/venv')\n"," venv.create(f'{_install_path}/{_ui_dir_name}/venv')\n"," run(f'tar -xf {venvPath} -C ./venv',cwd=f'{_install_path}/{_ui_dir_name}')\n"," if not Path(f'{_install_path}/{_ui_dir_name}/venv/bin/pip').exists():\n"," run('curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py')\n"," run(f'{_install_path}/{_ui_dir_name}/venv/bin/python3 get-pip.py')\n","\n"," get_ipython().system(f'''{_install_path}/{_ui_dir_name}/venv/bin/python3 -V''')\n"," get_ipython().system(f'''{_install_path}/{_ui_dir_name}/venv/bin/python3 -m pip -V''')\n","\n"," envInstalled = True\n"," print('安装需要的python环境 完成')\n"," \n","# 个性化配置 \n","def use_config():\n"," print('使用自定义配置 包括tag翻译 \\n')\n"," run(f'''mkdir -p {_install_path}/temp''')\n"," run(f'git clone {_sd_config_git_repu} sd-configs',cwd=f'{_install_path}/temp')\n"," run(f'cp -r -f -n {_install_path}/temp/sd-configs/dist/* {_install_path}/{_ui_dir_name}')\n"," if not Path(_ui_config_file).exists(): # ui配置文件\n"," run(f'''mkdir -p {_ui_config_file[:_ui_config_file.rfind('/')]}''')\n"," run(f'cp -f -n {_install_path}/{_ui_dir_name}/ui-config.json {_ui_config_file}')\n"," if not Path(_setting_file).exists(): # 设置配置文件\n"," run(f'''mkdir -p {_setting_file[:_setting_file.rfind('/')]}''')\n"," run(f'cp -f -n {_install_path}/{_ui_dir_name}/config.json {_setting_file}')\n","\n","def copy_last_log_to_images():\n"," if not Path(f'{_install_path}/{_ui_dir_name}/log/images').exists(): mkdirs(f'{_install_path}/{_ui_dir_name}/log/images')\n"," print('复制编号最大的一张收藏图到输出目录，用于保持编号，否则会出现收藏的图片被覆盖的情况')\n"," img_list = os.listdir(f'{_install_path}/{_ui_dir_name}/log/images')\n"," last_img_path = ''\n"," last_img_num = 0\n"," for img in img_list:\n"," if re.findall(r'^\\d+-',str(img)):\n"," num = int(re.findall(r'^\\d+-',str(img))[0][:-1])\n"," if num > last_img_num:\n"," last_img_path = img\n"," last_img_num = num\n"," print(f'{_install_path}/{_ui_dir_name}/log/images/{last_img_path} {_install_path}/{_ui_dir_name}/outputs/txt2img-images')\n"," run(f'''mkdir -p {_install_path}/{_ui_dir_name}/outputs/txt2img-images''')\n"," run(f'''cp -f {_install_path}/{_ui_dir_name}/log/images/{last_img_path} {_install_path}/{_ui_dir_name}/outputs/txt2img-images/''')\n"," \n"," print(f'{_install_path}/{_ui_dir_name}/log/images/{last_img_path} {_install_path}/{_ui_dir_name}/outputs/img2img-images')\n"," run(f'''mkdir -p {_install_path}/{_ui_dir_name}/outputs/img2img-images''')\n"," run(f'''cp -f {_install_path}/{_ui_dir_name}/log/images/{last_img_path} {_install_path}/{_ui_dir_name}/outputs/img2img-images/''')\n"," \n","def start_webui(i):\n"," # 只要不爆内存，其他方式关闭后会再次重启 访问地址会发生变化\n"," print(i,'--port',str(_server_port+1+i))\n"," if i>0:\n"," print(f'使用第{i+1}张显卡启动第{i+1}个服务，通过frpc或nrgok地址后加/{i}/进行访问（不能使用同一个浏览器）')\n"," if _useFrpc:\n"," restart_times = 0\n"," last_restart_time = time.time()\n"," while True:\n"," os.chdir(f'{_install_path}/{_ui_dir_name}')\n"," get_ipython().system(f'''venv/bin/python3 launch.py --device-id={i} --port {str(_server_port+1+i)} {'' if i ==0 else '--subpath=/1'}''')\n"," print('5秒后重启服务')\n"," if time.time() - last_restart_time < 30:\n"," restart_times = restart_times + 1\n"," else:\n"," restart_times = 0\n"," last_restart_time = time.time()\n"," if restart_times >3 :\n"," # 如果90秒内重启了3此，将不再自动重启\n"," break\n"," time.sleep(5)\n"," else:\n"," os.chdir(f'{_install_path}/{_ui_dir_name}')\n"," get_ipython().system(f'''venv/bin/python3 launch.py --device-id={i} --port {str(_server_port+1+i)} {'' if i ==0 else '--subpath=/1'}''')\n"," \n","# 启动\n","def start():\n"," print('启动')\n"," os.chdir(f'''{_install_path}/{_ui_dir_name}''')\n"," args = ''\n"," if _ui_config_file is not None and _ui_config_file != '' and Path(_ui_config_file).exists(): # ui配置文件\n"," args += ' --ui-config-file=' + _ui_config_file\n"," if _setting_file is not None and _setting_file != '' and Path(_setting_file).exists(): # 设置配置文件\n"," args += ' --ui-settings-file=' + _setting_file\n"," args += ' ' + otherArgs\n"," os.environ['COMMANDLINE_ARGS']=args\n"," run(f'''echo COMMANDLINE_ARGS=$COMMANDLINE_ARGS''')\n"," os.environ['REQS_FILE']='requirements.txt'\n","\n"," with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:\n"," for i in range(torch.cuda.device_count() if _multi_case else 1):\n"," executor.submit(start_webui,i)\n"," while not check_service('localhost',str(_server_port+1+i)): # 当当前服务启动完成才允许退出此次循环\n"," time.sleep(5)\n"," time.sleep(10)"]},{"cell_type":"markdown","metadata":{"id":"qLvsk8ByLCtF"},"source":["# 入口函数\n","---"]},{"cell_type":"code","execution_count":null,"metadata":{"_kg_hide-input":true,"id":"IOKjaMlcLCtF","trusted":true},"outputs":[],"source":["\n","# 启动非webui相关的的内容，加快启动速度\n","def main():\n"," global envInstalled\n"," global huggingface_is_init\n"," startTicks = time.time()\n"," stop_solo_threads()\n"," isInstall = True if os.getenv('IsInstall','False') == 'True' else False\n"," if isInstall is False or _reLoad: \n"," print('启动 安装和运行环境')\n"," install()\n"," link_dir()\n"," init_huggingface()\n"," threading.Thread(target = install_dependencies,daemon=True,name='solo_install_dependencies').start()\n"," threading.Thread(target = install_optimizing,daemon=True,name='solo_install_optimizing').start()\n"," threading.Thread(target = installProxyExe,daemon=True).start()\n"," link_or_download_flie(replace_path(_async_downloading), _link_instead_of_copy=_link_instead_of_copy,\n"," base_path=f'{_install_path}/{_ui_dir_name}')\n"," if huggingface_is_init:\n"," threading.Thread(target = download__huggingface_repo,daemon=True,\n"," args=([_huggingface_repo]),\n"," kwargs={\"callback\":copy_last_log_to_images},\n"," name='solo_download__huggingface_repo').start()\n"," \n"," link_or_download_flie(replace_path(_before_downloading), _link_instead_of_copy=_link_instead_of_copy,\n"," base_path=f'{_install_path}/{_ui_dir_name}',is_await=True)\n"," t = 0\n"," while not envInstalled:\n"," if t%10==0:\n"," print('等待python环境安装...')\n"," t = t+1\n"," time.sleep(1)\n"," use_config()\n"," localProxy()\n"," os.environ['IsInstall'] = 'True'\n"," else:\n"," envInstalled = True\n"," link_or_download_flie(replace_path(_before_start_sync_downloading), _link_instead_of_copy=_link_instead_of_copy,\n"," base_path=f'{_install_path}/{_ui_dir_name}',sync=True)\n"," threading.Thread(target = startProxy, daemon=True, name='solo_startProxy').start()\n"," if init_huggingface():\n"," start_sync_log_to_huggingface(_huggingface_repo)\n"," ticks = time.time()\n"," _on_before_start()\n"," print(\"加载耗时:\",(ticks - startTicks),\"秒\")\n"," start()\n"]},{"cell_type":"markdown","metadata":{"id":"0oaCRs2gLCtF"},"source":["# 执行区域\n","---"]},{"cell_type":"code","execution_count":null,"metadata":{"_kg_hide-output":true,"id":"O3DR0DWHLCtF","scrolled":true,"trusted":true},"outputs":[],"source":["# 启动\n","# _reLoad = True\n","# hidden_console_info = False\n","# run_by_none_device = True\n","# show_shell_info = True\n","\n","print(f'当前sd的安装路径是：{_install_path}/{_ui_dir_name}')\n","print(f'当前图片保存路径是：{_output_path}')\n","print(f'当前数据集路径是：{_input_path}')\n","\n","print(update_desc)\n","\n","if _skip_start:\n"," print('已跳过自动启动，可手动执行 main() 进行启动。')\n"," print('''推荐的启动代码：\n","try:\n"," check_gpu() # 检查是否存在gpu\n"," main()\n","except KeyboardInterrupt:\n"," stop_solo_threads() # 中断后自动停止后台线程 （有部分功能在后台线程中运行）\n"," ''')\n","else:\n"," try:\n"," check_gpu()\n"," main()\n"," except KeyboardInterrupt:\n"," stop_solo_threads()"]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.7"}},"nbformat":4,"nbformat_minor":4} +{"cells":[{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":[]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["update_desc = '''\n","欢迎使用 stable-diffusion-webui 便捷启动工具\n","\n","此脚本理论上可以在任何jupyter环境运行，但仅在 google colab 和 kaggle 测试。 已经无法在 google colab 免费实例上运行。\n","此脚本的【前置脚本】发布地址 https://www.kaggle.com/code/yiyiooo/stable-diffusion-webui-novelai-sdxl。\n","此脚本需配合 【前置脚本】 的脚本附带的配置项才能正常启动。\n","此脚本的内容会自动更新，你无需更新【前置脚本】就能获取到最新的功能。\n","增加一个说明：为解决加密图片的解密和浏览的问题，开发了一个独立应用，可以在这里下载 https://github.com/viyiviyi/encrypt_gallery/releases/\n","加密插件可解决生成nsfw图片后被平台封号问题\n","\n","路径说明\n","* 为了解决平台差异，所有的安装目录和文件输出目录都被重新指定，如果你需要在配置中访问这些目录，请查看以下说明\n","*\n","* 可以使用 $install_path 或 {install_path} 来访问安装目录，写在字符串内也会生效\n","* $output_path 或 {output_path} 可以访问输出目录\n","* 如果你需要安装在自定义目录，也可以设置这些值 如: install_path = '新的路径'\n","* 可自定义方法 on_before_start 并在方法内写上启动前需要的逻辑来实现在webui启动前执行自定义逻辑\n","* 可以增加配置 multi_case = True 来控制是否使用多卡\n","* 如果需要显示更多控制台输出 需配置 hidden_console_info = False\n","'''"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["from pathlib import Path\n","import os\n","import time\n","import re\n","import subprocess\n","import threading\n","import sys\n","import socket\n","import torch\n","from typing import List\n","import uuid\n","import asyncio\n","from urllib import request"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["# 内置参数默认值，当上下文有参数时可覆盖默认值\n","_runing = False\n","_useFrpc = locals().get('useFrpc') or globals().get('useFrpc') or True\n","\n","_useNgrok = locals().get('useNgrok') or globals().get('useNgrok') or True\n","\n","_reLoad = locals().get('reLoad') or globals().get('reLoad') or False\n"," \n","_before_downloading = locals().get('before_downloading') or globals().get('before_downloading') or ''\n","\n","_async_downloading = locals().get('async_downloading') or globals().get('async_downloading') or ''\n","\n","_before_start_sync_downloading = locals().get('before_start_sync_downloading') or globals().get('before_start_sync_downloading') or ''\n","\n","_server_port = locals().get('server_port') or globals().get('server_port') or 7860\n"," \n","_sd_git_repo = locals().get('sd_git_repo') or globals().get('sd_git_repo') or 'https://github.com/viyiviyi/stable-diffusion-webui.git -b local' \n","_sd_git_repo = _sd_git_repo\\\n"," .replace('{sdwui}','stable-diffusion-webui')\\\n"," .replace('{wui}',\"webui\")\n"," \n","_sd_config_git_repu = locals().get('sd_config_git_repu') or globals().get('sd_config_git_repu') or 'https://github.com/viyiviyi/sd-configs.git'\n","_sd_config_git_repu = _sd_config_git_repu\\\n"," .replace('{sdwui}','stable-diffusion-webui')\\\n"," .replace('{wui}',\"webui\")\n"," \n"," \n","_huggingface_token = locals().get('huggingface_token') or globals().get('huggingface_token') or '{input_path}/configs/huggingface_token.txt'\n","_huggingface_token = _huggingface_token\\\n"," .replace('{sdwui}','stable-diffusion-webui')\\\n"," .replace('{wui}',\"webui\")\n"," \n","_huggingface_repo = locals().get('huggingface_repo') or globals().get('huggingface_repo') or ''\n","_huggingface_repo = _huggingface_repo\\\n"," .replace('{sdwui}','stable-diffusion-webui')\\\n"," .replace('{wui}',\"webui\")\n","\n","_link_instead_of_copy = locals().get('link_instead_of_copy') or globals().get('link_instead_of_copy') or True\n"," \n","show_shell_info = locals().get('hidden_console_info') or globals().get('hidden_console_info')\n","if show_shell_info is None: show_shell_info = False\n","\n","_multi_case = locals().get('multi_case') or globals().get('multi_case') or False\n"," \n","_skip_start = locals().get('skip_start') or globals().get('skip_start') or True\n","\n","def before_start():\n"," ''''''\n","\n","_on_before_start = locals().get('on_before_start') or globals().get('on_before_start') or before_start \n"," \n","run_by_none_device = False\n","\n","_proxy_path = locals().get('proxy_path') or globals().get('proxy_path') or {}\n","\n","_sub_path = locals().get('sub_path') or globals().get('sub_path') or ['/','/1/']\n","if len(_sub_path) != 2:\n"," _sub_path = ['/','/1/']\n"," \n","_config_args:dict[str, str] = locals().get('config_args') or globals().get('config_args') or {}"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["\n","def run(command, cwd=None, desc=None, errdesc=None, custom_env=None,try_error:bool=True) -> str:\n"," global show_shell_info\n"," if desc is not None:\n"," print(desc)\n","\n"," run_kwargs = {\n"," \"args\": command,\n"," \"shell\": True,\n"," \"cwd\": cwd,\n"," \"env\": os.environ if custom_env is None else custom_env,\n"," \"encoding\": 'utf8',\n"," \"errors\": 'ignore',\n"," }\n","\n"," if not show_shell_info:\n"," run_kwargs[\"stdout\"] = run_kwargs[\"stderr\"] = subprocess.PIPE\n","\n"," result = subprocess.run(**run_kwargs)\n","\n"," if result.returncode != 0:\n"," error_bits = [\n"," f\"{errdesc or 'Error running command'}.\",\n"," f\"Command: {command}\",\n"," f\"Error code: {result.returncode}\",\n"," ]\n"," if result.stdout:\n"," error_bits.append(f\"stdout: {result.stdout}\")\n"," if result.stderr:\n"," error_bits.append(f\"stderr: {result.stderr}\")\n"," if try_error:\n"," print((RuntimeError(\"\\n\".join(error_bits))))\n"," else:\n"," raise RuntimeError(\"\\n\".join(error_bits))\n","\n"," if show_shell_info:\n"," print((result.stdout or \"\"))\n"," return (result.stdout or \"\")\n","\n","def mkdirs(path, exist_ok=True):\n"," if path and not Path(path).exists():\n"," os.makedirs(path,exist_ok=exist_ok)\n","\n","\n","# 检查网络\n","def check_service(host, port):\n"," try:\n"," socket.create_connection((host, port), timeout=5)\n"," return True\n"," except socket.error:\n"," return False\n","\n","\n","# 检查gpu是否存在\n","def check_gpu():\n"," if not run_by_none_device and torch.cuda.device_count() == 0:\n"," raise Exception('当前环境没有GPU')\n","\n","\n","def echoToFile(content:str,path:str):\n"," if path.find('/') >= 0:\n"," _path = '/'.join(path.split('/')[:-1])\n"," run(f'''mkdir -p {_path}''')\n"," with open(path,'w') as sh:\n"," sh.write(content)\n"," \n","def get_freefrp_confog(local_port):\n"," rd_str = uuid.uuid1()\n"," return (f'''\n","[common]\n","server_addr = frp.freefrp.net\n","server_port = 7000\n","token = freefrp.net\n","\n","[{rd_str}_http]\n","type = http\n","local_ip = 127.0.0.1\n","local_port = {local_port}\n","custom_domains = {rd_str}.frp.eaias.com\n","''',f'http://{rd_str}.frp.eaias.com')"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["\n","_install_path = f\"/kaggle\" if os.path.exists('/kaggle/') else f\"{os.environ['HOME']}/sd_webui\" # 安装目录\n","_output_path = '/kaggle/working' if os.path.exists('/kaggle/working/') else f\"{os.environ['HOME']}/.sdwui/Output\" # 输出目录 如果使用google云盘 会在google云盘增加sdwebui/Output\n","_input_path = '/kaggle/input' # 输入目录\n","_ui_dir_name = 'sd_main_dir'\n","\n","_install_path = locals().get('install_path') or globals().get('install_path') or _install_path\n","_output_path = locals().get('output_path') or globals().get('output_path') or _output_path\n","_input_path = locals().get('input_path') or globals().get('input_path') or _input_path\n","_ui_dir_name = locals().get('ui_dir_name') or globals().get('ui_dir_name') or _ui_dir_name\n","\n","install_path = _install_path\n","output_path = _output_path\n","input_path = _input_path\n","ui_dir_name = _ui_dir_name\n"," \n","google_drive = '' \n","\n","\n","_useGooglrDrive = locals().get('useGooglrDrive') or globals().get('useGooglrDrive') or True\n","\n","# 连接谷歌云\n","try:\n"," if _useGooglrDrive:\n"," from google.colab import drive\n"," drive.mount(f'~/google_drive')\n"," google_drive = f\"{os.environ['HOME']}/google_drive/MyDrive\"\n"," _output_path = f'{google_drive}/sdwebui/Output'\n"," _input_path = f'{google_drive}/sdwebui/Input'\n"," run(f'''mkdir -p {_input_path}''')\n"," print('''\n","已经链接到谷歌云盘\n","已在云盘创建Input和Output目录\n"," ''')\n","except:\n"," _useGooglrDrive = False\n","\n","run(f'''mkdir -p {_install_path}''')\n","run(f'''mkdir -p {_output_path}''')\n","\n","\n","os.environ['install_path'] = _install_path\n","os.environ['output_path'] = _output_path\n","os.environ['google_drive'] = google_drive\n","os.environ['input_path'] = _input_path\n","\n","def replace_path(input_str:str):\n"," if not input_str: return ''\n"," for key in _config_args:\n"," input_str = input_str.replace(key,_config_args[key])\n"," \n"," return input_str.replace('$install_path',_install_path)\\\n"," .replace('{install_path}',_install_path)\\\n"," .replace('$input_path',_input_path)\\\n"," .replace('{input_path}',_input_path)\\\n"," .replace('$output_path',_output_path)\\\n"," .replace('{output_path}',_output_path)\\\n"," .replace('{sdwui}','stable-diffusion-webui')\\\n"," .replace('{wui}',\"webui\")\n","\n","space_string = ' \\n\\r\\t\\'\\\",'\n","\n","def config_reader(conf:str):\n"," args = [replace_path(item.split('#')[0].strip(space_string)) for item in conf.split('\\n') if item.strip(space_string)]\n"," return [item.strip() for item in args if item.strip()]\n"]},{"cell_type":"code","execution_count":null,"metadata":{"_kg_hide-input":true,"id":"i3LhnwYHLCtC","trusted":true},"outputs":[],"source":["ngrokTokenFile = os.path.join(_input_path,'configs/ngrok_token.txt') # 非必填 存放ngrokToken的文件的路径\n","frpcConfigFile = os.path.join(_input_path,'configs/frpc_koishi.ini') # 非必填 frp 配置文件\n","# ss证书目录 下载nginx的版本，把pem格式改成crt格式\n","frpcSSLFFlies = [os.path.join(_input_path,'configs/koishi_ssl')]\n","if 'frp_ssl_dir' in locals() or 'frp_ssl_dir' in globals():\n"," frpcSSLFFlies = frpcSSLFFlies + config_reader(locals().get('frp_ssl_dir') or globals().get('frp_ssl_dir'))\n","# frpc 文件目录 如果目录不存在，会自动下载，也可以在数据集搜索 viyiviyi/utils 添加\n","frpcExePath = os.path.join(_input_path,'utils-tools/frpc')\n","# 其他需要加载的webui启动参数 写到【参数列表】这个配置去\n","otherArgs = '--xformers'\n","if 'sd_start_args' in locals() or 'sd_start_args' in globals():\n"," otherArgs = ' '.join([item for item in config_reader(locals().get('sd_start_args') or globals().get('sd_start_args')) if item != '--no-gradio-queue'])\n","venvPath = os.path.join(_input_path,'sd-webui-venv/venv.tar.bak') # 安装好的python环境 sd-webui-venv是一个公开是数据集 可以搜索添加\n","\n","# 用于使用kaggle api的token文件 参考 https://www.kaggle.com/docs/api\n","# 此文件用于自动上传koishi的相关配置 也可以用于保存重要的输出文件\n","kaggleApiTokenFile = os.path.join(_input_path,'configs/kaggle.json')\n","\n","requirements = []\n"]},{"cell_type":"code","execution_count":null,"metadata":{"_kg_hide-input":true,"id":"a_GtG2ayLCtD","trusted":true},"outputs":[],"source":["# 这下面的是用于初始化一些值或者环境变量的，轻易别改\n","_setting_file = replace_path(locals().get('setting_file') or globals().get('setting_file') or 'config.json')\n","\n","_ui_config_file = replace_path(locals().get('ui_config_file') or globals().get('ui_config_file') or 'ui-config.json')\n","\n","# 设置文件路径\n","if Path(f\"{os.environ['HOME']}/google_drive/MyDrive\").exists():\n"," if _setting_file == '/kaggle/working/configs/config.json':\n"," _setting_file = os.path.join(_output_path,'configs/config.json')\n"," if _ui_config_file == '/kaggle/working/configs/ui-config.json':\n"," _ui_config_file = os.path.join(_output_path,'configs/ui-config.json')\n"," \n","frpcStartArg = ''\n","freefrp_url = ''\n","_frp_temp_config_file = ''\n","_frp_config_or_file = replace_path(locals().get('frp_config_or_file') or globals().get('frp_config_or_file')) or frpcConfigFile\n","run(f'''mkdir -p {_install_path}/configFiles''')\n","if _frp_config_or_file:\n"," if '[common]' in _frp_config_or_file:\n"," echoToFile(_frp_config_or_file,f'{_install_path}/configFiles/temp_frpc_webui.ini')\n"," _frp_temp_config_file = f'{_install_path}/configFiles/temp_frpc_webui.ini'\n"," elif '.ini' in _frp_config_or_file:\n"," _frp_temp_config_file = _frp_config_or_file.strip()\n"," \n"," if _frp_temp_config_file:\n"," if Path(_frp_temp_config_file).exists():\n"," run(f'''cp -f {_frp_temp_config_file} {_install_path}/configFiles/frpc_webui.ini''')\n"," run(f'''sed -i \"s/local_port = .*/local_port = {_server_port}/g\" {_install_path}/configFiles/frpc_webui.ini''')\n"," frpcStartArg = f' -c {_install_path}/configFiles/frpc_webui.ini'\n"," elif _frp_config_or_file.strip().startswith('-f'):\n"," frpcStartArg = _frp_config_or_file.strip()\n"," \n","if not frpcStartArg:\n"," conf,url = get_freefrp_confog(_server_port)\n"," echoToFile(conf,f'{_install_path}/configFiles/frpc_webui.ini')\n"," freefrp_url = url\n"," frpcStartArg = f' -c {_install_path}/configFiles/frpc_webui.ini'\n","\n","ngrokToken=''\n","_ngrok_config_or_file = replace_path(locals().get('ngrok_config_or_file') or globals().get('ngrok_config_or_file')) or ngrokTokenFile\n","if _ngrok_config_or_file:\n"," if Path(_ngrok_config_or_file.strip()).exists():\n"," ngrokTokenFile = _ngrok_config_or_file.strip()\n"," if Path(ngrokTokenFile).exists():\n"," with open(ngrokTokenFile,encoding = \"utf-8\") as nkfile:\n"," ngrokToken = nkfile.readline()\n"," elif not _ngrok_config_or_file.strip().startswith('/'):\n"," ngrokToken=_ngrok_config_or_file.strip()\n"," \n","if not Path(venvPath).exists():\n"," venvPath = os.path.join(_input_path,'sd-webui-venv/venv.zip')\n"," \n","huggingface_headers:dict = None "]},{"cell_type":"markdown","metadata":{},"source":["## 文件下载工具\n","\n","---\n","\n","link_or_download_flie(config:str, skip_url:bool=False, _link_instead_of_copy:bool=True, base_path:str = '',sync:bool=False,thread_num:int=None)"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["import concurrent.futures\n","import importlib\n","import os\n","import pprint\n","import re\n","from pathlib import Path\n","from typing import List\n","\n","import requests\n","\n","show_shell_info = False\n","\n","def is_installed(package):\n"," try:\n"," spec = importlib.util.find_spec(package)\n"," except ModuleNotFoundError:\n"," return False\n","\n"," return spec is not None\n","\n","def download_file(url:str, filename:str, dist_path:str, cache_path = '',_link_instead_of_copy:bool=True,headers={}):\n"," # 获取文件的真实文件名\n"," if not filename:\n"," with requests.get(url, stream=True,headers=headers) as r:\n"," if 'Content-Disposition' in r.headers:\n"," filename = r.headers['Content-Disposition'].split('filename=')[1].strip('\"')\n"," r.close()\n"," if not filename and re.search(r'/[^/]+\\.[^/]+$',url):\n"," filename = url.split('/')[-1].split('?')[0]\n"," \n"," filename = re.sub(r'[\\\\/:*?\"<>|;]', '', filename)\n"," filename = re.sub(r'[\\s\\t]+', '_', filename)\n"," \n"," print(f'下载 {filename} url: {url} --> {dist_path}')\n"," \n"," # 创建目录\n"," if cache_path and not Path(cache_path).exists():\n"," os.makedirs(cache_path,exist_ok=True)\n"," if dist_path and not Path(dist_path).exists():\n"," os.makedirs(dist_path,exist_ok=True)\n"," \n"," # 拼接文件的完整路径\n"," filepath = os.path.join(dist_path, filename)\n","\n"," if cache_path:\n"," cache_path = os.path.join(cache_path, filename)\n"," \n"," # 判断文件是否已存在\n"," if Path(filepath).exists():\n"," print(f'文件 {filename} 已存在 {dist_path}')\n"," return\n"," \n"," if cache_path and Path(cache_path).exists():\n"," run(f'cp -n -r -f {\"-s\" if _link_instead_of_copy else \"\"} {cache_path} {dist_path}')\n"," print(f'文件缓存 {cache_path} --> {dist_path}')\n"," return\n"," # 下载文件\n"," request.urlretrieve(url, filepath, headers=headers)\n"," # with requests.get(url, stream=True, headers=headers) as r:\n"," # r.raise_for_status()\n"," # with open(cache_path or filepath, 'wb') as f:\n"," # for chunk in r.iter_content(chunk_size=1024):\n"," # if chunk:\n"," # f.write(chunk)\n"," # 如果使用了缓存目录 需要复制或链接文件到目标目录\n"," if cache_path:\n"," run(f'cp -n -r -f {\"-s\" if _link_instead_of_copy else \"\"} {cache_path} {dist_path}')\n"," print(f'下载完成 {filename} --> {dist_path}')\n"," \n","def download_git(url, dist_path, cache_path = '',_link_instead_of_copy:bool=True):\n"," if not Path(dist_path).exists():\n"," os.makedirs(dist_path,exist_ok=True)\n"," if show_shell_info:\n"," print(f'git 下载 {url} --> {dist_path}')\n"," if cache_path and not Path(cache_path).exists():\n"," os.makedirs(cache_path,exist_ok=True)\n"," run(f'git clone {url}',cwd = cache_path)\n"," if cache_path:\n"," run(f'cp -n -r -f {cache_path}/* {dist_path}')\n"," else:\n"," run(f'git clone {url}',cwd = dist_path)\n"," if show_shell_info:\n"," print(f'git 下载完成 {url} --> {dist_path}')\n"," \n"," \n","def download_huggingface(url:str, filename:str, dist_path, cache_path = '',_link_instead_of_copy:bool=True):\n"," fileReg = r'^https:\\/\\/huggingface.co(\\/([^\\/]+\\/)?[^\\/]+\\/[^\\/]+\\/(resolve|blob)\\/[^\\/]+\\/|[^\\.]+\\.[^\\.]+$|download=true)'\n"," def isFile(url:str):\n"," if re.match(fileReg,url):\n"," return True\n"," return False\n"," if isFile(url):\n"," download_file(url,filename,dist_path,cache_path,_link_instead_of_copy,headers=huggingface_headers)\n"," else:\n"," download_git(url,dist_path,cache_path,_link_instead_of_copy)\n"," \n","# 加入文件到下载列表\n","def pause_url(url:str,dist_path:str):\n"," file_name = ''\n"," if re.match(r'^[^:]+:(https?|ftps?)://', url, flags=0):\n"," file_name = re.findall(r'^[^:]+:',url)[0][:-1]\n"," url = url[len(file_name)+1:]\n"," if not re.match(r'^(https?|ftps?)://',url):\n"," return\n"," file_name = re.sub(r'\\s+','_',file_name or '')\n"," path_hash = str(hash(url)).replace('-','')\n"," \n"," return {'file_name':file_name,'path_hash':path_hash,'url':url,'dist_path':dist_path}\n","\n","def download_urls(download_list:List[dict],sync:bool=False,thread_num:int=5, \n"," cache_path:str=os.path.join(os.environ['HOME'],'.cache','download_util'),\n"," _link_instead_of_copy:bool=True,is_await:bool=False):\n"," if sync:\n"," for conf in download_list:\n"," cache_dir = os.path.join(cache_path,conf['path_hash'])\n"," if conf['url'].startswith('https://github.com'):\n"," try:\n"," download_git(conf['url'],conf['dist_path'],cache_path=cache_dir,_link_instead_of_copy=_link_instead_of_copy)\n"," except:\n"," pass\n"," continue\n"," if conf['url'].startswith('https://huggingface.co'):\n"," try:\n"," download_huggingface(conf['url'],conf['file_name'],conf['dist_path'],cache_path=cache_dir,_link_instead_of_copy=_link_instead_of_copy)\n"," except:\n"," pass\n"," continue\n"," if conf['url'].startswith('https://civitai.com'):\n"," if not re.search(r'token=.+', conf['url']):\n"," if conf['url'].find('?') == -1:\n"," conf['url'] = conf['url']+'?token=fee8bb78b75566eddfd04d061996185c'\n"," else:\n"," conf['url'] = conf['url']+'&token=fee8bb78b75566eddfd04d061996185c'\n"," try:\n"," download_file(conf['url'],conf['file_name'],conf['dist_path'],cache_path=cache_dir,_link_instead_of_copy=_link_instead_of_copy)\n"," except:\n"," pass\n"," else:\n"," executor = concurrent.futures.ThreadPoolExecutor(max_workers=thread_num)\n"," futures = []\n"," for conf in download_list:\n"," cache_dir = os.path.join(cache_path,conf['path_hash'])\n"," if conf['url'].startswith('https://github.com'):\n"," futures.append(executor.submit(download_git, conf['url'],conf['dist_path'],\n"," cache_path=cache_dir,_link_instead_of_copy=_link_instead_of_copy))\n"," continue\n"," if conf['url'].startswith('https://huggingface.co'):\n"," futures.append(executor.submit(download_huggingface,conf['url'],conf['file_name'],conf['dist_path'],cache_path=cache_dir,_link_instead_of_copy=_link_instead_of_copy))\n"," continue\n"," if conf['url'].startswith('https://civitai.com'):\n"," if not re.search(r'token=.+', conf['url']):\n"," if conf['url'].find('?') == -1:\n"," conf['url'] = conf['url']+'?token=fee8bb78b75566eddfd04d061996185c'\n"," else:\n"," conf['url'] = conf['url']+'&token=fee8bb78b75566eddfd04d061996185c'\n"," futures.append(executor.submit(download_file, conf['url'],conf['file_name'],conf['dist_path'],\n"," cache_path=cache_dir,_link_instead_of_copy=_link_instead_of_copy))\n"," if is_await:\n"," concurrent.futures.wait(futures)\n"," \n"," \n","def parse_config(config:str):\n"," space_string = ' \\n\\r\\t\\'\\\",'\n"," other_flie_list = [item.split('#')[0].strip(space_string) for item in config.split('\\n') if item.strip(space_string)]\n"," other_flie_list = [item.strip() for item in other_flie_list if item.strip()]\n"," other_flie_list_store = {}\n"," other_flie_list_store_name='default'\n"," other_flie_list_store_list_cache=[]\n"," \n"," for item in other_flie_list:\n"," if item.startswith('[') and item.endswith(']'):\n"," if not other_flie_list_store_name == 'default':\n"," other_flie_list_store[other_flie_list_store_name]=other_flie_list_store_list_cache\n"," other_flie_list_store_list_cache = []\n"," other_flie_list_store_name = item[1:-1]\n"," else:\n"," other_flie_list_store_list_cache.append(item)\n"," other_flie_list_store[other_flie_list_store_name]=other_flie_list_store_list_cache\n"," \n"," return other_flie_list_store\n","\n","\n","def link_or_download_flie(config:str, skip_url:bool=False, _link_instead_of_copy:bool=True, base_path:str = '',\n"," sync:bool=False,thread_num:int=None, is_await:bool=False):\n"," store:dict[str,List[str]] = parse_config(config)\n"," download_list = []\n"," for dist_dir in store.keys():\n"," dist_path = os.path.join(base_path,dist_dir)\n"," os.makedirs(dist_path,exist_ok=True)\n"," for path in store[dist_dir]:\n"," if 'https://' in path or 'http://' in path:\n"," if skip_url:\n"," continue\n"," if sync:\n"," try:\n"," download_urls([pause_url(path,dist_path)],_link_instead_of_copy = _link_instead_of_copy, sync=sync)\n"," except:\n"," pass\n"," continue\n"," download_list.append(pause_url(path,dist_path))\n"," else:\n"," run(f'cp -n -r -f {\"-s\" if _link_instead_of_copy else \"\"} {path} {dist_path}')\n"," if show_shell_info:\n"," print(f'{\"链接\" if _link_instead_of_copy else \"复制\"} {path} --> {dist_path}')\n"," run(f'rm -f {dist_path}/\\*.* ')\n"," if not skip_url:\n"," if show_shell_info:\n"," pprint.pprint(download_list)\n"," try:\n"," download_urls(download_list,_link_instead_of_copy = _link_instead_of_copy, sync=sync, thread_num=thread_num or 3,is_await=is_await)\n"," except:\n"," pass"]},{"cell_type":"markdown","metadata":{"id":"p0uS-BLULCtD"},"source":["## kaggle public API\n","\n","**不能使用%cd这种会改变当前工作目录的命令，会导致和其他线程冲突**\n","\n","---"]},{"cell_type":"code","execution_count":null,"metadata":{"_kg_hide-input":true,"id":"m8FJi4j0LCtD","trusted":true},"outputs":[],"source":["# 安装kaggle的api token文件\n","def initKaggleConfig():\n"," if Path('~/.kaggle/kaggle.json').exists():\n"," return True\n"," if Path(kaggleApiTokenFile).exists():\n"," run(f'''mkdir -p ~/.kaggle/''')\n"," run('cp '+kaggleApiTokenFile+' ~/.kaggle/kaggle.json')\n"," run(f'''chmod 600 ~/.kaggle/kaggle.json''')\n"," return True\n"," print('缺少kaggle的apiToken文件，访问：https://www.kaggle.com/你的kaggle用户名/account 获取')\n"," return False\n","\n","def getUserName():\n"," if not initKaggleConfig(): return\n"," import kaggle\n"," return kaggle.KaggleApi().read_config_file()['username']\n","\n","def createOrUpdateDataSet(path:str,datasetName:str):\n"," if not initKaggleConfig(): return\n"," print('创建或更新数据集 '+datasetName)\n"," import kaggle\n"," run(f'mkdir -p {_install_path}/kaggle_cache')\n"," run(f'rm -rf {_install_path}/kaggle_cache/*')\n"," datasetDirPath = _install_path+'/kaggle_cache/'+datasetName\n"," run('mkdir -p '+datasetDirPath)\n"," run('cp -f '+path+' '+datasetDirPath+'/')\n"," username = getUserName()\n"," print(\"kaggle username:\"+username)\n"," datasetPath = username+'/'+datasetName\n"," datasetList = kaggle.api.dataset_list(mine=True,search=datasetPath)\n"," print(datasetList)\n"," if len(datasetList) == 0 or datasetPath not in [str(d) for d in datasetList]: # 创建 create\n"," run('kaggle datasets init -p' + datasetDirPath)\n"," metadataFile = datasetDirPath+'/dataset-metadata.json'\n"," run('sed -i s/INSERT_TITLE_HERE/'+ datasetName + '/g ' + metadataFile)\n"," run('sed -i s/INSERT_SLUG_HERE/'+ datasetName + '/g ' + metadataFile)\n"," run('cat '+metadataFile)\n"," run('kaggle datasets create -p '+datasetDirPath)\n"," print('create database done')\n"," else:\n"," kaggle.api.dataset_metadata(datasetPath,datasetDirPath)\n"," kaggle.api.dataset_create_version(datasetDirPath, 'auto update',dir_mode='zip')\n"," print('upload database done')\n","\n","def downloadDatasetFiles(datasetName:str,outputPath:str):\n"," if not initKaggleConfig(): return\n"," print('下载数据集文件 '+datasetName)\n"," import kaggle\n"," username = getUserName()\n"," datasetPath = username+'/'+datasetName\n"," datasetList = kaggle.api.dataset_list(mine=True,search=datasetPath)\n"," if datasetPath not in [str(d) for d in datasetList]:\n"," return False\n"," run('mkdir -p '+outputPath)\n"," kaggle.api.dataset_download_files(datasetPath,path=outputPath,unzip=True)\n"," return True\n","\n"]},{"cell_type":"markdown","metadata":{},"source":["## 同步文件夹到 huggingface\n","\n","---"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":["# 文件夹与 huggingface 同步\n","if _huggingface_token and _huggingface_repo:\n"," if not is_installed('watchdog'):\n"," requirements.append('watchdog')\n"," if not is_installed('huggingface_hub'):\n"," requirements.append('huggingface_hub')\n"," else:\n"," try:\n"," from huggingface_hub import HfApi,login,snapshot_download\n"," except:\n"," requirements.append('huggingface_hub')\n","\n","huggingface_is_init = False\n","\n","def init_huggingface():\n"," if not _huggingface_token:\n"," return False\n","\n"," global huggingface_headers\n"," global huggingface_is_init\n"," \n"," from huggingface_hub import login\n"," token = replace_path(_huggingface_token)\n"," if not _huggingface_token.startswith('hf_') and Path(token).exists():\n"," with open(token,encoding = \"utf-8\") as nkfile:\n"," token = nkfile.readline()\n"," if not token.startswith('hf_'):\n"," print('huggingface token 不正确，请将 token 或 仅存放token 的txt文件路径填入 _huggingface_token 配置')\n"," return False\n"," login(token,add_to_git_credential=True)\n"," huggingface_headers = {'Authorization': 'Bearer '+token}\n"," print('huggingface token 已经加载，可以下载私有仓库或文件')\n"," \n"," if not _huggingface_repo:\n"," print('huggingface 同步收藏图片功能不会启动，可增加配置项 huggingface_token = \"token\" 和 huggingface_repo = “仓库id” 后启用 huggingface 同步收藏图片功能')\n"," return False\n"," huggingface_is_init = True\n"," return True\n","\n","\n","def download__huggingface_repo(repo_id:str,dist_directory:str=None,repo_type='dataset',callback=None):\n"," if not huggingface_is_init:\n"," print('huggingface 相关功能未初始化 请调用 init_huggingface() 初始化')\n"," \n"," if not dist_directory:\n"," dist_directory = f'{_install_path}/{_ui_dir_name}/log'\n"," \n"," print('下载收藏的图片')\n"," if not Path(f'{_install_path}/cache/huggingface/huggingface_repo').exists():\n"," mkdirs(f'{_install_path}/cache/huggingface')\n"," repo_path = ''\n"," if repo_type == 'dataset':\n"," repo_path = 'datasets'\n"," if repo_type == 'space':\n"," repo_path = 'spaces'\n"," if repo_path:\n"," run(f'git clone https://huggingface.co/{repo_path}/{repo_id} huggingface_repo',cwd=f'{_install_path}/cache/huggingface')\n"," else:\n"," run(f'git clone https://huggingface.co/{repo_id} huggingface_repo',cwd=f'{_install_path}/cache/huggingface')\n","\n"," run(f'cp -r -f -n -s {_install_path}/cache/huggingface/huggingface_repo/* {dist_directory}')\n"," if callback:\n"," callback()\n","\n","def start_sync_log_to_huggingface(repo_id:str,directory_to_watch:str=None,repo_type='dataset'):\n"," if not huggingface_is_init:\n"," print('huggingface 相关功能未初始化 请调用 init_huggingface() 初始化')\n"," \n"," from watchdog.observers import Observer\n"," from watchdog.events import FileSystemEventHandler\n"," from huggingface_hub import HfApi,login,snapshot_download\n"," \n"," # 配置监视的目录和 Hugging Face 仓库信息\n"," class FileChangeHandler(FileSystemEventHandler):\n"," def __init__(self, api, repo_id, repo_type,directory_to_watch):\n"," self.api = api\n"," self.repo_id = repo_id\n"," self.repo_type = repo_type\n"," self.directory_to_watch = directory_to_watch\n"," def on_created(self, event):\n"," if not event.is_directory:\n"," # 上传新文件到 Hugging Face 仓库\n"," file_path = event.src_path\n"," file_name:str = os.path.basename(file_path)\n"," print(file_name)\n"," if file_name[file_name.rindex('.'):] not in ['.png','.jpg','.txt','.webp','.jpeg']: return\n"," print(file_name,'>>','huggingface')\n"," try:\n"," self.api.upload_file(\n"," path_or_fileobj=file_path,\n"," path_in_repo=file_path.replace(self.directory_to_watch,''),\n"," repo_id=self.repo_id,\n"," repo_type=self.repo_type,\n"," )\n"," except IOError as error:\n"," print(error)\n","\n"," def on_deleted(self, event):\n"," if not event.is_directory:\n"," # 从 Hugging Face 仓库删除文件\n"," file_path = event.src_path\n"," file_name = os.path.basename(file_path)\n"," if file_name[file_name.rindex('.'):] not in ['.png','.jpg','.txt','.webp','.jpeg']: return\n"," try:\n"," self.api.delete_file(\n"," path_in_repo=file_path.replace(self.directory_to_watch,''),\n"," repo_id=self.repo_id,\n"," repo_type=self.repo_type,\n"," )\n"," except IOError as error:\n"," print(error)\n","\n"," def on_modified(self, event):\n"," if not event.is_directory:\n"," # 更新 Hugging Face 仓库中的文件\n"," file_path = event.src_path\n"," file_name = os.path.basename(file_path)\n"," if file_name[file_name.rindex('.'):] not in ['.png','.jpg','.txt','.webp','.jpeg']: return\n"," try:\n"," self.api.upload_file(\n"," path_or_fileobj=file_path,\n"," path_in_repo=file_path.replace(self.directory_to_watch,''),\n"," repo_id=self.repo_id,\n"," repo_type=self.repo_type,\n"," )\n"," except IOError as error:\n"," print(error)\n","\n"," def on_moved(self, event):\n"," if not event.is_directory:\n"," file_path = event.dest_path\n"," file_name = os.path.basename(file_path)\n"," if file_name[file_name.rindex('.'):] not in ['.png','.jpg','.txt','.webp','.jpeg']: return\n"," if event.dest_path.startswith(self.directory_to_watch):\n"," try:\n"," self.api.upload_file(\n"," path_or_fileobj=file_path,\n"," path_in_repo=file_path.replace(self.directory_to_watch,''),\n"," repo_id=self.repo_id,\n"," repo_type=self.repo_type,\n"," )\n"," except IOError as error:\n"," print(error)\n","\n"," api = HfApi()\n"," \n"," if not directory_to_watch:\n"," directory_to_watch = f'{_install_path}/{_ui_dir_name}/log'\n"," # 创建观察者对象并注册文件变化处理程序\n"," event_handler = FileChangeHandler(api,repo_id,repo_type,directory_to_watch)\n"," observer = Observer()\n"," observer.schedule(event_handler, directory_to_watch, recursive=True)\n","\n"," # 启动观察者\n"," observer.name = \"solo_directory_to_watch\"\n"," print(f'启动收藏图片文件夹监听，并自动同步到 huggingface {repo_type} : {repo_id}')\n"," observer.start()"]},{"cell_type":"markdown","metadata":{"id":"sswa04veLCtE"},"source":["## 工具函数\n","**不能使用%cd这种会改变当前工作目录的命令，会导致和其他线程冲突**\n","\n","---"]},{"cell_type":"code","execution_count":null,"metadata":{"_kg_hide-input":true,"trusted":true},"outputs":[],"source":["\n","def zipPath(path:str,zipName:str,format='tar'):\n"," if path.startswith('$install_path'):\n"," path = path.replace('$install_path',_install_path)\n"," if path.startswith('$output_path'):\n"," path = path.replace('$install_path',_output_path)\n"," if not path.startswith('/'):\n"," path = f'{_install_path}/{_ui_dir_name}/{path}'\n"," if Path(path).exists():\n"," if 'tar' == format:\n"," run(f'tar -cf {_output_path}/'+ zipName +'.tar -C '+ path +' . ')\n"," elif 'gz' == format:\n"," run(f'tar -czf {_output_path}/'+ zipName +'.tar.gz -C '+ path +' . ')\n"," return\n"," print('指定的目录不存在：'+path)\n"]},{"cell_type":"markdown","metadata":{},"source":["## 内网穿透\n","\n","---"]},{"cell_type":"code","execution_count":null,"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"id":"coqQvTSLLCtE","trusted":true},"outputs":[],"source":["def printUrl(url,name=''):\n"," print(f'{name} 访问地址：{url}')\n"," for key in sorted(_proxy_path.keys(), key=len)[::-1]:\n"," print(f'{name} 本地服务：{_proxy_path[key]} 访问地址：{url}{key}')\n","# ngrok\n","def startNgrok(ngrokToken:str,ngrokLocalPort:int):\n"," if not is_installed('pyngrok'):\n"," run('pip install pyngrok')\n"," from pyngrok import conf, ngrok\n"," try:\n"," conf.get_default().auth_token = ngrokToken\n"," conf.get_default().monitor_thread = False\n"," ssh_tunnels = ngrok.get_tunnels(conf.get_default())\n"," url = ''\n"," if len(ssh_tunnels) == 0:\n"," ssh_tunnel = ngrok.connect(ngrokLocalPort)\n"," url = ssh_tunnel.public_url\n"," print('ngrok 访问地址：'+ssh_tunnel.public_url)\n"," else:\n"," print('ngrok 访问地址：'+ssh_tunnels[0].public_url)\n"," url = ssh_tunnels[0].public_url\n"," printUrl(url,'ngrok')\n"," def auto_request_ngrok():\n"," if url:\n"," while(_runing):\n"," time.sleep(60*1)\n"," try:\n"," res = requests.get(url+'/',headers={\"ngrok-skip-browser-warning\" : \"1\"},timeout=10)\n"," except:\n"," ''\n"," # print('自动调用ngrok链接以保存链接不会断开',res.status_code)\n","\n"," # threading.Thread(target = auto_request_ngrok,daemon=True,name='solo_auto_request_ngrok').start()\n"," except:\n"," print('启动ngrok出错')\n"," \n","def startFrpc(name,configFile):\n"," if not Path(f'{_install_path}/frpc/frpc').exists():\n"," installFrpExe()\n"," if freefrp_url:\n"," printUrl(freefrp_url,'freefrp')\n"," echoToFile(f'''\n","cd {_install_path}/frpc/\n","{_install_path}/frpc/frpc {configFile}\n","''',f'{_install_path}/frpc/start.sh')\n"," get_ipython().system(f'''bash {_install_path}/frpc/start.sh''')\n"," \n","def installFrpExe():\n"," if _useFrpc:\n"," print('安装frpc')\n"," run(f'mkdir -p {_install_path}/frpc')\n"," if Path(frpcExePath).exists():\n"," run(f'cp -f -n {frpcExePath} {_install_path}/frpc/frpc')\n"," else:\n"," run(f'wget \"https://huggingface.co/datasets/ACCA225/Frp/resolve/main/frpc\" -O {_install_path}/frpc/frpc')\n"," \n"," for ssl in frpcSSLFFlies:\n"," if Path(ssl).exists():\n"," run(f'cp -f -n {ssl}/* {_install_path}/frpc/')\n"," run(f'chmod +x {_install_path}/frpc/frpc')\n"," run(f'{_install_path}/frpc/frpc -v')\n","\n","def startProxy():\n"," if _useNgrok:\n"," startNgrok(ngrokToken,_server_port)\n"," if _useFrpc:\n"," startFrpc('frpc_proxy',frpcStartArg)"]},{"cell_type":"markdown","metadata":{},"source":["## NGINX 反向代理\n","\n","---"]},{"cell_type":"code","execution_count":null,"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"outputs":[],"source":["\n","# nginx 反向代理配置文件\n","def localProxy():\n"," def getProxyLocation(subPath:str, localServer:str):\n"," return '''\n"," location '''+ subPath +'''\n"," {\n"," proxy_pass '''+ localServer +''';\n"," proxy_set_header REMOTE-HOST $remote_addr;\n"," proxy_buffering off;\n"," proxy_redirect off;\n"," proxy_connect_timeout 10m;\n"," proxy_read_timeout 10m;\n"," proxy_http_version 1.1;\n"," proxy_set_header Upgrade $http_upgrade;\n"," proxy_set_header Connection \"upgrade\";\n"," proxy_set_header Host $host;\n"," proxy_set_header X-Forwarded-Host $host;\n"," proxy_set_header X-Forwarded-Proto $scheme;\n"," proxy_set_header X-Real-IP $remote_addr;\n"," proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;\n"," }\n"," \n"," '''\n"," \n"," conf = '''\n","server\n","{\n"," listen '''+str(_server_port)+''';\n"," listen [::]:'''+str(_server_port)+''';\n"," server_name 127.0.0.1 localhost 0.0.0.0 \"\";\n"," \n"," if ($request_method = OPTIONS) {\n"," return 200;\n"," }\n"," fastcgi_send_timeout 10m;\n"," fastcgi_read_timeout 10m;\n"," fastcgi_connect_timeout 10m;\n"," \n"," '''+ ''.join([getProxyLocation(key,_proxy_path[key]) for key in sorted(_proxy_path.keys(), key=len)[::-1]]) +'''\n","}\n","'''\n"," echoToFile(conf,'/etc/nginx/conf.d/proxy_nginx.conf')\n"," if not check_service('localhost',_server_port):\n"," run(f'''nginx -c /etc/nginx/nginx.conf''')\n"," run(f'''nginx -s reload''')"]},{"cell_type":"markdown","metadata":{},"source":["## 线程清理工具\n","\n","---\n","\n","清理线程名以 solo_ 开头的所有线程"]},{"cell_type":"code","execution_count":null,"metadata":{"_kg_hide-input":true,"trusted":true},"outputs":[],"source":["import inspect\n","import ctypes\n","\n","def _async_raise(tid, exctype):\n"," \"\"\"raises the exception, performs cleanup if needed\"\"\"\n"," tid = ctypes.c_long(tid)\n"," if not inspect.isclass(exctype):\n"," exctype = type(exctype)\n"," res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype))\n"," if res == 0:\n"," raise ValueError(\"invalid thread id\")\n"," elif res != 1:\n"," # \"\"\"if it returns a number greater than one, you're in trouble,\n"," # and you should call it again with exc=NULL to revert the effect\"\"\"\n"," ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None)\n"," raise SystemError(\"PyThreadState_SetAsyncExc failed\")\n","\n","def stop_thread(thread):\n"," _async_raise(thread.ident, SystemExit)\n","\n","def stop_solo_threads():\n"," global _runing\n"," _runing = False\n"," # 获取当前所有活动的线程\n"," threads = threading.enumerate()\n"," # 关闭之前创建的子线程\n"," for thread in threads:\n"," if thread.name.startswith('solo_'):\n"," print(f'结束线程：{thread.name}')\n"," try:\n"," stop_thread(thread)\n"," except socket.error:\n"," print(f'结束线程：{thread.name} 执行失败')"]},{"cell_type":"markdown","metadata":{"id":"Ve3p8oOkLCtE"},"source":["# webui 安装和配置函数\n","---"]},{"cell_type":"code","execution_count":null,"metadata":{"_kg_hide-input":true,"id":"GTjyBJihLCtE","trusted":true},"outputs":[],"source":["envInstalled=False\n","quickStart = True\n","#安装\n","def install():\n"," print('安装')\n"," os.chdir(f'''{_install_path}''')\n"," run(f'''git lfs install''')\n"," run(f'''git config --global credential.helper store''')\n"," for requirement in requirements:\n"," run(f'pip install {requirement}')\n"," if _reLoad:\n"," run(f'''rm -rf {_install_path}/{_ui_dir_name}''')\n"," if Path(f\"{_ui_dir_name}\").exists():\n"," os.chdir(f'''{_install_path}/{_ui_dir_name}/''')\n"," run(f'''git checkout .''')\n"," run(f'''git pull''')\n"," else:\n"," run(f'''git clone --recursive {_sd_git_repo} {_ui_dir_name}''')\n"," if not Path(f'''{_install_path}/{_ui_dir_name}''').exists():\n"," print('sd-webui主程序安装失败，请检查 sd_git_repo 配置的值是否正确')\n"," sys.exit(0)\n"," os.chdir(f'''{_install_path}/{_ui_dir_name}''')\n"," print('安装 完成')\n","\n","# 链接输出目录\n","def link_dir():\n"," print('链接输出目录')\n"," # 链接图片输出目录\n"," run(f'''mkdir -p {_output_path}/outputs''')\n"," run(f'''rm -rf {_install_path}/{_ui_dir_name}/outputs''')\n"," run(f'''ln -s -r {_output_path}/outputs {_install_path}/{_ui_dir_name}/''')\n"," # 输出收藏目录\n"," run(f'''mkdir -p {_output_path}/log''')\n"," run(f'''rm -rf {_install_path}/{_ui_dir_name}/log''')\n"," run(f'''ln -s -r {_output_path}/log {_install_path}/{_ui_dir_name}/''')\n"," # 链接训练输出目录 文件夹链接会导致功能不能用\n"," run(f'''rm -rf {_install_path}/{_ui_dir_name}/textual_inversion''')\n"," run(f'''mkdir -p {_output_path}/textual_inversion/''')\n"," run(f'''ln -s -r {_output_path}/textual_inversion {_install_path}/{_ui_dir_name}/''')\n"," print('链接输出目录 完成') \n","\n","def install_optimizing():\n"," run('sudo apt update -y')\n"," run('sudo apt install nginx -y')\n"," \n","#安装依赖\n","def install_dependencies():\n"," print('安装需要的python环境')\n"," import venv\n"," global envInstalled\n"," global venvPath\n"," if Path(f'{_install_path}/{_ui_dir_name}/venv').exists():\n"," print('跳过安装python环境')\n"," envInstalled = True\n"," return\n"," \n","# run('add-apt-repository ppa:deadsnakes/ppa -y')\n","# run('apt update')\n","# run('apt install python3.10 -y')\n","# run('python3.10 -m venv venv',cwd=f'{_install_path}/{_ui_dir_name}')\n"," if quickStart:\n"," if not Path(venvPath).exists():\n"," mkdirs(f'{_install_path}/venv_cache',True)\n"," if not Path(f'{_install_path}/venv_cache/venv.tar.bak').exists():\n"," print('下载 venv.zip')\n"," download_file('https://huggingface.co/viyi/sdwui/resolve/main/venv.zip','venv.zip',f'{_install_path}/venv_cache')\n"," run(f'''unzip {_install_path}/venv_cache/venv.zip -d {_install_path}/venv_cache''')\n"," venvPath = f'{_install_path}/venv_cache/venv.tar.bak'\n"," run(f'''rm -rf {_install_path}/venv_cache/venv.zip''')\n"," elif venvPath.endswith('.zip'):\n"," mkdirs(f'{_install_path}/venv_cache',True)\n"," run(f'''unzip {venvPath} -d {_install_path}/venv_cache''')\n"," venvPath = f'{_install_path}/venv_cache/venv.tar.bak'\n"," print('解压环境')\n"," mkdirs(f'{_install_path}/{_ui_dir_name}/venv')\n","# run('python3.10 -m venv venv',cwd=f'{_install_path}/{_ui_dir_name}')\n"," run(f'tar -xf {venvPath} -C ./venv',cwd=f'{_install_path}/{_ui_dir_name}')\n"," run(f'rm -f {_install_path}/{_ui_dir_name}/venv/bin/pip*')\n"," run(f'rm -f {_install_path}/{_ui_dir_name}/venv/bin/python*')\n"," venv.create(f'{_install_path}/{_ui_dir_name}/venv')\n"," if not Path(f'{_install_path}/{_ui_dir_name}/venv/bin/pip').exists():\n"," run('curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py')\n"," run(f'{_install_path}/{_ui_dir_name}/venv/bin/python3 get-pip.py')\n","\n"," get_ipython().system(f'''{_install_path}/{_ui_dir_name}/venv/bin/python3 -V''')\n"," get_ipython().system(f'''{_install_path}/{_ui_dir_name}/venv/bin/python3 -m pip -V''')\n","\n"," envInstalled = True\n"," print('安装需要的python环境 完成')\n"," \n","# 个性化配置 \n","def use_config():\n"," print('使用自定义配置 包括tag翻译 \\n')\n"," run(f'''mkdir -p {_install_path}/temp''')\n"," run(f'git clone {_sd_config_git_repu} sd-configs',cwd=f'{_install_path}/temp')\n"," run(f'cp -r -f -n {_install_path}/temp/sd-configs/dist/* {_install_path}/{_ui_dir_name}')\n"," if not Path(_ui_config_file).exists() and _ui_config_file != 'ui-config.json': # ui配置文件\n"," run(f'''mkdir -p {_ui_config_file[:_ui_config_file.rfind('/')]}''')\n"," run(f'cp -f -n {_install_path}/{_ui_dir_name}/ui-config.json {_ui_config_file}')\n"," if not Path(_setting_file).exists() and _setting_file != 'config.json': # 设置配置文件\n"," run(f'''mkdir -p {_setting_file[:_setting_file.rfind('/')]}''')\n"," run(f'cp -f -n {_install_path}/{_ui_dir_name}/config.json {_setting_file}')\n","\n","def copy_last_log_to_images():\n"," if not Path(f'{_install_path}/{_ui_dir_name}/log/images').exists(): mkdirs(f'{_install_path}/{_ui_dir_name}/log/images')\n"," print('复制编号最大的一张收藏图到输出目录，用于保持编号，否则会出现收藏的图片被覆盖的情况')\n"," img_list = os.listdir(f'{_install_path}/{_ui_dir_name}/log/images')\n"," last_img_path = ''\n"," last_img_num = 0\n"," for img in img_list:\n"," if re.findall(r'^\\d+-',str(img)):\n"," num = int(re.findall(r'^\\d+-',str(img))[0][:-1])\n"," if num > last_img_num:\n"," last_img_path = img\n"," last_img_num = num\n"," \n"," if not last_img_path: return\n"," \n"," print(f'{_install_path}/{_ui_dir_name}/log/images/{last_img_path} {_install_path}/{_ui_dir_name}/outputs/txt2img-images')\n"," run(f'''mkdir -p {_install_path}/{_ui_dir_name}/outputs/txt2img-images''')\n"," run(f'''cp -f {_install_path}/{_ui_dir_name}/log/images/{last_img_path} {_install_path}/{_ui_dir_name}/outputs/txt2img-images/''')\n"," \n"," print(f'{_install_path}/{_ui_dir_name}/log/images/{last_img_path} {_install_path}/{_ui_dir_name}/outputs/img2img-images')\n"," run(f'''mkdir -p {_install_path}/{_ui_dir_name}/outputs/img2img-images''')\n"," run(f'''cp -f {_install_path}/{_ui_dir_name}/log/images/{last_img_path} {_install_path}/{_ui_dir_name}/outputs/img2img-images/''')\n"," \n","def start_webui(i):\n"," # 只要不爆内存，其他方式关闭后会再次重启 访问地址会发生变化\n"," print(i,'--port',str(_server_port+1+i))\n"," if i>0:\n"," print(f'使用第{i+1}张显卡启动第{i+1}个服务，通过frpc或nrgok地址后加{_sub_path[i]}进行访问')\n","\n"," restart_times = 0\n"," last_restart_time = time.time()\n"," while _runing:\n"," os.chdir(f'{_install_path}/{_ui_dir_name}')\n"," root_path = _sub_path[i]\n"," if root_path.endswith('/'): root_path = root_path[:-1]\n"," get_ipython().system(f'''venv/bin/python3 launch.py --device-id={i} --port {str(_server_port+1+i)} --subpath={_sub_path[i]}''')\n"," print('10秒后重启服务')\n"," if time.time() - last_restart_time < 60:\n"," restart_times = restart_times + 1\n"," else:\n"," restart_times = 0\n"," last_restart_time = time.time()\n"," if restart_times >3 :\n"," # 如果180秒内重启了3此，将不再自动重启\n"," break\n"," time.sleep(10)\n"," \n","# 启动\n","def start():\n"," print('启动webui')\n"," os.chdir(f'''{_install_path}/{_ui_dir_name}''')\n"," args = ''\n"," if _ui_config_file is not None and _ui_config_file != '' and Path(_ui_config_file).exists(): # ui配置文件\n"," args += ' --ui-config-file=' + _ui_config_file\n"," if _setting_file is not None and _setting_file != '' and Path(_setting_file).exists(): # 设置配置文件\n"," args += ' --ui-settings-file=' + _setting_file\n"," args += ' ' + otherArgs\n"," os.environ['COMMANDLINE_ARGS']=args\n"," run(f'''echo COMMANDLINE_ARGS=$COMMANDLINE_ARGS''')\n"," os.environ['REQS_FILE']='requirements.txt'\n","\n"," with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:\n"," for i in range(torch.cuda.device_count() if _multi_case else 1):\n"," executor.submit(start_webui,i)\n"," while _runing and not check_service('localhost',str(_server_port+1+i)): # 当当前服务启动完成才允许退出此次循环\n"," time.sleep(5)\n"," if not _runing: break\n"," time.sleep(10)"]},{"cell_type":"markdown","metadata":{"id":"qLvsk8ByLCtF"},"source":["# 入口函数\n","---"]},{"cell_type":"code","execution_count":null,"metadata":{"_kg_hide-input":true,"id":"IOKjaMlcLCtF","trusted":true},"outputs":[],"source":["\n","# 启动非webui相关的的内容，加快启动速度\n","def main():\n"," global envInstalled\n"," global huggingface_is_init\n"," global _runing\n"," stop_solo_threads()\n"," print('启动...')\n"," time.sleep(15)\n"," _runing = True\n"," startTicks = time.time()\n"," isInstall = True if os.getenv('IsInstall','False') == 'True' else False\n"," _proxy_path[_sub_path[0]] = f'http://127.0.0.1:{_server_port+1}/'\n"," _proxy_path[_sub_path[1]] = f'http://127.0.0.1:{_server_port+2}/'\n"," threading.Thread(target = startProxy, daemon=True, name='solo_startProxy').start()\n"," if isInstall is False or _reLoad: \n"," print('安装运行环境')\n"," install()\n"," link_dir()\n"," init_huggingface()\n"," threading.Thread(target = install_dependencies,daemon=True,name='solo_install_dependencies').start()\n"," threading.Thread(target = install_optimizing,daemon=True,name='solo_install_optimizing').start()\n"," link_or_download_flie(replace_path(_async_downloading), _link_instead_of_copy=_link_instead_of_copy,\n"," base_path=f'{_install_path}/{_ui_dir_name}')\n"," if huggingface_is_init:\n"," threading.Thread(target = download__huggingface_repo,daemon=True,\n"," args=([_huggingface_repo]),\n"," kwargs={\"callback\":copy_last_log_to_images},\n"," name='solo_download__huggingface_repo').start()\n"," \n"," link_or_download_flie(replace_path(_before_downloading), _link_instead_of_copy=_link_instead_of_copy,\n"," base_path=f'{_install_path}/{_ui_dir_name}',is_await=True,sync=True)\n"," t = 0\n"," while _runing and not envInstalled:\n"," if t%10==0:\n"," print('等待python环境安装...')\n"," t = t+1\n"," time.sleep(1)\n"," use_config()\n"," os.environ['IsInstall'] = 'True'\n"," else:\n"," envInstalled = True\n"," localProxy()\n"," link_or_download_flie(replace_path(_before_start_sync_downloading), _link_instead_of_copy=_link_instead_of_copy,\n"," base_path=f'{_install_path}/{_ui_dir_name}',sync=True)\n"," if init_huggingface():\n"," start_sync_log_to_huggingface(_huggingface_repo)\n"," ticks = time.time()\n"," _on_before_start()\n"," print(\"加载耗时:\",(ticks - startTicks),\"秒\")\n"," start()\n"]},{"cell_type":"markdown","metadata":{"id":"0oaCRs2gLCtF"},"source":["# 执行区域\n","---"]},{"cell_type":"code","execution_count":null,"metadata":{"_kg_hide-output":true,"id":"O3DR0DWHLCtF","scrolled":true,"trusted":true},"outputs":[],"source":["# 启动\n","# _reLoad = True\n","# hidden_console_info = False\n","# run_by_none_device = True\n","# show_shell_info = True\n","\n","print(f'当前sd的安装路径是：{_install_path}/{_ui_dir_name}')\n","print(f'当前图片保存路径是：{_output_path}')\n","print(f'当前数据集路径是：{_input_path}')\n","\n","print(update_desc)\n","\n","if _skip_start:\n"," print('已跳过自动启动，可手动执行 main() 进行启动。')\n"," print('''推荐的启动代码：\n","try:\n"," check_gpu() # 检查是否存在gpu\n"," main()\n","except KeyboardInterrupt:\n"," stop_solo_threads() # 中断后自动停止后台线程 （有部分功能在后台线程中运行）\n"," ''')\n","else:\n"," try:\n"," check_gpu()\n"," main()\n"," except KeyboardInterrupt:\n"," stop_solo_threads()"]},{"cell_type":"code","execution_count":null,"metadata":{"trusted":true},"outputs":[],"source":[]}],"metadata":{"kaggle":{"accelerator":"nvidiaTeslaT4","dataSources":[{"datasetId":2716934,"sourceId":6167400,"sourceType":"datasetVersion"},{"datasetId":3654544,"sourceId":6346544,"sourceType":"datasetVersion"},{"datasetId":2962375,"sourceId":6720235,"sourceType":"datasetVersion"},{"datasetId":3074484,"sourceId":6817788,"sourceType":"datasetVersion"}],"isGpuEnabled":true,"isInternetEnabled":true,"language":"python","sourceType":"notebook"},"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.10.13"}},"nbformat":4,"nbformat_minor":4}